From b07164e843c86586229f77a9393dec62e84faa87 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 03:22:29 +0000 Subject: [PATCH 01/42] Give codec work under the transition lock its own permit Five transitions run Avro work while holding the global transition lock: the rejected-signal command, signal arguments applied by a resumed task, cooperative cancellation requests, task repair and fenced deregistration. They shared the single codec permit with describe, query and listing reads, so under read load a transition waited on the lock for unrelated decodes. A per-call-site probe on the CPU deadline-health cell measured 0.58 to 0.80 s of such waiting in a cancellation request. These transitions now take a separate transition_codec permit. Reads keep signal_codec. Encodings, values and refusals are unchanged. New tests hold the read permit and require a cancellation request, a signal-resumed completion, a fenced deregistration and a range repair to finish within one second. They time out with the old wiring. Part of #325. Co-Authored-By: Claude Opus 5.5 --- docs/rust-server-transition-locking.md | 25 ++- rust/src/runtime/execution.rs | 41 +++-- rust/src/runtime/store.rs | 12 +- rust/src/runtime/transition_codec_tests.rs | 202 +++++++++++++++++++++ 4 files changed, 254 insertions(+), 26 deletions(-) create mode 100644 rust/src/runtime/transition_codec_tests.rs diff --git a/docs/rust-server-transition-locking.md b/docs/rust-server-transition-locking.md index e593eab1..89eeabe6 100644 --- a/docs/rust-server-transition-locking.md +++ b/docs/rust-server-transition-locking.md @@ -36,6 +36,21 @@ scheduler from sleeping after each pass to a fixed tick made no measurable difference. Deadline health and throughput under load are bounded by how long other transitions hold the lock. +A scratch build that logged lock wait and hold per call site then ran the same +cell on a host shared with other work, so absolute times are approximate: + +- A cancellation request held the lock 0.65 to 0.86 s, almost all of it + waiting for the single codec permit behind describe decodes of the dense + input (0.58 to 0.80 s). With a permit of its own the hold fell to 0.05 to + 0.09 s. Codec work under the lock now has its own permit. +- Activity status checks spent about 1.6 s per run waiting for the lock. They + no longer take it. +- Cancellation delivery and expiry, workflow completions, heartbeats and + activity expiry held the lock 0.2 to 0.9 s during the read wave without codec + work, several times longer than the same transitions before it. Their summary + projections took under 30 ms each. Lowering the decode thread's CPU priority + changed nothing measurable. This remaining cost is not yet attributed. + ## What lengthens lock hold time - **Summary projection at commit.** `Transaction::commit` reorders open waits @@ -43,10 +58,12 @@ other transitions hold the lock. `summary_history` pages through the run's entire history to do so (`terminal_summary.rs`). Every heartbeat and claim therefore costs time proportional to history length while holding the lock. -- **Codec work inside transactions.** Several paths take the single codec - permit and run Avro work on the blocking pool while the transaction is open, - for example the rejected-signal path, continue-as-new, cooperative requests - and repair. +- **Codec work inside transactions.** Five paths run Avro work on the blocking + pool while the transaction is open: the rejected-signal command, signal + arguments applied by a resumed workflow task, cooperative cancellation + requests, task repair and fenced worker deregistration. They take + `transition_codec`, never the read permit `signal_codec`, so they do not + queue behind reads. The CPU work itself still runs under the lock. - **Claim reads.** Workflow claims load the first history page (the requested page size, 100 by default) and compute the history budget under the lock. Query claims parse cached query JSON values. diff --git a/rust/src/runtime/execution.rs b/rust/src/runtime/execution.rs index edb69ac8..5715137b 100644 --- a/rust/src/runtime/execution.rs +++ b/rust/src/runtime/execution.rs @@ -25,6 +25,10 @@ enum Storage { #[path = "query_codec_tests.rs"] mod query_codec_tests; +#[cfg(test)] +#[path = "transition_codec_tests.rs"] +mod transition_codec_tests; + #[derive(Clone)] pub struct Runtime { storage: Arc, @@ -32,6 +36,10 @@ pub struct Runtime { pub(crate) role_tokens: [Option>; 3], scheduler: Arc, signal_codec: Arc, + // Codec work done while holding the transition lock takes this permit, + // never `signal_codec`, so a transition never queues behind reads for + // codec capacity. + transition_codec: Arc, pub(super) external: Arc, queries: Arc, } @@ -194,6 +202,7 @@ impl Runtime { role_tokens: [None, None, None], scheduler: Arc::new(TimerScheduler::new()), signal_codec: Arc::new(Semaphore::new(1)), + transition_codec: Arc::new(Semaphore::new(1)), external: Arc::new(external), queries: Arc::new(queries), }; @@ -262,6 +271,7 @@ impl Runtime { role_tokens: self.role_tokens.clone(), scheduler: self.scheduler.clone(), signal_codec: self.signal_codec.clone(), + transition_codec: self.transition_codec.clone(), external: self.external.clone(), queries: self.queries.clone(), } @@ -376,7 +386,7 @@ impl Runtime { name, body, prepared, - self.signal_codec.clone(), + self.transition_codec.clone(), ) .await } @@ -388,7 +398,7 @@ impl Runtime { name, body, prepared, - self.signal_codec.clone(), + self.transition_codec.clone(), ) .await } @@ -400,7 +410,7 @@ impl Runtime { name, body, prepared, - self.signal_codec.clone(), + self.transition_codec.clone(), ) .await } @@ -421,7 +431,7 @@ impl Runtime { run_id, body, context, - self.signal_codec.clone(), + self.transition_codec.clone(), ) .await } @@ -432,7 +442,7 @@ impl Runtime { run_id, body, context, - self.signal_codec.clone(), + self.transition_codec.clone(), ) .await } @@ -443,7 +453,7 @@ impl Runtime { run_id, body, context, - self.signal_codec.clone(), + self.transition_codec.clone(), ) .await } @@ -658,9 +668,8 @@ impl Runtime { ) -> Result<(Vec, Vec)> ); pub(crate) async fn activities(&self, workflow_id: &str, run_id: &str) -> Result { - // Writers take this permit while holding the writer transaction. Finish - // every read before taking it so a listing can never wait on a writer - // that is itself waiting on the permit. + // Finish every read before taking the codec permit, so a listing never + // holds codec capacity while it waits on the database. let (events, executions) = self.activity_metadata_input(workflow_id, run_id).await?; let permit = self .signal_codec @@ -700,7 +709,7 @@ impl Runtime { delegate!(worker_heartbeat(body: Value) -> Result); delegate!(deregister(worker_id: &str) -> Result); pub(crate) async fn deregister_fenced(&self, worker_id: &str, body: Value) -> Result { - let codec = self.signal_codec.clone(); + let codec = self.transition_codec.clone(); match &*self.storage { Storage::Sqlite(store) => store.deregister_fenced(worker_id, body, codec).await, Storage::Postgres(store) => store.deregister_fenced(worker_id, body, codec).await, @@ -761,17 +770,17 @@ impl Runtime { match &*self.storage { Storage::Sqlite(store) => { store - .complete_workflow(task_id, body, self.signal_codec.clone(), external) + .complete_workflow(task_id, body, self.transition_codec.clone(), external) .await } Storage::Postgres(store) => { store - .complete_workflow(task_id, body, self.signal_codec.clone(), external) + .complete_workflow(task_id, body, self.transition_codec.clone(), external) .await } Storage::MySql(store) => { store - .complete_workflow(task_id, body, self.signal_codec.clone(), external) + .complete_workflow(task_id, body, self.transition_codec.clone(), external) .await } } @@ -817,7 +826,7 @@ impl Runtime { run_id, reason, budget, - self.signal_codec.clone(), + self.transition_codec.clone(), ) .await } @@ -828,7 +837,7 @@ impl Runtime { run_id, reason, budget, - self.signal_codec.clone(), + self.transition_codec.clone(), ) .await } @@ -839,7 +848,7 @@ impl Runtime { run_id, reason, budget, - self.signal_codec.clone(), + self.transition_codec.clone(), ) .await } diff --git a/rust/src/runtime/store.rs b/rust/src/runtime/store.rs index 6490d0f9..24ae515a 100644 --- a/rust/src/runtime/store.rs +++ b/rust/src/runtime/store.rs @@ -402,7 +402,7 @@ where name: &str, body: Value, mut prepared: PreparedControlArguments, - signal_codec: Arc, + transition_codec: Arc, ) -> Result { reject_fields(&body, &["input", "request_id"])?; if name.trim().is_empty() || name.len() > 255 { @@ -505,7 +505,7 @@ where "message":format!("Workflow signal [{name}] is unknown for workflow type [{workflow_type}]: the durable command contract does not declare that handler. Command contract source [durable_history], declared signals [{labels}].")}); } prepared.command = prepare_rejected_signal_command( - signal_codec, + transition_codec, name.to_owned(), prepared.values.clone(), validation_errors.clone(), @@ -696,7 +696,7 @@ where run_id: &str, task_id: &str, task: &DB::Row, - signal_codec: Arc, + transition_codec: Arc, ) -> Result<()> { let payload = DB::document_row(task, "payload")?; if payload["resume_source_kind"] != "workflow_signal" { @@ -723,7 +723,7 @@ where return Err(refuse(StatusCode::CONFLICT, "signal_wait_mismatch")); } let prepared = prepare_control_arguments( - signal_codec, + transition_codec, name.clone(), DB::string(&signal, "arguments")?, "invalid_signal_arguments", @@ -1499,7 +1499,7 @@ where &self, task_id: &str, body: Value, - signal_codec: Arc, + transition_codec: Arc, external: super::external_resolution::PreparedCommands, ) -> Result { let commands = body["commands"] @@ -1699,7 +1699,7 @@ where return Err(refuse(StatusCode::CONFLICT, "update_task_mismatch")); } let mut sequence = Self::authored_sequence(&mut tx, &run_id).await?; - Self::apply_task_signal(&mut tx, &run_id, task_id, &task, signal_codec).await?; + Self::apply_task_signal(&mut tx, &run_id, task_id, &task, transition_codec).await?; for (index, command) in commands.iter().enumerate() { if command["type"] == "fail_workflow" { // Published parseCommands applies every nonterminal command diff --git a/rust/src/runtime/transition_codec_tests.rs b/rust/src/runtime/transition_codec_tests.rs new file mode 100644 index 00000000..becf90d8 --- /dev/null +++ b/rust/src/runtime/transition_codec_tests.rs @@ -0,0 +1,202 @@ +//! Transitions that encode or decode while holding the transition lock use +//! their own codec permit. Each test holds the read codec permit, as a long +//! describe decode would, and requires the transition to finish anyway. +use super::*; +use crate::codec::{Value as Payload, ValueCodec}; +use serde_json::json; + +fn envelope(value: Payload) -> Value { + json!({"codec":"avro","blob":ValueCodec::new().unwrap().encode(&value).unwrap()}) +} + +const DECISIVE: Duration = Duration::from_secs(1); + +async fn runtime(directory: &tempfile::TempDir) -> Runtime { + let runtime = Runtime::open( + directory.path().join("runtime.sqlite").to_str().unwrap(), + "test-token".into(), + ) + .await + .unwrap(); + runtime + .register(json!({"worker_id":"codec-worker","task_queue":"test","runtime":"php", + "supported_workflow_types":["echo"],"supported_activity_types":[], + "workflow_command_contracts":{"echo":{"signals":["payload"],"signal_contracts":[{"name":"payload", + "parameters":[{"name":"value","position":0,"type":"array","required":true,"variadic":false, + "default_available":false,"allows_null":false}]}]}}})) + .await + .unwrap(); + runtime +} + +async fn start(runtime: &Runtime, workflow_id: &str) { + runtime + .start(json!({"workflow_id":workflow_id,"workflow_type":"echo","task_queue":"test", + "input":envelope(Payload::Array(vec![]))})) + .await + .unwrap(); +} + +async fn poll(runtime: &Runtime) -> Value { + let task = runtime + .poll( + json!({"worker_id":"codec-worker","task_queue":"test","timeout_seconds":0}), + "workflow", + "1.20", + ) + .await + .unwrap()["task"] + .clone(); + assert!(!task.is_null(), "workflow task is ready"); + task +} + +async fn complete(runtime: &Runtime, task: &Value, commands: Value) -> Result { + runtime + .complete_workflow( + task["task_id"].as_str().unwrap(), + json!({"lease_owner":task["lease_owner"],"workflow_task_attempt":task["workflow_task_attempt"], + "commands":commands}), + ) + .await +} + +#[tokio::test] +async fn cancellation_request_does_not_wait_for_read_codec_capacity() { + let directory = tempfile::tempdir().unwrap(); + let runtime = runtime(&directory).await; + start(&runtime, "codec-cancel").await; + let _held = runtime.signal_codec.clone().acquire_owned().await.unwrap(); + let (status, accepted) = tokio::time::timeout( + DECISIVE, + runtime.request_cancellation( + "codec-cancel", + None, + json!({"reason":"held read permit","cleanup_timeout_seconds":30}), + ), + ) + .await + .expect("the cancellation encode must not wait for read codec capacity") + .unwrap(); + assert_eq!(status, StatusCode::ACCEPTED, "{accepted}"); + runtime.close().await; +} + +#[tokio::test] +async fn signal_resumed_completion_does_not_wait_for_read_codec_capacity() { + let directory = tempfile::tempdir().unwrap(); + let runtime = runtime(&directory).await; + start(&runtime, "codec-signal").await; + let waiting = poll(&runtime).await; + complete( + &runtime, + &waiting, + json!([{"type":"open_signal_wait","signal_name":"payload"}]), + ) + .await + .unwrap(); + runtime + .signal( + "codec-signal", + None, + "payload", + json!({"input":envelope(Payload::Array(vec![Payload::Map( + std::collections::BTreeMap::from([("count".into(), Payload::Long(7))]), + )]))}), + ) + .await + .unwrap(); + let resumed = poll(&runtime).await; + let _held = runtime.signal_codec.clone().acquire_owned().await.unwrap(); + let completed = tokio::time::timeout( + DECISIVE, + complete( + &runtime, + &resumed, + json!([{"type":"complete_workflow","result":envelope(Payload::Long(7))}]), + ), + ) + .await + .expect("applying the signal must not wait for read codec capacity") + .unwrap(); + assert_eq!(completed["recorded"], true, "{completed}"); + runtime.close().await; +} + +#[tokio::test] +async fn fenced_deregistration_repair_does_not_wait_for_read_codec_capacity() { + let directory = tempfile::tempdir().unwrap(); + let runtime = Runtime::open( + directory.path().join("runtime.sqlite").to_str().unwrap(), + "test-token".into(), + ) + .await + .unwrap(); + let registration = runtime + .register(json!({"worker_id":"codec-worker","task_queue":"test","runtime":"php", + "supported_workflow_types":["echo"],"supported_activity_types":[]})) + .await + .unwrap(); + start(&runtime, "codec-repair").await; + poll(&runtime).await; + let _held = runtime.signal_codec.clone().acquire_owned().await.unwrap(); + let deregistered = tokio::time::timeout( + DECISIVE, + runtime.deregister_fenced( + "codec-worker", + json!({"registration_token":registration["registration_token"]}), + ), + ) + .await + .expect("repairing the leased task must not wait for read codec capacity") + .unwrap(); + assert_eq!(deregistered["recovered_workflow_task_count"], 1, "{deregistered}"); + runtime.close().await; +} + +#[tokio::test] +async fn range_repair_does_not_wait_for_read_codec_capacity() { + use axum::{ + body::{Body, to_bytes}, + http::Request, + }; + use tower::ServiceExt; + let directory = tempfile::tempdir().unwrap(); + let runtime = runtime(&directory).await; + start(&runtime, "codec-range").await; + let task = poll(&runtime).await; + let reported = runtime + .fail_workflow_task( + task["task_id"].as_str().unwrap(), + json!({"lease_owner":task["lease_owner"],"workflow_task_attempt":task["workflow_task_attempt"], + "failure":{"message":"Recorded version 1 is outside supported range 2..2.", + "type":"DurableWorkflow\\Exception\\NonDeterministicWorkflow", + "reason":"version_marker_incompatible_range","sequence":1}}), + ) + .await + .unwrap(); + assert_eq!(reported["recorded"], true, "{reported}"); + // The repair context comes from the authenticated request, so use the route. + let request = Request::builder() + .method("POST") + .uri("/api/workflows/codec-range/repair") + .header("authorization", "Bearer test-token") + .header("content-type", "application/json") + .header("x-namespace", "default") + .header("x-durable-workflow-control-plane-version", "2") + .body(Body::empty()) + .unwrap(); + let _held = runtime.signal_codec.clone().acquire_owned().await.unwrap(); + let response = tokio::time::timeout( + DECISIVE, + super::super::router(runtime.clone()).oneshot(request), + ) + .await + .expect("the repair payload encode must not wait for read codec capacity") + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let repaired: Value = + serde_json::from_slice(&to_bytes(response.into_body(), 1 << 20).await.unwrap()).unwrap(); + assert_eq!(repaired["outcome"], "repair_dispatched", "{repaired}"); + runtime.close().await; +} From 095bf3e0055af6425d770e9a1e7f5bec962f7e44 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Thu, 15 Oct 2026 06:06:28 +0000 Subject: [PATCH 02/42] Reproduce native empty and malformed command declaration boundaries --- rust/tests/execution.rs | 3 + rust/tests/execution/worker_contracts.rs | 106 +++++++++++++++++++++++ 2 files changed, 109 insertions(+) create mode 100644 rust/tests/execution/worker_contracts.rs diff --git a/rust/tests/execution.rs b/rust/tests/execution.rs index 8c2b7ba5..0c1f6a19 100644 --- a/rust/tests/execution.rs +++ b/rust/tests/execution.rs @@ -49,6 +49,9 @@ mod selected_run_signals; #[path = "execution/worker_incarnations.rs"] mod worker_incarnations; +#[path = "execution/worker_contracts.rs"] +mod worker_contracts; + #[path = "execution/worker_deregistration.rs"] mod worker_deregistration; diff --git a/rust/tests/execution/worker_contracts.rs b/rust/tests/execution/worker_contracts.rs new file mode 100644 index 00000000..72f730ee --- /dev/null +++ b/rust/tests/execution/worker_contracts.rs @@ -0,0 +1,106 @@ +use super::*; + +fn definition() -> Value { + json!({"worker_id":"contract-worker","task_queue":"test","runtime":"python", + "supported_workflow_types":["echo"],"supported_activity_types":[]}) +} + +async fn register_contract(app: &Router, declaration: Option) -> (StatusCode, Value) { + let mut body = definition(); + if let Some(declaration) = declaration { + body["workflow_command_contracts"] = declaration; + } + request(app, "POST", "/api/worker/register", body).await +} + +async fn snapshot(app: &Router, started: &Value) -> Value { + let history = run_history(app, &started["workflow_id"], &started["run_id"]).await; + history.as_array().unwrap().iter() + .find(|event| event["event_type"] == "WorkflowStarted").unwrap()["payload"].clone() +} + +fn assert_empty(payload: &Value) { + for field in ["declared_queries", "declared_signals", "declared_updates", + "declared_query_contracts", "declared_signal_contracts", "declared_update_contracts"] { + assert_eq!(payload.get(field), Some(&json!([])), "{field}: {payload}"); + } +} + +#[tokio::test] +async fn explicit_empty_declarations_survive_cold_reads_and_later_registration() { + let database = TestDatabase::new().await; + let runtime = database.open().await.unwrap(); + let app = router(runtime.clone()); + let mut originals = Vec::new(); + for (index, declaration) in [json!({}), json!([]), json!({"queries":[],"signals":[], + "updates":[],"query_contracts":[],"signal_contracts":[],"update_contracts":[]})].into_iter().enumerate() { + let registered = register_contract(&app, Some(json!({"echo":declaration}))).await; + assert_eq!(registered.0, StatusCode::CREATED, "{}", registered.1); + let started = start(&app, &format!("empty-contract-{index}")).await; + let before = snapshot(&app, &started).await; + assert_empty(&before); + originals.push((started, before)); + } + assert_eq!(register_contract(&app, Some(json!({"echo":{"query_contracts":[ + {"name":"state","parameters":[]}],"signals":["z","a","z"]}}))).await.0, StatusCode::CREATED); + let later = start(&app, "named-contract").await; + let named = snapshot(&app, &later).await; + assert_eq!(named["declared_queries"], json!(["state"])); + assert_eq!(named["declared_signals"], json!(["a","z"])); + assert_eq!(register_contract(&app, None).await.0, StatusCode::CREATED); + let omitted = start(&app, "reregister-omitted").await; + assert_eq!(snapshot(&app, &omitted).await["declared_queries"], json!(["state"])); + drop(app); + drop(runtime); + let cold = router(database.open().await.unwrap()); + for (started, before) in originals { + assert_eq!(snapshot(&cold, &started).await, before); + } + drop(cold); + database.remove().await; +} + +#[tokio::test] +async fn legacy_missing_entries_remain_unknown_instead_of_becoming_empty() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + for (index, declaration) in [None, Some(Value::Null), Some(json!({})), + Some(json!([])), Some(json!({"other":{}}))].into_iter().enumerate() { + let registered = register_contract(&app, declaration).await; + assert_eq!(registered.0, StatusCode::CREATED, "{}", registered.1); + let original = start(&app, &format!("legacy-contract-{index}")).await; + let before = snapshot(&app, &original).await; + assert!(before.get("declared_queries").is_none(), "{before}"); + assert_eq!(register_contract(&app, Some(json!({"echo":{}}))).await.0, StatusCode::CREATED); + assert_eq!(snapshot(&app, &original).await, before); + assert_eq!(register_contract(&app, Some(json!({}))).await.0, StatusCode::CREATED); + } + drop(app); + database.remove().await; +} + +#[tokio::test] +async fn invalid_declarations_refuse_before_registration_rotation_or_lease_release() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + let accepted = register_contract(&app, Some(json!({"echo":{}}))).await; + assert_eq!(accepted.0, StatusCode::CREATED); + let started = start(&app, "contract-lease").await; + let task = poll(&app, "contract-worker", "workflow").await; + let before = snapshot(&app, &started).await; + for declaration in [Value::Null, json!("bad"), json!({"queries":null}), json!({"queries":"state"}), + json!({"queries":[17]}), json!({"signals":[" "]}), json!({"updates":{"key":"set"}}), + json!({"update_validators":null}), json!({"query_contracts":"bad"}), + json!({"signal_contracts":[null]}), json!({"query_contracts":[{}]}), + json!({"update_contracts":[{"name":17}]})] { + let refused = register_contract(&app, Some(json!({"echo":declaration}))).await; + assert_eq!(refused.0, StatusCode::UNPROCESSABLE_ENTITY, "{}", refused.1); + assert_eq!(snapshot(&app, &started).await, before); + } + let completed = finish_task(&app, &task, json!([{"type":"complete_workflow", + "result":envelope(Payload::Long(9007199254740993))}])).await; + assert_eq!(completed.0, StatusCode::OK, "{}", completed.1); + assert_eq!(completed.1["recorded"], true); + drop(app); + database.remove().await; +} From 1a739d4c46bd1e74846c6110e0b6e870d24927ee Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Thu, 15 Oct 2026 06:15:16 +0000 Subject: [PATCH 03/42] Preserve explicit worker command declaration snapshots --- rust/src/runtime/children.rs | 9 +- rust/src/runtime/mod.rs | 1 + rust/src/runtime/store.rs | 36 ++++---- rust/src/runtime/worker_contracts.rs | 93 +++++++++++++++++++ rust/tests/execution/worker_contracts.rs | 109 ++++++++++++++++++----- 5 files changed, 209 insertions(+), 39 deletions(-) create mode 100644 rust/src/runtime/worker_contracts.rs diff --git a/rust/src/runtime/children.rs b/rust/src/runtime/children.rs index 73fc654b..c6d194fe 100644 --- a/rust/src/runtime/children.rs +++ b/rust/src/runtime/children.rs @@ -155,6 +155,11 @@ where { let contracts = DB::document_row(&worker, "workflow_command_contracts")?; if let Some(contract) = contracts.get(workflow_type) { + // Older development registrations may predate validation. + // Never turn a malformed entry into authoritative empties. + let Ok(contract) = super::worker_contracts::declaration(contract) else { + continue; + }; for (source, target) in [ ("signals", "declared_signals"), ("signal_contracts", "declared_signal_contracts"), @@ -164,7 +169,9 @@ where ("update_contracts", "declared_update_contracts"), ("update_validators", "declared_update_validators"), ] { - started[target] = contract[source].clone(); + if let Some(value) = contract.get(source) { + started[target] = value.clone(); + } } break; } diff --git a/rust/src/runtime/mod.rs b/rust/src/runtime/mod.rs index 0d051c04..2fa6aa8f 100644 --- a/rust/src/runtime/mod.rs +++ b/rust/src/runtime/mod.rs @@ -63,6 +63,7 @@ pub mod upgrade; mod visibility; mod wait_projection; mod wait_timeouts; +mod worker_contracts; mod worker_incarnations; mod workflow_failures; diff --git a/rust/src/runtime/store.rs b/rust/src/runtime/store.rs index 6490d0f9..04349dd6 100644 --- a/rust/src/runtime/store.rs +++ b/rust/src/runtime/store.rs @@ -813,27 +813,29 @@ where } } let runtime = text(&body, "runtime")?; - let contracts = body + let supplied_contracts = body .get("workflow_command_contracts") - .cloned() - .unwrap_or_else(|| json!({})); - // PHP encodes an empty associative map as []; activity-only published - // workers send this shape. Preserve a map internally for lookups. - let contracts = if contracts.as_array().is_some_and(|values| values.is_empty()) { - json!({}) - } else { - contracts - }; - if !contracts.is_object() { - return Err(refuse( - StatusCode::UNPROCESSABLE_ENTITY, - "invalid_worker_definition", - )); - } + .map(|contracts| { + super::worker_contracts::normalize(contracts, &body["supported_workflow_types"]) + }) + .transpose()?; let token = super::worker_incarnations::token().await?; let mut tx = self.begin().await?; - let previous = Self::query("SELECT worker_id FROM workflow_worker_registrations WHERE namespace=$1 AND worker_id=$2") + let previous = Self::query("SELECT worker_id,workflow_command_contracts FROM workflow_worker_registrations WHERE namespace=$1 AND worker_id=$2") .bind(self.namespace.as_ref()).bind(worker_id).fetch_optional(&mut *tx).await?; + let contracts = match supplied_contracts { + Some(contracts) => contracts, + None => { + let existing = match &previous { + Some(worker) => { + DB::optional_document_row(worker, "workflow_command_contracts")? + .unwrap_or(Value::Null) + } + None => Value::Null, + }; + super::worker_contracts::normalize(&existing, &body["supported_workflow_types"])? + } + }; if previous.is_some() { // Match PHP's registration boundary: return original workflow leases // to ready without a repair command/history event. Token rotation diff --git a/rust/src/runtime/worker_contracts.rs b/rust/src/runtime/worker_contracts.rs new file mode 100644 index 00000000..871249dc --- /dev/null +++ b/rust/src/runtime/worker_contracts.rs @@ -0,0 +1,93 @@ +//! Registration declarations become immutable admission snapshots. A missing +//! workflow entry stays unknown; a valid empty entry declares empty lists. +use super::{Result, refuse}; +use axum::http::StatusCode; +use serde_json::{Value, json}; +use std::collections::BTreeSet; + +fn invalid() -> super::RuntimeError { + refuse( + StatusCode::UNPROCESSABLE_ENTITY, + "invalid_worker_definition", + ) +} + +fn name(value: &Value) -> Result<&str> { + let text = value.as_str().ok_or_else(invalid)?; + let text = text.trim_matches([' ', '\t', '\n', '\r', '\0', '\u{b}']); + if text.is_empty() || text.chars().count() > 255 { + return Err(invalid()); + } + Ok(text) +} + +fn list<'a>(entry: &'a Value, field: &str) -> Result<&'a [Value]> { + match entry.get(field) { + None => Ok(&[]), + Some(Value::Array(values)) => Ok(values), + Some(_) => Err(invalid()), + } +} + +pub(super) fn declaration(entry: &Value) -> Result { + if !entry.is_object() && !entry.as_array().is_some_and(Vec::is_empty) { + return Err(invalid()); + } + let mut result = json!({}); + for (names, handlers) in [ + ("queries", "query_contracts"), + ("signals", "signal_contracts"), + ("updates", "update_contracts"), + ] { + let mut declared = BTreeSet::new(); + for value in list(entry, names)? { + declared.insert(name(value)?.to_owned()); + } + let mut seen = BTreeSet::new(); + let mut contracts = Vec::new(); + for handler in list(entry, handlers)? { + let handler_name = name(&handler["name"])?; + declared.insert(handler_name.to_owned()); + if seen.insert(handler_name.to_owned()) { + contracts.push( + json!({"name":handler_name,"parameters":handler.get("parameters") + .filter(|value| value.is_array()).cloned().unwrap_or_else(|| json!([]))}), + ); + } + } + result[names] = json!(declared); + result[handlers] = json!(contracts); + } + if entry.get("update_validators").is_some() { + let mut validators = BTreeSet::new(); + for value in list(entry, "update_validators")? { + validators.insert(name(value)?.to_owned()); + } + result["update_validators"] = json!(validators); + } + Ok(result) +} + +pub(super) fn normalize(value: &Value, supported: &Value) -> Result { + if value.is_null() || value.as_array().is_some_and(Vec::is_empty) { + return Ok(json!({})); + } + let entries = value.as_object().ok_or_else(invalid)?; + let mut result = json!({}); + for (workflow_type, entry) in entries { + // Validate even an unadvertised declaration before filtering it out. + // Otherwise malformed metadata could rotate an existing registration. + let entry = declaration(entry)?; + let workflow_type = workflow_type.trim_matches([' ', '\t', '\n', '\r', '\0', '\u{b}']); + if !workflow_type.is_empty() + && supported.as_array().is_some_and(|types| { + types + .iter() + .any(|value| value.as_str() == Some(workflow_type)) + }) + { + result[workflow_type] = entry; + } + } + Ok(result) +} diff --git a/rust/tests/execution/worker_contracts.rs b/rust/tests/execution/worker_contracts.rs index 72f730ee..9dac8ad6 100644 --- a/rust/tests/execution/worker_contracts.rs +++ b/rust/tests/execution/worker_contracts.rs @@ -15,13 +15,24 @@ async fn register_contract(app: &Router, declaration: Option) -> (StatusC async fn snapshot(app: &Router, started: &Value) -> Value { let history = run_history(app, &started["workflow_id"], &started["run_id"]).await; - history.as_array().unwrap().iter() - .find(|event| event["event_type"] == "WorkflowStarted").unwrap()["payload"].clone() + history + .as_array() + .unwrap() + .iter() + .find(|event| event["event_type"] == "WorkflowStarted") + .unwrap()["payload"] + .clone() } fn assert_empty(payload: &Value) { - for field in ["declared_queries", "declared_signals", "declared_updates", - "declared_query_contracts", "declared_signal_contracts", "declared_update_contracts"] { + for field in [ + "declared_queries", + "declared_signals", + "declared_updates", + "declared_query_contracts", + "declared_signal_contracts", + "declared_update_contracts", + ] { assert_eq!(payload.get(field), Some(&json!([])), "{field}: {payload}"); } } @@ -32,8 +43,15 @@ async fn explicit_empty_declarations_survive_cold_reads_and_later_registration() let runtime = database.open().await.unwrap(); let app = router(runtime.clone()); let mut originals = Vec::new(); - for (index, declaration) in [json!({}), json!([]), json!({"queries":[],"signals":[], - "updates":[],"query_contracts":[],"signal_contracts":[],"update_contracts":[]})].into_iter().enumerate() { + for (index, declaration) in [ + json!({}), + json!([]), + json!({"queries":[],"signals":[], + "updates":[],"query_contracts":[],"signal_contracts":[],"update_contracts":[]}), + ] + .into_iter() + .enumerate() + { let registered = register_contract(&app, Some(json!({"echo":declaration}))).await; assert_eq!(registered.0, StatusCode::CREATED, "{}", registered.1); let started = start(&app, &format!("empty-contract-{index}")).await; @@ -41,15 +59,26 @@ async fn explicit_empty_declarations_survive_cold_reads_and_later_registration() assert_empty(&before); originals.push((started, before)); } - assert_eq!(register_contract(&app, Some(json!({"echo":{"query_contracts":[ - {"name":"state","parameters":[]}],"signals":["z","a","z"]}}))).await.0, StatusCode::CREATED); + assert_eq!( + register_contract( + &app, + Some(json!({"echo":{"query_contracts":[ + {"name":"state","parameters":[]}],"signals":["z","a","z"]}})) + ) + .await + .0, + StatusCode::CREATED + ); let later = start(&app, "named-contract").await; let named = snapshot(&app, &later).await; assert_eq!(named["declared_queries"], json!(["state"])); - assert_eq!(named["declared_signals"], json!(["a","z"])); + assert_eq!(named["declared_signals"], json!(["a", "z"])); assert_eq!(register_contract(&app, None).await.0, StatusCode::CREATED); let omitted = start(&app, "reregister-omitted").await; - assert_eq!(snapshot(&app, &omitted).await["declared_queries"], json!(["state"])); + assert_eq!( + snapshot(&app, &omitted).await["declared_queries"], + json!(["state"]) + ); drop(app); drop(runtime); let cold = router(database.open().await.unwrap()); @@ -64,16 +93,30 @@ async fn explicit_empty_declarations_survive_cold_reads_and_later_registration() async fn legacy_missing_entries_remain_unknown_instead_of_becoming_empty() { let database = TestDatabase::new().await; let app = router(database.open().await.unwrap()); - for (index, declaration) in [None, Some(Value::Null), Some(json!({})), - Some(json!([])), Some(json!({"other":{}}))].into_iter().enumerate() { + for (index, declaration) in [ + None, + Some(Value::Null), + Some(json!({})), + Some(json!([])), + Some(json!({"other":{}})), + ] + .into_iter() + .enumerate() + { let registered = register_contract(&app, declaration).await; assert_eq!(registered.0, StatusCode::CREATED, "{}", registered.1); let original = start(&app, &format!("legacy-contract-{index}")).await; let before = snapshot(&app, &original).await; assert!(before.get("declared_queries").is_none(), "{before}"); - assert_eq!(register_contract(&app, Some(json!({"echo":{}}))).await.0, StatusCode::CREATED); + assert_eq!( + register_contract(&app, Some(json!({"echo":{}}))).await.0, + StatusCode::CREATED + ); assert_eq!(snapshot(&app, &original).await, before); - assert_eq!(register_contract(&app, Some(json!({}))).await.0, StatusCode::CREATED); + assert_eq!( + register_contract(&app, Some(json!({}))).await.0, + StatusCode::CREATED + ); } drop(app); database.remove().await; @@ -88,19 +131,43 @@ async fn invalid_declarations_refuse_before_registration_rotation_or_lease_relea let started = start(&app, "contract-lease").await; let task = poll(&app, "contract-worker", "workflow").await; let before = snapshot(&app, &started).await; - for declaration in [Value::Null, json!("bad"), json!({"queries":null}), json!({"queries":"state"}), - json!({"queries":[17]}), json!({"signals":[" "]}), json!({"updates":{"key":"set"}}), - json!({"update_validators":null}), json!({"query_contracts":"bad"}), - json!({"signal_contracts":[null]}), json!({"query_contracts":[{}]}), - json!({"update_contracts":[{"name":17}]})] { + for declaration in [ + Value::Null, + json!("bad"), + json!({"queries":null}), + json!({"queries":"state"}), + json!({"queries":[17]}), + json!({"signals":[" "]}), + json!({"updates":{"key":"set"}}), + json!({"update_validators":null}), + json!({"query_contracts":"bad"}), + json!({"signal_contracts":[null]}), + json!({"query_contracts":[{}]}), + json!({"update_contracts":[{"name":17}]}), + ] { let refused = register_contract(&app, Some(json!({"echo":declaration}))).await; assert_eq!(refused.0, StatusCode::UNPROCESSABLE_ENTITY, "{}", refused.1); assert_eq!(snapshot(&app, &started).await, before); } - let completed = finish_task(&app, &task, json!([{"type":"complete_workflow", - "result":envelope(Payload::Long(9007199254740993))}])).await; + let completed = finish_task( + &app, + &task, + json!([{"type":"complete_workflow", + "result":envelope(Payload::Long(9007199254740993))}]), + ) + .await; assert_eq!(completed.0, StatusCode::OK, "{}", completed.1); assert_eq!(completed.1["recorded"], true); + let retired = request_protocol( + &app, + "POST", + "/api/worker/registrations/contract-worker/deregister", + json!({"registration_token":accepted.1["registration_token"]}), + "1.20", + ) + .await; + assert_eq!(retired.0, StatusCode::OK, "{}", retired.1); + assert_eq!(retired.1["recovered_workflow_task_count"], 0); drop(app); database.remove().await; } From c4e37fd283adb43d94dea30561d661f8b2cd7054 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 03:58:38 +0000 Subject: [PATCH 04/42] Report leased and empty poll statuses like the worker protocol The worker protocol requires poll_status "leased" on a workflow task claim, and published PHP (2.5.14 and 2.8) reports "leased" and "empty" for workflow, activity and query polls alike. Native reported "task" and "timeout". It now reports the same two values. The wake hint task_queue_changed and every refusal status are unchanged. The cross-kind poll contract, which accepted either claim status, now requires "leased". Part of #325. Co-Authored-By: Claude Opus 5.5 --- rust/src/runtime/queries.rs | 4 ++-- rust/src/runtime/store.rs | 4 ++-- rust/tests/execution/cross_kind_poll.rs | 6 +++--- .../conformance/server-parity/cross-kind-poll-contract.mjs | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/rust/src/runtime/queries.rs b/rust/src/runtime/queries.rs index f91c4e04..861532c3 100644 --- a/rust/src/runtime/queries.rs +++ b/rust/src/runtime/queries.rs @@ -357,14 +357,14 @@ where loop { let wake = self.wake.notified(); match self.claim_query(&body, config, &proof).await? { - QueryClaim::Task(task) => return Ok(poll_result(Some(task), "task")), + QueryClaim::Task(task) => return Ok(poll_result(Some(task), "leased")), QueryClaim::StaleWorker => { return Ok(poll_result(None, "stale_worker_registration")); } QueryClaim::Empty => {} } if tokio::time::Instant::now() >= deadline { - return Ok(poll_result(None, "timeout")); + return Ok(poll_result(None, "empty")); } if self .poll_queue_changed(&body, "query", &mut cross_kind) diff --git a/rust/src/runtime/store.rs b/rust/src/runtime/store.rs index 6490d0f9..2f25c77c 100644 --- a/rust/src/runtime/store.rs +++ b/rust/src/runtime/store.rs @@ -939,12 +939,12 @@ where let wake = self.wake.notified(); if let Some(task) = self.claim(&body, kind, protocol_version).await? { return Ok( - json!({"task": task, "poll_status": "task", "server_capabilities": capabilities()}), + json!({"task": task, "poll_status": "leased", "server_capabilities": capabilities()}), ); } if tokio::time::Instant::now() >= deadline { return Ok( - json!({"task": null, "poll_status": "timeout", "server_capabilities": capabilities()}), + json!({"task": null, "poll_status": "empty", "server_capabilities": capabilities()}), ); } if self diff --git a/rust/tests/execution/cross_kind_poll.rs b/rust/tests/execution/cross_kind_poll.rs index 754af04f..5986b0ce 100644 --- a/rust/tests/execution/cross_kind_poll.rs +++ b/rust/tests/execution/cross_kind_poll.rs @@ -177,7 +177,7 @@ async fn scoped_hints_require_original_opt_in_and_a_new_ready_identity() { let result = held.await.unwrap(); assert_eq!(result.0, StatusCode::OK, "{scenario}: {}", result.1); assert_eq!( - result.1["poll_status"], "timeout", + result.1["poll_status"], "empty", "{scenario}: {}", result.1 ); @@ -216,7 +216,7 @@ async fn activity_admission_wakes_other_polls_but_same_kind_claim_wins() { json!({"lease_owner":"mixed","activity_attempt_id":activity["activity_attempt_id"],"result":envelope(Payload::Null)})).await.0,StatusCode::OK); let result = held.await.unwrap(); assert_eq!(result.0, StatusCode::OK, "{}", result.1); - assert_eq!(result.1["poll_status"], "task"); + assert_eq!(result.1["poll_status"], "leased"); assert!(result.1["task"].is_object()); runtime.close().await; database.remove().await; @@ -311,7 +311,7 @@ async fn zero_timeout_polls_keep_the_existing_empty_reply() { ) .await; assert_eq!(result.0, StatusCode::OK, "{}", result.1); - assert_eq!(result.1["poll_status"], "timeout"); + assert_eq!(result.1["poll_status"], "empty"); assert_eq!(result.1.get("task"), Some(&Value::Null)); } runtime.close().await; diff --git a/scripts/conformance/server-parity/cross-kind-poll-contract.mjs b/scripts/conformance/server-parity/cross-kind-poll-contract.mjs index 5dceb69a..adf89f2d 100644 --- a/scripts/conformance/server-parity/cross-kind-poll-contract.mjs +++ b/scripts/conformance/server-parity/cross-kind-poll-contract.mjs @@ -44,7 +44,7 @@ export function checkCrossKindPollWake(record, profile) { } const sweep = following[sweepIndex]; assert.equal(sweep.http, 200); - assert(['task', 'leased'].includes(sweep.poll_status)); + assert.equal(sweep.poll_status, 'leased', 'a claim reports leased, as the worker protocol requires'); assert.equal(sweep.task_is_null, false); assert.equal(typeof sweep.task_id, 'string'); assert(sweep.task_id.length > 0, 'the ordinary poll separately grants a task'); From d5d880fd754b044f1c13ba48cee4fcfd87f054df Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 07:41:11 +0000 Subject: [PATCH 05/42] Let held long polls wait without the transition lock Each held native poll used to repeat its full claim every 50 ms: a transaction under the global transition lock that read the registration, deleted expired poll receipts and selected candidates. With no work, idle cost grew by about 0.035 cores per waiting poll, two-thirds of it in MySQL, and every probe queued behind, and delayed, real transitions. A held poll now claims under the lock on its first probe. Afterwards it claims only when a lock-free read finds something it could claim or must refuse: an eligible task or query, a receipt for its own poll request, or a removed, moved or replaced registration. The read runs at once when a commit on the same node wakes the poll, and otherwise every 50 ms, which is how a poll still observes other server processes. A false positive costs one empty claim, and a commit racing a read is seen on the next read. New tests hold the transition lock from outside the runtime and check that idle workflow, activity and query polls still end at their deadline (each fails with the previous probing), and that a held poll still claims a task or query, and refuses a removed registration, committed by another node. Co-Authored-By: Claude Opus 5.5 --- docs/rust-server-transition-locking.md | 9 ++ rust/src/runtime/backend.rs | 4 +- rust/src/runtime/mod.rs | 1 + rust/src/runtime/poll_readiness.rs | 97 +++++++++++++++++ rust/src/runtime/queries.rs | 22 ++-- rust/src/runtime/store.rs | 17 ++- rust/tests/execution.rs | 2 + rust/tests/execution/lock_free_waiting.rs | 123 ++++++++++++++++++++++ rust/tests/execution/read_snapshots.rs | 6 +- 9 files changed, 265 insertions(+), 16 deletions(-) create mode 100644 rust/src/runtime/poll_readiness.rs create mode 100644 rust/tests/execution/lock_free_waiting.rs diff --git a/docs/rust-server-transition-locking.md b/docs/rust-server-transition-locking.md index e593eab1..4e6b6bad 100644 --- a/docs/rust-server-transition-locking.md +++ b/docs/rust-server-transition-locking.md @@ -26,6 +26,15 @@ read-only snapshot (`Store::begin_read()`, `READ_SNAPSHOT_BEGIN` in `backend.rs`). Control-plane reads such as describe and run history query the pool without a transaction. +A held long poll takes the lock for its first probe, and afterwards only when +a lock-free read finds something to claim or refuse: an eligible task (or +query), a receipt for the poll's own request ID, or a removed, moved or +replaced registration (`poll_readiness.rs`). The read runs at once when a +commit on the same node wakes the poll, and otherwise every 50 ms, which is +how a poll observes another server process. An idle poll therefore neither +waits for the lock nor delays transitions, and an unrelated commit does not +send every waiting poll to the lock. + ## Measured cost Under the CPU deadline-health profile on a local MySQL 8.0 cell, per-step diff --git a/rust/src/runtime/backend.rs b/rust/src/runtime/backend.rs index 7aaa6940..7dc40438 100644 --- a/rust/src/runtime/backend.rs +++ b/rust/src/runtime/backend.rs @@ -337,8 +337,8 @@ impl Backend for Postgres { async fn begin(pool: &Pool) -> Result> { let mut tx = pool.begin().await?; // The first single-namespace slice serializes its durable transitions - // across independent pools/processes. Empty long polls release this - // transaction before waiting. Throughput/idle cost remain unqualified. + // across independent pools/processes. Held long polls wait without it + // (poll_readiness.rs). Throughput/idle cost remain unqualified. sqlx::query("SELECT pg_advisory_xact_lock(4924501473315676163)") .execute(&mut *tx) .await?; diff --git a/rust/src/runtime/mod.rs b/rust/src/runtime/mod.rs index 0d051c04..e83d15aa 100644 --- a/rust/src/runtime/mod.rs +++ b/rust/src/runtime/mod.rs @@ -35,6 +35,7 @@ mod memos; pub mod mysql; mod namespaces; pub mod postgres; +mod poll_readiness; mod previews; mod queries; mod query_authority; diff --git a/rust/src/runtime/poll_readiness.rs b/rust/src/runtime/poll_readiness.rs new file mode 100644 index 00000000..f0727bf3 --- /dev/null +++ b/rust/src/runtime/poll_readiness.rs @@ -0,0 +1,97 @@ +//! Lock-free waiting for held long polls. +//! +//! A held poll takes the transition lock only to claim or refuse. After its +//! first claim it reads, without the lock, whether anything now exists that it +//! could claim or must refuse: an eligible task, a receipt for its own request, +//! or a changed registration. A same-node commit wakes it to read at once; +//! timed reads observe other server processes. A false positive costs one locked +//! claim that finds nothing, and a commit racing a read is seen on the next one. +use super::{ + Result, backend::Backend, query_authority::{self, QueryTaskConfig}, store::Store, text, +}; +use serde_json::Value; +use sqlx::{ColumnIndex, Decode, Encode, Executor, IntoArguments, Type}; +use std::time::Duration; + +/// How often a held poll reads for work committed by another server process. +pub(super) const READ_INTERVAL: Duration = Duration::from_millis(50); + +impl Store +where + for<'c> &'c mut DB::Connection: Executor<'c, Database = DB>, + DB::Arguments: IntoArguments, + for<'q> &'q str: Encode<'q, DB> + Type, + for<'q> Option<&'q str>: Encode<'q, DB>, + for<'q> Option: Encode<'q, DB>, + for<'q> String: Encode<'q, DB> + Type, + for<'q> i64: Encode<'q, DB> + Type, + for<'r> i64: Decode<'r, DB>, + usize: ColumnIndex, +{ + /// Whether a held workflow or activity poll should claim again. + pub(super) async fn task_poll_ready(&self, body: &Value, kind: &str) -> Result { + let worker_id = text(body, "worker_id")?; + let queue = text(body, "task_queue")?; + // The claim refuses a removed or moved registration. + let registration = Self::query( + "SELECT task_queue FROM workflow_worker_registrations WHERE worker_id=$1 AND namespace=$2", + ) + .bind(worker_id) + .bind(self.namespace.as_ref()) + .fetch_optional(&self.pool) + .await?; + match registration { + Some(row) if DB::string(&row, "task_queue")? == queue => {} + _ => return Ok(true), + } + // A concurrent poll with the same request ID may have claimed already; + // the claim replays its receipt. + if let Some(request_id) = body.get("poll_request_id").and_then(Value::as_str) + && Self::query("SELECT task_id FROM dw_poll_receipts WHERE worker_id=$1 AND kind=$2 AND request_id=$3 AND namespace=$4") + .bind(worker_id).bind(kind).bind(request_id).bind(self.namespace.as_ref()) + .fetch_optional(&self.pool).await?.is_some() + { + return Ok(true); + } + let at = super::store::now(); + Ok(Self::query(DB::TASK_CANDIDATES_SQL) + .bind(self.namespace.as_ref()) + .bind(queue) + .bind(kind) + .bind(DB::bind_time(at)) + .bind(DB::bind_time(at)) + .fetch_optional(&self.pool) + .await? + .is_some()) + } + + /// Whether a held query poll should claim again. `proof` is the + /// registration this poll belongs to. + pub(super) async fn query_poll_ready( + &self, + body: &Value, + config: &QueryTaskConfig, + proof: &str, + ) -> Result { + let owner = text(body, "worker_id")?; + // The claim refuses a replaced, removed or stale registration. + if Self::query_worker_proof(&self.pool, self.namespace.as_ref(), owner, config) + .await? + .as_deref() + != Some(proof) + { + return Ok(true); + } + let at = super::store::now(); + let query_now = query_authority::wire(at); + Ok(Self::query(DB::POLL_WAKE_QUERIES_SQL) + .bind(at.timestamp()) + .bind(self.namespace.as_ref()) + .bind(text(body, "task_queue")?) + .bind(query_now.clone()) + .bind(query_now) + .fetch_optional(&self.pool) + .await? + .is_some()) + } +} diff --git a/rust/src/runtime/queries.rs b/rust/src/runtime/queries.rs index f91c4e04..ed9a5588 100644 --- a/rust/src/runtime/queries.rs +++ b/rust/src/runtime/queries.rs @@ -354,14 +354,19 @@ where } else { None }; + // Only the first probe claims under the lock unconditionally; later + // probes claim after a lock-free read finds something to claim or refuse. + let mut first = true; loop { let wake = self.wake.notified(); - match self.claim_query(&body, config, &proof).await? { - QueryClaim::Task(task) => return Ok(poll_result(Some(task), "task")), - QueryClaim::StaleWorker => { - return Ok(poll_result(None, "stale_worker_registration")); + if std::mem::take(&mut first) || self.query_poll_ready(&body, config, &proof).await? { + match self.claim_query(&body, config, &proof).await? { + QueryClaim::Task(task) => return Ok(poll_result(Some(task), "task")), + QueryClaim::StaleWorker => { + return Ok(poll_result(None, "stale_worker_registration")); + } + QueryClaim::Empty => {} } - QueryClaim::Empty => {} } if tokio::time::Instant::now() >= deadline { return Ok(poll_result(None, "timeout")); @@ -375,11 +380,14 @@ where json!({"task":null,"poll_status":"task_queue_changed","server_capabilities":super::store::capabilities()}), ); } - tokio::select! { _ = wake => {}, _ = tokio::time::sleep(Duration::from_millis(50)) => {} } + tokio::select! { + _ = wake => {}, + _ = tokio::time::sleep(super::poll_readiness::READ_INTERVAL) => {}, + } } } - async fn query_worker_proof<'e, E>( + pub(super) async fn query_worker_proof<'e, E>( executor: E, namespace: &str, owner: &str, diff --git a/rust/src/runtime/store.rs b/rust/src/runtime/store.rs index 6490d0f9..1dddc5da 100644 --- a/rust/src/runtime/store.rs +++ b/rust/src/runtime/store.rs @@ -933,11 +933,17 @@ where } else { None }; + // Only the first probe claims under the lock unconditionally. Later + // probes, on a same-node wake or a timed read that also observes + // other server processes, claim only after a lock-free read finds + // something to claim or refuse. + let mut first = true; loop { - // Register the notification before probing; periodic probing also - // observes work committed by an independent process. + // Register the notification before probing. let wake = self.wake.notified(); - if let Some(task) = self.claim(&body, kind, protocol_version).await? { + if (std::mem::take(&mut first) || self.task_poll_ready(&body, kind).await?) + && let Some(task) = self.claim(&body, kind, protocol_version).await? + { return Ok( json!({"task": task, "poll_status": "task", "server_capabilities": capabilities()}), ); @@ -956,7 +962,10 @@ where json!({"task":null,"poll_status":"task_queue_changed","server_capabilities":capabilities()}), ); } - tokio::select! { _ = wake => {}, _ = tokio::time::sleep(Duration::from_millis(50)) => {} } + tokio::select! { + _ = wake => {}, + _ = tokio::time::sleep(super::poll_readiness::READ_INTERVAL) => {}, + } } } diff --git a/rust/tests/execution.rs b/rust/tests/execution.rs index 8c2b7ba5..7eeceb17 100644 --- a/rust/tests/execution.rs +++ b/rust/tests/execution.rs @@ -61,6 +61,8 @@ mod version_markers; #[path = "execution/activity_deadlines.rs"] mod activity_deadlines; +#[path = "execution/lock_free_waiting.rs"] +mod lock_free_waiting; #[path = "execution/read_snapshots.rs"] mod read_snapshots; diff --git a/rust/tests/execution/lock_free_waiting.rs b/rust/tests/execution/lock_free_waiting.rs new file mode 100644 index 00000000..bd22759d --- /dev/null +++ b/rust/tests/execution/lock_free_waiting.rs @@ -0,0 +1,123 @@ +//! A held long poll waits without the transition lock, so idle polls neither +//! wait behind transitions nor delay them. It still claims work, and refuses +//! a removed registration, committed by another server process. +use super::read_snapshots::hold_transition_lock; +use super::*; +use std::time::Duration; + +fn held_poll( + app: &Router, + worker: &str, + kind: &str, + seconds: u64, +) -> tokio::task::JoinHandle<(StatusCode, Value)> { + let app = app.clone(); + let path = format!("/api/worker/{kind}-tasks/poll"); + let body = json!({"worker_id":worker,"task_queue":"test","timeout_seconds":seconds}); + tokio::spawn(async move { request(&app, "POST", &path, body).await }) +} + +#[tokio::test] +async fn held_polls_reach_their_deadline_while_a_transition_holds_the_lock() { + let database = TestDatabase::new().await; + let runtime = database.open().await.unwrap(); + let app = router(runtime.clone()); + register(&app, "waiting-worker", json!(["echo"]), json!(["echo-activity"])).await; + query_worker(&app, "waiting-query-worker").await; + let polls = [ + held_poll(&app, "waiting-worker", "workflow", 2), + held_poll(&app, "waiting-worker", "activity", 2), + held_poll(&app, "waiting-query-worker", "query", 2), + ]; + // Each poll's first claim takes the lock; let them finish and wait. + tokio::time::sleep(Duration::from_millis(500)).await; + let writer = hold_transition_lock(&database).await; + for poll in polls { + // The polls end about 1.5 s from now. A probe that took the lock + // would wait for it, and the lock is held past this bound. + let (status, body) = tokio::time::timeout(Duration::from_secs(3), poll) + .await + .expect("an idle poll must reach its deadline without the transition lock") + .unwrap(); + assert_eq!(status, StatusCode::OK, "{body}"); + assert!(body["task"].is_null(), "{body}"); + } + writer.release().await; + runtime.close().await; + database.remove().await; +} + +#[tokio::test] +async fn held_polls_observe_work_and_removed_registrations_from_another_node() { + let database = TestDatabase::new().await; + let node_a = database.open().await.unwrap(); + let node_b = database.open().await.unwrap(); + let app_a = router(node_a.clone()); + let app_b = router(node_b.clone()); + register(&app_a, "observing-worker", json!(["echo"]), json!([])).await; + query_worker(&app_a, "observing-query-worker").await; + + // Node B's commits cannot wake node A's poll in process. + let waiting = held_poll(&app_a, "observing-worker", "workflow", 20); + tokio::time::sleep(Duration::from_millis(300)).await; + let started = start(&app_b, "observed-elsewhere").await; + let (status, claimed) = tokio::time::timeout(Duration::from_secs(5), waiting) + .await + .expect("a held poll claims work another node committed") + .unwrap(); + assert_eq!(status, StatusCode::OK, "{claimed}"); + assert_eq!(claimed["task"]["run_id"], started["run_id"], "{claimed}"); + let completed = request( + &app_a, + "POST", + &format!( + "/api/worker/workflow-tasks/{}/complete", + claimed["task"]["task_id"].as_str().unwrap() + ), + completion( + &claimed["task"], + json!([{"type":"complete_workflow","result":envelope(Payload::Null)}]), + ), + ) + .await; + assert_eq!(completed.0, StatusCode::OK, "{}", completed.1); + + let waiting = held_poll(&app_a, "observing-query-worker", "query", 5); + tokio::time::sleep(Duration::from_millis(300)).await; + let query = { + let app = app_b.clone(); + let path = format!( + "/api/workflows/observed-elsewhere/runs/{}/query/state", + started["run_id"].as_str().unwrap() + ); + let input = json!({"input":envelope(Payload::Array(vec![Payload::Long(1)]))}); + tokio::spawn(async move { request(&app, "POST", &path, input).await }) + }; + let (status, claimed) = tokio::time::timeout(Duration::from_secs(3), waiting) + .await + .expect("a held query poll claims a query another node admitted") + .unwrap(); + assert_eq!(status, StatusCode::OK, "{claimed}"); + assert_eq!(claimed["task"]["query_name"], "state", "{claimed}"); + query.abort(); + + let waiting = held_poll(&app_a, "observing-worker", "workflow", 20); + tokio::time::sleep(Duration::from_millis(300)).await; + let removed = request( + &app_b, + "DELETE", + "/api/worker/registrations/observing-worker", + Value::Null, + ) + .await; + assert_eq!(removed.0, StatusCode::OK, "{}", removed.1); + let (status, refused) = tokio::time::timeout(Duration::from_secs(5), waiting) + .await + .expect("a held poll refuses a registration another node removed") + .unwrap(); + assert_eq!(status, StatusCode::CONFLICT, "{refused}"); + assert_eq!(refused["reason"], "worker_not_registered", "{refused}"); + node_a.close().await; + node_b.close().await; + database.remove().await; +} diff --git a/rust/tests/execution/read_snapshots.rs b/rust/tests/execution/read_snapshots.rs index 8dcb0d81..2c0fdcfb 100644 --- a/rust/tests/execution/read_snapshots.rs +++ b/rust/tests/execution/read_snapshots.rs @@ -3,7 +3,7 @@ use super::*; use sqlx::Connection; -enum Writer { +pub(super) enum Writer { Sqlite(sqlx::SqliteConnection), Postgres(sqlx::PgConnection), MySql(sqlx::MySqlConnection), @@ -11,7 +11,7 @@ enum Writer { // Takes the lock a native transition takes, on a connection outside the // runtime's pool, as a transition from another server process would. -async fn hold_transition_lock(database: &TestDatabase) -> Writer { +pub(super) async fn hold_transition_lock(database: &TestDatabase) -> Writer { match database { TestDatabase::Sqlite(dir) => { let mut connection = sqlx::SqliteConnection::connect(&format!( @@ -56,7 +56,7 @@ async fn hold_transition_lock(database: &TestDatabase) -> Writer { } impl Writer { - async fn release(self) { + pub(super) async fn release(self) { match self { Self::Sqlite(mut connection) => { sqlx::raw_sql("ROLLBACK") From 7e4919313a0d5ac0512e22c5e181988a4ddb9902 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 07:50:39 +0000 Subject: [PATCH 06/42] Import the codec test names explicitly Co-Authored-By: Claude Opus 5.5 --- rust/src/runtime/transition_codec_tests.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/rust/src/runtime/transition_codec_tests.rs b/rust/src/runtime/transition_codec_tests.rs index becf90d8..9f02385d 100644 --- a/rust/src/runtime/transition_codec_tests.rs +++ b/rust/src/runtime/transition_codec_tests.rs @@ -1,9 +1,11 @@ //! Transitions that encode or decode while holding the transition lock use //! their own codec permit. Each test holds the read codec permit, as a long //! describe decode would, and requires the transition to finish anyway. -use super::*; +use super::{Result, Runtime}; use crate::codec::{Value as Payload, ValueCodec}; -use serde_json::json; +use axum::http::StatusCode; +use serde_json::{Value, json}; +use std::time::Duration; fn envelope(value: Payload) -> Value { json!({"codec":"avro","blob":ValueCodec::new().unwrap().encode(&value).unwrap()}) From 0ac1b8feacb57b38b7cbfdfa08cf0c8e8e92b3bd Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 07:56:11 +0000 Subject: [PATCH 07/42] Retry the PHP reference's lock-pressure refusals below the SDK transport The published PHP SQLite reference shares its database file with its maintenance loop and sometimes refuses a request with its documented HTTP 503 backend_lock_pressure reply: retryable, nothing committed, worker mutations fenced by task and attempt. Probe runs failed on these refusals for polls, worker operations and a Rust SDK actor's deregistration. - Parity probes now build their SDK clients through parityClient(), whose default transport is the SDK's own Psr18Transport over Guzzle with an innermost retry middleware. The transport keeps every capability the SDK checks (bounded requests and payload transfer), unlike the earlier wrapper. Observed-transport stacks add the same middleware innermost, so recording middleware sees only the final reply. - lockPressureOperationRetryDelay retries any non-poll request only on status 503, reason backend_lock_pressure and retryable true, within 15 retries, 30 s of waiting and the request's own timeout. Polls keep the stricter rule that no task was handed out. Bodies that cannot be replayed are never retried. Retries are reported once on stderr. - The Rust SDK actor retries its deregistration on the same refusal, under the same bounds. Co-Authored-By: Claude Opus 5.5 --- .../server-parity/activity-deadline-probe.php | 3 +- .../server-parity/admission-probe.php | 3 +- .../server-parity/cancellation-probe.php | 5 +- .../child-cancellation-probe.php | 3 +- .../server-parity/cooperative-probe.php | 6 +- .../server-parity/cooperative-restart.php | 7 +- .../rust/src/lock_pressure.rs | 94 ++++++++++++++++++ .../corrected-markers/rust/src/main.rs | 6 +- .../corrected-markers/rust/src/memo_author.rs | 5 +- .../lock-pressure-middleware.php | 95 +++++++++++++++++++ .../server-parity/lock-pressure-retry.php | 42 ++++++++ .../server-parity/namespace-probe.php | 4 +- .../server-parity/patch-deployment-probe.php | 6 +- scripts/conformance/server-parity/probe.php | 8 +- .../server-parity/sdk-direction-probe.php | 5 +- .../sdk-namespace-signal-probe.php | 5 +- .../sdk-query-direction-probe.php | 5 +- .../sdk-selected-run-signal-probe.php | 5 +- .../sdk-update-direction-probe.php | 5 +- .../server-parity/task-retry-probe.php | 3 +- .../server-parity/waiting-history-probe.php | 3 +- .../ServerParityLockPressureRetryTest.php | 24 +++++ 22 files changed, 312 insertions(+), 30 deletions(-) create mode 100644 scripts/conformance/server-parity/corrected-markers/rust/src/lock_pressure.rs create mode 100644 scripts/conformance/server-parity/lock-pressure-middleware.php diff --git a/scripts/conformance/server-parity/activity-deadline-probe.php b/scripts/conformance/server-parity/activity-deadline-probe.php index b4bd61df..4728f626 100644 --- a/scripts/conformance/server-parity/activity-deadline-probe.php +++ b/scripts/conformance/server-parity/activity-deadline-probe.php @@ -3,6 +3,7 @@ declare(strict_types=1); use DurableWorkflow\Client; +require_once __DIR__.'/lock-pressure-middleware.php'; require_once __DIR__.'/namespace-probe.php'; require_once __DIR__.'/activity-deadline-poll.php'; @@ -34,7 +35,7 @@ function activityDeadlineHttpObservation(array $fixture, array $options): array if (!preg_match('#^http://127\.0\.0\.1:[0-9]+$#', $url)) { throw new RuntimeException('Activity deadline recorder did not bind its local port'); } - $client = new Client($url, namespace:'default', token:getenv('DW_PARITY_TOKEN') ?: null); + $client = parityClient($url, namespace:'default', token:getenv('DW_PARITY_TOKEN') ?: null); $checkBudget = static function () use ($deadline): void { if (microtime(true) >= $deadline) { throw new RuntimeException('Original activity deadline observation expired'); diff --git a/scripts/conformance/server-parity/admission-probe.php b/scripts/conformance/server-parity/admission-probe.php index 0e681a1d..337bd956 100644 --- a/scripts/conformance/server-parity/admission-probe.php +++ b/scripts/conformance/server-parity/admission-probe.php @@ -8,6 +8,7 @@ use Workflow\V2\StartOptions; use Workflow\V2\WorkflowStub; use Workflow\WorkflowOptions; +require_once __DIR__.'/lock-pressure-middleware.php'; function parityAdmissionToken(string $auth): string { @@ -86,7 +87,7 @@ function finishHttpAdmission(Client $client, array $fixture, string $workflowId, throw $error; } finally { if (isset($definition['cleanup_auth'])) { - $cleanupClient = new Client($url, namespace: $namespace, token: parityAdmissionToken($definition['cleanup_auth'])); + $cleanupClient = parityClient($url, namespace: $namespace, token: parityAdmissionToken($definition['cleanup_auth'])); $state['cleanup_auth'] = $definition['cleanup_auth']; $state['cleanup'] = $cleanupClient->cancelWorkflow($peerId, $definition['cleanup_reason'], $peer->selectedRunId); } else { diff --git a/scripts/conformance/server-parity/cancellation-probe.php b/scripts/conformance/server-parity/cancellation-probe.php index 0292c6ab..98f31d05 100644 --- a/scripts/conformance/server-parity/cancellation-probe.php +++ b/scripts/conformance/server-parity/cancellation-probe.php @@ -17,6 +17,7 @@ use Workflow\V2\WorkflowStub; use Workflow\V2\StartOptions; use Workflow\WorkflowOptions; +require_once __DIR__.'/lock-pressure-middleware.php'; // Application callbacks use only the published APIs. These observations never // mutate engine rows or synthesize a claim, cancellation or terminal outcome. @@ -40,7 +41,7 @@ function cancellationReceipt(callable $submit): array function httpCancellationObservation(array $fixture, string $workflowId, string $namespace, string $queue, string $url): array { $polls = $completions = $activityPolls = $activityOutcomes = []; - $client = new Client($url, namespace: $namespace, token: getenv('DW_PARITY_TOKEN') ?: null, + $client = parityClient($url, namespace: $namespace, token: getenv('DW_PARITY_TOKEN') ?: null, transport: observedChildTransport($polls, $completions, $activityPolls, $activityOutcomes)); $phase = $fixture['immediate_cancellation']['phase']; $reason = $fixture['immediate_cancellation']['reason']; @@ -113,7 +114,7 @@ function httpCancellationObservation(array $fixture, string $workflowId, string $task['activity_attempt_id'], $task['lease_owner'], $fixture['input'])); } // A separate published Client reloads the same durable run and history. - $fresh = new Client($url, namespace: $namespace, token: getenv('DW_PARITY_TOKEN') ?: null); + $fresh = parityClient($url, namespace: $namespace, token: getenv('DW_PARITY_TOKEN') ?: null); $inspectionWorker = $workflowId.':inspection'; $fresh->registerWorker($inspectionWorker, $queue, [$fixture['workflow_type']], ['parity.v1.cancel_activity']); $remaining = ['workflow' => $fresh->pollWorkflowTask($inspectionWorker, $queue, 0), diff --git a/scripts/conformance/server-parity/child-cancellation-probe.php b/scripts/conformance/server-parity/child-cancellation-probe.php index b548be1b..7b8ee444 100644 --- a/scripts/conformance/server-parity/child-cancellation-probe.php +++ b/scripts/conformance/server-parity/child-cancellation-probe.php @@ -5,6 +5,7 @@ use DurableWorkflow\Client; use Workflow\V2\Models\WorkflowRun; use Workflow\V2\WorkflowStub; +require_once __DIR__.'/lock-pressure-middleware.php'; final class ChildCancellationProbeState { @@ -54,7 +55,7 @@ function finishHttpChildCancellation(Client $client, string $parentId, string $p $state['terminal_duplicate'] = cancellationReceipt(static fn (): array => $client->cancelWorkflow($id, 'post-parent-completion replacement', $runId)); $state['child_history_after_terminal_duplicate'] = httpHistory($client, $id, $runId); $state['parent_history_after_terminal_duplicate'] = httpHistory($client, $parentId, $parentRunId); - $fresh = new Client($url, namespace: $namespace, token: getenv('DW_PARITY_TOKEN') ?: null); + $fresh = parityClient($url, namespace: $namespace, token: getenv('DW_PARITY_TOKEN') ?: null); $state['fresh_child'] = $fresh->describeWorkflow($id, $runId)->raw; $state['fresh_child_history'] = httpHistory($fresh, $id, $runId); $state['fresh_parent'] = $fresh->describeWorkflow($parentId, $parentRunId)->raw; diff --git a/scripts/conformance/server-parity/cooperative-probe.php b/scripts/conformance/server-parity/cooperative-probe.php index cb2e19d8..b856b308 100644 --- a/scripts/conformance/server-parity/cooperative-probe.php +++ b/scripts/conformance/server-parity/cooperative-probe.php @@ -21,10 +21,12 @@ use Workflow\V2\WorkflowStub; use Workflow\V2\StartOptions; use Workflow\WorkflowOptions; +require_once __DIR__.'/lock-pressure-middleware.php'; function cooperativeObservedTransport(array &$traffic): Psr18Transport { $stack = HandlerStack::create(); + $stack->unshift(LockPressurePollRetries::middleware(), 'lock_pressure_poll_retry'); $stack->push(static function (callable $handler) use (&$traffic): callable { return static function (RequestInterface $request, array $options) use ($handler, &$traffic) { return $handler($request, $options)->then(static function (ResponseInterface $response) use ($request, &$traffic): ResponseInterface { @@ -75,7 +77,7 @@ function httpCooperativeObservation(array $fixture, string $workflowId, string $ { $traffic = []; $diagnostics = []; - $client = new Client($url, namespace: $namespace, token: getenv('DW_PARITY_TOKEN') ?: null, + $client = parityClient($url, namespace: $namespace, token: getenv('DW_PARITY_TOKEN') ?: null, transport: cooperativeObservedTransport($traffic), workerProtocolVersion: '1.20'); $phase = $fixture['cooperative_cancellation']['phase']; $reason = $fixture['cooperative_cancellation']['reason']; @@ -150,7 +152,7 @@ static function (string $event, array $details) use (&$handle, &$diagnostics, $c } catch (WorkflowCancelled $error) { $outcome = ['type' => $error::class, 'message' => $error->getMessage()]; } - $fresh = new Client($url, namespace: $namespace, token: getenv('DW_PARITY_TOKEN') ?: null, workerProtocolVersion: '1.20'); + $fresh = parityClient($url, namespace: $namespace, token: getenv('DW_PARITY_TOKEN') ?: null, workerProtocolVersion: '1.20'); return ['execution' => $execution->raw, 'workflow_id' => $execution->workflowId, 'run_id' => $execution->runId, 'workflow_type' => $execution->workflowType, 'namespace' => $execution->namespace, 'task_queue' => $execution->taskQueue, diff --git a/scripts/conformance/server-parity/cooperative-restart.php b/scripts/conformance/server-parity/cooperative-restart.php index 6fd9950b..2e4b81d3 100644 --- a/scripts/conformance/server-parity/cooperative-restart.php +++ b/scripts/conformance/server-parity/cooperative-restart.php @@ -8,6 +8,7 @@ use DurableWorkflow\Worker\CancellationDelivery; require __DIR__.'/vendor/autoload.php'; +require_once __DIR__.'/lock-pressure-middleware.php'; require __DIR__.'/observation-values.php'; require __DIR__.'/cooperative-probe.php'; @@ -56,7 +57,7 @@ function cooperativeEvent(array $events, string $type): array $workflowId = 'root-restart-'.$outcome; $queue = 'root-restart-'.$outcome; $traffic = []; - $client = new Client($url, namespace: 'default', token: getenv('DW_PARITY_TOKEN') ?: null, + $client = parityClient($url, namespace: 'default', token: getenv('DW_PARITY_TOKEN') ?: null, transport: cooperativeObservedTransport($traffic), workerProtocolVersion: '1.20'); $worker = $handle = $accepted = null; $guard = microtime(true) + 20; @@ -118,7 +119,7 @@ static function (string $event) use (&$handle, $client, $fixture, $workflowId, $ foreach (['deadline_expired', 'completed'] as $outcome) { $receipt = $receipts[$outcome]; $traffic = []; - $client = new Client($url, namespace: 'default', token: getenv('DW_PARITY_TOKEN') ?: null, + $client = parityClient($url, namespace: 'default', token: getenv('DW_PARITY_TOKEN') ?: null, transport: cooperativeObservedTransport($traffic), workerProtocolVersion: '1.20'); $handle = $client->workflowHandle($receipt['workflow_id'], $receipt['run_id']); $before = httpHistory($client, $receipt['workflow_id'], $receipt['run_id']); @@ -188,7 +189,7 @@ static function () use (&$worker, $handle, $guard): float { } $duplicate = $client->requestWorkflowCancellation($receipt['workflow_id'], 'post-terminal replacement', 3600, $receipt['run_id']); assertCooperativeRecovery($duplicate['duplicate'] === true && $duplicate['cancellation_request']['cleanup_deadline_at'] === $receipt['context']['cleanup_deadline_at'], 'Terminal duplicate changed the original budget.'); - $fresh = new Client($url, namespace: 'default', token: getenv('DW_PARITY_TOKEN') ?: null, workerProtocolVersion: '1.20'); + $fresh = parityClient($url, namespace: 'default', token: getenv('DW_PARITY_TOKEN') ?: null, workerProtocolVersion: '1.20'); assertCooperativeRecovery(sameCooperativeValue($events, httpHistory($fresh, $receipt['workflow_id'], $receipt['run_id'])), 'Late calls or fresh reads changed terminal history.'); try { $fresh->workflowHandle($receipt['workflow_id'], $receipt['run_id'])->resultOfSelectedRun(1); diff --git a/scripts/conformance/server-parity/corrected-markers/rust/src/lock_pressure.rs b/scripts/conformance/server-parity/corrected-markers/rust/src/lock_pressure.rs new file mode 100644 index 00000000..5f9f9f9b --- /dev/null +++ b/scripts/conformance/server-parity/corrected-markers/rust/src/lock_pressure.rs @@ -0,0 +1,94 @@ +//! The published PHP SQLite reference can refuse a worker operation with its +//! documented HTTP 503 `backend_lock_pressure` reply and `retryable: true` +//! after its transaction failed, so nothing was committed. Only that refusal is +//! retried, within 15 retries and 30 seconds of waiting, as the PHP probes' +//! transport does (`../lock-pressure-retry.php`). Every other outcome is +//! returned unchanged. +use durable_workflow::{Error, Result}; +use serde_json::Value; +use std::{future::Future, time::Duration}; + +const MAX_RETRIES: u32 = 15; +const MAX_WAIT: Duration = Duration::from_secs(30); + +fn delay_for(status: u16, body: &str, retries: u32, waited: Duration) -> Option { + if status != 503 || retries >= MAX_RETRIES { + return None; + } + let body: Value = serde_json::from_str(body).ok()?; + if body.get("reason")?.as_str()? != "backend_lock_pressure" + || body.get("retryable") != Some(&Value::Bool(true)) + { + return None; + } + let seconds = match body.get("retry_after_seconds") { + None => 1.0, + Some(value) => value.as_f64()?, + }; + let delay = Duration::from_secs_f64(seconds.clamp(0.1, 5.0)); + (waited + delay <= MAX_WAIT).then_some(delay) +} + +/// Runs `call`, repeating it only after the documented lock-pressure refusal. +pub async fn retrying(what: &str, mut call: F) -> Result +where + F: FnMut() -> Fut, + Fut: Future>, +{ + let (mut retries, mut waited) = (0, Duration::ZERO); + loop { + let error = match call().await { + Err(error) => error, + done => return done, + }; + let delay = match &error { + Error::Http { status, body } => delay_for(status.as_u16(), body, retries, waited), + _ => None, + }; + let Some(delay) = delay else { + return Err(error); + }; + eprintln!("{what}: retrying backend_lock_pressure refusal after {delay:?}"); + tokio::time::sleep(delay).await; + waited += delay; + retries += 1; + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const REFUSAL: &str = r#"{"reason":"backend_lock_pressure","operation":"destroy","outcome":"deferred","retryable":true,"retry_after_seconds":1}"#; + + #[test] + fn documented_refusal_waits_for_the_advertised_interval() { + assert_eq!(delay_for(503, REFUSAL, 0, Duration::ZERO), Some(Duration::from_secs(1))); + let long = REFUSAL.replace(r#""retry_after_seconds":1"#, r#""retry_after_seconds":60"#); + assert_eq!(delay_for(503, &long, 0, Duration::ZERO), Some(Duration::from_secs(5))); + let absent = REFUSAL.replace(r#","retry_after_seconds":1"#, ""); + assert_eq!(delay_for(503, &absent, 0, Duration::ZERO), Some(Duration::from_secs(1))); + } + + #[test] + fn other_statuses_reasons_and_bodies_are_returned() { + for (status, body) in [ + (409, REFUSAL.to_owned()), + (500, REFUSAL.to_owned()), + (503, REFUSAL.replace("backend_lock_pressure", "backend_unavailable")), + (503, REFUSAL.replace(r#""retryable":true"#, r#""retryable":false"#)), + (503, REFUSAL.replace(r#","retryable":true"#, "")), + (503, REFUSAL.replace(r#""retry_after_seconds":1"#, r#""retry_after_seconds":"soon""#)), + (503, "not json".to_owned()), + ] { + assert_eq!(delay_for(status, &body, 0, Duration::ZERO), None, "{status} {body}"); + } + } + + #[test] + fn retries_stop_at_the_attempt_and_wait_budgets() { + assert_eq!(delay_for(503, REFUSAL, 15, Duration::ZERO), None); + assert_eq!(delay_for(503, REFUSAL, 2, Duration::from_millis(29_500)), None); + assert_eq!(delay_for(503, REFUSAL, 14, Duration::from_secs(29)), Some(Duration::from_secs(1))); + } +} diff --git a/scripts/conformance/server-parity/corrected-markers/rust/src/main.rs b/scripts/conformance/server-parity/corrected-markers/rust/src/main.rs index c5f7bdb7..3a1d863c 100644 --- a/scripts/conformance/server-parity/corrected-markers/rust/src/main.rs +++ b/scripts/conformance/server-parity/corrected-markers/rust/src/main.rs @@ -11,6 +11,7 @@ use durable_workflow::{ use serde_json::{json, Value}; mod legacy_range; +mod lock_pressure; mod memo_author; const WORKFLOW: &str = "parity.v1.corrected_markers"; @@ -221,7 +222,10 @@ async fn main() -> Result<()> { } }) .await; - let deregistered = client.deregister_worker_registration(&worker_id).await; + let deregistered = lock_pressure::retrying("deregister", || { + client.deregister_worker_registration(&worker_id) + }) + .await; let result = match outcome { Ok(result) => result, Err(_) => Err(Error::WorkerLoop( diff --git a/scripts/conformance/server-parity/corrected-markers/rust/src/memo_author.rs b/scripts/conformance/server-parity/corrected-markers/rust/src/memo_author.rs index 8520ac90..d2c4c04c 100644 --- a/scripts/conformance/server-parity/corrected-markers/rust/src/memo_author.rs +++ b/scripts/conformance/server-parity/corrected-markers/rust/src/memo_author.rs @@ -90,7 +90,10 @@ pub async fn run(args: &[String]) -> Result<()> { }) .await .map_err(|_| Error::WorkerLoop("Original memo phase observation expired".into()))?; - let retired = client.deregister_worker_registration(&worker_id).await; + let retired = super::lock_pressure::retrying("deregister", || { + client.deregister_worker_registration(&worker_id) + }) + .await; outcome?; retired?; println!( diff --git a/scripts/conformance/server-parity/lock-pressure-middleware.php b/scripts/conformance/server-parity/lock-pressure-middleware.php new file mode 100644 index 00000000..11cf1d5e --- /dev/null +++ b/scripts/conformance/server-parity/lock-pressure-middleware.php @@ -0,0 +1,95 @@ + */ + public static array $retries = []; + private static bool $reporting = false; + + public static function middleware(): callable + { + if (!self::$reporting) { + self::$reporting = true; + register_shutdown_function(static function (): void { + if (self::$retries !== []) { + fwrite(STDERR, json_encode(['lock_pressure_poll_retries' => self::$retries], JSON_THROW_ON_ERROR).PHP_EOL); + } + }); + } + + return static fn (callable $handler): callable => static function (RequestInterface $request, array $options) use ($handler) { + $started = microtime(true); + $budget = (float) ($options['timeout'] ?? 0); + $waited = 0.0; + $count = 0; + while (true) { + try { + $response = $handler($request, $options)->wait(); + } catch (\Throwable $error) { + return Create::rejectionFor($error); + } + // A request body that cannot be replayed exactly is never retried. + if (!$response instanceof ResponseInterface || $response->getStatusCode() !== 503 + || (!$request->getBody()->isSeekable() && $request->getBody()->getSize() !== 0)) { + return Create::promiseFor($response); + } + $text = (string) $response->getBody(); + $response = $response->withBody(Utils::streamFor($text)); + $decoded = json_decode($text, true); + $delay = lockPressureOperationRetryDelay($request->getMethod(), (string) $request->getUri(), 503, + is_array($decoded) ? $decoded : null, $count, $waited); + if ($delay === null || ($budget > 0 && microtime(true) - $started + $delay >= $budget)) { + return Create::promiseFor($response); + } + self::$retries[] = ['path' => $request->getUri()->getPath(), 'retry_after_seconds' => $delay]; + usleep((int) round($delay * 1_000_000)); + $waited += $delay; + $count++; + if ($request->getBody()->isSeekable()) { + $request->getBody()->rewind(); + } + } + }; + } + + /** The SDK's default transport, with this retry innermost. */ + public static function transport(): Psr18Transport + { + $stack = HandlerStack::create(); + $stack->unshift(self::middleware(), 'lock_pressure_poll_retry'); + + return new Psr18Transport(client: new GuzzleClient(['http_errors' => false, 'handler' => $stack])); + } +} + +/** An SDK client for a parity probe. A caller-supplied transport is used unchanged. */ +function parityClient(mixed ...$arguments): Client +{ + $arguments['transport'] ??= LockPressurePollRetries::transport(); + + return new Client(...$arguments); +} diff --git a/scripts/conformance/server-parity/lock-pressure-retry.php b/scripts/conformance/server-parity/lock-pressure-retry.php index fa7c28af..112299d7 100644 --- a/scripts/conformance/server-parity/lock-pressure-retry.php +++ b/scripts/conformance/server-parity/lock-pressure-retry.php @@ -38,3 +38,45 @@ function lockPressurePollRetryDelay( return $waitedSeconds + $delay <= $maxWaitSeconds ? $delay : null; } + +/** + * Decide whether any other request refused for backend lock pressure may be + * retried. + * + * Server returns this refusal as HTTP 503 with `reason: backend_lock_pressure` + * and `retryable: true` only after its transaction failed with a concurrency + * error, so nothing was committed. Worker mutations remain fenced by their + * task and attempt identifiers (`outcome: deferred`), so repeating the request + * cannot record a second outcome. Polls keep lockPressurePollRetryDelay's + * stricter check that no task was handed out. + * + * Returns the delay in seconds before the next attempt, or null to return the + * refusal to the caller unchanged. + */ +function lockPressureOperationRetryDelay( + string $method, + string $uri, + ?int $status, + mixed $response, + int $retriesSoFar, + float $waitedSeconds, + int $maxRetries = 15, + float $maxWaitSeconds = 30.0, +): ?float { + $path = parse_url($uri, PHP_URL_PATH); + if (! is_string($path) || str_ends_with($path, '/poll')) { + return lockPressurePollRetryDelay($method, $uri, $status, $response, $retriesSoFar, $waitedSeconds, + $maxRetries, $maxWaitSeconds); + } + if ($status !== 503 || ! is_array($response) || ($response['reason'] ?? null) !== 'backend_lock_pressure' + || ($response['retryable'] ?? null) !== true || $retriesSoFar >= $maxRetries) { + return null; + } + $retryAfter = $response['retry_after_seconds'] ?? 1; + if (! is_int($retryAfter) && ! is_float($retryAfter)) { + return null; + } + $delay = min(max((float) $retryAfter, 0.1), 5.0); + + return $waitedSeconds + $delay <= $maxWaitSeconds ? $delay : null; +} diff --git a/scripts/conformance/server-parity/namespace-probe.php b/scripts/conformance/server-parity/namespace-probe.php index 7422ec26..4f756671 100644 --- a/scripts/conformance/server-parity/namespace-probe.php +++ b/scripts/conformance/server-parity/namespace-probe.php @@ -20,6 +20,7 @@ use Workflow\V2\WorkflowStub; use Workflow\WorkflowOptions; use Workflow\Serializers\Serializer; +require_once __DIR__.'/lock-pressure-middleware.php'; function namespaceReceipt(string $id, callable $request): array { @@ -45,6 +46,7 @@ function namespaceHttpRun(Client $client, string $workflowId, string $runId): ar function namespaceTransport(array &$receipts, ?Closure $beforeCompletion = null): Psr18Transport { $stack = HandlerStack::create(); + $stack->unshift(LockPressurePollRetries::middleware(), 'lock_pressure_poll_retry'); $stack->push(static function (callable $handler) use (&$receipts, $beforeCompletion): callable { return static function (RequestInterface $request, array $options) use ($handler, &$receipts, $beforeCompletion) { $path = $request->getUri()->getPath(); @@ -98,7 +100,7 @@ function finishHttpNamespaces(Client $admin, array $fixture, string $workflowId, 'before' => $before, 'after' => namespaceHttpRun($clients[0], $ids[0], $handles[0]->selectedRunId)]; }; foreach ($names as $index => $name) { - $clients[] = new Client($url, namespace: $name, token: getenv('DW_PARITY_TOKEN') ?: null, + $clients[] = parityClient($url, namespace: $name, token: getenv('DW_PARITY_TOKEN') ?: null, transport: namespaceTransport($state['worker_receipts'][$index], $index === 0 ? $beforeCompletion : null)); $state['registrations'][] = $clients[$index]->registerWorker($state['worker_id'], $queue, [$definition['peer_workflow_type']], ['parity.v1.echo_activity']); diff --git a/scripts/conformance/server-parity/patch-deployment-probe.php b/scripts/conformance/server-parity/patch-deployment-probe.php index 03294413..204b073c 100644 --- a/scripts/conformance/server-parity/patch-deployment-probe.php +++ b/scripts/conformance/server-parity/patch-deployment-probe.php @@ -20,6 +20,7 @@ use Workflow\V2\StartOptions; use Workflow\V2\WorkflowStub; use Workflow\WorkflowOptions; +require_once __DIR__.'/lock-pressure-middleware.php'; final class PatchDeploymentState { @@ -100,6 +101,7 @@ function patchHttpPhase(array $fixture, array $options): array $worker = $handle = null; $requests = $diagnostics = []; $stack = HandlerStack::create(); + $stack->unshift(LockPressurePollRetries::middleware(), 'lock_pressure_poll_retry'); $stack->push(static function (callable $handler) use (&$worker, &$requests, $phase, $fixture): callable { return static function (RequestInterface $request, array $transportOptions) use ($handler, &$worker, &$requests, $phase, $fixture) { return $handler($request, $transportOptions)->then(static function (ResponseInterface $response) use ($request, &$worker, &$requests, $phase, $fixture): ResponseInterface { @@ -126,7 +128,7 @@ function patchHttpPhase(array $fixture, array $options): array }); }; }); - $client = new Client($options['url'], namespace: 'default', token: getenv('DW_PARITY_TOKEN') ?: null, + $client = parityClient($options['url'], namespace: 'default', token: getenv('DW_PARITY_TOKEN') ?: null, transport: new Psr18Transport(client: new GuzzleClient(['http_errors' => false, 'handler' => $stack]))); if ($phase === 'replacement') { $handle = $client->workflowHandle($workflowId, $options['run-id']); @@ -279,7 +281,7 @@ function patchPublishedSdkPhase(array $fixture, array $options): array if ($result['language'] !== $consumer['language'] || $result['sdk_version'] !== $consumer['version']) { throw new RuntimeException('Actual published worker differs from the selected consumer.'); } - $client = new Client($options['url'], namespace: 'default', token: getenv('DW_PARITY_TOKEN') ?: null); + $client = parityClient($options['url'], namespace: 'default', token: getenv('DW_PARITY_TOKEN') ?: null); $runId = $original ? ($result['run_id'] ?? null) : $options['run-id']; if (! is_string($runId) || $runId === '') { throw new RuntimeException('Actual published author returned no original run identity.'); diff --git a/scripts/conformance/server-parity/probe.php b/scripts/conformance/server-parity/probe.php index d63bc0f8..9c078e33 100644 --- a/scripts/conformance/server-parity/probe.php +++ b/scripts/conformance/server-parity/probe.php @@ -64,12 +64,14 @@ require __DIR__.'/observation-values.php'; require __DIR__.'/probe-failure.php'; +require_once __DIR__.'/lock-pressure-middleware.php'; function observedChildTransport(array &$polls, array &$completions, array &$activityPolls, array &$activityOutcomes, ?Closure $afterCompletion = null, ?array &$scheduleReceipts = null): Psr18Transport { // Observe real published transport I/O. No response or request is changed, // and authentication headers are never retained. $stack = HandlerStack::create(); + $stack->unshift(LockPressurePollRetries::middleware(), 'lock_pressure_poll_retry'); $stack->push(static function (callable $handler) use (&$polls, &$completions, &$activityPolls, &$activityOutcomes, $afterCompletion, &$scheduleReceipts): callable { return static function (RequestInterface $request, array $options) use ($handler, &$polls, &$completions, &$activityPolls, &$activityOutcomes, $afterCompletion, &$scheduleReceipts) { return $handler($request, $options)->then(static function (ResponseInterface $response) use ($request, &$polls, &$completions, &$activityPolls, &$activityOutcomes, $afterCompletion, &$scheduleReceipts): ResponseInterface { @@ -114,7 +116,7 @@ function observedChildTransport(array &$polls, array &$completions, array &$acti // client in an independent PHP process so this probe's worker keeps polling. if ($mode === 'query') { try { - $client = new Client($options['url'], namespace: $namespace, token: getenv('DW_PARITY_TOKEN') ?: null); + $client = parityClient($options['url'], namespace: $namespace, token: getenv('DW_PARITY_TOKEN') ?: null); $result = $client->queryWorkflow($workflowId, $fixture['query_name'], $fixture['query_arguments'], $options['run-id']); echo json_encode(['result' => $result, 'typed_result' => typedValue($result)], JSON_THROW_ON_ERROR | JSON_UNESCAPED_UNICODE)."\n"; exit(0); @@ -142,9 +144,9 @@ function httpObservation(array $fixture, string $workflowId, string $namespace, } : null; $transport = isset($fixture['child_count']) || isset($fixture['child_cancellation']) || isset($fixture['retry_policy']) || isset($fixture['schedule']) ? observedChildTransport($workflowPolls, $workflowCompletions, $activityPolls, $activityOutcomes, $afterCompletion, $scheduleReceipts) : null; - $client = new Client($url, namespace: $namespace, transport: $transport, token: getenv('DW_PARITY_TOKEN') ?: null); + $client = parityClient($url, namespace: $namespace, transport: $transport, token: getenv('DW_PARITY_TOKEN') ?: null); $workerClient = ($fixture['auth_profile'] ?? null) === 'role_tokens' - ? new Client($url, namespace: $namespace, token: parityAdmissionToken('worker')) : $client; + ? parityClient($url, namespace: $namespace, token: parityAdmissionToken('worker')) : $client; $handle = null; $deliveries = []; $queries = []; diff --git a/scripts/conformance/server-parity/sdk-direction-probe.php b/scripts/conformance/server-parity/sdk-direction-probe.php index b7fa2df5..1a582e4a 100644 --- a/scripts/conformance/server-parity/sdk-direction-probe.php +++ b/scripts/conformance/server-parity/sdk-direction-probe.php @@ -5,6 +5,7 @@ use DurableWorkflow\Client; use DurableWorkflow\Worker; use DurableWorkflow\Worker\WorkflowContext; +require_once __DIR__.'/lock-pressure-middleware.php'; function sdkDirectionResponseSummary(mixed $response, bool $workflowHistory = false): mixed { @@ -73,7 +74,7 @@ function sdkDirectionHttpObservation(array $fixture, array $options, ?Closure $a $spec = $fixture[$directionKey]; $childWorkflowId = $childRunId = ''; $namespace = $fixture['namespace'] ?? 'default'; - $client = new Client($options['url'], namespace: $namespace, token: getenv('DW_PARITY_TOKEN') ?: null); + $client = parityClient($options['url'], namespace: $namespace, token: getenv('DW_PARITY_TOKEN') ?: null); $deadline = $scenarioDeadline ?? microtime(true) + 35; $observerStart = null; try { @@ -81,7 +82,7 @@ function sdkDirectionHttpObservation(array $fixture, array $options, ?Closure $a if (! preg_match('#^http://127\\.0\\.0\\.1:[0-9]+$#', $url)) { throw new RuntimeException('SDK observer did not bind its local port.'); } - $startClient = $signalDirection ? new Client($url, namespace: $namespace, token: getenv('DW_PARITY_TOKEN') ?: null) : null; + $startClient = $signalDirection ? parityClient($url, namespace: $namespace, token: getenv('DW_PARITY_TOKEN') ?: null) : null; foreach ($spec['phase_order'] as $phase) { $language = $signalDirection ? $spec[in_array($phase, ['signal-1', 'signal-2'], true) ? 'client' : 'workflow'] : ($timerAuthor ? $spec['language'] : ($childDirection ? $spec[$phase === 'child' ? 'child' : 'parent'] diff --git a/scripts/conformance/server-parity/sdk-namespace-signal-probe.php b/scripts/conformance/server-parity/sdk-namespace-signal-probe.php index 082f7364..1f7e411d 100644 --- a/scripts/conformance/server-parity/sdk-namespace-signal-probe.php +++ b/scripts/conformance/server-parity/sdk-namespace-signal-probe.php @@ -11,6 +11,7 @@ use Workflow\V2\StartOptions; use Workflow\V2\WorkflowStub; use Workflow\WorkflowOptions; +require_once __DIR__.'/lock-pressure-middleware.php'; function namespaceSignalRunFixture(array $fixture, bool $peer): array { @@ -158,7 +159,7 @@ function namespaceSignalHttpObservation(array $fixture, array $options): array throw new RuntimeException('Namespace control observer did not bind its local port.'); } $checkpoint('client_setup'); - $admin = new Client($url, namespace: 'default', token: getenv('DW_PARITY_TOKEN') ?: null); + $admin = parityClient($url, namespace: 'default', token: getenv('DW_PARITY_TOKEN') ?: null); foreach ($fixtures as $index => $runFixture) { try { $checkpoint('namespace_describe', $runFixture['namespace']); @@ -175,7 +176,7 @@ function namespaceSignalHttpObservation(array $fixture, array $options): array $state['administration'][] = ['created' => $created, 'namespace' => $namespace->raw]; $paths[$index] = tempnam(sys_get_temp_dir(), 'parity-namespace-fixture-'); file_put_contents($paths[$index], json_encode($runFixture, JSON_THROW_ON_ERROR | JSON_UNESCAPED_UNICODE)); - $clients[$index] = new Client($options['url'], namespace: $runFixture['namespace'], token: getenv('DW_PARITY_TOKEN') ?: null); + $clients[$index] = parityClient($options['url'], namespace: $runFixture['namespace'], token: getenv('DW_PARITY_TOKEN') ?: null); $start = microtime(true); $deadline = $start + 35; $state['run_budgets'][$index] = ['started_at_unix' => $start, 'deadline_at_unix' => $deadline]; diff --git a/scripts/conformance/server-parity/sdk-query-direction-probe.php b/scripts/conformance/server-parity/sdk-query-direction-probe.php index 18dce942..1beb8c23 100644 --- a/scripts/conformance/server-parity/sdk-query-direction-probe.php +++ b/scripts/conformance/server-parity/sdk-query-direction-probe.php @@ -4,6 +4,7 @@ use Composer\InstalledVersions; use DurableWorkflow\Client; +require_once __DIR__.'/lock-pressure-middleware.php'; function sdkQueryActorOpen(string $language, string $phase, array $fixture, array $options, string $runId, float $deadline): array { @@ -242,7 +243,7 @@ function sdkQueryDirectionHttpObservation(array $fixture, array $options): array throw new RuntimeException('SDK query direction requires its evidence directory.'); } $spec = $fixture['sdk_query_direction']; - $client = new Client($options['url'], namespace: 'default', token: getenv('DW_PARITY_TOKEN') ?: null); + $client = parityClient($options['url'], namespace: 'default', token: getenv('DW_PARITY_TOKEN') ?: null); $deadline = microtime(true) + 35; $runId = ''; $phases = $actors = []; @@ -262,7 +263,7 @@ function sdkQueryDirectionHttpObservation(array $fixture, array $options): array if ($request['method'] === 'POST' && $request['path'] === '/api/worker/register' && $request['status'] === 201 && ($request['response']['registered'] ?? null) === true && ($request['request']['worker_id'] ?? null) === $options['workflow-id'].':original') { - $startClient = new Client($author['url'], namespace: 'default', token: getenv('DW_PARITY_TOKEN') ?: null); + $startClient = parityClient($author['url'], namespace: 'default', token: getenv('DW_PARITY_TOKEN') ?: null); $runId = $startClient->startWorkflow($fixture['workflow_type'], $options['workflow-id'], $spec['task_queue'], [$fixture['input'], $fixture['signal_count']])->selectedRunId; $observerStart = ['role' => 'observer', 'language' => 'php', 'pid' => getmypid(), diff --git a/scripts/conformance/server-parity/sdk-selected-run-signal-probe.php b/scripts/conformance/server-parity/sdk-selected-run-signal-probe.php index a97a41a0..cd11ce1c 100644 --- a/scripts/conformance/server-parity/sdk-selected-run-signal-probe.php +++ b/scripts/conformance/server-parity/sdk-selected-run-signal-probe.php @@ -10,6 +10,7 @@ use Workflow\V2\StartOptions; use Workflow\V2\WorkflowStub; use Workflow\WorkflowOptions; +require_once __DIR__.'/lock-pressure-middleware.php'; require __DIR__.'/selected-run-signal-state.php'; @@ -70,7 +71,7 @@ function selectedRunSignalHttpObservation(array $fixture, array $options): array if (! preg_match('#^http://127\.0\.0\.1:[0-9]+$#', $url)) { throw new RuntimeException('Selected-run control observer failed to bind.'); } - $admin = new Client($url, namespace: 'default', token: getenv('DW_PARITY_TOKEN') ?: null); + $admin = parityClient($url, namespace: 'default', token: getenv('DW_PARITY_TOKEN') ?: null); foreach ($fixtures as $index => $runFixture) { $checkpoint('namespace_setup_'.$index); try { @@ -84,7 +85,7 @@ function selectedRunSignalHttpObservation(array $fixture, array $options): array $state['administration'][] = ['created' => $created, 'namespace' => $namespace->raw]; $paths[$index] = tempnam(sys_get_temp_dir(), 'parity-selected-run-'); file_put_contents($paths[$index], json_encode($runFixture, JSON_THROW_ON_ERROR | JSON_UNESCAPED_UNICODE)); - $clients[$index] = new Client($options['url'], namespace: $runFixture['namespace'], token: getenv('DW_PARITY_TOKEN') ?: null); + $clients[$index] = parityClient($options['url'], namespace: $runFixture['namespace'], token: getenv('DW_PARITY_TOKEN') ?: null); $start = microtime(true); $deadline = $start + 35; $state['run_budgets'][$index] = ['started_at_unix' => $start, 'deadline_at_unix' => $deadline]; diff --git a/scripts/conformance/server-parity/sdk-update-direction-probe.php b/scripts/conformance/server-parity/sdk-update-direction-probe.php index 9fe3f7bb..e8ce522c 100644 --- a/scripts/conformance/server-parity/sdk-update-direction-probe.php +++ b/scripts/conformance/server-parity/sdk-update-direction-probe.php @@ -4,6 +4,7 @@ use Composer\InstalledVersions; use DurableWorkflow\Client; +require_once __DIR__.'/lock-pressure-middleware.php'; function sdkUpdateActorOpen(string $language, string $phase, array $fixture, array $options, string $runId, float $deadline): array { @@ -78,7 +79,7 @@ function sdkUpdateDirectionHttpObservation(array $fixture, array $options): arra throw new RuntimeException('SDK update direction requires its evidence directory.'); } $spec = $fixture['sdk_update_direction']; - $client = new Client($options['url'], namespace: 'default', token: getenv('DW_PARITY_TOKEN') ?: null); + $client = parityClient($options['url'], namespace: 'default', token: getenv('DW_PARITY_TOKEN') ?: null); $deadline = microtime(true) + 35; $workflowId = $options['workflow-id']; $runId = ''; @@ -113,7 +114,7 @@ function sdkUpdateDirectionHttpObservation(array $fixture, array $options): arra if ($request['method'] === 'POST' && $request['path'] === '/api/worker/register' && $request['status'] === 201 && ($request['response']['registered'] ?? null) === true && ($request['request']['worker_id'] ?? null) === $workflowId.':original') { - $start = new Client($actor['url'], namespace: 'default', token: getenv('DW_PARITY_TOKEN') ?: null); + $start = parityClient($actor['url'], namespace: 'default', token: getenv('DW_PARITY_TOKEN') ?: null); $runId = $start->startWorkflow($fixture['workflow_type'], $workflowId, $spec['task_queue'], [$fixture['input'], $fixture['signal_count']])->selectedRunId; $observerStart = ['role' => 'observer', 'language' => 'php', 'pid' => getmypid(), diff --git a/scripts/conformance/server-parity/task-retry-probe.php b/scripts/conformance/server-parity/task-retry-probe.php index 42c81935..fa05a5fd 100644 --- a/scripts/conformance/server-parity/task-retry-probe.php +++ b/scripts/conformance/server-parity/task-retry-probe.php @@ -3,12 +3,13 @@ declare(strict_types=1); use DurableWorkflow\Client; +require_once __DIR__.'/lock-pressure-middleware.php'; function finishHttpTaskRetry(Client $admin, array $fixture, string $workflowId, string $queue, string $url): array { $definition = $fixture['workflow_task_retry']; $requests = []; - $client = new Client($url, namespace: 'default', token: getenv('DW_PARITY_TOKEN') ?: null, + $client = parityClient($url, namespace: 'default', token: getenv('DW_PARITY_TOKEN') ?: null, transport: namespaceTransport($requests)); $workerId = $workflowId.'-retry-worker'; $peerId = $workflowId.$definition['peer_suffix']; diff --git a/scripts/conformance/server-parity/waiting-history-probe.php b/scripts/conformance/server-parity/waiting-history-probe.php index 6d21e002..26d4efd9 100644 --- a/scripts/conformance/server-parity/waiting-history-probe.php +++ b/scripts/conformance/server-parity/waiting-history-probe.php @@ -3,12 +3,13 @@ declare(strict_types=1); use DurableWorkflow\Client; +require_once __DIR__.'/lock-pressure-middleware.php'; function finishHttpWaitingHistory(Client $admin, array $fixture, string $workflowId, string $queue, string $url): array { $definition = $fixture['waiting_for_history']; $requests = []; - $client = new Client($url, namespace: 'default', token: getenv('DW_PARITY_TOKEN') ?: null, + $client = parityClient($url, namespace: 'default', token: getenv('DW_PARITY_TOKEN') ?: null, transport: namespaceTransport($requests)); $workerId = $workflowId.'-waiting-worker'; $peerId = $workflowId.$definition['peer_suffix']; diff --git a/tests/Unit/ServerParityLockPressureRetryTest.php b/tests/Unit/ServerParityLockPressureRetryTest.php index e29e3beb..849e1b31 100644 --- a/tests/Unit/ServerParityLockPressureRetryTest.php +++ b/tests/Unit/ServerParityLockPressureRetryTest.php @@ -50,6 +50,30 @@ public function test_other_requests_statuses_and_bodies_are_never_retried(): voi } } + public function test_documented_operation_refusals_are_retried_and_polls_keep_their_rules(): void + { + $deferred = ['reason' => 'backend_lock_pressure', 'outcome' => 'deferred', 'retryable' => true, 'retry_after_seconds' => 1]; + $complete = 'http://parity-server:8080/api/worker/workflow-tasks/task/complete'; + $start = 'http://parity-server:8080/api/workflows'; + self::assertSame(1.0, \lockPressureOperationRetryDelay('POST', $complete, 503, $deferred, 0, 0.0)); + self::assertSame(1.0, \lockPressureOperationRetryDelay('POST', $start, 503, + ['reason' => 'backend_lock_pressure', 'retryable' => true, 'retry_after_seconds' => 1], 0, 0.0)); + self::assertSame(1.0, \lockPressureOperationRetryDelay('POST', self::POLL, 503, $this->refusal(), 0, 0.0)); + foreach ([ + [$complete, 503, array_replace($deferred, ['retryable' => false])], + [$complete, 503, array_diff_key($deferred, ['retryable' => true])], + [$complete, 503, array_replace($deferred, ['reason' => 'backend_unavailable'])], + [$complete, 409, $deferred], + [$complete, 503, null], + // A poll refusal still needs task null, even through this entry point. + [self::POLL, 503, $deferred], + ] as $index => [$uri, $status, $response]) { + self::assertNull(\lockPressureOperationRetryDelay('POST', $uri, $status, $response, 0, 0.0), (string) $index); + } + self::assertNull(\lockPressureOperationRetryDelay('POST', $complete, 503, $deferred, 15, 0.0)); + self::assertNull(\lockPressureOperationRetryDelay('POST', $complete, 503, $deferred, 2, 29.5)); + } + public function test_retries_stop_at_the_attempt_and_wait_budgets(): void { self::assertNull(\lockPressurePollRetryDelay('POST', self::POLL, 503, $this->refusal(), 15, 0.0)); From e737397b72124d192fcdc0b007d73cea0c95fb79 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 08:02:08 +0000 Subject: [PATCH 08/42] Design the backup-first native takeover The storage audit lists the takeover gates, and the inspection commands keep their backup, quiescence, compatibility and takeover flags false. This note proposes what turns each one true: - a fingerprint of the database's complete logical contents, compared between the stopped source and a restored backup, then checked again inside the takeover's own exclusive transaction before the marker is written; - per-backend writer exclusion and atomicity, with a marker step journal where DDL commits implicitly; - resumption of an interrupted takeover from the marker, refusal of a different fingerprint, and the per-backend upgrade fixtures and checks. It records what is not yet known, such as MySQL's exclusion method. Co-Authored-By: Claude Opus 5.5 --- docs/rust-server-storage-audit.md | 4 +- docs/rust-server-upgrade-takeover.md | 124 +++++++++++++++++++++++++++ 2 files changed, 127 insertions(+), 1 deletion(-) create mode 100644 docs/rust-server-upgrade-takeover.md diff --git a/docs/rust-server-storage-audit.md b/docs/rust-server-storage-audit.md index b1cfd921..cd97df38 100644 --- a/docs/rust-server-storage-audit.md +++ b/docs/rust-server-storage-audit.md @@ -96,5 +96,7 @@ and takeover flag stays false. There is no mixed PHP/Rust fleet or PHP rollback after upgrade. Restoring a pre-upgrade backup is a separate recovery action with its own data boundary. -All gates above remain open. HTTP export/import remains a supported deployment +All gates above remain open. The [takeover design](rust-server-upgrade-takeover.md) +proposes how the takeover command proves the backup, excludes other writers and +resumes after interruption. HTTP export/import remains a supported deployment transfer surface, but it does not substitute for in-place upgrade qualification. diff --git a/docs/rust-server-upgrade-takeover.md b/docs/rust-server-upgrade-takeover.md new file mode 100644 index 00000000..7795cb75 --- /dev/null +++ b/docs/rust-server-upgrade-takeover.md @@ -0,0 +1,124 @@ +# Native upgrade takeover + +This note designs the one-way, backup-first takeover of an existing PHP Server +database by the native runtime. It is a design input for +[Server #325](https://github.com/durable-workflow/server/issues/325), not a +qualified procedure. The gates it must satisfy are in the +[storage and upgrade audit](rust-server-storage-audit.md). Today +`upgrade-inspect`, `upgrade-queue-inspect` and `upgrade-payload-inspect` only +read, and their `backup_verified`, `php_quiescence_verified`, +`stored_representation_compatibility_verified` and `takeover_allowed` flags are +always false. This note says what makes each one true. + +## Shape + +Takeover is an explicit command, never a side effect of starting the server. +Native startup keeps refusing a PHP or unknown database. + +1. The operator stops every PHP write-capable role: HTTP, bootstrap and + migration, queue workers and the scheduler. +2. `upgrade-fingerprint` prints a digest of the database's complete logical + contents (below). +3. The operator takes a backup with the + [existing procedure](self-hosted-backup-and-restore.md), restores it into a + separate database, and runs `upgrade-fingerprint` there. Equal digests show + that the backup restores to exactly the stopped database. +4. `upgrade-takeover` runs with `DW_UPGRADE_BACKUP_FINGERPRINT` set to that + digest. It refuses unless the live database still has that digest inside + the same exclusive transaction that writes the ownership marker. A write + after the backup, by PHP or anything else, changes the digest and stops the + takeover before any mutation. +5. Native starts normally on the upgraded database. PHP refuses it from the + marker onward ([PHP ownership checks](php-database-ownership.md)). + +There is no PHP rollback after step 4. Recovery restores the verified backup +into a new database for a PHP installation, with the data boundary of that +backup. + +## Fingerprint + +The digest covers every relation in the database, not only the ones PHP +knows: the sorted relation names, each relation's column names and declared +types, and every row in primary-key order (all columns when there is no +primary key). Values are encoded canonically, so a dump and restore reproduces +the same bytes: integers in decimal, text and JSON as stored text, timestamps +at their stored precision, binary as hex, NULL distinct from empty. It is read +in one read-only snapshot (`READ_SNAPSHOT_BEGIN`). The command reports the +digest, the per-relation row counts and the elapsed time, never row contents. + +Takeover reads every row once more, inside its exclusive transaction. That +cost is accepted for a one-time operation. Its duration is reported so an +operator can plan the stop window from the fingerprint run. + +External payload objects are not in the database. Until a qualified object +check exists, takeover refuses when `upgrade-payload-inspect` reports any +external reference, and the operator's object manifest from the backup +procedure remains the record for those bytes. + +## Exclusive transaction per backend + +The marker, the fingerprint check and the schema changes must not interleave +with any other writer. + +| Backend | Exclusion | Atomicity | +| --- | --- | --- | +| SQLite | `BEGIN EXCLUSIVE` on the database file | Fingerprint, marker, native relations and conversions commit in one transaction. | +| PostgreSQL | `LOCK TABLE … IN EXCLUSIVE MODE` on every relation, then the native advisory key | DDL is transactional, so everything commits in one transaction. | +| MySQL and MariaDB | To be qualified: `LOCK TABLES … WRITE` on every relation, or a check that no other session uses the database, repeated after the marker is written | DDL commits implicitly, so the marker records a step journal (below). | + +On SQLite and PostgreSQL, a PHP connection that slipped past the stop blocks on +these locks, and a write it committed earlier changes the fingerprint. Neither +leaves a mixed state. + +## Marker and interruption + +The marker is written first in each takeover transaction, with state +`takeover`, the fingerprint and the step list. Native startup refuses state +`takeover`. Each step is idempotent and records its completion in the marker. +Rerunning `upgrade-takeover` with the same fingerprint resumes after the last +recorded step; a different fingerprint is refused. The final step sets state +`ready` and the native schema version. + +On SQLite and PostgreSQL an interruption rolls the whole transaction back, so +the database is either unchanged PHP or fully native. On MySQL and MariaDB the +step journal makes each committed DDL step visible, and resumption continues +from it. The PHP fence refuses the database from the first step, because the +marker exists. + +## Steps + +1. Refuse unless every inspection command reports `qualified` for this + database and the source catalog matches a qualified published PHP release. +2. Check the fingerprint (above) and write the marker. +3. Create the native relations and indexes (`upgrade.rs`, `native_object`). +4. Convert or adopt stored representations the inspections qualified: + poll-request bindings in task payloads, worker registrations (an + incarnation per active registration), cached query state, and queued + timer wakeups mapped to their durable task rows. Anything not qualified + was already refused by step 1. +5. Record the native migration ledger and set the marker to `ready`. + +## Tests + +Each backend gets an upgrade fixture written by the last published PHP release +with running and waiting workflows, a pending timer, a leased activity, an +acknowledged signal and an expired lease. The tests check: + +- refusal without a fingerprint, with a stale one, and after one extra row; +- interruption after each step, then resumption to the same final state; +- native completes the original work with its original histories, deadlines + and identities, and refuses the stale attempt; +- PHP refuses the upgraded database at startup, bootstrap, migration, worker + and scheduler entry points; +- a backup restored into a new database fingerprints equal to the stopped + source. + +## Open questions + +- Whether MySQL and MariaDB can create the native relations while + `LOCK TABLES` holds every existing one, or need the session check instead. +- Whether the declared column types print identically after a restore into + another minor server version. Until that is shown, the restore check uses + the same server version as the source. +- How long the fingerprint takes on a large installation, measured on a + disposable host before the stop window is documented. From eb8604ae877dcce61d23f8ede09466cb02de098b Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 08:02:45 +0000 Subject: [PATCH 09/42] Require a fenced PHP release before takeover Co-Authored-By: Claude Opus 5.5 --- docs/rust-server-upgrade-takeover.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/rust-server-upgrade-takeover.md b/docs/rust-server-upgrade-takeover.md index 7795cb75..1b1b7d16 100644 --- a/docs/rust-server-upgrade-takeover.md +++ b/docs/rust-server-upgrade-takeover.md @@ -15,8 +15,10 @@ always false. This note says what makes each one true. Takeover is an explicit command, never a side effect of starting the server. Native startup keeps refusing a PHP or unknown database. -1. The operator stops every PHP write-capable role: HTTP, bootstrap and - migration, queue workers and the scheduler. +1. The installation runs Server 2.5.14 or later, the first release with the + ownership fence, so any PHP role that restarts later refuses the upgraded + database. The operator then stops every PHP write-capable role: HTTP, + bootstrap and migration, queue workers and the scheduler. 2. `upgrade-fingerprint` prints a digest of the database's complete logical contents (below). 3. The operator takes a backup with the @@ -29,7 +31,8 @@ Native startup keeps refusing a PHP or unknown database. after the backup, by PHP or anything else, changes the digest and stops the takeover before any mutation. 5. Native starts normally on the upgraded database. PHP refuses it from the - marker onward ([PHP ownership checks](php-database-ownership.md)). + marker onward ([PHP ownership checks](php-database-ownership.md)). Earlier + PHP releases lack that check, which is why step 1 requires 2.5.14. There is no PHP rollback after step 4. Recovery restores the verified backup into a new database for a PHP installation, with the data boundary of that From 98ef1af7af18bb13c12f15c55452a68b37071399 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 08:09:56 +0000 Subject: [PATCH 10/42] Back off a held poll's timed reads up to 250 ms On the October 20 profile host, lock-free reads every 50 ms cut MySQL's idle cost at 24 waiting polls only from about 0.53 to 0.46 cores, and the native process stayed near 0.28 cores. The cost is the read rate, 20 per second per poll, not the lock. A held poll now reads 50 ms after a probe and then at doubling intervals up to 250 ms. A same-node wake still reads at once and starts again from 50 ms, so same-node work is unaffected, and work committed by another server process is observed within 250 ms. No wait runs past the poll's deadline. Co-Authored-By: Claude Opus 5.5 --- docs/rust-server-transition-locking.md | 9 +++++---- rust/src/runtime/poll_readiness.rs | 28 ++++++++++++++++++++++++-- rust/src/runtime/queries.rs | 6 ++---- rust/src/runtime/store.rs | 6 ++---- 4 files changed, 35 insertions(+), 14 deletions(-) diff --git a/docs/rust-server-transition-locking.md b/docs/rust-server-transition-locking.md index 4e6b6bad..9bd74a9c 100644 --- a/docs/rust-server-transition-locking.md +++ b/docs/rust-server-transition-locking.md @@ -30,10 +30,11 @@ A held long poll takes the lock for its first probe, and afterwards only when a lock-free read finds something to claim or refuse: an eligible task (or query), a receipt for the poll's own request ID, or a removed, moved or replaced registration (`poll_readiness.rs`). The read runs at once when a -commit on the same node wakes the poll, and otherwise every 50 ms, which is -how a poll observes another server process. An idle poll therefore neither -waits for the lock nor delays transitions, and an unrelated commit does not -send every waiting poll to the lock. +commit on the same node wakes the poll. Otherwise it runs 50 ms after a probe +and then at doubling intervals up to 250 ms, so a poll observes work committed +by another server process within 250 ms. An idle poll therefore neither waits +for the lock nor delays transitions, and an unrelated commit does not send +every waiting poll to the lock. ## Measured cost diff --git a/rust/src/runtime/poll_readiness.rs b/rust/src/runtime/poll_readiness.rs index f0727bf3..459db2fe 100644 --- a/rust/src/runtime/poll_readiness.rs +++ b/rust/src/runtime/poll_readiness.rs @@ -12,9 +12,33 @@ use super::{ use serde_json::Value; use sqlx::{ColumnIndex, Decode, Encode, Executor, IntoArguments, Type}; use std::time::Duration; +use tokio::{sync::futures::Notified, time::Instant}; -/// How often a held poll reads for work committed by another server process. -pub(super) const READ_INTERVAL: Duration = Duration::from_millis(50); +/// Paces a held poll's timed reads, which are how it observes another server +/// process. The first read comes 50 ms after a probe, and each later one waits +/// twice as long, up to 250 ms. A same-node wake reads at once and starts again +/// from 50 ms. No wait runs past the poll's deadline. +pub(super) struct ReadPace { + next: Duration, +} + +impl ReadPace { + const FIRST: Duration = Duration::from_millis(50); + const LONGEST: Duration = Duration::from_millis(250); + + pub(super) fn new() -> Self { + Self { next: Self::FIRST } + } + + /// Returns at a same-node wake or when the next timed read is due. + pub(super) async fn wait(&mut self, wake: Notified<'_>, deadline: Instant) { + let pause = self.next.min(deadline.saturating_duration_since(Instant::now())); + tokio::select! { + _ = wake => self.next = Self::FIRST, + _ = tokio::time::sleep(pause) => self.next = (self.next * 2).min(Self::LONGEST), + } + } +} impl Store where diff --git a/rust/src/runtime/queries.rs b/rust/src/runtime/queries.rs index ed9a5588..cdca5859 100644 --- a/rust/src/runtime/queries.rs +++ b/rust/src/runtime/queries.rs @@ -357,6 +357,7 @@ where // Only the first probe claims under the lock unconditionally; later // probes claim after a lock-free read finds something to claim or refuse. let mut first = true; + let mut pace = super::poll_readiness::ReadPace::new(); loop { let wake = self.wake.notified(); if std::mem::take(&mut first) || self.query_poll_ready(&body, config, &proof).await? { @@ -380,10 +381,7 @@ where json!({"task":null,"poll_status":"task_queue_changed","server_capabilities":super::store::capabilities()}), ); } - tokio::select! { - _ = wake => {}, - _ = tokio::time::sleep(super::poll_readiness::READ_INTERVAL) => {}, - } + pace.wait(wake, deadline).await; } } diff --git a/rust/src/runtime/store.rs b/rust/src/runtime/store.rs index 1dddc5da..9b045bee 100644 --- a/rust/src/runtime/store.rs +++ b/rust/src/runtime/store.rs @@ -938,6 +938,7 @@ where // other server processes, claim only after a lock-free read finds // something to claim or refuse. let mut first = true; + let mut pace = super::poll_readiness::ReadPace::new(); loop { // Register the notification before probing. let wake = self.wake.notified(); @@ -962,10 +963,7 @@ where json!({"task":null,"poll_status":"task_queue_changed","server_capabilities":capabilities()}), ); } - tokio::select! { - _ = wake => {}, - _ = tokio::time::sleep(super::poll_readiness::READ_INTERVAL) => {}, - } + pace.wait(wake, deadline).await; } } From eb272676d352c75fc79c08b0d929170ff3e19402 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 08:46:36 +0000 Subject: [PATCH 11/42] Keep the readiness reads beside the claims they mirror The task read now sits next to the task claim in store.rs and the query read next to the query claim in queries.rs, inside their existing Store impls. poll_readiness.rs keeps only the read pacing. No behavior changes. Co-Authored-By: Claude Opus 5.5 --- docs/rust-server-transition-locking.md | 2 +- rust/src/runtime/backend.rs | 2 +- rust/src/runtime/poll_readiness.rs | 92 ++------------------------ rust/src/runtime/queries.rs | 32 ++++++++- rust/src/runtime/store.rs | 37 +++++++++++ 5 files changed, 74 insertions(+), 91 deletions(-) diff --git a/docs/rust-server-transition-locking.md b/docs/rust-server-transition-locking.md index 9bd74a9c..392f7743 100644 --- a/docs/rust-server-transition-locking.md +++ b/docs/rust-server-transition-locking.md @@ -29,7 +29,7 @@ pool without a transaction. A held long poll takes the lock for its first probe, and afterwards only when a lock-free read finds something to claim or refuse: an eligible task (or query), a receipt for the poll's own request ID, or a removed, moved or -replaced registration (`poll_readiness.rs`). The read runs at once when a +replaced registration (`Store::task_poll_ready` and `Store::query_poll_ready`). The read runs at once when a commit on the same node wakes the poll. Otherwise it runs 50 ms after a probe and then at doubling intervals up to 250 ms, so a poll observes work committed by another server process within 250 ms. An idle poll therefore neither waits diff --git a/rust/src/runtime/backend.rs b/rust/src/runtime/backend.rs index 7dc40438..b31a8349 100644 --- a/rust/src/runtime/backend.rs +++ b/rust/src/runtime/backend.rs @@ -338,7 +338,7 @@ impl Backend for Postgres { let mut tx = pool.begin().await?; // The first single-namespace slice serializes its durable transitions // across independent pools/processes. Held long polls wait without it - // (poll_readiness.rs). Throughput/idle cost remain unqualified. + // (Store::task_poll_ready). Throughput/idle cost remain unqualified. sqlx::query("SELECT pg_advisory_xact_lock(4924501473315676163)") .execute(&mut *tx) .await?; diff --git a/rust/src/runtime/poll_readiness.rs b/rust/src/runtime/poll_readiness.rs index 459db2fe..6c026582 100644 --- a/rust/src/runtime/poll_readiness.rs +++ b/rust/src/runtime/poll_readiness.rs @@ -3,14 +3,10 @@ //! A held poll takes the transition lock only to claim or refuse. After its //! first claim it reads, without the lock, whether anything now exists that it //! could claim or must refuse: an eligible task, a receipt for its own request, -//! or a changed registration. A same-node commit wakes it to read at once; -//! timed reads observe other server processes. A false positive costs one locked -//! claim that finds nothing, and a commit racing a read is seen on the next one. -use super::{ - Result, backend::Backend, query_authority::{self, QueryTaskConfig}, store::Store, text, -}; -use serde_json::Value; -use sqlx::{ColumnIndex, Decode, Encode, Executor, IntoArguments, Type}; +//! or a changed registration (`Store::task_poll_ready`, `Store::query_poll_ready`). +//! A same-node commit wakes it to read at once; timed reads observe other +//! server processes. A false positive costs one locked claim that finds +//! nothing, and a commit racing a read is seen on the next one. use std::time::Duration; use tokio::{sync::futures::Notified, time::Instant}; @@ -39,83 +35,3 @@ impl ReadPace { } } } - -impl Store -where - for<'c> &'c mut DB::Connection: Executor<'c, Database = DB>, - DB::Arguments: IntoArguments, - for<'q> &'q str: Encode<'q, DB> + Type, - for<'q> Option<&'q str>: Encode<'q, DB>, - for<'q> Option: Encode<'q, DB>, - for<'q> String: Encode<'q, DB> + Type, - for<'q> i64: Encode<'q, DB> + Type, - for<'r> i64: Decode<'r, DB>, - usize: ColumnIndex, -{ - /// Whether a held workflow or activity poll should claim again. - pub(super) async fn task_poll_ready(&self, body: &Value, kind: &str) -> Result { - let worker_id = text(body, "worker_id")?; - let queue = text(body, "task_queue")?; - // The claim refuses a removed or moved registration. - let registration = Self::query( - "SELECT task_queue FROM workflow_worker_registrations WHERE worker_id=$1 AND namespace=$2", - ) - .bind(worker_id) - .bind(self.namespace.as_ref()) - .fetch_optional(&self.pool) - .await?; - match registration { - Some(row) if DB::string(&row, "task_queue")? == queue => {} - _ => return Ok(true), - } - // A concurrent poll with the same request ID may have claimed already; - // the claim replays its receipt. - if let Some(request_id) = body.get("poll_request_id").and_then(Value::as_str) - && Self::query("SELECT task_id FROM dw_poll_receipts WHERE worker_id=$1 AND kind=$2 AND request_id=$3 AND namespace=$4") - .bind(worker_id).bind(kind).bind(request_id).bind(self.namespace.as_ref()) - .fetch_optional(&self.pool).await?.is_some() - { - return Ok(true); - } - let at = super::store::now(); - Ok(Self::query(DB::TASK_CANDIDATES_SQL) - .bind(self.namespace.as_ref()) - .bind(queue) - .bind(kind) - .bind(DB::bind_time(at)) - .bind(DB::bind_time(at)) - .fetch_optional(&self.pool) - .await? - .is_some()) - } - - /// Whether a held query poll should claim again. `proof` is the - /// registration this poll belongs to. - pub(super) async fn query_poll_ready( - &self, - body: &Value, - config: &QueryTaskConfig, - proof: &str, - ) -> Result { - let owner = text(body, "worker_id")?; - // The claim refuses a replaced, removed or stale registration. - if Self::query_worker_proof(&self.pool, self.namespace.as_ref(), owner, config) - .await? - .as_deref() - != Some(proof) - { - return Ok(true); - } - let at = super::store::now(); - let query_now = query_authority::wire(at); - Ok(Self::query(DB::POLL_WAKE_QUERIES_SQL) - .bind(at.timestamp()) - .bind(self.namespace.as_ref()) - .bind(text(body, "task_queue")?) - .bind(query_now.clone()) - .bind(query_now) - .fetch_optional(&self.pool) - .await? - .is_some()) - } -} diff --git a/rust/src/runtime/queries.rs b/rust/src/runtime/queries.rs index cdca5859..27b78179 100644 --- a/rust/src/runtime/queries.rs +++ b/rust/src/runtime/queries.rs @@ -385,7 +385,7 @@ where } } - pub(super) async fn query_worker_proof<'e, E>( + async fn query_worker_proof<'e, E>( executor: E, namespace: &str, owner: &str, @@ -437,6 +437,36 @@ where delivered } + /// Whether a held query poll should claim again. `proof` is the + /// registration this poll belongs to. + async fn query_poll_ready( + &self, + body: &Value, + config: &QueryTaskConfig, + proof: &str, + ) -> Result { + let owner = text(body, "worker_id")?; + // The claim refuses a replaced, removed or stale registration. + if Self::query_worker_proof(&self.pool, self.namespace.as_ref(), owner, config) + .await? + .as_deref() + != Some(proof) + { + return Ok(true); + } + let at = super::store::now(); + let query_now = authority::wire(at); + Ok(Self::query(DB::POLL_WAKE_QUERIES_SQL) + .bind(at.timestamp()) + .bind(self.namespace.as_ref()) + .bind(text(body, "task_queue")?) + .bind(query_now.clone()) + .bind(query_now) + .fetch_optional(&self.pool) + .await? + .is_some()) + } + async fn claim_query( &self, body: &Value, diff --git a/rust/src/runtime/store.rs b/rust/src/runtime/store.rs index 9b045bee..c593973b 100644 --- a/rust/src/runtime/store.rs +++ b/rust/src/runtime/store.rs @@ -967,6 +967,43 @@ where } } + /// Whether a held workflow or activity poll should claim again. + async fn task_poll_ready(&self, body: &Value, kind: &str) -> Result { + let worker_id = text(body, "worker_id")?; + let queue = text(body, "task_queue")?; + // The claim refuses a removed or moved registration. + let registration = Self::query( + "SELECT task_queue FROM workflow_worker_registrations WHERE worker_id=$1 AND namespace=$2", + ) + .bind(worker_id) + .bind(self.namespace.as_ref()) + .fetch_optional(&self.pool) + .await?; + match registration { + Some(row) if DB::string(&row, "task_queue")? == queue => {} + _ => return Ok(true), + } + // A concurrent poll with the same request ID may have claimed already; + // the claim replays its receipt. + if let Some(request_id) = body.get("poll_request_id").and_then(Value::as_str) + && Self::query("SELECT task_id FROM dw_poll_receipts WHERE worker_id=$1 AND kind=$2 AND request_id=$3 AND namespace=$4") + .bind(worker_id).bind(kind).bind(request_id).bind(self.namespace.as_ref()) + .fetch_optional(&self.pool).await?.is_some() + { + return Ok(true); + } + let at = now(); + Ok(Self::query(DB::TASK_CANDIDATES_SQL) + .bind(self.namespace.as_ref()) + .bind(queue) + .bind(kind) + .bind(DB::bind_time(at)) + .bind(DB::bind_time(at)) + .fetch_optional(&self.pool) + .await? + .is_some()) + } + async fn claim( &self, body: &Value, From 00b8b65386a9e235cd84646b841d082ce167aba9 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 08:49:05 +0000 Subject: [PATCH 12/42] Record that native's schema must first converge on the source release Co-Authored-By: Claude Opus 5.5 --- docs/rust-server-upgrade-takeover.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/rust-server-upgrade-takeover.md b/docs/rust-server-upgrade-takeover.md index 1b1b7d16..52a8c03e 100644 --- a/docs/rust-server-upgrade-takeover.md +++ b/docs/rust-server-upgrade-takeover.md @@ -101,6 +101,16 @@ marker exists. was already refused by step 1. 5. Record the native migration ledger and set the marker to `ready`. +Native's frozen physical layout predates the current PHP release, so step 3 +alone does not yield a database native accepts. PHP 2.8.0 adds its own +`workflow_worker_registration_incarnations` table, a `call_deadline_at` +column on `workflow_service_calls` and a changed `workflow_worker_registrations` +(`rust/upgrade/php-2.8.0/`). Before takeover is implemented, native's schema +moves to the layout of the release takeover starts from, and native reads +PHP's incarnation table instead of keeping a second one. A fresh native +database then has the same PHP layout, and takeover only adds native relations +and converts data. + ## Tests Each backend gets an upgrade fixture written by the last published PHP release From 026e25879f36b49815141a28a0d2dc63544f925b Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 09:04:05 +0000 Subject: [PATCH 13/42] Reproduce default and opt-out workflow definition upgrade stalls --- .../WorkflowDefinitionEvolutionTest.php | 109 ++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100644 tests/Feature/WorkflowDefinitionEvolutionTest.php diff --git a/tests/Feature/WorkflowDefinitionEvolutionTest.php b/tests/Feature/WorkflowDefinitionEvolutionTest.php new file mode 100644 index 00000000..93ff05ac --- /dev/null +++ b/tests/Feature/WorkflowDefinitionEvolutionTest.php @@ -0,0 +1,109 @@ +createNamespace('default'); + config(['server.polling.timeout' => 0, 'workflows.v2.compatibility.pin_to_recorded_fingerprint' => true]); + } + + public function test_default_replacement_can_claim_when_the_recorded_definition_is_gone(): void + { + [$run, $task, $started] = $this->original(); + $original = [$started->getRawOriginal('payload'), $run->run_deadline_at, $run->execution_deadline_at]; + $this->deleteJson('/api/workers/original-worker', [], $this->apiHeaders())->assertOk(); + $this->worker('replacement-worker', 'definition-v2'); + + $this->poll('replacement-worker')->assertOk() + ->assertJsonPath('task.run_id', $run->id) + ->assertJsonPath('task.task_id', $task->id); + + self::assertSame($original, [ + $started->fresh()->getRawOriginal('payload'), $run->fresh()->run_deadline_at, $run->fresh()->execution_deadline_at, + ]); + self::assertSame(1, $task->fresh()->attempt_count); + self::assertSame(0, $task->fresh()->repair_count); + } + + public function test_explicit_opt_out_does_not_wait_for_a_matching_peer(): void + { + [$run, $task] = $this->original(); + $this->worker('replacement-worker', 'definition-v2'); + config(['workflows.v2.compatibility.pin_to_recorded_fingerprint' => false]); + + $this->poll('replacement-worker')->assertOk() + ->assertJsonPath('task.run_id', $run->id) + ->assertJsonPath('task.task_id', $task->id); + } + + public function test_default_keeps_the_matching_registered_peer_preference(): void + { + [, $task] = $this->original(); + $this->worker('replacement-worker', 'definition-v2'); + + $this->poll('replacement-worker')->assertOk()->assertJsonPath('task', null); + self::assertSame(TaskStatus::Ready, $task->fresh()->status); + self::assertNull($task->fresh()->lease_owner); + self::assertSame(0, $task->fresh()->attempt_count); + + $this->poll('original-worker')->assertOk()->assertJsonPath('task.task_id', $task->id); + } + + private function original(): array + { + $this->worker('original-worker', 'definition-v1'); + $start = $this->postJson('/api/workflows', [ + 'workflow_id' => 'definition-original', 'workflow_type' => self::TYPE, 'task_queue' => self::QUEUE, + 'run_timeout_seconds' => 60, 'execution_timeout_seconds' => 120, + ], $this->apiHeaders())->assertCreated(); + $run = WorkflowRun::query()->findOrFail($start->json('run_id')); + $task = WorkflowTask::query()->where('workflow_run_id', $run->id)->firstOrFail(); + $started = WorkflowHistoryEvent::query()->where('workflow_run_id', $run->id) + ->where('event_type', 'WorkflowStarted')->firstOrFail(); + self::assertSame('worker', $started->payload['workflow_definition_fingerprint_source']); + self::assertSame('definition-v1', $started->payload['workflow_definition_fingerprint']); + + return [$run, $task, $started]; + } + + private function worker(string $id, string $fingerprint): void + { + $this->postJson('/api/worker/register', [ + 'worker_id' => $id, 'task_queue' => self::QUEUE, 'runtime' => 'python', 'sdk_version' => '2.6.1', + 'capability_manifest' => $this->portableWorkerAffinityRefusalManifest(), + 'supported_workflow_types' => [self::TYPE], + 'workflow_definition_fingerprints' => [self::TYPE => $fingerprint], + 'workflow_command_contracts' => [self::TYPE => ['signals' => [], 'queries' => [], 'updates' => [], 'update_validators' => []]], + ], $this->workerHeaders())->assertCreated(); + } + + private function poll(string $id): \Illuminate\Testing\TestResponse + { + return $this->postJson('/api/worker/workflow-tasks/poll', [ + 'worker_id' => $id, 'task_queue' => self::QUEUE, 'timeout_seconds' => 0, + ], $this->workerHeaders()); + } +} From e03bd5fc39166723a8a80430d6fb47e5b1629376 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 09:21:23 +0000 Subject: [PATCH 14/42] Allow strict replay by replacement workflow definitions and explain routing --- .../Controllers/Api/TaskQueueController.php | 38 ++-- app/Http/Controllers/Api/WorkerController.php | 10 +- .../Controllers/Api/WorkflowController.php | 1 + app/Support/ServerWorkflowControlPlane.php | 26 ++- app/Support/WorkflowDefinitionRouting.php | 179 ++++++++++++++++++ app/Support/WorkflowReplayEligibility.php | 40 ++++ app/Support/WorkflowRunDiagnostics.php | 13 ++ app/Support/WorkflowTaskPoller.php | 68 ++----- config/dw-contract.php | 2 +- docs/server-reference.md | 20 +- .../WorkflowDefinitionEvolutionTest.php | 178 ++++++++++++++++- 11 files changed, 501 insertions(+), 74 deletions(-) create mode 100644 app/Support/WorkflowDefinitionRouting.php create mode 100644 app/Support/WorkflowReplayEligibility.php diff --git a/app/Http/Controllers/Api/TaskQueueController.php b/app/Http/Controllers/Api/TaskQueueController.php index f27274fd..907d18fe 100644 --- a/app/Http/Controllers/Api/TaskQueueController.php +++ b/app/Http/Controllers/Api/TaskQueueController.php @@ -10,6 +10,7 @@ use App\Support\TaskQueueAdmission; use App\Support\TaskQueueBuildIdRolloutSnapshot; use App\Support\TaskQueuePriorityFairnessSurface; +use App\Support\WorkflowDefinitionRouting; use App\Support\WorkflowQueryTaskBroker; use Illuminate\Http\JsonResponse; use Illuminate\Http\Request; @@ -28,6 +29,7 @@ public function __construct( private readonly TaskQueueBuildIdRolloutSnapshot $buildIdRollouts, private readonly TaskQueuePriorityFairnessSurface $priorityFairness, private readonly DeploymentLifecycleService $deployments, + private readonly WorkflowDefinitionRouting $definitionRouting, ) {} public function index(Request $request): JsonResponse @@ -67,22 +69,26 @@ public function show(Request $request, string $taskQueue): JsonResponse $namespace = (string) $request->attributes->get('namespace'); - return ControlPlaneProtocol::json( - $this->withAdmission( + $payload = $this->withAdmission( + $namespace, + $this->withRecentTaskFlow( $namespace, - $this->withRecentTaskFlow( + $taskQueue, + StandaloneWorkerVisibility::queueDetail( $namespace, $taskQueue, - StandaloneWorkerVisibility::queueDetail( - $namespace, - $taskQueue, - WorkerRegistration::class, - now(), - $this->workerStaleAfterSeconds(), - )->toArray(), - ), + WorkerRegistration::class, + now(), + $this->workerStaleAfterSeconds(), + )->toArray(), ), ); + $payload['definition_routing'] = $this->definitionRouting->forQueue($namespace, $taskQueue); + if ($payload['definition_routing']['blocked_sample_count'] > 0) { + $payload['repair']['needs_attention'] = true; + } + + return ControlPlaneProtocol::json($payload); } /** @@ -263,7 +269,7 @@ private function setBuildIdDrainIntent(Request $request, string $taskQueue, stri } /** - * @param array $deployment + * @param array $deployment * @return array */ private function buildIdLifecyclePayload( @@ -351,7 +357,7 @@ private function normalizeDeploymentLifecycleTimestamps(array $deployment): arra } /** - * @param list> $blockages + * @param list> $blockages */ private function orderBlockages(array &$blockages): void { @@ -630,6 +636,7 @@ private function activityAttemptStateCounts(string $namespace, string $taskQueue if ($attemptStatus === ActivityAttemptStatus::Running->value) { $counts['running_count'] += $count; $counts['open_count'] += $count; + continue; } @@ -639,6 +646,7 @@ private function activityAttemptStateCounts(string $namespace, string $taskQueue ) { $counts['retrying_count'] += $count; $counts['open_count'] += $count; + continue; } @@ -646,6 +654,7 @@ private function activityAttemptStateCounts(string $namespace, string $taskQueue $counts['timed_out_count'] += $count; $counts['expired_count'] += $count; $counts['terminal_count'] += $count; + continue; } @@ -656,18 +665,21 @@ private function activityAttemptStateCounts(string $namespace, string $taskQueue ) { $counts['timed_out_count'] += $count; $counts['terminal_count'] += $count; + continue; } if ($attemptStatus === ActivityAttemptStatus::Failed->value) { $counts['failed_count'] += $count; $counts['terminal_count'] += $count; + continue; } if ($attemptStatus === ActivityAttemptStatus::Completed->value) { $counts['completed_count'] += $count; $counts['terminal_count'] += $count; + continue; } diff --git a/app/Http/Controllers/Api/WorkerController.php b/app/Http/Controllers/Api/WorkerController.php index 9d9e3c6e..a90ca6cc 100644 --- a/app/Http/Controllers/Api/WorkerController.php +++ b/app/Http/Controllers/Api/WorkerController.php @@ -46,6 +46,7 @@ use App\Support\WorkflowHistoryPageToken; use App\Support\WorkflowMetadataCapabilityPolicy; use App\Support\WorkflowQueryTaskBroker; +use App\Support\WorkflowReplayEligibility; use App\Support\WorkflowStreamCommandProcessor; use App\Support\WorkflowStreamService; use App\Support\WorkflowTaskLeaseRecovery; @@ -407,7 +408,10 @@ public function register(Request $request): JsonResponse $this->validateWorkerTaskCapacity($maxWorkflowTasks, $maxActivityTasks); $maxWorkerSessions = $validated['max_concurrent_worker_sessions'] ?? 10; $taskSlots = is_array($validated['task_slots'] ?? null) ? $validated['task_slots'] : []; - $processMetrics = $this->normalizeProcessMetrics($validated['process_metrics'] ?? null); + $processMetrics = [ + ...($this->normalizeProcessMetrics($validated['process_metrics'] ?? null) ?? []), + WorkflowReplayEligibility::PROTOCOL_METRIC => WorkerProtocol::requestVersion($request), + ]; $releaseLeasesForRegistration = $this->shouldReleaseLeasesForWorkerRegistration($existing, $processMetrics); try { @@ -935,6 +939,10 @@ public function heartbeat(Request $request): JsonResponse if (array_key_exists('process_metrics', $validated)) { $update['process_metrics'] = $this->normalizeProcessMetrics($validated['process_metrics']); } + $update['process_metrics'] = [ + ...($update['process_metrics'] ?? $worker->process_metrics ?? []), + WorkflowReplayEligibility::PROTOCOL_METRIC => WorkerProtocol::requestVersion($request), + ]; if (array_key_exists('heartbeat_interval_seconds', $validated) && $validated['heartbeat_interval_seconds'] !== null) { diff --git a/app/Http/Controllers/Api/WorkflowController.php b/app/Http/Controllers/Api/WorkflowController.php index 7af65aa5..d2c32382 100644 --- a/app/Http/Controllers/Api/WorkflowController.php +++ b/app/Http/Controllers/Api/WorkflowController.php @@ -1250,6 +1250,7 @@ private function formatRun(WorkflowRun $run, string $namespace, array $descripti 'compatibility_supported_in_fleet' => $this->compatibilitySupportedInFleet($namespace, $run), 'compatibility_fleet_reason' => $this->compatibilityFleetReason($namespace, $run), 'payload_codec' => $run->payload_codec, + 'definition_routing' => $description['definition_routing'] ?? null, 'execution_timeout_seconds' => $description['execution_timeout_seconds'] ?? null, 'run_timeout_seconds' => $runDescription['run_timeout_seconds'] ?? null, 'execution_deadline_at' => $runDescription['execution_deadline_at'] ?? null, diff --git a/app/Support/ServerWorkflowControlPlane.php b/app/Support/ServerWorkflowControlPlane.php index 16273570..3094a947 100644 --- a/app/Support/ServerWorkflowControlPlane.php +++ b/app/Support/ServerWorkflowControlPlane.php @@ -26,6 +26,7 @@ public function __construct( private readonly ControlPlaneFailureDiagnostics $failureDiagnostics, private readonly NamespaceDurableStateQuota $durableStateQuota, private readonly WorkflowStartVersionPin $versionPin, + private readonly WorkflowDefinitionRouting $definitionRouting, ) {} public function start(string $workflowType, ?string $instanceId = null, array $options = []): array @@ -148,10 +149,12 @@ public function terminate(string $instanceId, array $options = []): array public function repair(string $instanceId, array $options = []): array { - return $this->mutateCommand( + $result = $this->mutateCommand( $instanceId, 'repair', HistoryEventType::RepairRequested, $options, fn (): array => $this->inner->repair($instanceId, $options), ); + + return $this->withDefinitionRouting($instanceId, $options, $result); } public function redrive(string $instanceId, string $failedRunId, array $options = []): array @@ -309,7 +312,26 @@ private function recoveredCommandResult(Throwable $exception, WorkflowCommand $c public function describe(string $instanceId, array $options = []): array { - return $this->inner->describe($instanceId, $options); + return $this->withDefinitionRouting($instanceId, $options, $this->inner->describe($instanceId, $options)); + } + + private function withDefinitionRouting(string $instanceId, array $options, array $result): array + { + $namespace = $this->namespace($options); + if ($namespace === null || ($result['found'] ?? true) === false + || in_array($result['reason'] ?? null, ['instance_not_found', 'run_not_found'], true)) { + return $result; + } + + $runId = data_get($result, 'run.run_id') ?? $result['workflow_run_id'] ?? $options['run_id'] ?? null; + $run = WorkflowRun::query()->where('namespace', $namespace)->where('workflow_instance_id', $instanceId) + ->when(is_string($runId), static fn ($query) => $query->whereKey($runId)) + ->orderByDesc('run_number')->first(); + if ($run instanceof WorkflowRun) { + $result['definition_routing'] = $this->definitionRouting->inspect($run); + } + + return $result; } /** diff --git a/app/Support/WorkflowDefinitionRouting.php b/app/Support/WorkflowDefinitionRouting.php new file mode 100644 index 00000000..cb1f264f --- /dev/null +++ b/app/Support/WorkflowDefinitionRouting.php @@ -0,0 +1,179 @@ + */ + public function forWorker( + WorkflowRun $run, + WorkerRegistration $worker, + ?string $taskQueue = null, + ?string $requiredCompatibility = null, + ): array { + $decision = $this->context($run, $taskQueue, $requiredCompatibility); + $recorded = $decision['recorded_fingerprint']; + $advertised = self::stringValue($worker->workflow_definition_fingerprints[$run->workflow_type] ?? null); + $decision['advertised_fingerprint'] = $advertised; + + if (! $decision['worker_pinned']) { + return [...$decision, 'allowed' => true, 'status' => 'not_worker_pinned']; + } + if (! $decision['pinning_enabled']) { + return [...$decision, 'allowed' => true, 'status' => 'pinning_disabled']; + } + if ($recorded === null || $advertised === null) { + return [...$decision, 'allowed' => false, 'status' => $recorded === null + ? 'recorded_definition_missing' : 'worker_definition_missing']; + } + if (hash_equals($recorded, $advertised)) { + return [...$decision, 'allowed' => true, 'status' => 'matching_definition']; + } + + foreach ($this->eligibleWorkers($run, $decision['task_queue'], $decision['required_build_id']) as $peer) { + if (self::stringValue($peer->workflow_definition_fingerprints[$run->workflow_type] ?? null) === $recorded) { + return [...$decision, 'allowed' => false, 'status' => 'matching_definition_preferred', + 'preferred_worker_id' => $peer->worker_id]; + } + } + + // Match the embedded resolver: prefer the recorded source if it is + // available, otherwise let the current definition validate replay. + // This never changes the run's original fingerprint or build pin. + return [...$decision, 'allowed' => true, 'status' => 'replay_required']; + } + + /** @return array */ + public function inspect(WorkflowRun $run, ?string $taskQueue = null, ?string $requiredCompatibility = null): array + { + $task = WorkflowTask::query()->where('workflow_run_id', $run->id) + ->where('task_type', TaskType::Workflow->value)->where('status', TaskStatus::Ready->value) + ->where(static fn ($query) => $query->whereNull('available_at')->orWhere('available_at', '<=', now()->format('Y-m-d H:i:s.u'))) + ->when($taskQueue !== null, static fn ($query) => $query->where('queue', $taskQueue)) + ->orderBy('id')->first(); + if ($task instanceof WorkflowTask) { + $taskQueue ??= $task->queue; + $requiredCompatibility ??= TaskCompatibility::resolve($task, $run); + } + $context = $this->context($run, $taskQueue, $requiredCompatibility); + $matching = $changed = $missing = 0; + $examples = []; + + foreach ($this->eligibleWorkers($run, (string) $context['task_queue'], $context['required_build_id']) as $worker) { + $fingerprint = self::stringValue($worker->workflow_definition_fingerprints[$run->workflow_type] ?? null); + if ($fingerprint === null) { + $missing++; + } elseif ($fingerprint === $context['recorded_fingerprint']) { + $matching++; + } else { + $changed++; + } + if (count($examples) < 10) { + $examples[] = ['worker_id' => $worker->worker_id, 'fingerprint' => $fingerprint]; + } + } + + $eligible = $matching + $changed + $missing; + $status = match (true) { + ! $context['worker_pinned'] => 'not_worker_pinned', + ! $context['pinning_enabled'] => 'pinning_disabled', + $context['recorded_fingerprint'] === null => 'recorded_definition_missing', + $matching > 0 => 'matching_definition_available', + $changed > 0 => 'replay_required', + $missing > 0 => 'worker_definition_missing', + default => 'no_eligible_worker', + }; + $runnable = ! $run->status->isTerminal() && $task instanceof WorkflowTask; + $blocked = $runnable && $context['worker_pinned'] && $context['pinning_enabled'] + && in_array($status, ['recorded_definition_missing', 'worker_definition_missing', 'no_eligible_worker'], true); + + return [...$context, 'status' => $status, 'blocked' => $blocked, 'runnable_task' => $runnable, + 'eligible_worker_count' => $eligible, 'matching_worker_count' => $matching, + 'changed_worker_count' => $changed, 'missing_fingerprint_count' => $missing, + 'workers' => $examples, 'workers_truncated' => $eligible > count($examples), + 'resolution' => $blocked + ? 'Register a fresh worker for the original namespace, task queue, workflow type, build and replay capabilities with its definition fingerprint. Do not rewrite the run history.' + : ($status === 'replay_required' + ? 'Current workers may replay the original history. Use versioned branches for command changes; incompatible replay still refuses.' + : 'Recorded build, namespace, type and replay-capability guards remain in force.'), + ]; + } + + /** Bounded examples of due ready runs, never a repair or a payload read. */ + public function forQueue(string $namespace, string $taskQueue): array + { + $runs = WorkflowRun::query()->where('namespace', $namespace)->where('queue', $taskQueue) + ->whereIn('id', WorkflowTask::query()->select('workflow_run_id') + ->where('task_type', TaskType::Workflow->value)->where('status', TaskStatus::Ready->value) + ->where('queue', $taskQueue) + ->where(static fn ($query) => $query->whereNull('available_at')->orWhere('available_at', '<=', now()->format('Y-m-d H:i:s.u')))) + ->orderBy('id')->limit(11)->get(); + $examples = []; + foreach ($runs->take(10) as $run) { + if (! $run->status->isTerminal()) { + $examples[] = ['workflow_id' => $run->workflow_instance_id, 'run_id' => $run->id, + ...$this->inspect($run, $taskQueue)]; + } + } + + return ['sample_limit' => 10, 'has_more_ready_runs' => $runs->count() > 10, + 'blocked_sample_count' => count(array_filter($examples, static fn (array $row): bool => $row['blocked'])), + 'runs' => $examples]; + } + + /** @return array */ + private function context(WorkflowRun $run, ?string $taskQueue, ?string $requiredCompatibility): array + { + $started = WorkflowHistoryEvent::query()->where('workflow_run_id', $run->id) + ->where('event_type', 'WorkflowStarted')->orderBy('sequence')->first(); + $recorded = self::stringValue($started?->payload['workflow_definition_fingerprint'] ?? null); + + return [ + 'pinning_enabled' => (bool) config('workflows.v2.compatibility.pin_to_recorded_fingerprint', true), + 'worker_pinned' => ($started?->payload['workflow_definition_fingerprint_source'] ?? null) === 'worker', + 'recorded_fingerprint' => $recorded, + 'workflow_type' => $run->workflow_type, + 'task_queue' => $taskQueue ?? $run->queue, + 'required_build_id' => $requiredCompatibility ?? self::stringValue($run->compatibility), + ]; + } + + /** @return Generator */ + private function eligibleWorkers( + WorkflowRun $run, + string $taskQueue, + ?string $requiredCompatibility, + ): Generator { + $workers = WorkerRegistration::query()->where('namespace', $run->namespace) + ->where('task_queue', $taskQueue)->where('status', 'active'); + if ($requiredCompatibility !== null) { + $workers->where('build_id', $requiredCompatibility); + } + + foreach ($workers->cursor(['namespace', 'worker_id', 'task_queue', 'build_id', 'status', 'last_heartbeat_at', + 'workflow_definition_fingerprints', 'supported_workflow_types', 'max_concurrent_workflow_tasks', + 'capabilities', 'process_metrics']) as $worker) { + if (! in_array($run->workflow_type, $worker->supported_workflow_types ?? [], true) + || ($worker->max_concurrent_workflow_tasks !== null && (int) $worker->max_concurrent_workflow_tasks < 1)) { + continue; + } + if (WorkflowReplayEligibility::allows($worker, $run, WorkflowReplayEligibility::recordedProtocol($worker))) { + yield $worker; + } + } + } + + private static function stringValue(mixed $value): ?string + { + return is_string($value) && trim($value) !== '' ? trim($value) : null; + } +} diff --git a/app/Support/WorkflowReplayEligibility.php b/app/Support/WorkflowReplayEligibility.php new file mode 100644 index 00000000..172dc4b0 --- /dev/null +++ b/app/Support/WorkflowReplayEligibility.php @@ -0,0 +1,40 @@ +namespace !== $run->namespace || ! WorkerPollFence::isFresh($worker)) { + return false; + } + + $capabilities = is_array($worker->capabilities) ? $worker->capabilities : []; + + if (is_string($run->cancellation_request_command_id) + && ! CooperativeCancellationPolicy::supports($capabilities, $protocolVersion)) { + return false; + } + if (is_string($run->cancellation_request_command_id) + && NexusCooperativeCancellationPolicy::requiresCapabilityForReplay($run) + && ! NexusCooperativeCancellationPolicy::supports($capabilities, $protocolVersion)) { + return false; + } + + return PreparedLocalActivityPolicy::canReplayRun($run->id, $capabilities, $protocolVersion) + && WorkflowMetadataCapabilityPolicy::canReplayRun($run->id, $capabilities, $protocolVersion); + } + + public static function recordedProtocol(WorkerRegistration $worker): ?string + { + $version = $worker->process_metrics[self::PROTOCOL_METRIC] ?? null; + + return is_string($version) && $version !== '' ? $version : null; + } +} diff --git a/app/Support/WorkflowRunDiagnostics.php b/app/Support/WorkflowRunDiagnostics.php index 6f649056..810d3f1b 100644 --- a/app/Support/WorkflowRunDiagnostics.php +++ b/app/Support/WorkflowRunDiagnostics.php @@ -41,6 +41,8 @@ class WorkflowRunDiagnostics private const LAST_EVENT_PAYLOAD_KEY_LIMIT = 20; + public function __construct(private readonly WorkflowDefinitionRouting $definitionRouting) {} + /** * @return array */ @@ -84,6 +86,7 @@ public function forRun(string $namespace, WorkflowRun $run, bool $includeLastEve 'recent_failures_truncated' => $failureRows->count() > self::FAILURE_LIMIT, 'latest_workflow_task_failure' => $latestWorkflowTaskFailure, 'compatibility' => $this->compatibility($namespace, $run, $summary, $taskQueue), + 'definition_routing' => $this->definitionRouting->inspect($run), 'cancellation_cascade_supported' => class_exists(CancellationCascadeView::class), 'cancellation_cascade' => class_exists(CancellationCascadeView::class) && $run->namespace === $namespace ? CancellationCascadeView::forRun($run) @@ -95,6 +98,16 @@ public function forRun(string $namespace, WorkflowRun $run, bool $includeLastEve ...$this->pendingWorkflowRoutingFindings($namespace, $run, $taskRows->all(), $taskQueue), ]; + if ($payload['definition_routing']['blocked']) { + $payload['findings'][] = [ + 'severity' => 'warning', 'code' => 'workflow_definition_routing_blocked', + 'message' => 'A ready workflow task has no eligible definition route.', + 'routing_status' => $payload['definition_routing']['status'], + 'expected_resolution' => $payload['definition_routing']['resolution'], + ]; + $payload['diagnostic_status'] = 'needs_attention'; + } + return $payload; } diff --git a/app/Support/WorkflowTaskPoller.php b/app/Support/WorkflowTaskPoller.php index 6c21da0b..4cc91734 100644 --- a/app/Support/WorkflowTaskPoller.php +++ b/app/Support/WorkflowTaskPoller.php @@ -12,7 +12,6 @@ use Workflow\V2\Enums\TaskStatus; use Workflow\V2\Enums\TaskType; use Workflow\V2\Jobs\RunTimerTask; -use Workflow\V2\Models\WorkflowHistoryEvent; use Workflow\V2\Models\WorkflowRun; use Workflow\V2\Models\WorkflowSignal; use Workflow\V2\Models\WorkflowTask; @@ -42,6 +41,7 @@ public function __construct( private readonly PollRequestLeaseBinding $pollLeaseBindings, private readonly WorkflowUpdateValidationTaskBroker $updateValidationTasks, private readonly WorkflowTaskKindSelector $taskKindSelector, + private readonly WorkflowDefinitionRouting $definitionRouting, ) {} /** @@ -1041,9 +1041,11 @@ private function claimReadyTask( } if (! $this->matchesRecordedWorkflowDefinition( + $namespace, + $leaseOwner, $runId, - $readyTask['workflow_type'] ?? null, - $workflowDefinitionFingerprints, + $taskQueue, + $effectiveCompatibility, )) { continue; } @@ -2325,63 +2327,25 @@ private function workerCanReplayRun( ->where('worker_id', $workerId) ->first(); - if (! $worker instanceof WorkerRegistration || ! WorkerPollFence::isFresh($worker)) { - return false; - } - - $capabilities = is_array($worker->capabilities) - ? array_values(array_filter( - $worker->capabilities, - static fn (mixed $capability): bool => is_string($capability) && trim($capability) !== '', - )) - : []; - $run = WorkflowRun::query()->where('namespace', $namespace)->find($runId); - if ($run instanceof WorkflowRun && is_string($run->cancellation_request_command_id) - && ! CooperativeCancellationPolicy::supports($capabilities, $protocolVersion)) { - return false; - } - if ($run instanceof WorkflowRun && is_string($run->cancellation_request_command_id) - && NexusCooperativeCancellationPolicy::requiresCapabilityForReplay($run) - && ! NexusCooperativeCancellationPolicy::supports($capabilities, $protocolVersion)) { - return false; - } - return PreparedLocalActivityPolicy::canReplayRun($runId, $capabilities, $protocolVersion) - && WorkflowMetadataCapabilityPolicy::canReplayRun( - $runId, - $capabilities, - $protocolVersion, - ); + return $worker instanceof WorkerRegistration && $run instanceof WorkflowRun + && WorkflowReplayEligibility::allows($worker, $run, $protocolVersion); } - /** @param array $workflowDefinitionFingerprints */ private function matchesRecordedWorkflowDefinition( + string $namespace, + string $workerId, string $runId, - mixed $workflowType, - array $workflowDefinitionFingerprints, + string $taskQueue, + ?string $requiredCompatibility, ): bool { - $started = WorkflowHistoryEvent::query() - ->where('workflow_run_id', $runId) - ->where('event_type', 'WorkflowStarted') - ->orderBy('sequence') - ->first(); - - if (($started?->payload['workflow_definition_fingerprint_source'] ?? null) !== 'worker') { - return true; - } - - $recorded = $this->nonEmptyString($started?->payload['workflow_definition_fingerprint'] ?? null); - - if ($recorded === null) { - return false; - } - - $advertised = is_string($workflowType) - ? $this->nonEmptyString($workflowDefinitionFingerprints[$workflowType] ?? null) - : null; + $run = WorkflowRun::query()->where('namespace', $namespace)->find($runId); + $worker = WorkerRegistration::query()->where('namespace', $namespace) + ->where('worker_id', $workerId)->first(); - return $advertised !== null && hash_equals($recorded, $advertised); + return $run instanceof WorkflowRun && $worker instanceof WorkerRegistration + && $this->definitionRouting->forWorker($run, $worker, $taskQueue, $requiredCompatibility)['allowed']; } /** diff --git a/config/dw-contract.php b/config/dw-contract.php index f9f23786..2e8abf2f 100644 --- a/config/dw-contract.php +++ b/config/dw-contract.php @@ -889,7 +889,7 @@ 'legacy' => 'WORKFLOW_V2_COMPATIBILITY_HEARTBEAT_TTL', ], 'DW_V2_PIN_TO_RECORDED_FINGERPRINT' => [ - 'description' => 'When true (default), in-flight runs resolve their workflow class from the fingerprint recorded at WorkflowStarted.', + 'description' => 'Prefer an eligible worker matching the recorded definition. If none remains, current code must validate strict replay. False removes only this source preference, not build pins or replay checks.', 'default' => 'true', 'since' => '2.0.0', 'legacy' => 'WORKFLOW_V2_PIN_TO_RECORDED_FINGERPRINT', diff --git a/docs/server-reference.md b/docs/server-reference.md index 8d836aa7..d793ee96 100644 --- a/docs/server-reference.md +++ b/docs/server-reference.md @@ -418,6 +418,24 @@ Workers that omit fingerprints during re-registration cannot clear previously stored fingerprints for workflow types they still advertise; the server keeps the stored value until a new worker id is used. +Existing runs retain their original definition fingerprint, build pin, history +and deadlines. By default, the Server prefers a fresh eligible worker with the +recorded definition. When that definition is no longer available in the run's +namespace, queue, workflow type and build, a replacement definition can claim +the original task and must pass the SDK's strict replay checks. Use versioned +branches such as `get_version` for changes to durable commands. A different +source fingerprint alone is not proof of incompatible replay. + +`DW_V2_PIN_TO_RECORDED_FINGERPRINT=false` removes the matching-source preference. +It does not remove build pins, namespace or queue isolation, workflow-type and +capability checks, immutable registration identity, lease fences or strict replay. +Run detail, run debug, repair responses and queue detail expose +`definition_routing`. `replay_required` is an allowed fallback, while +`blocked=true` means a due ready workflow task has no eligible definition route. +Repair can fix transport state; it cannot manufacture a compatible worker or +rewrite the original definition. Queue observations contain at most ten run +examples and explicitly report whether more ready runs exist. + ### 5. Poll for Workflow Tasks The server holds the connection open (long-poll) until a task is ready or @@ -1715,7 +1733,7 @@ inside the package's `config/workflows.php` via | `DW_V2_SUPPORTED_COMPATIBILITIES` | (unset) | Comma-separated marker list the worker accepts, or `*` for any. | | `DW_V2_COMPATIBILITY_NAMESPACE` | (unset) | Compatibility namespace for independent fleets sharing one database. | | `DW_V2_COMPATIBILITY_HEARTBEAT_TTL` | `30` | Seconds a worker-compatibility heartbeat remains valid. | -| `DW_V2_PIN_TO_RECORDED_FINGERPRINT` | `true` | Resolve in-flight runs from the fingerprint recorded at WorkflowStarted. | +| `DW_V2_PIN_TO_RECORDED_FINGERPRINT` | `true` | Prefer an eligible recorded-definition worker; otherwise require strict replay by current code. False removes this preference, not build pins or replay checks. | | `DW_V2_CONTINUE_AS_NEW_EVENT_THRESHOLD` | `10000` | History event count at which the package signals continue-as-new. | | `DW_V2_CONTINUE_AS_NEW_SIZE_BYTES_THRESHOLD` | `5242880` | Serialized-history byte count at which the package signals continue-as-new. | | `DW_V2_HISTORY_EVENT_WARNING_THRESHOLD` | `8000` | Event count at which history pressure becomes `approaching`. | diff --git a/tests/Feature/WorkflowDefinitionEvolutionTest.php b/tests/Feature/WorkflowDefinitionEvolutionTest.php index 93ff05ac..3bbd0ac7 100644 --- a/tests/Feature/WorkflowDefinitionEvolutionTest.php +++ b/tests/Feature/WorkflowDefinitionEvolutionTest.php @@ -2,11 +2,16 @@ namespace Tests\Feature; +use App\Models\WorkerRegistration; +use App\Support\WorkflowReplayEligibility; use Illuminate\Foundation\Testing\RefreshDatabase; use Illuminate\Support\Facades\Queue; +use Illuminate\Testing\TestResponse; +use PHPUnit\Framework\Attributes\DataProvider; use Tests\Feature\Concerns\ServerTestHelpers; use Tests\TestCase; use Workflow\V2\Enums\TaskStatus; +use Workflow\V2\Models\WorkflowCommand; use Workflow\V2\Models\WorkflowHistoryEvent; use Workflow\V2\Models\WorkflowRun; use Workflow\V2\Models\WorkflowTask; @@ -33,7 +38,7 @@ protected function setUp(): void public function test_default_replacement_can_claim_when_the_recorded_definition_is_gone(): void { [$run, $task, $started] = $this->original(); - $original = [$started->getRawOriginal('payload'), $run->run_deadline_at, $run->execution_deadline_at]; + $original = [$started->getRawOriginal('payload'), $run->getRawOriginal('run_deadline_at'), $run->getRawOriginal('execution_deadline_at')]; $this->deleteJson('/api/workers/original-worker', [], $this->apiHeaders())->assertOk(); $this->worker('replacement-worker', 'definition-v2'); @@ -42,7 +47,7 @@ public function test_default_replacement_can_claim_when_the_recorded_definition_ ->assertJsonPath('task.task_id', $task->id); self::assertSame($original, [ - $started->fresh()->getRawOriginal('payload'), $run->fresh()->run_deadline_at, $run->fresh()->execution_deadline_at, + $started->fresh()->getRawOriginal('payload'), $run->fresh()->getRawOriginal('run_deadline_at'), $run->fresh()->getRawOriginal('execution_deadline_at'), ]); self::assertSame(1, $task->fresh()->attempt_count); self::assertSame(0, $task->fresh()->repair_count); @@ -72,6 +77,169 @@ public function test_default_keeps_the_matching_registered_peer_preference(): vo $this->poll('original-worker')->assertOk()->assertJsonPath('task.task_id', $task->id); } + #[DataProvider('ineligibleRecordedWorkers')] + public function test_an_ineligible_recorded_peer_does_not_strand_a_replacement(array $attributes): void + { + [, $task] = $this->original(); + WorkerRegistration::query()->where('worker_id', 'original-worker')->firstOrFail() + ->forceFill($attributes)->save(); + $this->worker('replacement-worker', 'definition-v2'); + + $this->poll('replacement-worker')->assertOk()->assertJsonPath('task.task_id', $task->id); + self::assertSame(1, $task->fresh()->attempt_count); + self::assertSame(0, $task->fresh()->repair_count); + } + + public static function ineligibleRecordedWorkers(): array + { + return [ + 'stale heartbeat' => [['last_heartbeat_at' => '2000-01-01 00:00:00']], + 'missing heartbeat' => [['last_heartbeat_at' => null]], + 'draining' => [['status' => 'draining']], + 'other namespace' => [['namespace' => 'other']], + 'other queue' => [['task_queue' => 'other']], + 'other type' => [['supported_workflow_types' => ['other']]], + 'no workflow capacity' => [['max_concurrent_workflow_tasks' => 0]], + 'missing definition' => [['workflow_definition_fingerprints' => []]], + ]; + } + + public function test_busy_matching_peer_retains_preference_without_inventing_slot_authority(): void + { + [, $task] = $this->original(); + WorkerRegistration::query()->where('worker_id', 'original-worker')->update(['available_workflow_slots' => 0]); + $this->worker('replacement-worker', 'definition-v2'); + + $this->poll('replacement-worker')->assertOk()->assertJsonPath('task', null); + self::assertSame(TaskStatus::Ready, $task->fresh()->status); + self::assertSame(0, $task->fresh()->attempt_count); + } + + public function test_fallback_does_not_remove_the_original_build_pin(): void + { + [$run, $task] = $this->original(); + $run->forceFill(['compatibility' => 'build-v1'])->save(); + $task->forceFill(['compatibility' => 'build-v1'])->save(); + $this->deleteJson('/api/workers/original-worker', [], $this->apiHeaders())->assertOk(); + $this->worker('replacement-worker', 'definition-v2', ['build_id' => 'build-v2']); + + $this->poll('replacement-worker', ['build_id' => 'build-v2'])->assertOk()->assertJsonPath('task', null); + self::assertSame(TaskStatus::Ready, $task->fresh()->status); + self::assertSame(0, $task->fresh()->attempt_count); + self::assertSame('build-v1', $run->fresh()->compatibility); + } + + public function test_a_matching_peer_on_another_build_cannot_suppress_a_pinned_replacement(): void + { + [$run, $task] = $this->original(); + $run->forceFill(['compatibility' => 'build-v1'])->save(); + $task->forceFill(['compatibility' => 'build-v1'])->save(); + WorkerRegistration::query()->where('worker_id', 'original-worker')->update(['build_id' => 'build-v2']); + $this->worker('replacement-worker', 'definition-v2', ['build_id' => 'build-v1']); + + $this->poll('replacement-worker', ['build_id' => 'build-v1'])->assertOk()->assertJsonPath('task.task_id', $task->id); + self::assertSame('build-v1', $run->fresh()->compatibility); + } + + public function test_matching_peer_must_support_the_recorded_replay_protocol(): void + { + [$run, $task] = $this->original(); + $worker = WorkerRegistration::query()->where('worker_id', 'original-worker')->firstOrFail(); + $worker->forceFill(['process_metrics' => [WorkflowReplayEligibility::PROTOCOL_METRIC => '1.16']])->save(); + WorkflowHistoryEvent::query()->create([ + 'workflow_run_id' => $run->id, + 'sequence' => WorkflowHistoryEvent::query()->where('workflow_run_id', $run->id)->max('sequence') + 1, + 'event_type' => 'TimerScheduled', + 'payload' => ['condition_wait_occurrence_id' => 'original-condition-occurrence'], + ]); + $this->worker('replacement-worker', 'definition-v2'); + + $this->poll('replacement-worker')->assertOk()->assertJsonPath('task.task_id', $task->id); + } + + public function test_client_process_metrics_cannot_spoof_the_saved_protocol(): void + { + $this->worker('original-worker', 'definition-v1', [ + 'process_metrics' => [WorkflowReplayEligibility::PROTOCOL_METRIC => '999.0', 'memory_bytes' => 123], + ]); + $worker = WorkerRegistration::query()->where('worker_id', 'original-worker')->firstOrFail(); + self::assertSame($this->workerHeaders()['X-Durable-Workflow-Protocol-Version'], WorkflowReplayEligibility::recordedProtocol($worker)); + self::assertSame(123, $worker->process_metrics['memory_bytes']); + + $this->postJson('/api/worker/heartbeat', [ + 'worker_id' => 'original-worker', 'process_metrics' => [WorkflowReplayEligibility::PROTOCOL_METRIC => '999.0', 'memory_bytes' => 456], + ], $this->workerHeaders())->assertOk(); + self::assertSame($this->workerHeaders()['X-Durable-Workflow-Protocol-Version'], WorkflowReplayEligibility::recordedProtocol($worker->fresh())); + self::assertSame(456, $worker->fresh()->process_metrics['memory_bytes']); + } + + public function test_run_debug_queue_and_repair_distinguish_fallback_from_a_real_block(): void + { + [$run, $task] = $this->original(); + $this->deleteJson('/api/workers/original-worker', [], $this->apiHeaders())->assertOk(); + $this->worker('replacement-worker', 'definition-v2'); + $this->getJson('/api/workflows/definition-original', $this->apiHeaders())->assertOk() + ->assertJsonPath('definition_routing.status', 'replay_required') + ->assertJsonPath('definition_routing.blocked', false) + ->assertJsonPath('definition_routing.recorded_fingerprint', 'definition-v1'); + $this->getJson('/api/workflows/definition-original/runs/'.$run->id, $this->apiHeaders())->assertOk() + ->assertJsonPath('definition_routing.status', 'replay_required'); + $this->getJson('/api/workflows/definition-original/debug', $this->apiHeaders())->assertOk() + ->assertJsonPath('definition_routing.blocked', false) + ->assertJsonMissing(['code' => 'workflow_definition_routing_blocked']); + + WorkerRegistration::query()->where('worker_id', 'replacement-worker')->firstOrFail() + ->forceFill(['workflow_definition_fingerprints' => []])->save(); + $this->getJson('/api/workflows/definition-original', $this->apiHeaders())->assertOk() + ->assertJsonPath('definition_routing.status', 'worker_definition_missing') + ->assertJsonPath('definition_routing.blocked', true); + $this->getJson('/api/workflows/definition-original/debug', $this->apiHeaders())->assertOk() + ->assertJsonPath('diagnostic_status', 'needs_attention') + ->assertJsonFragment(['code' => 'workflow_definition_routing_blocked']); + $this->getJson('/api/task-queues/'.self::QUEUE, $this->apiHeaders())->assertOk() + ->assertJsonPath('definition_routing.blocked_sample_count', 1) + ->assertJsonPath('definition_routing.runs.0.run_id', $run->id) + ->assertJsonPath('repair.needs_attention', true); + $repair = $this->postJson('/api/workflows/definition-original/repair', [ + 'request_id' => 'definition-routing-repair', + ], $this->apiHeaders())->assertOk() + ->assertJsonPath('definition_routing.status', 'worker_definition_missing') + ->assertJsonPath('definition_routing.blocked', true); + $command = WorkflowCommand::query()->findOrFail($repair->json('command_id')); + self::assertSame($command->commandReason(), $repair->json('reason')); + self::assertSame($command->outcome->value, $repair->json('outcome')); + self::assertSame(0, $task->fresh()->attempt_count); + self::assertSame(0, $task->fresh()->repair_count); + + $this->createNamespace('other'); + $this->getJson('/api/task-queues/'.self::QUEUE, $this->apiHeaders('other'))->assertOk() + ->assertJsonPath('definition_routing.runs', []); + } + + #[DataProvider('notRunnableTasks')] + public function test_definition_drift_is_not_a_ready_task_block_when_delivery_is_not_due(string $case): void + { + [$run, $task] = $this->original(); + $this->deleteJson('/api/workers/original-worker', [], $this->apiHeaders())->assertOk(); + if ($case === 'terminal') { + $run->forceFill(['status' => 'completed', 'closed_at' => now()])->save(); + } elseif ($case === 'leased') { + $task->forceFill(['status' => TaskStatus::Leased, 'lease_owner' => 'original-worker'])->save(); + } else { + $task->forceFill(['available_at' => now()->addMinutes(1)])->save(); + } + $this->getJson('/api/workflows/definition-original', $this->apiHeaders())->assertOk() + ->assertJsonPath('definition_routing.runnable_task', false) + ->assertJsonPath('definition_routing.blocked', false); + $this->getJson('/api/workflows/definition-original/debug', $this->apiHeaders())->assertOk() + ->assertJsonMissing(['code' => 'workflow_definition_routing_blocked']); + } + + public static function notRunnableTasks(): array + { + return [['terminal'], ['leased'], ['future']]; + } + private function original(): array { $this->worker('original-worker', 'definition-v1'); @@ -89,7 +257,7 @@ private function original(): array return [$run, $task, $started]; } - private function worker(string $id, string $fingerprint): void + private function worker(string $id, string $fingerprint, array $overrides = []): void { $this->postJson('/api/worker/register', [ 'worker_id' => $id, 'task_queue' => self::QUEUE, 'runtime' => 'python', 'sdk_version' => '2.6.1', @@ -97,13 +265,15 @@ private function worker(string $id, string $fingerprint): void 'supported_workflow_types' => [self::TYPE], 'workflow_definition_fingerprints' => [self::TYPE => $fingerprint], 'workflow_command_contracts' => [self::TYPE => ['signals' => [], 'queries' => [], 'updates' => [], 'update_validators' => []]], + ...$overrides, ], $this->workerHeaders())->assertCreated(); } - private function poll(string $id): \Illuminate\Testing\TestResponse + private function poll(string $id, array $overrides = []): TestResponse { return $this->postJson('/api/worker/workflow-tasks/poll', [ 'worker_id' => $id, 'task_queue' => self::QUEUE, 'timeout_seconds' => 0, + ...$overrides, ], $this->workerHeaders()); } } From cd7d110c724ba0a45c03041267c5879981e5983e Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 09:21:27 +0000 Subject: [PATCH 15/42] Convert each PHP release layout to native's instead of chasing it Co-Authored-By: Claude Opus 5.5 --- docs/rust-server-upgrade-takeover.md | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/docs/rust-server-upgrade-takeover.md b/docs/rust-server-upgrade-takeover.md index 52a8c03e..ca8ddd6f 100644 --- a/docs/rust-server-upgrade-takeover.md +++ b/docs/rust-server-upgrade-takeover.md @@ -103,13 +103,24 @@ marker exists. Native's frozen physical layout predates the current PHP release, so step 3 alone does not yield a database native accepts. PHP 2.8.0 adds its own -`workflow_worker_registration_incarnations` table, a `call_deadline_at` -column on `workflow_service_calls` and a changed `workflow_worker_registrations` -(`rust/upgrade/php-2.8.0/`). Before takeover is implemented, native's schema -moves to the layout of the release takeover starts from, and native reads -PHP's incarnation table instead of keeping a second one. A fresh native -database then has the same PHP layout, and takeover only adds native relations -and converts data. +`workflow_worker_registration_incarnations` table, a `registration_token` +column on `workflow_worker_registrations` and a `call_deadline_at` column on +`workflow_service_calls` (`rust/upgrade/php-2.8.0/`). Server 2.9.7, with Core +2.7.2, also adds `workflow_tasks.queue_delivery_reference`. Native keeps its +own layout, and step 4 converts each qualified PHP release's layout to it +with a recorded recipe: + +- PHP's registration incarnations and tokens are copied into native's + incarnation table, so a running SDK worker's fenced deregistration still + matches. The PHP table and column are then removed. +- `queue_delivery_reference` describes PHP's database queue delivery, which + native does not use. It is removed after its values are checked against + the queue inspection's timer bindings. +- Service calls are not part of native yet, so a source with service calls + is refused before any change, and `call_deadline_at` is removed only from + an empty table. + +A release whose layout has no recipe is refused by step 1. ## Tests From 57de9ec39b4107f32d2a7bcb6b070a4158cf3dc1 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 09:28:33 +0000 Subject: [PATCH 16/42] Bind routing diagnostics to the canonical selected or current run --- app/Support/ServerWorkflowControlPlane.php | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/app/Support/ServerWorkflowControlPlane.php b/app/Support/ServerWorkflowControlPlane.php index 3094a947..caa2da4f 100644 --- a/app/Support/ServerWorkflowControlPlane.php +++ b/app/Support/ServerWorkflowControlPlane.php @@ -323,10 +323,11 @@ private function withDefinitionRouting(string $instanceId, array $options, array return $result; } - $runId = data_get($result, 'run.run_id') ?? $result['workflow_run_id'] ?? $options['run_id'] ?? null; - $run = WorkflowRun::query()->where('namespace', $namespace)->where('workflow_instance_id', $instanceId) - ->when(is_string($runId), static fn ($query) => $query->whereKey($runId)) - ->orderByDesc('run_number')->first(); + $runId = data_get($result, 'run.run_id') ?? $result['resolved_run_id'] ?? $result['run_id'] + ?? $result['workflow_run_id'] ?? $options['run_id'] ?? null; + $run = is_string($runId) + ? NamespaceWorkflowScope::run($namespace, $instanceId, $runId) + : NamespaceWorkflowScope::currentRun($namespace, $instanceId); if ($run instanceof WorkflowRun) { $result['definition_routing'] = $this->definitionRouting->inspect($run); } From a8e324a548f7af06142c54756b4d29fd045bdc37 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 09:54:15 +0000 Subject: [PATCH 17/42] Exercise configured PostgreSQL bound queries through transaction poolers --- tests/Feature/PostgresPoolerTest.php | 83 ++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 tests/Feature/PostgresPoolerTest.php diff --git a/tests/Feature/PostgresPoolerTest.php b/tests/Feature/PostgresPoolerTest.php new file mode 100644 index 00000000..8be5876e --- /dev/null +++ b/tests/Feature/PostgresPoolerTest.php @@ -0,0 +1,83 @@ +markTestSkipped('Requires the isolated DW_POOLER_TEST_HOST endpoint.'); + } + $base = config('database.connections.pgsql'); + foreach (['pooler_first', 'pooler_second'] as $name) { + config(['database.connections.'.$name => [...$base, + 'host' => getenv('DW_POOLER_TEST_HOST'), + 'port' => getenv('DW_POOLER_TEST_PORT') ?: '6432', + 'database' => getenv('DW_POOLER_TEST_DATABASE') ?: 'dw_pooler_probe', + 'username' => getenv('DW_POOLER_TEST_USER') ?: 'pooler_probe', + 'password' => getenv('DW_POOLER_TEST_PASSWORD') ?: '', + ]]); + } + } + + protected function tearDown(): void + { + foreach (['pooler_first', 'pooler_second'] as $name) { + DB::purge($name); + } + parent::tearDown(); + } + + public function test_clients_keep_their_bound_values_and_types_when_backend_sessions_change(): void + { + $first = DB::connection('pooler_first'); + $second = DB::connection('pooler_second'); + + // Occupy two server sessions using simple protocol commands first. + // The probe itself then exercises Laravel's configured PDO strategy. + $first->getPdo()->exec('BEGIN'); + $first->getPdo()->exec('SELECT 1'); + $second->getPdo()->exec('BEGIN'); + $second->getPdo()->exec('SELECT 1'); + $first->getPdo()->exec('ROLLBACK'); + $second->getPdo()->exec('ROLLBACK'); + + for ($iteration = 0; $iteration < 20; $iteration++) { + $marker = 'first-'.$iteration.'-'.bin2hex(random_bytes(8)); + $row = $first->selectOne('SELECT CAST(? AS TEXT) AS marker, CAST(? AS BIGINT) AS number, CAST(? AS TEXT) AS absent', + [$marker, '9223372036854775807', null]); + self::assertSame($marker, $row->marker); + self::assertSame('9223372036854775807', (string) $row->number); + self::assertNull($row->absent); + + $other = 'second-'.$iteration.'-Київ'; + $row = $second->selectOne('SELECT CAST(? AS TEXT) AS marker', [$other]); + self::assertSame($other, $row->marker); + } + } + + public function test_bound_statements_do_not_abort_a_multi_query_transaction(): void + { + $connection = DB::connection('pooler_first'); + $connection->beginTransaction(); + try { + foreach (['first', 'second', 'third'] as $value) { + self::assertSame($value, $connection->selectOne('SELECT CAST(? AS TEXT) AS marker', [$value])->marker); + self::assertSame(2, (int) $connection->selectOne('SELECT 1 + 1 AS result')->result); + } + } finally { + $connection->rollBack(); + } + self::assertSame('after-rollback', $connection->selectOne('SELECT CAST(? AS TEXT) AS marker', ['after-rollback'])->marker); + } +} From 0447dd5bc6208602329d31552916fdae1fd2deb9 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 10:01:37 +0000 Subject: [PATCH 18/42] Reproduce readiness passing a connected database with broken bound reads --- tests/Feature/HealthControllerTest.php | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tests/Feature/HealthControllerTest.php b/tests/Feature/HealthControllerTest.php index 46919084..cb0d629b 100644 --- a/tests/Feature/HealthControllerTest.php +++ b/tests/Feature/HealthControllerTest.php @@ -11,6 +11,7 @@ use App\Support\ServerReadiness; use App\Support\ServerTopology; use Illuminate\Foundation\Testing\RefreshDatabase; +use Illuminate\Database\SQLiteConnection; use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Schema; use RuntimeException; @@ -25,6 +26,30 @@ class HealthControllerTest extends TestCase use RefreshDatabase; use ServerTestHelpers; + public function test_readiness_refuses_a_connected_database_that_cannot_execute_bound_reads(): void + { + $pdo = DB::connection()->getPdo(); + DB::extend('broken_bound_reads', static fn (array $config): SQLiteConnection => new class($pdo, ':memory:', '', $config) extends SQLiteConnection + { + public function select($query, $bindings = [], $useReadPdo = true) + { + if ($bindings !== []) { + throw new \PDOException('The connected database cannot execute bound statements.'); + } + + return parent::select($query, $bindings, $useReadPdo); + } + }); + config([ + 'database.connections.broken_bound_reads' => ['driver' => 'broken_bound_reads'], + 'database.default' => 'broken_bound_reads', + ]); + + $this->getJson('/api/ready') + ->assertStatus(503) + ->assertJsonPath('checks.database.status', 'unavailable'); + } + public function test_health_check_returns_serving_when_database_is_available(): void { $response = $this->getJson('/api/health'); From ba06b0f8fff4c4c8e32cd041bd58cc7b35b27199 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 10:05:27 +0000 Subject: [PATCH 19/42] Use native unnamed PostgreSQL binding and probe primary query readiness --- app/Support/ServerReadiness.php | 19 ++++++++++++++----- config/database.php | 5 +++++ tests/Feature/HealthControllerTest.php | 6 ++++-- 3 files changed, 23 insertions(+), 7 deletions(-) diff --git a/app/Support/ServerReadiness.php b/app/Support/ServerReadiness.php index 43a4ce09..93bd425f 100644 --- a/app/Support/ServerReadiness.php +++ b/app/Support/ServerReadiness.php @@ -40,7 +40,7 @@ public function snapshot(): array : $this->cacheCheck(); $checks = [ - 'database' => $this->databaseCheck(), + 'database' => $this->databaseCheck(probeQuery: true), 'migrations' => $this->migrationCheck(), 'queue' => $queue, 'default_namespace' => $this->defaultNamespaceCheck(), @@ -93,16 +93,25 @@ public function bootstrapStatus(): array ])); } - private function databaseCheck(): array + private function databaseCheck(bool $probeQuery = false): array { try { - DB::connection()->getPdo(); + $connection = DB::connection(); + $connection->getPdo(); + if ($probeQuery) { + $row = $connection->selectOne('SELECT ? AS readiness_probe', [1], useReadPdo: false); + if ((string) ($row->readiness_probe ?? '') !== '1') { + throw new \RuntimeException('The database did not return the readiness probe value.'); + } + } return [ 'status' => 'ok', - 'check_scope' => 'connection_only', + 'check_scope' => $probeQuery ? 'bound_read_query' : 'connection_only', 'write_capacity_verified' => false, - 'message' => 'The database connection is available. Write capacity has not been verified.', + 'message' => $probeQuery + ? 'The primary database connection can execute a bound read query. Write capacity has not been verified.' + : 'The database connection is available. Write capacity has not been verified.', ]; } catch (\Throwable $exception) { return [ diff --git a/config/database.php b/config/database.php index 6fafd209..c80ae54b 100644 --- a/config/database.php +++ b/config/database.php @@ -98,6 +98,11 @@ 'prefix_indexes' => true, 'search_path' => 'public', 'sslmode' => 'prefer', + // Keep native parameter binding without session-scoped statement + // names, so transaction poolers can safely change backend sessions. + 'options' => extension_loaded('pdo_pgsql') ? [ + (PHP_VERSION_ID >= 80500 ? Pdo\Pgsql::ATTR_DISABLE_PREPARES : PDO::PGSQL_ATTR_DISABLE_PREPARES) => true, + ] : [], ], 'sqlsrv' => [ diff --git a/tests/Feature/HealthControllerTest.php b/tests/Feature/HealthControllerTest.php index cb0d629b..361b763e 100644 --- a/tests/Feature/HealthControllerTest.php +++ b/tests/Feature/HealthControllerTest.php @@ -31,13 +31,13 @@ public function test_readiness_refuses_a_connected_database_that_cannot_execute_ $pdo = DB::connection()->getPdo(); DB::extend('broken_bound_reads', static fn (array $config): SQLiteConnection => new class($pdo, ':memory:', '', $config) extends SQLiteConnection { - public function select($query, $bindings = [], $useReadPdo = true) + public function select($query, $bindings = [], $useReadPdo = true, array $fetchUsing = []) { if ($bindings !== []) { throw new \PDOException('The connected database cannot execute bound statements.'); } - return parent::select($query, $bindings, $useReadPdo); + return parent::select($query, $bindings, $useReadPdo, $fetchUsing); } }); config([ @@ -147,6 +147,8 @@ public function test_readiness_check_returns_ready_when_bootstrap_state_is_avail $response->assertOk() ->assertJsonPath('status', 'ready') ->assertJsonPath('checks.database.status', 'ok') + ->assertJsonPath('checks.database.check_scope', 'bound_read_query') + ->assertJsonPath('checks.database.write_capacity_verified', false) ->assertJsonPath('checks.migrations.status', 'ok') ->assertJsonPath('checks.migrations.operator_surface.available', true) ->assertJsonPath('checks.migrations.readiness_contract.version', 1) From 2e59f1b38275581d58e0aeb86e4ff8fba858342b Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 10:13:55 +0000 Subject: [PATCH 20/42] Qualify PostgreSQL transaction pools in ordinary PHP CI --- .github/workflows/phpunit-feature.yml | 21 +++++++++++++ app/Support/ServerReadiness.php | 4 ++- config/database.php | 8 +++-- docs/server-reference.md | 31 +++++++++++++++++++ tests/Feature/HealthControllerTest.php | 5 ++-- tests/Feature/PostgresPoolerTest.php | 41 +++++++++++++++++++++----- 6 files changed, 96 insertions(+), 14 deletions(-) diff --git a/.github/workflows/phpunit-feature.yml b/.github/workflows/phpunit-feature.yml index 16b58a36..d403c73e 100644 --- a/.github/workflows/phpunit-feature.yml +++ b/.github/workflows/phpunit-feature.yml @@ -220,6 +220,21 @@ jobs: done [ "$health" = healthy ] done + for mode in session transaction0 transaction200; do + pool_mode=transaction + prepared_statements=0 + [ "$mode" != session ] || pool_mode=session + [ "$mode" != transaction200 ] || prepared_statements=200 + docker run --detach \ + --name "dw-server-pooler-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }}-$mode" \ + --network "dw-server-feature-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }}" \ + --memory 128m --memory-swap 128m \ + -e DB_HOST="dw-server-backup-pgsql-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }}" \ + -e DB_NAME=postgres -e DB_USER=postgres -e DB_PASSWORD=pooler-disposable-password \ + -e AUTH_TYPE=trust -e "POOL_MODE=$pool_mode" -e "MAX_PREPARED_STATEMENTS=$prepared_statements" \ + -e DEFAULT_POOL_SIZE=2 -e MIN_POOL_SIZE=2 -e SERVER_ROUND_ROBIN=1 \ + edoburu/pgbouncer@sha256:b17551c776ef7e5769ef80b956d20f85e2fd25dd8912d31d58f782aad495b711 + done tar \ --exclude=vendor \ --exclude='*/vendor' \ @@ -236,6 +251,9 @@ jobs: -e DW_TEST_COMPLETION_MYSQL_HOST="dw-server-backup-mysql-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }}" \ -e DW_TEST_BACKUP_PGSQL_HOST="dw-server-backup-pgsql-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }}" \ -e DW_TEST_COMPLETION_PGSQL_HOST="dw-server-backup-pgsql-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }}" \ + -e DW_POOLER_TEST_HOST="dw-server-backup-pgsql-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }}" \ + -e DW_POOLER_TEST_PORT=5432 -e DW_POOLER_TEST_DATABASE=postgres -e DW_POOLER_TEST_USER=postgres \ + -e DW_POOLER_TEST_PREFIX="dw-server-pooler-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }}" \ -w /app \ --entrypoint sh \ durable-workflow-server-corpus-validator \ @@ -283,6 +301,9 @@ jobs: - name: Clean up test services if: always() run: | + for mode in session transaction0 transaction200; do + docker rm -f "dw-server-pooler-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }}-$mode" >/dev/null 2>&1 || true + done docker rm -f \ "dw-server-s3-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }}" \ "dw-server-backup-mysql-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }}" \ diff --git a/app/Support/ServerReadiness.php b/app/Support/ServerReadiness.php index 93bd425f..f11d9c7c 100644 --- a/app/Support/ServerReadiness.php +++ b/app/Support/ServerReadiness.php @@ -95,10 +95,12 @@ public function bootstrapStatus(): array private function databaseCheck(bool $probeQuery = false): array { + $queryAttempted = false; try { $connection = DB::connection(); $connection->getPdo(); if ($probeQuery) { + $queryAttempted = true; $row = $connection->selectOne('SELECT ? AS readiness_probe', [1], useReadPdo: false); if ((string) ($row->readiness_probe ?? '') !== '1') { throw new \RuntimeException('The database did not return the readiness probe value.'); @@ -117,7 +119,7 @@ private function databaseCheck(bool $probeQuery = false): array return [ 'status' => 'unavailable', 'message' => $exception->getMessage(), - 'reason' => BackendUnavailable::is($exception) ? 'backend_unavailable' : null, + 'reason' => BackendUnavailable::is($exception) ? 'backend_unavailable' : ($queryAttempted ? 'database_query_failed' : null), ]; } } diff --git a/config/database.php b/config/database.php index c80ae54b..aebce8f6 100644 --- a/config/database.php +++ b/config/database.php @@ -1,6 +1,8 @@ true, 'engine' => null, 'options' => extension_loaded('pdo_mysql') ? array_filter([ - (PHP_VERSION_ID >= 80500 ? Pdo\Mysql::ATTR_SSL_CA : PDO::MYSQL_ATTR_SSL_CA) => env('MYSQL_ATTR_SSL_CA'), + (PHP_VERSION_ID >= 80500 ? Mysql::ATTR_SSL_CA : PDO::MYSQL_ATTR_SSL_CA) => env('MYSQL_ATTR_SSL_CA'), ]) : [], ], @@ -81,7 +83,7 @@ 'strict' => true, 'engine' => null, 'options' => extension_loaded('pdo_mysql') ? array_filter([ - (PHP_VERSION_ID >= 80500 ? Pdo\Mysql::ATTR_SSL_CA : PDO::MYSQL_ATTR_SSL_CA) => env('MYSQL_ATTR_SSL_CA'), + (PHP_VERSION_ID >= 80500 ? Mysql::ATTR_SSL_CA : PDO::MYSQL_ATTR_SSL_CA) => env('MYSQL_ATTR_SSL_CA'), ]) : [], ], @@ -101,7 +103,7 @@ // Keep native parameter binding without session-scoped statement // names, so transaction poolers can safely change backend sessions. 'options' => extension_loaded('pdo_pgsql') ? [ - (PHP_VERSION_ID >= 80500 ? Pdo\Pgsql::ATTR_DISABLE_PREPARES : PDO::PGSQL_ATTR_DISABLE_PREPARES) => true, + (PHP_VERSION_ID >= 80500 ? Pgsql::ATTR_DISABLE_PREPARES : PDO::PGSQL_ATTR_DISABLE_PREPARES) => true, ] : [], ], diff --git a/docs/server-reference.md b/docs/server-reference.md index 8d836aa7..7c6ca9c1 100644 --- a/docs/server-reference.md +++ b/docs/server-reference.md @@ -1599,6 +1599,37 @@ overlay or with `kubectl scale`. ### Configuration +#### PostgreSQL connection pooling + +The PostgreSQL connection uses native bound parameters without session-scoped +prepared-statement names. This lets PgBouncer transaction pooling change the +physical database session between transactions. Direct and session-pooled +connections use the same binding path. Statement pooling cannot support the +Server's multi-statement transactions. + +The PHP feature job runs the same alternating-client and rollback controls +against direct PostgreSQL, session pooling, and transaction pooling with +`max_prepared_statements=0` and `200`. The transaction controls require actual +backend PID changes. To run against an isolated existing endpoint: + +```bash +DW_POOLER_TEST_HOST=pooler DW_POOLER_TEST_PORT=6432 \ +DW_POOLER_TEST_DATABASE=isolated_test DW_POOLER_TEST_USER=test_user \ +vendor/bin/phpunit tests/Feature/PostgresPoolerTest.php --fail-on-skipped +``` + +These controls do not validate custom session settings, temporary tables, +advisory-lock policies, or arbitrary proxy configurations. Qualify the exact +published image, pooler version, authentication, schema and TLS settings used +by your deployment before routing production traffic. + +`/api/ready` executes a bound read on the primary database connection and +reports `checks.database.check_scope=bound_read_query`. A failed query reports +`database_query_failed`, or `backend_unavailable` when the backend is known to +be unavailable. This remains a read probe: `write_capacity_verified=false`. +Use an actual workflow canary to verify writes and the complete worker path. +The runtime's lightweight bootstrap guard remains connection-only. + All operator-facing configuration is via `DW_*` environment variables. `config/dw-contract.php` is the authoritative machine-checkable contract; CI (`tests/Unit/EnvContractTest.php`) diffs it against `.env.example`, diff --git a/tests/Feature/HealthControllerTest.php b/tests/Feature/HealthControllerTest.php index 361b763e..d40ef79f 100644 --- a/tests/Feature/HealthControllerTest.php +++ b/tests/Feature/HealthControllerTest.php @@ -10,8 +10,8 @@ use App\Support\RedisReadinessProcess; use App\Support\ServerReadiness; use App\Support\ServerTopology; -use Illuminate\Foundation\Testing\RefreshDatabase; use Illuminate\Database\SQLiteConnection; +use Illuminate\Foundation\Testing\RefreshDatabase; use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Schema; use RuntimeException; @@ -47,7 +47,8 @@ public function select($query, $bindings = [], $useReadPdo = true, array $fetchU $this->getJson('/api/ready') ->assertStatus(503) - ->assertJsonPath('checks.database.status', 'unavailable'); + ->assertJsonPath('checks.database.status', 'unavailable') + ->assertJsonPath('checks.database.reason', 'database_query_failed'); } public function test_health_check_returns_serving_when_database_is_available(): void diff --git a/tests/Feature/PostgresPoolerTest.php b/tests/Feature/PostgresPoolerTest.php index 8be5876e..b04fbe2b 100644 --- a/tests/Feature/PostgresPoolerTest.php +++ b/tests/Feature/PostgresPoolerTest.php @@ -3,6 +3,7 @@ namespace Tests\Feature; use Illuminate\Support\Facades\DB; +use PHPUnit\Framework\Attributes\DataProvider; use Tests\TestCase; /** @@ -12,17 +13,29 @@ */ class PostgresPoolerTest extends TestCase { - protected function setUp(): void + public static function endpoints(): array { - parent::setUp(); - if (! is_string(getenv('DW_POOLER_TEST_HOST')) || getenv('DW_POOLER_TEST_HOST') === '') { + $endpoints = ['configured' => [getenv('DW_POOLER_TEST_HOST') ?: null, (int) (getenv('DW_POOLER_TEST_PORT') ?: 6432), getenv('DW_POOLER_TEST_EXPECT_SWITCH') === '1']]; + $prefix = getenv('DW_POOLER_TEST_PREFIX'); + if (is_string($prefix) && $prefix !== '') { + foreach (['session', 'transaction0', 'transaction200'] as $mode) { + $endpoints[$mode] = [$prefix.'-'.$mode, 5432, $mode !== 'session']; + } + } + + return $endpoints; + } + + private function configureEndpoint(?string $host, int $port): void + { + if ($host === null) { $this->markTestSkipped('Requires the isolated DW_POOLER_TEST_HOST endpoint.'); } $base = config('database.connections.pgsql'); foreach (['pooler_first', 'pooler_second'] as $name) { config(['database.connections.'.$name => [...$base, - 'host' => getenv('DW_POOLER_TEST_HOST'), - 'port' => getenv('DW_POOLER_TEST_PORT') ?: '6432', + 'host' => $host, + 'port' => $port, 'database' => getenv('DW_POOLER_TEST_DATABASE') ?: 'dw_pooler_probe', 'username' => getenv('DW_POOLER_TEST_USER') ?: 'pooler_probe', 'password' => getenv('DW_POOLER_TEST_PASSWORD') ?: '', @@ -38,8 +51,10 @@ protected function tearDown(): void parent::tearDown(); } - public function test_clients_keep_their_bound_values_and_types_when_backend_sessions_change(): void + #[DataProvider('endpoints')] + public function test_clients_keep_their_bound_values_and_types_when_backend_sessions_change(?string $host, int $port, bool $expectSwitch): void { + $this->configureEndpoint($host, $port); $first = DB::connection('pooler_first'); $second = DB::connection('pooler_second'); @@ -52,10 +67,12 @@ public function test_clients_keep_their_bound_values_and_types_when_backend_sess $first->getPdo()->exec('ROLLBACK'); $second->getPdo()->exec('ROLLBACK'); + $backends = []; for ($iteration = 0; $iteration < 20; $iteration++) { $marker = 'first-'.$iteration.'-'.bin2hex(random_bytes(8)); - $row = $first->selectOne('SELECT CAST(? AS TEXT) AS marker, CAST(? AS BIGINT) AS number, CAST(? AS TEXT) AS absent', + $row = $first->selectOne('SELECT CAST(? AS TEXT) AS marker, CAST(? AS BIGINT) AS number, CAST(? AS TEXT) AS absent, pg_backend_pid() AS backend', [$marker, '9223372036854775807', null]); + $backends[] = $row->backend; self::assertSame($marker, $row->marker); self::assertSame('9223372036854775807', (string) $row->number); self::assertNull($row->absent); @@ -63,11 +80,19 @@ public function test_clients_keep_their_bound_values_and_types_when_backend_sess $other = 'second-'.$iteration.'-Київ'; $row = $second->selectOne('SELECT CAST(? AS TEXT) AS marker', [$other]); self::assertSame($other, $row->marker); + // An odd number of commands per iteration makes the next first + // client's query use another backend in the round-robin pool. + self::assertSame(1, (int) $first->selectOne('SELECT 1 AS value')->value); + } + if ($expectSwitch) { + self::assertGreaterThan(1, count(array_unique($backends)), 'The control must change this client\'s physical backend.'); } } - public function test_bound_statements_do_not_abort_a_multi_query_transaction(): void + #[DataProvider('endpoints')] + public function test_bound_statements_do_not_abort_a_multi_query_transaction(?string $host, int $port, bool $expectSwitch): void { + $this->configureEndpoint($host, $port); $connection = DB::connection('pooler_first'); $connection->beginTransaction(); try { From 75d6f3801c8a00bcef016045afe4ecc8d6e942b3 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 10:34:49 +0000 Subject: [PATCH 21/42] Clarify replay routing decisions and preserve explicit metrics clearing --- app/Http/Controllers/Api/WorkerController.php | 2 +- app/Support/WorkflowDefinitionRouting.php | 33 +++++++++---------- app/Support/WorkflowReplayEligibility.php | 9 ++--- app/Support/WorkflowTaskPoller.php | 3 -- .../WorkflowDefinitionEvolutionTest.php | 12 +++++++ 5 files changed, 32 insertions(+), 27 deletions(-) diff --git a/app/Http/Controllers/Api/WorkerController.php b/app/Http/Controllers/Api/WorkerController.php index a90ca6cc..4da9a14e 100644 --- a/app/Http/Controllers/Api/WorkerController.php +++ b/app/Http/Controllers/Api/WorkerController.php @@ -940,7 +940,7 @@ public function heartbeat(Request $request): JsonResponse $update['process_metrics'] = $this->normalizeProcessMetrics($validated['process_metrics']); } $update['process_metrics'] = [ - ...($update['process_metrics'] ?? $worker->process_metrics ?? []), + ...(array_key_exists('process_metrics', $update) ? ($update['process_metrics'] ?? []) : ($worker->process_metrics ?? [])), WorkflowReplayEligibility::PROTOCOL_METRIC => WorkerProtocol::requestVersion($request), ]; diff --git a/app/Support/WorkflowDefinitionRouting.php b/app/Support/WorkflowDefinitionRouting.php index cb1f264f..2719ee01 100644 --- a/app/Support/WorkflowDefinitionRouting.php +++ b/app/Support/WorkflowDefinitionRouting.php @@ -25,18 +25,16 @@ public function forWorker( $advertised = self::stringValue($worker->workflow_definition_fingerprints[$run->workflow_type] ?? null); $decision['advertised_fingerprint'] = $advertised; - if (! $decision['worker_pinned']) { - return [...$decision, 'allowed' => true, 'status' => 'not_worker_pinned']; - } - if (! $decision['pinning_enabled']) { - return [...$decision, 'allowed' => true, 'status' => 'pinning_disabled']; - } - if ($recorded === null || $advertised === null) { - return [...$decision, 'allowed' => false, 'status' => $recorded === null - ? 'recorded_definition_missing' : 'worker_definition_missing']; - } - if (hash_equals($recorded, $advertised)) { - return [...$decision, 'allowed' => true, 'status' => 'matching_definition']; + $immediate = match (true) { + ! $decision['worker_pinned'] => ['allowed' => true, 'status' => 'not_worker_pinned'], + ! $decision['pinning_enabled'] => ['allowed' => true, 'status' => 'pinning_disabled'], + $recorded === null => ['allowed' => false, 'status' => 'recorded_definition_missing'], + $advertised === null => ['allowed' => false, 'status' => 'worker_definition_missing'], + hash_equals($recorded, $advertised) => ['allowed' => true, 'status' => 'matching_definition'], + default => null, + }; + if ($immediate !== null) { + return [...$decision, ...$immediate]; } foreach ($this->eligibleWorkers($run, $decision['task_queue'], $decision['required_build_id']) as $peer) { @@ -95,16 +93,17 @@ public function inspect(WorkflowRun $run, ?string $taskQueue = null, ?string $re $runnable = ! $run->status->isTerminal() && $task instanceof WorkflowTask; $blocked = $runnable && $context['worker_pinned'] && $context['pinning_enabled'] && in_array($status, ['recorded_definition_missing', 'worker_definition_missing', 'no_eligible_worker'], true); + $resolution = match (true) { + $blocked => 'Register a fresh worker for the original namespace, task queue, workflow type, build and replay capabilities with its definition fingerprint. Do not rewrite the run history.', + $status === 'replay_required' => 'Current workers may replay the original history. Use versioned branches for command changes; incompatible replay still refuses.', + default => 'Recorded build, namespace, type and replay-capability guards remain in force.', + }; return [...$context, 'status' => $status, 'blocked' => $blocked, 'runnable_task' => $runnable, 'eligible_worker_count' => $eligible, 'matching_worker_count' => $matching, 'changed_worker_count' => $changed, 'missing_fingerprint_count' => $missing, 'workers' => $examples, 'workers_truncated' => $eligible > count($examples), - 'resolution' => $blocked - ? 'Register a fresh worker for the original namespace, task queue, workflow type, build and replay capabilities with its definition fingerprint. Do not rewrite the run history.' - : ($status === 'replay_required' - ? 'Current workers may replay the original history. Use versioned branches for command changes; incompatible replay still refuses.' - : 'Recorded build, namespace, type and replay-capability guards remain in force.'), + 'resolution' => $resolution, ]; } diff --git a/app/Support/WorkflowReplayEligibility.php b/app/Support/WorkflowReplayEligibility.php index 172dc4b0..da4744b2 100644 --- a/app/Support/WorkflowReplayEligibility.php +++ b/app/Support/WorkflowReplayEligibility.php @@ -18,12 +18,9 @@ public static function allows(WorkerRegistration $worker, WorkflowRun $run, ?str $capabilities = is_array($worker->capabilities) ? $worker->capabilities : []; if (is_string($run->cancellation_request_command_id) - && ! CooperativeCancellationPolicy::supports($capabilities, $protocolVersion)) { - return false; - } - if (is_string($run->cancellation_request_command_id) - && NexusCooperativeCancellationPolicy::requiresCapabilityForReplay($run) - && ! NexusCooperativeCancellationPolicy::supports($capabilities, $protocolVersion)) { + && (! CooperativeCancellationPolicy::supports($capabilities, $protocolVersion) + || (NexusCooperativeCancellationPolicy::requiresCapabilityForReplay($run) + && ! NexusCooperativeCancellationPolicy::supports($capabilities, $protocolVersion)))) { return false; } diff --git a/app/Support/WorkflowTaskPoller.php b/app/Support/WorkflowTaskPoller.php index 4cc91734..f21fb687 100644 --- a/app/Support/WorkflowTaskPoller.php +++ b/app/Support/WorkflowTaskPoller.php @@ -749,7 +749,6 @@ function () use ( historyPageSize: $historyPageSize, acceptHistoryEncoding: $acceptHistoryEncoding, supportedWorkflowTypes: $supportedWorkflowTypes, - workflowDefinitionFingerprints: $workflowDefinitionFingerprints, workerPollFence: $workerPollFence, ), ), @@ -798,7 +797,6 @@ function () use ( historyPageSize: $historyPageSize, acceptHistoryEncoding: $acceptHistoryEncoding, supportedWorkflowTypes: $supportedWorkflowTypes, - workflowDefinitionFingerprints: $workflowDefinitionFingerprints, workerPollFence: $workerPollFence, ), ), @@ -968,7 +966,6 @@ private function claimReadyTask( ?int $historyPageSize = null, ?string $acceptHistoryEncoding = null, array $supportedWorkflowTypes = [], - array $workflowDefinitionFingerprints = [], array $workerPollFence = [], ): ?array { $readyTasks = $this->pollReplayEligibleReadyTasks( diff --git a/tests/Feature/WorkflowDefinitionEvolutionTest.php b/tests/Feature/WorkflowDefinitionEvolutionTest.php index 3bbd0ac7..f8398e18 100644 --- a/tests/Feature/WorkflowDefinitionEvolutionTest.php +++ b/tests/Feature/WorkflowDefinitionEvolutionTest.php @@ -173,6 +173,18 @@ public function test_client_process_metrics_cannot_spoof_the_saved_protocol(): v self::assertSame(456, $worker->fresh()->process_metrics['memory_bytes']); } + public function test_explicit_null_clears_client_metrics_without_clearing_server_protocol(): void + { + $this->worker('original-worker', 'definition-v1', ['process_metrics' => ['memory_bytes' => 123]]); + $this->postJson('/api/worker/heartbeat', [ + 'worker_id' => 'original-worker', 'process_metrics' => null, + ], $this->workerHeaders())->assertOk(); + $worker = WorkerRegistration::query()->where('worker_id', 'original-worker')->firstOrFail(); + self::assertSame([ + WorkflowReplayEligibility::PROTOCOL_METRIC => $this->workerHeaders()['X-Durable-Workflow-Protocol-Version'], + ], $worker->process_metrics); + } + public function test_run_debug_queue_and_repair_distinguish_fallback_from_a_real_block(): void { [$run, $task] = $this->original(); From d867e0f43955e591874b5ea9da28964f7578781b Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 10:39:48 +0000 Subject: [PATCH 22/42] Print failed Rust tests in the failing step's own log The parity jobs write Rust test output to evidence files, so a failed cargo test left only its exit code in the step log. An ERR trap now prints the panics and the failed test names from those files. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/server-parity.yml | 3 +++ .../conformance/server-parity/rust-test-failures.sh | 13 +++++++++++++ 2 files changed, 16 insertions(+) create mode 100755 scripts/conformance/server-parity/rust-test-failures.sh diff --git a/.github/workflows/server-parity.yml b/.github/workflows/server-parity.yml index c4bb53b6..38d3bae9 100644 --- a/.github/workflows/server-parity.yml +++ b/.github/workflows/server-parity.yml @@ -377,6 +377,7 @@ jobs: - name: Bootstrap independent published PHP and native MySQL databases run: | set -euo pipefail + trap 'bash scripts/conformance/server-parity/rust-test-failures.sh' ERR mkdir -p cargo-home rust-target parity-evidence mysql-state chmod 0777 parity-evidence docker network create "$RESOURCE_SCOPE" @@ -856,6 +857,7 @@ jobs: POSTGRES_IMAGE: ${{ matrix.image }} run: | set -euo pipefail + trap 'bash scripts/conformance/server-parity/rust-test-failures.sh' ERR mkdir -p cargo-home rust-target parity-evidence pg-state chmod 0777 parity-evidence docker network create "$RESOURCE_SCOPE" @@ -1364,6 +1366,7 @@ jobs: - name: Execute official Rust codec and cross-decode with published PHP run: | set -euo pipefail + trap 'bash scripts/conformance/server-parity/rust-test-failures.sh' ERR mkdir -p cargo-home rust-target parity-evidence # Compiling the library tests peaks near 2.5 GB; a 2 GB limit OOM-killed rustc (exit 101). docker run --rm --user="$(id -u):$(id -g)" --cpus=2 --memory=4g --memory-swap=4g \ diff --git a/scripts/conformance/server-parity/rust-test-failures.sh b/scripts/conformance/server-parity/rust-test-failures.sh new file mode 100755 index 00000000..6e83f9e9 --- /dev/null +++ b/scripts/conformance/server-parity/rust-test-failures.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# Print the failures in this job's Rust test output, so a failed test step is +# diagnosable from its own log. Reads parity-evidence; changes nothing. +set -uo pipefail +for output in parity-evidence/*tests*.txt; do + [[ -f $output ]] && grep -q '^test result: FAILED' "$output" || continue + echo "::group::Rust test failures in ${output#parity-evidence/}" + grep -E -A3 'panicked at' "$output" | cut -c1-600 | head -n 120 + # The closing summary names every failed test. + sed -n '/^failures:$/,$p' "$output" | grep -E '^failures:$|^ [A-Za-z0-9_:]+$|^test result:' | tail -n 60 + echo "::endgroup::" +done +exit 0 From bac6c80cb59379b532fd459118ac9620188ec963 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 10:45:15 +0000 Subject: [PATCH 23/42] Describe queue delivery references as the lookup metadata they are PHP grants queue_delivery_reference no task authority, and the queue inspection already binds every queued job to its task row, so the conversion removes the column and reports how many references it held. Co-Authored-By: Claude Opus 5.5 --- docs/rust-server-upgrade-takeover.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/rust-server-upgrade-takeover.md b/docs/rust-server-upgrade-takeover.md index ca8ddd6f..3f03af10 100644 --- a/docs/rust-server-upgrade-takeover.md +++ b/docs/rust-server-upgrade-takeover.md @@ -113,9 +113,11 @@ with a recorded recipe: - PHP's registration incarnations and tokens are copied into native's incarnation table, so a running SDK worker's fenced deregistration still matches. The PHP table and column are then removed. -- `queue_delivery_reference` describes PHP's database queue delivery, which - native does not use. It is removed after its values are checked against - the queue inspection's timer bindings. +- `queue_delivery_reference` is optional lookup metadata for PHP's database + queue delivery, and PHP itself grants it no task authority. Native does not + use PHP's queue, and the queue inspection has already bound every queued + job to its durable task row, so the column is removed and the takeover + report counts the references it held. - Service calls are not part of native yet, so a source with service calls is refused before any change, and `call_deadline_at` is removed only from an empty table. From c14032cea3fb36538de2abd80a0df60aab207e7e Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 11:01:31 +0000 Subject: [PATCH 24/42] Take over a stopped PHP PostgreSQL database backup-first upgrade-takeover now also accepts DB_CONNECTION=pgsql. One read-committed transaction takes native's bootstrap lock, locks every public table in ACCESS EXCLUSIVE mode (refusing with upgrade_source_in_use when a session still holds one after the 5 s lock wait), runs the inspections and the fingerprint against the operator's verified backup digest, adds the native relations, converts the recorded PHP release layout, records the native migration and runs native's startup verification before committing. The layout conversion and ledger record are now shared by SQLite and PostgreSQL through the Backend trait. Inspection and fingerprint reads can run inside the takeover's transaction, and PostgreSQL fingerprint reads are bounded by the client deadline instead of the 30 s statement limit. A takeover killed before its commit leaves nothing committed. On SQLite it leaves a rollback journal that read-only inspection refuses; a takeover rerun with the same digest lets SQLite roll it back, and the fingerprint check then proves the file is the backed-up one. Both backends have a test that SIGKILLs a real takeover process just before its commit and reruns it. The upgrade source cohort takes over a restored copy of the published-PHP PostgreSQL source too, resumes the originals under native and checks PHP's refusal, as it does for SQLite. Co-Authored-By: Claude Opus 5.5 --- rust/README.md | 29 +- rust/src/main.rs | 11 +- rust/src/runtime/postgres.rs | 304 +++++++++++++++- rust/src/runtime/schema.rs | 37 +- rust/src/runtime/upgrade.rs | 101 ++++-- rust/src/runtime/upgrade/fingerprint.rs | 81 ++--- rust/src/runtime/upgrade/takeover.rs | 334 +++++++++++++----- rust/src/runtime/upgrade_tests.rs | 65 ++++ .../server-parity/upgrade-source-cleanup.sh | 6 + .../server-parity/upgrade-source-profile.sh | 44 ++- 10 files changed, 797 insertions(+), 215 deletions(-) diff --git a/rust/README.md b/rust/README.md index 1e4aa7db..0876ce23 100644 --- a/rust/README.md +++ b/rust/README.md @@ -412,31 +412,38 @@ version-1/version-2/version-3 development schemas are refused without conversion needed old development data separately; this unpublished slice supplies no older-development converter. Use a new isolated file for version 4. -This refusal is not the upgrade mechanism. A stopped PHP SQLite database can -be taken over with the development `upgrade-takeover` command, following the -[takeover design](../docs/rust-server-upgrade-takeover.md): +This refusal is not the upgrade mechanism. A stopped PHP SQLite or PostgreSQL +database can be taken over with the development `upgrade-takeover` command, +following the [takeover design](../docs/rust-server-upgrade-takeover.md): 1. Run PHP Server 2.5.14 or later, which refuses a Rust-owned database, and stop every PHP role. -2. Run `upgrade-fingerprint` on the source, mounted read-only. Back the file - up, restore the backup to a separate path, and fingerprint that too. The - digests must be equal. +2. Run `upgrade-fingerprint` on the source: a SQLite file mounted read-only, + or PostgreSQL through a read-only role. Back the database up (a file copy, + or `pg_dump`), restore the backup separately, and fingerprint that too. + The digests must be equal. 3. Run `upgrade-takeover` with `DW_UPGRADE_BACKUP_FINGERPRINT` set to that digest and the original queue settings (`DW_RUST_UPGRADE_QUEUE_*`, as for - `upgrade-queue-inspect`). + `upgrade-queue-inspect`). PostgreSQL uses the owner's `DB_*` settings. Inspection, the fingerprint check, the conversion of the recorded PHP release layout, the native relations, the migration ledger and native's own startup -verification all happen in one exclusive transaction. A refusal or an -interruption leaves the PHP file unchanged. The command refuses: +verification all happen in one transaction. SQLite holds the file +exclusively; PostgreSQL takes native's bootstrap lock and locks every table +before reading, and refuses with `upgrade_source_in_use` if a session still +holds one after five seconds. A refusal leaves the PHP database unchanged. An +interrupted takeover commits nothing: rerun it with the same digest. On +SQLite, the rerun lets SQLite roll back the journal the interrupted run left, +and the fingerprint check then proves the file is the backed-up one. The +command refuses: - a digest that differs, so any write after the backup stops it; - unrecognized queued jobs, external payloads and service calls; - a layout other than the frozen one, PHP 2.8.0 or PHP 2.9.7, or a database native already owns. -PostgreSQL and MySQL/MariaDB takeover, nested payload columns and multi-node -qualification remain open. **Never connect PHP to a Rust development database.** +MySQL/MariaDB takeover, nested payload columns and multi-node qualification +remain open. **Never connect PHP to a Rust development database.** Named namespace isolation and configured worker/operator/admin roles have representative shared qualification. Principal/custom-provider credentials and the full authorization contract remain open. Broader timer/signal semantics, failure/retry policy, cancellation and the rest of the API also diff --git a/rust/src/main.rs b/rust/src/main.rs index 79365e07..22c7793f 100644 --- a/rust/src/main.rs +++ b/rust/src/main.rs @@ -143,11 +143,10 @@ async fn main() -> Result<(), Box> { use durable_workflow_server::runtime::upgrade::{ queue_binding::QueueSourceConfig, takeover, }; - if backend != "sqlite" { + if backend != "sqlite" && backend != "pgsql" { eprintln!("upgrade_takeover_backend_unqualified"); return Err("upgrade takeover refused".into()); } - let path = env::var("DB_DATABASE").map_err(|_| "DB_DATABASE is required")?; let fingerprint = env::var("DW_UPGRADE_BACKUP_FINGERPRINT") .map_err(|_| "DW_UPGRADE_BACKUP_FINGERPRINT is required")?; // The original Laravel queue transport, as for upgrade-queue-inspect. @@ -164,7 +163,13 @@ async fn main() -> Result<(), Box> { default_connection: &default_connection, default_queue: &default_queue, }; - return match takeover::take_over_sqlite(&path, &fingerprint, &queues).await { + let result = if backend == "pgsql" { + takeover::take_over_postgres(postgres_options()?, &fingerprint, &queues).await + } else { + let path = env::var("DB_DATABASE").map_err(|_| "DB_DATABASE is required")?; + takeover::take_over_sqlite(&path, &fingerprint, &queues).await + }; + return match result { Ok(report) => { println!("{report}"); Ok(()) diff --git a/rust/src/runtime/postgres.rs b/rust/src/runtime/postgres.rs index fa763e30..06af1a9c 100644 --- a/rust/src/runtime/postgres.rs +++ b/rust/src/runtime/postgres.rs @@ -39,6 +39,23 @@ fn migrator() -> &'static Migrator { }) } +/// The single native migration, which a takeover records as applied once +/// conversion has produced its schema. +pub(super) fn migration() -> &'static Migration { + migrator().iter().next().expect("one native PostgreSQL migration") +} + +/// The native relations, indexes and ownership marker that a fresh +/// bootstrap adds after the frozen PHP layout. Takeover applies them to a +/// converted PHP database instead of running the whole migration. +pub(super) fn native_relations_sql() -> Result { + super::upgrade::native_relations( + include_str!("../../migrations/postgres/0003_full_schema.sql"), + "CREATE TABLE public.dw_server_schema ", + include_str!("../../migrations/postgres/0004_worker_incarnations.sql"), + ) +} + fn options(options: PgConnectOptions) -> PgConnectOptions { options .application_name("durable-workflow-rust-development") @@ -156,7 +173,7 @@ pub(super) async fn verify_history(connection: &mut PgConnection) -> Result<()> Ok(()) } -async fn verify(connection: &mut PgConnection) -> Result<()> { +pub(super) async fn verify(connection: &mut PgConnection) -> Result<()> { verify_scope(connection).await?; verify_history(connection).await?; // A canonical catalog is captured with PostgreSQL itself, not inferred @@ -712,4 +729,289 @@ mod tests { db.remove().await; } } + + use crate::runtime::upgrade::{fingerprint, queue_binding::QueueSourceConfig, takeover}; + + const TAKEOVER_QUEUES: QueueSourceConfig<'static> = QueueSourceConfig { + connection: "database", + default_connection: "database", + default_queue: "default", + }; + + /// The frozen PHP layout moved to PHP 2.9.7's by the changes PHP's own + /// migrations make (rust/upgrade/php-2.8.0 and php-2.9.7). Inspection + /// must call it php-2.9.7. + async fn php_297(db: &Database) -> PgConnection { + let frozen = include_str!("../../migrations/postgres/0003_full_schema.sql"); + let php = &frozen[..frozen.find("-- Native bookkeeping").unwrap()]; + let mut connection = db.connection().await; + sqlx::raw_sql(php).execute(&mut connection).await.unwrap(); + sqlx::raw_sql( + "ALTER TABLE workflow_worker_registrations ADD COLUMN registration_token varchar(32); + CREATE TABLE workflow_worker_registration_incarnations (token varchar(32) NOT NULL, namespace varchar(255) NOT NULL, worker_id varchar(255) NOT NULL, status varchar(32) NOT NULL, recovered_workflow_task_count integer NOT NULL DEFAULT 0, finished_at timestamp(6) without time zone, created_at timestamp(6) without time zone, updated_at timestamp(6) without time zone, PRIMARY KEY (token)); + CREATE INDEX workflow_worker_registration_incarnations_finished_at_index ON workflow_worker_registration_incarnations (finished_at); + CREATE INDEX dw_worker_incarnation_scope_idx ON workflow_worker_registration_incarnations (namespace, worker_id); + ALTER TABLE workflow_service_calls ADD COLUMN call_deadline_at timestamp(6) without time zone; + CREATE INDEX wf_service_calls_deadline_idx ON workflow_service_calls (status, call_deadline_at); + ALTER TABLE workflow_tasks ADD COLUMN queue_delivery_reference json;", + ) + .execute(&mut connection) + .await + .unwrap(); + connection + } + + async fn backup_fingerprint(db: &Database) -> String { + fingerprint::fingerprint_postgres(db.options.clone()) + .await + .unwrap()["fingerprint"] + .as_str() + .unwrap() + .to_owned() + } + + /// Every row and the physical schema, for byte-for-byte comparisons. + async fn source_state(db: &Database) -> (Catalog, String) { + let mut connection = db.connection().await; + let state = ( + catalog(&mut connection).await.unwrap(), + backup_fingerprint(db).await, + ); + connection.close().await.unwrap(); + state + } + + fn takeover_refused(result: Result, expected: &str) { + match result { + Err(super::super::RuntimeError::Refused { reason, .. }) => assert_eq!(reason, expected), + Err(error) => panic!("unexpected takeover error: {error}"), + Ok(report) => panic!("takeover was accepted: {report}"), + } + } + + #[tokio::test] + #[ignore = "requires a disposable PostgreSQL; run the qualification_ tests explicitly"] + async fn qualification_takeover_converts_php_297_and_native_opens_it() { + let db = Database::new().await; + let mut connection = php_297(&db).await; + sqlx::raw_sql( + "INSERT INTO workflow_worker_registrations(worker_id,namespace,task_queue,runtime,registration_token,created_at,updated_at) VALUES ('current','default','q','php','0123456789abcdef0123456789abcdef','2026-10-20 09:00:00','2026-10-20 09:00:00'); + INSERT INTO workflow_worker_registrations(worker_id,namespace,task_queue,runtime,created_at,updated_at) VALUES ('older','default','q','php','2026-10-20 09:00:00','2026-10-20 09:00:00'); + INSERT INTO workflow_worker_registration_incarnations VALUES ('0123456789abcdef0123456789abcdef','default','current','active',0,NULL,'2026-10-20 09:00:00','2026-10-20 09:00:00'); + INSERT INTO workflow_worker_registration_incarnations VALUES ('11111111111111111111111111111111','default','current','superseded',0,'2026-10-20 08:00:00','2026-10-20 07:00:00','2026-10-20 08:00:00'); + INSERT INTO workflow_worker_registration_incarnations VALUES ('22222222222222222222222222222222','default','gone','deregistered',3,'2026-10-20 08:30:00','2026-10-20 07:00:00','2026-10-20 08:30:00');", + ) + .execute(&mut connection) + .await + .unwrap(); + connection.close().await.unwrap(); + let fingerprint = backup_fingerprint(&db).await; + let report = takeover::take_over_postgres(db.options.clone(), &fingerprint, &TAKEOVER_QUEUES) + .await + .unwrap(); + assert_eq!(report["status"], "taken_over", "{report}"); + assert_eq!(report["backend"], "pgsql"); + assert_eq!(report["source_layout"], "php-2.9.7"); + assert_eq!(report["converted"]["incarnations_copied"], 3, "{report}"); + assert_eq!(report["converted"]["incarnations_created"], 1, "{report}"); + let mut connection = db.connection().await; + let incarnations: Vec<(String, String, String, Option, bool)> = sqlx::query_as( + "SELECT token,worker_id,status,recovered_workflow_task_count,finished_at IS NOT NULL FROM dw_worker_registration_incarnations ORDER BY worker_id,status", + ) + .fetch_all(&mut connection) + .await + .unwrap(); + assert_eq!(incarnations.len(), 4, "{incarnations:?}"); + // The current worker keeps its PHP token; the retired ones keep their state. + assert!(incarnations.contains(&( + "0123456789abcdef0123456789abcdef".into(), + "current".into(), + "active".into(), + None, + false + ))); + assert!(incarnations.contains(&( + "22222222222222222222222222222222".into(), + "gone".into(), + "deregistered".into(), + Some(3), + true + ))); + assert!(incarnations.iter().any(|row| row.1 == "current" && row.2 == "superseded" && row.3.is_none() && row.4)); + assert!(incarnations.iter().any(|row| row.1 == "older" && row.2 == "active" && row.0.len() == 32)); + connection.close().await.unwrap(); + let storage = PostgresStorage::open(db.options.clone()).await.unwrap(); + storage.close().await; + PostgresStorage::check(db.options.clone()).await.unwrap(); + // A second takeover finds a native database. + takeover_refused( + takeover::take_over_postgres(db.options.clone(), &fingerprint, &TAKEOVER_QUEUES).await, + "upgrade_source_already_owned", + ); + db.remove().await; + } + + #[tokio::test] + #[ignore = "requires a disposable PostgreSQL; run the qualification_ tests explicitly"] + async fn qualification_takeover_of_the_frozen_layout_adds_only_native_relations() { + let db = Database::new().await; + let frozen = include_str!("../../migrations/postgres/0003_full_schema.sql"); + let mut connection = db.connection().await; + sqlx::raw_sql(&frozen[..frozen.find("-- Native bookkeeping").unwrap()]) + .execute(&mut connection) + .await + .unwrap(); + connection.close().await.unwrap(); + let fingerprint = backup_fingerprint(&db).await; + let report = takeover::take_over_postgres(db.options.clone(), &fingerprint, &TAKEOVER_QUEUES) + .await + .unwrap(); + assert_eq!(report["source_layout"], "frozen", "{report}"); + PostgresStorage::check(db.options.clone()).await.unwrap(); + db.remove().await; + } + + #[tokio::test] + #[ignore = "requires a disposable PostgreSQL; run the qualification_ tests explicitly"] + async fn qualification_takeover_refusals_leave_the_php_database_unchanged() { + let db = Database::new().await; + php_297(&db).await.close().await.unwrap(); + let fingerprint = backup_fingerprint(&db).await; + let original = source_state(&db).await; + takeover_refused( + takeover::take_over_postgres(db.options.clone(), &format!("sha256:{}", "0".repeat(64)), &TAKEOVER_QUEUES).await, + "upgrade_backup_fingerprint_mismatch", + ); + takeover_refused( + takeover::take_over_postgres(db.options.clone(), "not-a-fingerprint", &TAKEOVER_QUEUES).await, + "upgrade_backup_fingerprint_required", + ); + assert_eq!(source_state(&db).await, original); + // A PHP connection still holding a table refuses the takeover within + // the lock wait instead of stalling it. + let mut php = db.connection().await; + sqlx::raw_sql("BEGIN; LOCK TABLE jobs IN ROW EXCLUSIVE MODE") + .execute(&mut php) + .await + .unwrap(); + takeover_refused( + takeover::take_over_postgres(db.options.clone(), &fingerprint, &TAKEOVER_QUEUES).await, + "upgrade_source_in_use", + ); + sqlx::raw_sql("ROLLBACK").execute(&mut php).await.unwrap(); + // A write after the backup changes the digest, so the old one is refused. + sqlx::raw_sql("INSERT INTO migrations(migration,batch) VALUES ('after-backup',9)") + .execute(&mut php) + .await + .unwrap(); + let written = source_state(&db).await; + takeover_refused( + takeover::take_over_postgres(db.options.clone(), &fingerprint, &TAKEOVER_QUEUES).await, + "upgrade_backup_fingerprint_mismatch", + ); + assert_eq!(source_state(&db).await, written); + php.close().await.unwrap(); + db.remove().await; + for (statement, expected) in [ + ( + "INSERT INTO workflow_service_calls(id,endpoint_name,service_name,operation_name,status,operation_mode) VALUES ('call','e','s','o','accepted','sync')", + "upgrade_source_service_calls_unsupported", + ), + ( + "INSERT INTO runtime_external_payloads(id,namespace,storage_uri,storage_uri_sha256,codec,sha256,size_bytes) VALUES ('p','default','s3://b/k','x','avro','y',1)", + "upgrade_source_external_payloads_unsupported", + ), + ] { + let db = Database::new().await; + let mut connection = php_297(&db).await; + sqlx::raw_sql(statement).execute(&mut connection).await.unwrap(); + connection.close().await.unwrap(); + let fingerprint = backup_fingerprint(&db).await; + let before = source_state(&db).await; + takeover_refused( + takeover::take_over_postgres(db.options.clone(), &fingerprint, &TAKEOVER_QUEUES).await, + expected, + ); + assert_eq!(source_state(&db).await, before, "{expected}"); + db.remove().await; + } + } + + #[tokio::test] + #[ignore = "only invoked as a real subprocess by the takeover interruption qualification"] + async fn takeover_child() { + let url = std::env::var("DW_TAKEOVER_TEST_DATABASE").unwrap(); + let fingerprint = std::env::var("DW_TAKEOVER_TEST_FINGERPRINT").unwrap(); + // Parks just before the commit, where the parent kills this process. + let report = takeover::take_over_postgres( + PgConnectOptions::from_str(&url).unwrap(), + &fingerprint, + &TAKEOVER_QUEUES, + ) + .await; + panic!("takeover returned instead of parking: {report:?}"); + } + + #[cfg(unix)] + #[tokio::test] + #[ignore = "requires a disposable PostgreSQL; run the qualification_ tests explicitly"] + async fn qualification_takeover_killed_before_commit_leaves_php_and_retries() { + use std::os::unix::process::ExitStatusExt; + let db = Database::new().await; + php_297(&db).await.close().await.unwrap(); + let fingerprint = backup_fingerprint(&db).await; + let original = source_state(&db).await; + let directory = tempfile::tempdir().unwrap(); + let receipt = directory.path().join("boundary"); + let mut child = Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "runtime::postgres::tests::takeover_child", + "--ignored", + "--nocapture", + ]) + .env("DW_TAKEOVER_TEST_DATABASE", &db.url) + .env("DW_TAKEOVER_TEST_FINGERPRINT", &fingerprint) + .env("DW_TAKEOVER_TEST_RECEIPT", &receipt) + .spawn() + .unwrap(); + let deadline = tokio::time::Instant::now() + Duration::from_secs(60); + while !receipt.exists() && tokio::time::Instant::now() < deadline { + if let Some(status) = child.try_wait().unwrap() { + panic!("child exited before the commit point: {status}"); + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + if !receipt.exists() { + child.kill().unwrap(); + child.wait().unwrap(); + panic!("child did not reach the commit point"); + } + assert_eq!(std::fs::read_to_string(&receipt).unwrap(), "before-commit"); + child.kill().unwrap(); + assert_eq!(child.wait().unwrap().signal(), Some(9)); + // PostgreSQL ends the killed session's transaction and its locks. + let released = tokio::time::Instant::now() + Duration::from_secs(15); + loop { + let mut connection = db.connection().await; + let sessions: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM pg_stat_activity WHERE datname=current_database() AND pid<>pg_backend_pid()") + .fetch_one(&mut connection) + .await + .unwrap(); + connection.close().await.unwrap(); + if sessions == 0 { + break; + } + assert!(tokio::time::Instant::now() < released, "killed takeover session survived"); + tokio::time::sleep(Duration::from_millis(50)).await; + } + assert_eq!(source_state(&db).await, original, "partial takeover survived"); + // The same verified backup fingerprint still takes the database over. + let report = takeover::take_over_postgres(db.options.clone(), &fingerprint, &TAKEOVER_QUEUES) + .await + .unwrap(); + assert_eq!(report["status"], "taken_over", "{report}"); + PostgresStorage::check(db.options.clone()).await.unwrap(); + db.remove().await; + } } diff --git a/rust/src/runtime/schema.rs b/rust/src/runtime/schema.rs index 6b6ebaf0..9b602f31 100644 --- a/rust/src/runtime/schema.rs +++ b/rust/src/runtime/schema.rs @@ -35,36 +35,17 @@ fn migrator() -> &'static Migrator { /// bootstrap adds after the frozen PHP layout. Takeover applies them to a /// converted PHP database instead of running the whole migration. pub(super) fn native_relations_sql() -> Result { - let frozen = include_str!("../../migrations/sqlite/0003_full_schema.sql"); - let start = frozen - .find("CREATE TABLE dw_server_schema ") - .ok_or_else(|| refuse(StatusCode::CONFLICT, "native_schema_catalog_mismatch"))?; - Ok(format!( - "{}\n{}", - &frozen[start..], - include_str!("../../migrations/sqlite/0004_worker_incarnations.sql") - )) + super::upgrade::native_relations( + include_str!("../../migrations/sqlite/0003_full_schema.sql"), + "CREATE TABLE dw_server_schema ", + include_str!("../../migrations/sqlite/0004_worker_incarnations.sql"), + ) } -/// Records the native migration as applied in the caller's transaction, for a -/// takeover whose schema came from conversion. `verify` then checks the -/// marker, this ledger and the complete catalog as for any native database. -pub(super) async fn record_migration(connection: &mut SqliteConnection) -> Result<()> { - use sqlx::migrate::Migrate; - connection - .ensure_migrations_table("_sqlx_migrations") - .await?; - let migration = migrator() - .iter() - .next() - .ok_or_else(|| refuse(StatusCode::CONFLICT, "native_migration_history_mismatch"))?; - sqlx::query("INSERT INTO _sqlx_migrations(version,description,success,checksum,execution_time) VALUES ($1,$2,TRUE,$3,0)") - .bind(migration.version) - .bind(migration.description.as_ref()) - .bind(migration.checksum.as_ref()) - .execute(&mut *connection) - .await?; - Ok(()) +/// The single native migration, which a takeover records as applied once +/// conversion has produced its schema. +pub(super) fn migration() -> &'static Migration { + migrator().iter().next().expect("one native SQLite migration") } type Catalog = Vec<(String, String, String, String)>; diff --git a/rust/src/runtime/upgrade.rs b/rust/src/runtime/upgrade.rs index 068fa201..d9029d1c 100644 --- a/rust/src/runtime/upgrade.rs +++ b/rust/src/runtime/upgrade.rs @@ -452,7 +452,16 @@ pub(super) async fn open_sqlite_file( "upgrade_source_sqlite_journal_unqualified", )); } - for suffix in ["-wal", "-shm", "-journal"] { + // A killed takeover leaves its rollback journal. Takeover opens the + // file writable, so SQLite rolls that journal back before anything is + // read, and the fingerprint then proves the result is the backed-up + // PHP database. Read-only opens never attempt that recovery. + let sidecars: &[&str] = if writable { + &["-wal", "-shm"] + } else { + &["-wal", "-shm", "-journal"] + }; + for suffix in sidecars { if tokio::fs::try_exists(format!("{path}{suffix}")).await? { return Err(refuse( StatusCode::CONFLICT, @@ -566,47 +575,77 @@ async fn inspect_postgres_inner( ) -> Result { queue_config(queues)?; let deadline = Instant::now() + BUDGET; - let mut connection = timeout_at( + let connection = timeout_at( deadline, PgConnection::connect_with(&postgres::source_options(options)), ) .await .map_err(|_| expired())??; - let result = timeout_at(deadline, async { + bounded(connection, deadline, async |connection| { let mut tx = connection .begin_with("BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY") .await?; - let (catalog, layout) = postgres::php_source_catalog(&mut tx).await?; - let tables: Vec = catalog - .iter() - .filter(|(kind, _, data)| kind == "relation" && data.0["kind"] == "r") - .map(|(_, name, _)| name.clone()) - .collect(); - let counts = count_tables!(tx, tables, '"'); - let queue_inspection = match queues { - Some(config) => { - Some(queue_scan::inspect::(&mut *tx, config, counts["jobs"]).await?) - } - None => None, - }; - let payload_inspection = if payloads { - Some(payload_scan::inspect::(&mut *tx, &counts).await?) - } else { - None - }; + let report = postgres_inventory(&mut tx, queues, payloads).await; tx.rollback().await?; - report( - "pgsql", - &catalog, - layout, - counts, - queue_inspection, - payload_inspection, - ) + report }) .await - .map_err(|_| expired()) - .and_then(|result| result); +} + +/// The PostgreSQL inspection report, read inside the caller's transaction. +pub(super) async fn postgres_inventory( + tx: &mut PgConnection, + queues: Option<&QueueSourceConfig<'_>>, + payloads: bool, +) -> Result { + let (catalog, layout) = postgres::php_source_catalog(&mut *tx).await?; + let tables: Vec = catalog + .iter() + .filter(|(kind, _, data)| kind == "relation" && data.0["kind"] == "r") + .map(|(_, name, _)| name.clone()) + .collect(); + let counts = count_tables!(tx, tables, '"'); + let queue_inspection = match queues { + Some(config) => { + Some(queue_scan::inspect::(&mut *tx, config, counts["jobs"]).await?) + } + None => None, + }; + let payload_inspection = if payloads { + Some(payload_scan::inspect::(&mut *tx, &counts).await?) + } else { + None + }; + report( + "pgsql", + &catalog, + layout, + counts, + queue_inspection, + payload_inspection, + ) +} + +/// The frozen migration from its ownership marker on, then the incarnation +/// migration: what a fresh native bootstrap adds after the frozen PHP layout. +pub(super) fn native_relations(frozen: &str, marker: &str, incarnations: &str) -> Result { + let start = frozen + .find(marker) + .ok_or_else(|| refuse(StatusCode::CONFLICT, "native_schema_catalog_mismatch"))?; + Ok(format!("{}\n{incarnations}", &frozen[start..])) +} + +/// Runs work on a dedicated source connection within the deadline, then +/// closes the connection whatever the work returned. +pub(super) async fn bounded( + mut connection: C, + deadline: Instant, + work: impl AsyncFnOnce(&mut C) -> Result, +) -> Result { + let result = timeout_at(deadline, work(&mut connection)) + .await + .map_err(|_| expired()) + .and_then(|result| result); close(connection.close()).await?; result } diff --git a/rust/src/runtime/upgrade/fingerprint.rs b/rust/src/runtime/upgrade/fingerprint.rs index a13ed89e..384995ff 100644 --- a/rust/src/runtime/upgrade/fingerprint.rs +++ b/rust/src/runtime/upgrade/fingerprint.rs @@ -8,7 +8,7 @@ //! per-table row counts leave the database. The fingerprint grants nothing: //! every compatibility, backup, quiescence and takeover flag stays false. use super::{ - Result, close, expired, mysql, mysql_source, open_sqlite_source, postgres, readonly_source, + Result, bounded, expired, mysql, mysql_source, open_sqlite_source, postgres, readonly_source, refuse, schema, }; use axum::http::StatusCode; @@ -80,7 +80,7 @@ impl Digest { /// Names come from a catalog equal to a qualified PHP layout. Anything else is /// refused rather than quoted. -fn identifier(name: &str) -> Result<&str> { +pub(super) fn identifier(name: &str) -> Result<&str> { if name.is_empty() || !name.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'_') { return Err(refuse( StatusCode::CONFLICT, @@ -123,21 +123,6 @@ fn row_select( )) } -/// Runs a read on a dedicated source connection within the deadline, then -/// closes the connection whatever the read returned. -async fn bounded_read( - mut connection: C, - deadline: Instant, - read: impl AsyncFnOnce(&mut C) -> Result, -) -> Result { - let result = timeout_at(deadline, read(&mut connection)) - .await - .map_err(|_| expired()) - .and_then(|result| result); - close(connection.close()).await?; - result -} - // Every rendered value is text, framed in column order. macro_rules! hash_rows { ($tx:ident, $digest:ident, $table:expr, $sql:expr, $width:expr) => {{ @@ -169,7 +154,7 @@ pub async fn fingerprint_sqlite_cli(path: &str) -> Result { pub async fn fingerprint_sqlite(path: &str) -> Result { let deadline = Instant::now() + BUDGET; let connection = open_sqlite_source(path, deadline).await?; - bounded_read(connection, deadline, async |connection| { + bounded(connection, deadline, async |connection| { // One deferred read transaction keeps one snapshot of the file. let mut tx = connection.begin().await?; let report = sqlite_digest(&mut tx).await; @@ -214,45 +199,53 @@ pub async fn fingerprint_postgres(options: PgConnectOptions) -> Result { ) .await .map_err(|_| expired())??; - bounded_read(connection, deadline, async |connection| { + bounded(connection, deadline, async |connection| { let mut tx = connection .begin_with("BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY") .await?; - // Fixed text forms for timestamps, intervals, floats and bytes. - sqlx::raw_sql("SET LOCAL TimeZone='UTC'; SET LOCAL DateStyle='ISO, YMD'; SET LOCAL IntervalStyle='postgres'; SET LOCAL extra_float_digits=1; SET LOCAL bytea_output='hex'") - .execute(&mut *tx) - .await?; - let (catalog, layout) = postgres::php_source_catalog(&mut tx).await?; - let mut digest = Digest::new("pgsql", &catalog)?; - let tables: Vec = catalog - .iter() - .filter(|(kind, _, data)| kind == "relation" && data.0["kind"] == "r") - .map(|(_, name, _)| name.clone()) - .collect(); - for table in &tables { - let table = identifier(table)?; - let columns: Vec = sqlx::query_scalar("SELECT column_name::text FROM information_schema.columns WHERE table_schema='public' AND table_name=$1 ORDER BY ordinal_position") - .bind(table).fetch_all(&mut *tx).await?; - let key: Vec = sqlx::query_scalar("SELECT a.attname::text FROM pg_index i CROSS JOIN LATERAL unnest(i.indkey::int2[]) WITH ORDINALITY AS k(attnum, position) JOIN pg_attribute a ON a.attrelid=i.indrelid AND a.attnum=k.attnum WHERE i.indrelid=('public.'||quote_ident($1))::regclass AND i.indisprimary ORDER BY k.position") - .bind(table).fetch_all(&mut *tx).await?; - digest.table(table, &columns); - let sql = row_select(&format!("public.\"{table}\""), &columns, &key, '"', |c, _| { - format!("quote_nullable(\"{c}\")") - })?; - hash_rows!(tx, digest, table, sql, columns.len()); - } + let report = postgres_digest(&mut tx).await; tx.rollback().await?; - Ok(digest.report("pgsql", layout)) + report }) .await } +/// The PostgreSQL fingerprint report, read inside the caller's transaction. +/// Its settings last until that transaction ends. +pub(super) async fn postgres_digest(tx: &mut PgConnection) -> Result { + // Fixed text forms for timestamps, intervals, floats and bytes. Long table + // reads are expected, so only the client deadline bounds them. + sqlx::raw_sql("SET LOCAL TimeZone='UTC'; SET LOCAL DateStyle='ISO, YMD'; SET LOCAL IntervalStyle='postgres'; SET LOCAL extra_float_digits=1; SET LOCAL bytea_output='hex'; SET LOCAL statement_timeout=0") + .execute(&mut *tx) + .await?; + let (catalog, layout) = postgres::php_source_catalog(&mut *tx).await?; + let mut digest = Digest::new("pgsql", &catalog)?; + let tables: Vec = catalog + .iter() + .filter(|(kind, _, data)| kind == "relation" && data.0["kind"] == "r") + .map(|(_, name, _)| name.clone()) + .collect(); + for table in &tables { + let table = identifier(table)?; + let columns: Vec = sqlx::query_scalar("SELECT column_name::text FROM information_schema.columns WHERE table_schema='public' AND table_name=$1 ORDER BY ordinal_position") + .bind(table).fetch_all(&mut *tx).await?; + let key: Vec = sqlx::query_scalar("SELECT a.attname::text FROM pg_index i CROSS JOIN LATERAL unnest(i.indkey::int2[]) WITH ORDINALITY AS k(attnum, position) JOIN pg_attribute a ON a.attrelid=i.indrelid AND a.attnum=k.attnum WHERE i.indrelid=('public.'||quote_ident($1))::regclass AND i.indisprimary ORDER BY k.position") + .bind(table).fetch_all(&mut *tx).await?; + digest.table(table, &columns); + let sql = row_select(&format!("public.\"{table}\""), &columns, &key, '"', |c, _| { + format!("quote_nullable(\"{c}\")") + })?; + hash_rows!(tx, digest, table, sql, columns.len()); + } + Ok(digest.report("pgsql", layout)) +} + pub async fn fingerprint_mysql(options: MySqlConnectOptions) -> Result { let deadline = Instant::now() + BUDGET; // TIMESTAMP values render in UTC. Long table reads are expected, but lock // waits stay short; the client deadline bounds the whole read. let (connection, backend) = mysql_source(options, deadline, true, 0).await?; - bounded_read(connection, deadline, async |connection| { + bounded(connection, deadline, async |connection| { let mut tx = connection .begin_with("START TRANSACTION WITH CONSISTENT SNAPSHOT, READ ONLY") .await?; diff --git a/rust/src/runtime/upgrade/takeover.rs b/rust/src/runtime/upgrade/takeover.rs index a0e71b9b..b7dbcb1c 100644 --- a/rust/src/runtime/upgrade/takeover.rs +++ b/rust/src/runtime/upgrade/takeover.rs @@ -1,30 +1,78 @@ -//! Backup-first, one-way takeover of a stopped PHP SQLite database, following +//! Backup-first, one-way takeover of a stopped PHP database, following //! docs/rust-server-upgrade-takeover.md. //! -//! Everything happens in one `BEGIN EXCLUSIVE` transaction: the inspections, -//! the fingerprint check against the operator's verified backup, the layout -//! conversion, the native relations, the migration ledger and native's own -//! startup verification. An interrupted or refused takeover therefore leaves -//! the PHP database exactly as it was, and a committed one is a native -//! database that PHP 2.5.14 and later refuse to open. +//! On SQLite and PostgreSQL everything happens in one transaction that no +//! other writer can interleave with: the inspections, the fingerprint check +//! against the operator's verified backup, the layout conversion, the native +//! relations, the migration ledger and native's own startup verification. An +//! interrupted or refused takeover therefore leaves the PHP database exactly +//! as it was, and a committed one is a native database that PHP 2.5.14 and +//! later refuse to open. use super::{ - Result, close, expired, fingerprint, open_sqlite_file, queue_binding::QueueSourceConfig, - queue_config, refuse, schema, sqlite_inventory, + Result, bounded, fingerprint, open_sqlite_file, postgres, postgres_inventory, + queue_binding::QueueSourceConfig, queue_config, refuse, schema, sqlite_inventory, }; +use crate::runtime::backend::Backend; use axum::http::StatusCode; use serde_json::{Value, json}; -use sqlx::{AssertSqlSafe, Connection, SqliteConnection}; +use sqlx::{ + AssertSqlSafe, Connection, Encode, Executor, IntoArguments, PgConnection, Postgres, Sqlite, + SqliteConnection, Transaction, Type, + migrate::{Migrate, Migration}, + postgres::PgConnectOptions, +}; use std::time::Duration; use tokio::time::{Instant, timeout_at}; /// Takeover reads every row twice (inspection and fingerprint). const BUDGET: Duration = Duration::from_secs(3600); +/// Native's PostgreSQL bootstrap key, so a native node cannot open the +/// database while it is taken over. +const POSTGRES_BOOTSTRAP_KEY: i64 = 4924501473315676162; pub async fn take_over_sqlite( path: &str, backup_fingerprint: &str, queues: &QueueSourceConfig<'_>, ) -> Result { + preconditions(backup_fingerprint, queues)?; + let deadline = Instant::now() + BUDGET; + let connection = open_sqlite_file(path, deadline, true).await?; + bounded(connection, deadline, async |connection| { + let mut tx = connection.begin_with("BEGIN EXCLUSIVE").await?; + let outcome = take_over_sqlite_in(&mut tx, backup_fingerprint, queues).await; + settle(tx, outcome).await + }) + .await +} + +/// Takes over with the owner's credentials. The connection keeps native's +/// short lock wait, so a PHP process still holding a table refuses the +/// takeover instead of stalling it. +pub async fn take_over_postgres( + options: PgConnectOptions, + backup_fingerprint: &str, + queues: &QueueSourceConfig<'_>, +) -> Result { + preconditions(backup_fingerprint, queues)?; + let deadline = Instant::now() + BUDGET; + let connection = timeout_at( + deadline, + PgConnection::connect_with(&postgres::source_options(options)), + ) + .await + .map_err(|_| super::expired())??; + bounded(connection, deadline, async |connection| { + // Read committed: once every table is locked, each statement reads + // the latest committed state and nothing else can change it. + let mut tx = connection.begin().await?; + let outcome = take_over_postgres_in(&mut tx, backup_fingerprint, queues).await; + settle(tx, outcome).await + }) + .await +} + +fn preconditions(backup_fingerprint: &str, queues: &QueueSourceConfig<'_>) -> Result<()> { queue_config(Some(queues))?; if !backup_fingerprint.starts_with("sha256:") || backup_fingerprint.len() != 71 { return Err(refuse( @@ -32,68 +80,141 @@ pub async fn take_over_sqlite( "upgrade_backup_fingerprint_required", )); } - let deadline = Instant::now() + BUDGET; - let mut connection = open_sqlite_file(path, deadline, true).await?; - let result = timeout_at(deadline, async { - let mut tx = connection.begin_with("BEGIN EXCLUSIVE").await?; - match take_over(&mut tx, backup_fingerprint, queues).await { - Ok(report) => { - tx.commit().await?; - Ok(report) - } - Err(error) => { - tx.rollback().await?; - Err(error) - } + Ok(()) +} + +/// Commits a completed takeover, and rolls back anything else. +async fn settle(tx: Transaction<'_, DB>, outcome: Result) -> Result { + match outcome { + Ok(report) => { + tx.commit().await?; + Ok(report) } - }) - .await - .map_err(|_| expired()) - .and_then(|result| result); - close(connection.close()).await?; - result + Err(error) => { + tx.rollback().await?; + Err(error) + } + } } -async fn take_over( +async fn take_over_sqlite_in( tx: &mut SqliteConnection, backup_fingerprint: &str, queues: &QueueSourceConfig<'_>, ) -> Result { let inventory = sqlite_inventory(&mut *tx, Some(queues), true).await?; - qualified(&inventory)?; + let layout = qualified(&inventory)?; // The same digest the operator compared with the restored backup. Any // write after that backup changes it, so nothing below can run. let digest = fingerprint::sqlite_digest(&mut *tx).await?; - if digest["fingerprint"] != backup_fingerprint { + matches_backup(&digest, backup_fingerprint)?; + sqlx::raw_sql(AssertSqlSafe(schema::native_relations_sql()?)) + .execute(&mut *tx) + .await?; + let converted = convert::(&mut *tx, layout).await?; + record_migration::(&mut *tx, schema::migration()).await?; + // Exactly the check native startup makes: marker, ledger and catalog. + schema::verify(&mut *tx).await?; + #[cfg(test)] + interruption_point(); + Ok(report("sqlite", layout, &digest, &inventory, converted)) +} + +async fn take_over_postgres_in( + tx: &mut PgConnection, + backup_fingerprint: &str, + queues: &QueueSourceConfig<'_>, +) -> Result { + // Whole-table reads and index builds are expected; the client deadline + // bounds them, while lock waits stay short. + sqlx::raw_sql("SET LOCAL statement_timeout=0") + .execute(&mut *tx) + .await?; + sqlx::query("SELECT pg_advisory_xact_lock($1)") + .bind(POSTGRES_BOOTSTRAP_KEY) + .execute(&mut *tx) + .await?; + // The later DDL needs ACCESS EXCLUSIVE anyway; taking it first means no + // lock is ever upgraded while another session waits. + let tables: Vec = sqlx::query_scalar("SELECT c.relname::text FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname='public' AND c.relkind IN ('r','p') ORDER BY c.relname") + .fetch_all(&mut *tx) + .await?; + if tables.is_empty() { return Err(refuse( StatusCode::CONFLICT, - "upgrade_backup_fingerprint_mismatch", + "upgrade_source_catalog_unqualified", )); } - let layout = inventory["source_layout"] - .as_str() - .ok_or_else(|| refuse(StatusCode::CONFLICT, "upgrade_source_catalog_unqualified"))?; - sqlx::raw_sql(AssertSqlSafe(schema::native_relations_sql()?)) + let tables = tables + .iter() + .map(|table| fingerprint::identifier(table).map(|table| format!("public.\"{table}\""))) + .collect::>>()? + .join(","); + sqlx::raw_sql(AssertSqlSafe(format!( + "LOCK TABLE {tables} IN ACCESS EXCLUSIVE MODE" + ))) + .execute(&mut *tx) + .await + .map_err(|error| match &error { + sqlx::Error::Database(database) if database.code().as_deref() == Some("55P03") => { + refuse(StatusCode::CONFLICT, "upgrade_source_in_use") + } + _ => error.into(), + })?; + let inventory = postgres_inventory(&mut *tx, Some(queues), true).await?; + let layout = qualified(&inventory)?; + let digest = fingerprint::postgres_digest(&mut *tx).await?; + matches_backup(&digest, backup_fingerprint)?; + sqlx::raw_sql(AssertSqlSafe(postgres::native_relations_sql()?)) .execute(&mut *tx) .await?; - let converted = convert(&mut *tx, layout).await?; - schema::record_migration(&mut *tx).await?; - // Exactly the check native startup makes: marker, ledger and catalog. - schema::verify(&mut *tx).await?; - Ok(json!({ + let converted = convert::(&mut *tx, layout).await?; + record_migration::(&mut *tx, postgres::migration()).await?; + postgres::verify(&mut *tx).await?; + #[cfg(test)] + interruption_point(); + Ok(report("pgsql", layout, &digest, &inventory, converted)) +} + +/// Lets the interruption qualification kill a real takeover process after +/// every step has run, just before the commit. +#[cfg(test)] +fn interruption_point() { + if let Ok(receipt) = std::env::var("DW_TAKEOVER_TEST_RECEIPT") { + std::fs::write(format!("{receipt}.partial"), "before-commit").unwrap(); + std::fs::rename(format!("{receipt}.partial"), &receipt).unwrap(); + loop { + std::thread::park(); + } + } +} + +fn matches_backup(digest: &Value, backup_fingerprint: &str) -> Result<()> { + if digest["fingerprint"] != backup_fingerprint { + return Err(refuse( + StatusCode::CONFLICT, + "upgrade_backup_fingerprint_mismatch", + )); + } + Ok(()) +} + +fn report(backend: &str, layout: &str, digest: &Value, inventory: &Value, converted: Value) -> Value { + json!({ "schema":"durable-workflow.rust-upgrade-takeover.v1", - "status":"taken_over", "backend":"sqlite", "source_layout":layout, + "status":"taken_over", "backend":backend, "source_layout":layout, "fingerprint":digest["fingerprint"], "table_rows":inventory["table_rows"], "converted":converted, "backup_fingerprint_matched":true, // What the inspections cannot yet establish stays explicit. "nested_payloads_verified":false, "other_root_payload_columns_verified":false, "stored_representation_compatibility_verified":false - })) + }) } /// Takeover needs every inspection result it relies on, and refuses what /// native cannot represent yet: external payloads and service calls. -fn qualified(inventory: &Value) -> Result<()> { +/// Returns the qualified source layout. +fn qualified(inventory: &Value) -> Result<&str> { if inventory["queue_inspection"]["status"] != "known_subset_bound" { return Err(refuse( StatusCode::CONFLICT, @@ -126,14 +247,52 @@ fn qualified(inventory: &Value) -> Result<()> { "upgrade_source_external_payloads_unsupported", )); } + inventory["source_layout"] + .as_str() + .ok_or_else(|| refuse(StatusCode::CONFLICT, "upgrade_source_catalog_unqualified")) +} + +fn query(sql: &'static str) -> sqlx::query::Query<'static, DB, DB::Arguments> +where + DB::Arguments: IntoArguments, +{ + sqlx::query(AssertSqlSafe(DB::statement(sql))) +} + +/// Records the native migration as applied, for a schema that came from +/// conversion. Native verification then checks this ledger like any other. +async fn record_migration( + tx: &mut DB::Connection, + migration: &'static Migration, +) -> Result<()> +where + DB::Connection: Migrate, + for<'c> &'c mut DB::Connection: Executor<'c, Database = DB>, + DB::Arguments: IntoArguments, + for<'q> i64: Encode<'q, DB> + Type, + for<'q> &'q str: Encode<'q, DB> + Type, + for<'q> &'q [u8]: Encode<'q, DB> + Type, +{ + tx.ensure_migrations_table("_sqlx_migrations").await?; + query::("INSERT INTO _sqlx_migrations(version,description,success,checksum,execution_time) VALUES ($1,$2,TRUE,$3,0)") + .bind(migration.version) + .bind(migration.description.as_ref()) + .bind(migration.checksum.as_ref()) + .execute(&mut *tx) + .await?; Ok(()) } /// Converts a recorded PHP release layout to native's (the design's recipes). -async fn convert(tx: &mut SqliteConnection, layout: &str) -> Result { - let (incarnation_table, token_column) = match layout { - "frozen" => (false, false), - "php-2.8.0" | "php-2.9.7" => (true, true), +async fn convert(tx: &mut DB::Connection, layout: &str) -> Result +where + for<'c> &'c mut DB::Connection: Executor<'c, Database = DB>, + DB::Arguments: IntoArguments, + for<'q> String: Encode<'q, DB> + Type, +{ + let php_incarnations = match layout { + "frozen" => false, + "php-2.8.0" | "php-2.9.7" => true, _ => { return Err(refuse( StatusCode::CONFLICT, @@ -141,31 +300,30 @@ async fn convert(tx: &mut SqliteConnection, layout: &str) -> Result { )); } }; - let calls: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM workflow_service_calls") - .fetch_one(&mut *tx) - .await?; - if calls != 0 { + if count::(tx, "SELECT COUNT(*) AS n FROM workflow_service_calls").await? != 0 { return Err(refuse( StatusCode::CONFLICT, "upgrade_source_service_calls_unsupported", )); } - let now = super::super::store::now() - .format("%Y-%m-%d %H:%M:%S%.6f") - .to_string(); + let now = super::super::store::now(); let mut copied = 0; - if incarnation_table { + if php_incarnations { // Same statuses as native. Native keeps the recovered count only on // deregistration and a finish time on every retired incarnation. - copied = sqlx::query("INSERT INTO dw_worker_registration_incarnations(token,namespace,worker_id,status,recovered_workflow_task_count,finished_at,created_at,updated_at) SELECT token,namespace,worker_id,status,CASE WHEN status='deregistered' THEN recovered_workflow_task_count END,CASE WHEN status='active' THEN NULL ELSE COALESCE(finished_at,updated_at,created_at,$1) END,COALESCE(created_at,$1),COALESCE(updated_at,created_at,$1) FROM workflow_worker_registration_incarnations WHERE status IN ('active','superseded','deregistered')") - .bind(&now) - .execute(&mut *tx) - .await? - .rows_affected(); - let total: i64 = - sqlx::query_scalar("SELECT COUNT(*) FROM workflow_worker_registration_incarnations") - .fetch_one(&mut *tx) - .await?; + copied = DB::affected( + query::("INSERT INTO dw_worker_registration_incarnations(token,namespace,worker_id,status,recovered_workflow_task_count,finished_at,created_at,updated_at) SELECT token,namespace,worker_id,status,CASE WHEN status='deregistered' THEN recovered_workflow_task_count END,CASE WHEN status='active' THEN NULL ELSE COALESCE(finished_at,updated_at,created_at,$1) END,COALESCE(created_at,$2),COALESCE(updated_at,created_at,$3) FROM workflow_worker_registration_incarnations WHERE status IN ('active','superseded','deregistered')") + .bind(DB::bind_time(now)) + .bind(DB::bind_time(now)) + .bind(DB::bind_time(now)) + .execute(&mut *tx) + .await?, + ); + let total = count::( + tx, + "SELECT COUNT(*) AS n FROM workflow_worker_registration_incarnations", + ) + .await?; if copied != total as u64 { return Err(refuse( StatusCode::CONFLICT, @@ -175,27 +333,28 @@ async fn convert(tx: &mut SqliteConnection, layout: &str) -> Result { } // Every active registration gets the authority native expects. A PHP // 2.8.0+ worker keeps the token it was given; an older one gets a new token. - let missing: Vec<(String, String, Option)> = sqlx::query_as(AssertSqlSafe(format!( - "SELECT r.namespace,r.worker_id,{} FROM workflow_worker_registrations r WHERE r.status='active' AND NOT EXISTS (SELECT 1 FROM dw_worker_registration_incarnations i WHERE i.namespace=r.namespace AND i.worker_id=r.worker_id AND i.status='active') ORDER BY r.namespace,r.worker_id", - if token_column { "r.registration_token" } else { "NULL" } - ))) + let missing = query::(if php_incarnations { + "SELECT r.namespace,r.worker_id,r.registration_token AS token FROM workflow_worker_registrations r WHERE r.status='active' AND NOT EXISTS (SELECT 1 FROM dw_worker_registration_incarnations i WHERE i.namespace=r.namespace AND i.worker_id=r.worker_id AND i.status='active') ORDER BY r.namespace,r.worker_id" + } else { + "SELECT r.namespace,r.worker_id,NULL AS token FROM workflow_worker_registrations r WHERE r.status='active' AND NOT EXISTS (SELECT 1 FROM dw_worker_registration_incarnations i WHERE i.namespace=r.namespace AND i.worker_id=r.worker_id AND i.status='active') ORDER BY r.namespace,r.worker_id" + }) .fetch_all(&mut *tx) .await?; - for (namespace, worker_id, token) in &missing { - let token = match token { - Some(token) => token.clone(), + for row in &missing { + let token = match DB::optional_string(row, "token")? { + Some(token) => token, None => super::super::worker_incarnations::token().await?, }; - sqlx::query("INSERT INTO dw_worker_registration_incarnations(token,namespace,worker_id,status,created_at,updated_at) VALUES ($1,$2,$3,'active',$4,$5)") + query::("INSERT INTO dw_worker_registration_incarnations(token,namespace,worker_id,status,created_at,updated_at) VALUES ($1,$2,$3,'active',$4,$5)") .bind(token) - .bind(namespace) - .bind(worker_id) - .bind(&now) - .bind(&now) + .bind(DB::string(row, "namespace")?) + .bind(DB::string(row, "worker_id")?) + .bind(DB::bind_time(now)) + .bind(DB::bind_time(now)) .execute(&mut *tx) .await?; } - if incarnation_table { + if php_incarnations { sqlx::raw_sql( "DROP TABLE workflow_worker_registration_incarnations; \ ALTER TABLE workflow_worker_registrations DROP COLUMN registration_token; \ @@ -206,11 +365,12 @@ async fn convert(tx: &mut SqliteConnection, layout: &str) -> Result { .await?; } let delivery_references = if layout == "php-2.9.7" { - // PHP's own queue delivery bookkeeping; native does not use the jobs. - let references: i64 = sqlx::query_scalar( - "SELECT COUNT(*) FROM workflow_tasks WHERE queue_delivery_reference IS NOT NULL", + // PHP's own queue delivery lookup metadata; native does not use PHP's + // queue, and the queue inspection bound every job to its task row. + let references = count::( + tx, + "SELECT COUNT(*) AS n FROM workflow_tasks WHERE queue_delivery_reference IS NOT NULL", ) - .fetch_one(&mut *tx) .await?; sqlx::raw_sql("ALTER TABLE workflow_tasks DROP COLUMN queue_delivery_reference") .execute(&mut *tx) @@ -224,3 +384,11 @@ async fn convert(tx: &mut SqliteConnection, layout: &str) -> Result { "queue_delivery_references_removed":delivery_references })) } + +async fn count(tx: &mut DB::Connection, sql: &'static str) -> Result +where + for<'c> &'c mut DB::Connection: Executor<'c, Database = DB>, + DB::Arguments: IntoArguments, +{ + DB::number(&query::(sql).fetch_one(&mut *tx).await?, "n") +} diff --git a/rust/src/runtime/upgrade_tests.rs b/rust/src/runtime/upgrade_tests.rs index d45f6009..a140ec2b 100644 --- a/rust/src/runtime/upgrade_tests.rs +++ b/rust/src/runtime/upgrade_tests.rs @@ -935,6 +935,71 @@ async fn takeover_converts_a_php_297_database_that_native_then_opens() { connection.close().await.unwrap(); } +#[tokio::test] +#[ignore = "only invoked as a real subprocess by the takeover interruption test"] +async fn sqlite_takeover_child() { + let path = std::env::var("DW_TAKEOVER_TEST_DATABASE").unwrap(); + let fingerprint = std::env::var("DW_TAKEOVER_TEST_FINGERPRINT").unwrap(); + // Parks just before the commit, where the parent kills this process. + let report = takeover::take_over_sqlite(&path, &fingerprint, &queue_config()).await; + panic!("takeover returned instead of parking: {report:?}"); +} + +#[cfg(unix)] +#[tokio::test] +async fn takeover_killed_before_commit_leaves_php_data_and_can_be_rerun() { + use std::os::unix::process::ExitStatusExt; + let (directory, path) = php297_layout().await; + let fingerprint = backup_fingerprint(&path).await; + let original = std::fs::read(&path).unwrap(); + let receipt = directory.path().join("boundary"); + let mut child = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "runtime::upgrade::tests::sqlite_takeover_child", + "--ignored", + "--nocapture", + ]) + .env("DW_TAKEOVER_TEST_DATABASE", &path) + .env("DW_TAKEOVER_TEST_FINGERPRINT", &fingerprint) + .env("DW_TAKEOVER_TEST_RECEIPT", &receipt) + .spawn() + .unwrap(); + let deadline = tokio::time::Instant::now() + Duration::from_secs(60); + while !receipt.exists() && tokio::time::Instant::now() < deadline { + if let Some(status) = child.try_wait().unwrap() { + panic!("child exited before the commit point: {status}"); + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + if !receipt.exists() { + child.kill().unwrap(); + child.wait().unwrap(); + panic!("child did not reach the commit point"); + } + child.kill().unwrap(); + assert_eq!(child.wait().unwrap().signal(), Some(9)); + // Either SQLite had not written the file yet, or its rollback journal + // still holds the original pages. Read-only inspection never recovers it. + if Path::new(&format!("{path}-journal")).exists() { + reason( + inspect_sqlite(&path).await, + "upgrade_source_sqlite_journal_unqualified", + ); + } else { + assert_eq!(std::fs::read(&path).unwrap(), original); + } + // Rerunning with the same verified fingerprint rolls the journal back, + // finds the backed-up PHP database and takes it over. + let report = takeover::take_over_sqlite(&path, &fingerprint, &queue_config()) + .await + .unwrap(); + assert_eq!(report["status"], "taken_over", "{report}"); + assert_eq!(report["fingerprint"], fingerprint.as_str()); + let runtime = crate::runtime::Runtime::open(&path, "test-token".into()).await.unwrap(); + runtime.close().await; +} + #[tokio::test] async fn takeover_of_the_frozen_layout_adds_only_native_relations() { let (_directory, path) = php_layout().await; diff --git a/scripts/conformance/server-parity/upgrade-source-cleanup.sh b/scripts/conformance/server-parity/upgrade-source-cleanup.sh index fb8f53b3..418448a3 100755 --- a/scripts/conformance/server-parity/upgrade-source-cleanup.sh +++ b/scripts/conformance/server-parity/upgrade-source-cleanup.sh @@ -53,6 +53,12 @@ case "$backend" in *) ;; esac fi +if [[ $backend == pgsql && -f "$evidence/takeover-owned.txt" && "$(cat "$evidence/takeover-owned.txt")" == "$RESOURCE_SCOPE" ]]; then + docker run --rm --user=1000:1000 --network "$RESOURCE_SCOPE" -e PGPASSWORD="$password" \ + --entrypoint psql "$POSTGRES_IMAGE" -h "${DW_PARITY_UPGRADE_DB_HOST:-parity-pg}" -U parity -d postgres -v ON_ERROR_STOP=1 \ + -c "DROP DATABASE IF EXISTS upgrade_takeover_ref${label:+_$label}" \ + >> "$evidence/database-cleanup.txt" 2>&1 +fi if test -f "$evidence/roles-owned.txt" && test "$(cat "$evidence/roles-owned.txt")" = "$RESOURCE_SCOPE"; then if test "$backend" = mysql; then docker run --rm --user=1000:1000 --network "$RESOURCE_SCOPE" --entrypoint mysql "$MYSQL_IMAGE" \ diff --git a/scripts/conformance/server-parity/upgrade-source-profile.sh b/scripts/conformance/server-parity/upgrade-source-profile.sh index 409a3b69..b50f7c58 100755 --- a/scripts/conformance/server-parity/upgrade-source-profile.sh +++ b/scripts/conformance/server-parity/upgrade-source-profile.sh @@ -52,9 +52,10 @@ else chmod 0666 "$state/database.sqlite" fi printf '%s\n' "$RESOURCE_SCOPE" > "$evidence/database-owned.txt" -# SQLite also takes over a copy and resumes the originals, so its timer is short. +# SQLite and PostgreSQL also take over a copy and resume the originals, so +# their timer is short. timer_seconds=600 -if test "$backend" = sqlite; then timer_seconds=60; fi +if [[ $backend == sqlite || $backend == pgsql ]]; then timer_seconds=60; fi configuration=("${admin[@]}" -e DB_DATABASE="$database" -e DW_AUTH_TOKEN=upgrade-fixture-token -e CACHE_STORE=file -e QUEUE_CONNECTION=database -e DB_SQLITE_JOURNAL_MODE=DELETE) php_run() { @@ -205,28 +206,44 @@ change_restored fingerprint "$restored" > "$evidence/changed-fingerprint.json" 2>> "$evidence/fingerprint.stderr" changed_fingerprint=$(jq -r .fingerprint "$evidence/changed-fingerprint.json") [[ $changed_fingerprint =~ ^sha256:[0-9a-f]{64}$ && $changed_fingerprint != "$source_fingerprint" ]] -if test "$backend" = sqlite; then - # Backup-first takeover of a copy of the stopped source, as an operator's - # live database: its digest is the one the restored backup matched. Native - # then resumes the originals, and PHP must refuse the upgraded file. - cp "$state/database.sqlite" "$state/takeover.sqlite" +# Backup-first takeover of a copy of the stopped source, as an operator's +# live database: its digest is the one the restored backup matched. Native +# then resumes the originals, and PHP must refuse the upgraded database. +target=() +case "$backend" in + sqlite) + cp "$state/database.sqlite" "$state/takeover.sqlite" + target=(-e DB_CONNECTION=sqlite -e DB_DATABASE=/state/takeover.sqlite) + target_network=none ;; + pgsql) + # The same backup restored once more, taken over with the owner role. + takeover_database=upgrade_takeover_ref${label:+_$label} + printf '%s\n' "$RESOURCE_SCOPE" > "$evidence/takeover-owned.txt" + pg_admin createdb "$takeover_database" 2>> "$evidence/backup.stderr" + pg_admin pg_restore --no-owner --exit-on-error -d "$takeover_database" < "$state/backup.dump" 2>> "$evidence/backup.stderr" + target=(-e DB_CONNECTION=pgsql -e DB_HOST="$host" -e DB_DATABASE="$takeover_database" + -e DB_USERNAME=parity -e DB_PASSWORD="$password") + target_network=$RESOURCE_SCOPE ;; + *) ;; +esac +if (( ${#target[@]} )); then docker run --rm --name "$resource-takeover" --user=1000:1000 --cpus=1 --memory=256m --memory-swap=256m \ - --network none -e DB_CONNECTION=sqlite -e DB_DATABASE=/state/takeover.sqlite -e DW_RUST_EXPERIMENTAL=1 \ + --network "$target_network" "${target[@]}" -e DW_RUST_EXPERIMENTAL=1 \ -e DW_UPGRADE_BACKUP_FINGERPRINT="$source_fingerprint" \ -e DW_RUST_UPGRADE_QUEUE_CONNECTION=database -e DW_RUST_UPGRADE_QUEUE_DEFAULT_CONNECTION=database \ -e DW_RUST_UPGRADE_QUEUE_DEFAULT_QUEUE=default \ -v "$PWD/$state:/state" -v "$PWD/rust-target:/target:ro" \ --entrypoint /target/debug/durable-workflow-server "$rust_image" upgrade-takeover \ > "$evidence/takeover.json" 2> "$evidence/takeover.stderr" - jq -e --arg layout "$layout" --arg fingerprint "$source_fingerprint" '.status=="taken_over" - and .source_layout==$layout and .fingerprint==$fingerprint and .backup_fingerprint_matched==true + jq -e --arg backend "$backend" --arg layout "$layout" --arg fingerprint "$source_fingerprint" '.status=="taken_over" + and .backend==$backend and .source_layout==$layout and .fingerprint==$fingerprint + and .backup_fingerprint_matched==true and .converted.incarnations_copied==(if $layout=="frozen" then 0 else 1 end) and .stored_representation_compatibility_verified==false' "$evidence/takeover.json" >/dev/null docker run -d --name "$resource-native" --network "$RESOURCE_SCOPE" --network-alias parity-upgrade-native \ --user=1000:1000 --cpus=1 --memory=512m --memory-swap=512m -e DW_RUST_EXPERIMENTAL=1 \ -e DW_BIND_ADDRESS=0.0.0.0:8080 -e DW_AUTH_DRIVER=token -e DW_AUTH_TOKEN=upgrade-fixture-token \ - -e DB_CONNECTION=sqlite -e DB_DATABASE=/state/takeover.sqlite \ - -v "$PWD/$state:/state" -v "$PWD/rust-target:/target:ro" \ + "${target[@]}" -v "$PWD/$state:/state" -v "$PWD/rust-target:/target:ro" \ --entrypoint /target/debug/durable-workflow-server "$rust_image" > "$evidence/native-container.txt" ready=0 for attempt in $(seq 1 30); do @@ -244,11 +261,10 @@ if test "$backend" = sqlite; then and .timer.events[-1]=="WorkflowCompleted"' "$evidence/resume.json" >/dev/null docker stop --signal SIGTERM --time 10 "$resource-native" >/dev/null if docker run --rm --name "$resource-control" --user=1000:1000 --cpus=1 --memory=512m --memory-swap=512m \ - --network none -e DB_CONNECTION=sqlite -e DB_DATABASE=/state/takeover.sqlite -e DW_AUTH_TOKEN=upgrade-fixture-token \ + --network "$target_network" "${target[@]}" -e DW_AUTH_TOKEN=upgrade-fixture-token \ -e CACHE_STORE=file -e QUEUE_CONNECTION=database -v "$PWD/$state:/state" --entrypoint php "$image" \ artisan server:assert-php-database > "$evidence/php-refusal.txt" 2>&1; then exit 1; fi grep -q php_database_refused "$evidence/php-refusal.txt" - sqlite_takeover=true fi snapshot > "$evidence/after.json" cmp "$evidence/before.json" "$evidence/after.json" From 6cc5d3d56957c7be6f5a69698daf041d12f70c0c Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 11:02:03 +0000 Subject: [PATCH 25/42] Match the design to the SQLite and PostgreSQL takeovers as built PostgreSQL locks every table in ACCESS EXCLUSIVE mode after native's bootstrap key, since the later DDL needs that mode anyway. On SQLite and PostgreSQL the marker commits with everything else, so an interrupted takeover is simply rerun; only MySQL and MariaDB need the step journal. Co-Authored-By: Claude Opus 5.5 --- docs/rust-server-upgrade-takeover.md | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/docs/rust-server-upgrade-takeover.md b/docs/rust-server-upgrade-takeover.md index 3f03af10..0aa1f821 100644 --- a/docs/rust-server-upgrade-takeover.md +++ b/docs/rust-server-upgrade-takeover.md @@ -66,7 +66,7 @@ with any other writer. | Backend | Exclusion | Atomicity | | --- | --- | --- | | SQLite | `BEGIN EXCLUSIVE` on the database file | Fingerprint, marker, native relations and conversions commit in one transaction. | -| PostgreSQL | `LOCK TABLE … IN EXCLUSIVE MODE` on every relation, then the native advisory key | DDL is transactional, so everything commits in one transaction. | +| PostgreSQL | Native's bootstrap advisory key, then `LOCK TABLE … IN ACCESS EXCLUSIVE MODE` on every table, with a short lock wait. The later DDL needs that mode anyway, so no lock is upgraded while another session waits. | DDL is transactional, so everything commits in one transaction. | | MySQL and MariaDB | To be qualified: `LOCK TABLES … WRITE` on every relation, or a check that no other session uses the database, repeated after the marker is written | DDL commits implicitly, so the marker records a step journal (below). | On SQLite and PostgreSQL, a PHP connection that slipped past the stop blocks on @@ -75,18 +75,19 @@ leaves a mixed state. ## Marker and interruption -The marker is written first in each takeover transaction, with state -`takeover`, the fingerprint and the step list. Native startup refuses state -`takeover`. Each step is idempotent and records its completion in the marker. -Rerunning `upgrade-takeover` with the same fingerprint resumes after the last -recorded step; a different fingerprint is refused. The final step sets state -`ready` and the native schema version. - -On SQLite and PostgreSQL an interruption rolls the whole transaction back, so -the database is either unchanged PHP or fully native. On MySQL and MariaDB the -step journal makes each committed DDL step visible, and resumption continues -from it. The PHP fence refuses the database from the first step, because the -marker exists. +On SQLite and PostgreSQL the marker commits with everything else, so an +interruption leaves the database unchanged PHP and nothing needs a journal. +The operator reruns `upgrade-takeover` with the same fingerprint. A killed +SQLite takeover leaves a rollback journal; read-only inspection refuses it, +and the rerun lets SQLite roll it back before the fingerprint check. + +MySQL and MariaDB commit DDL implicitly, so there the marker is written first, +with state `takeover`, the fingerprint and the step list. Native startup +refuses state `takeover`. Each step is idempotent and records its completion +in the marker. Rerunning `upgrade-takeover` with the same fingerprint resumes +after the last recorded step; a different fingerprint is refused. The final +step sets state `ready` and the native schema version. The PHP fence refuses +the database from the first step, because the marker exists. ## Steps From d94b1180e24576f33dae03c64bb9adfed11ffaa8 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 11:03:51 +0000 Subject: [PATCH 26/42] Call the Rust test failure summary through a named trap function The Docker workflow isolation policy only accepts repository helpers in statically known positions, as the container log tails already use. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/server-parity.yml | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/.github/workflows/server-parity.yml b/.github/workflows/server-parity.yml index 38d3bae9..31788a26 100644 --- a/.github/workflows/server-parity.yml +++ b/.github/workflows/server-parity.yml @@ -377,7 +377,11 @@ jobs: - name: Bootstrap independent published PHP and native MySQL databases run: | set -euo pipefail - trap 'bash scripts/conformance/server-parity/rust-test-failures.sh' ERR + # Failed Rust tests are printed from their evidence files into this step's log. + rust_test_failures() { + bash scripts/conformance/server-parity/rust-test-failures.sh + } + trap rust_test_failures ERR mkdir -p cargo-home rust-target parity-evidence mysql-state chmod 0777 parity-evidence docker network create "$RESOURCE_SCOPE" @@ -857,7 +861,11 @@ jobs: POSTGRES_IMAGE: ${{ matrix.image }} run: | set -euo pipefail - trap 'bash scripts/conformance/server-parity/rust-test-failures.sh' ERR + # Failed Rust tests are printed from their evidence files into this step's log. + rust_test_failures() { + bash scripts/conformance/server-parity/rust-test-failures.sh + } + trap rust_test_failures ERR mkdir -p cargo-home rust-target parity-evidence pg-state chmod 0777 parity-evidence docker network create "$RESOURCE_SCOPE" @@ -1366,7 +1374,11 @@ jobs: - name: Execute official Rust codec and cross-decode with published PHP run: | set -euo pipefail - trap 'bash scripts/conformance/server-parity/rust-test-failures.sh' ERR + # Failed Rust tests are printed from their evidence files into this step's log. + rust_test_failures() { + bash scripts/conformance/server-parity/rust-test-failures.sh + } + trap rust_test_failures ERR mkdir -p cargo-home rust-target parity-evidence # Compiling the library tests peaks near 2.5 GB; a 2 GB limit OOM-killed rustc (exit 101). docker run --rm --user="$(id -u):$(id -g)" --cpus=2 --memory=4g --memory-swap=4g \ From e6e349cafa881d59047d3ddc7943dca45326da71 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 11:21:31 +0000 Subject: [PATCH 27/42] fix: restore readiness probe fixture transaction ownership --- app/Support/ServerReadiness.php | 9 +++++-- config/database.php | 24 ++++++++++++------- .../Feature/DatabaseStorageExhaustedTest.php | 2 +- tests/Feature/HealthControllerTest.php | 16 +++++++++---- 4 files changed, 35 insertions(+), 16 deletions(-) diff --git a/app/Support/ServerReadiness.php b/app/Support/ServerReadiness.php index f11d9c7c..9dbedf53 100644 --- a/app/Support/ServerReadiness.php +++ b/app/Support/ServerReadiness.php @@ -103,7 +103,7 @@ private function databaseCheck(bool $probeQuery = false): array $queryAttempted = true; $row = $connection->selectOne('SELECT ? AS readiness_probe', [1], useReadPdo: false); if ((string) ($row->readiness_probe ?? '') !== '1') { - throw new \RuntimeException('The database did not return the readiness probe value.'); + throw new \UnexpectedValueException('The database did not return the readiness probe value.'); } } @@ -116,10 +116,15 @@ private function databaseCheck(bool $probeQuery = false): array : 'The database connection is available. Write capacity has not been verified.', ]; } catch (\Throwable $exception) { + $reason = $queryAttempted ? 'database_query_failed' : null; + if (BackendUnavailable::is($exception)) { + $reason = 'backend_unavailable'; + } + return [ 'status' => 'unavailable', 'message' => $exception->getMessage(), - 'reason' => BackendUnavailable::is($exception) ? 'backend_unavailable' : ($queryAttempted ? 'database_query_failed' : null), + 'reason' => $reason, ]; } } diff --git a/config/database.php b/config/database.php index aebce8f6..fd1f198a 100644 --- a/config/database.php +++ b/config/database.php @@ -4,6 +4,18 @@ use Pdo\Mysql; use Pdo\Pgsql; +$mysqlOptions = []; +if (extension_loaded('pdo_mysql')) { + $sslCaAttribute = PHP_VERSION_ID >= 80500 ? Mysql::ATTR_SSL_CA : PDO::MYSQL_ATTR_SSL_CA; + $mysqlOptions = array_filter([$sslCaAttribute => env('MYSQL_ATTR_SSL_CA')]); +} + +$pgsqlOptions = []; +if (extension_loaded('pdo_pgsql')) { + $disablePreparesAttribute = PHP_VERSION_ID >= 80500 ? Pgsql::ATTR_DISABLE_PREPARES : PDO::PGSQL_ATTR_DISABLE_PREPARES; + $pgsqlOptions = [$disablePreparesAttribute => true]; +} + return [ /* @@ -62,9 +74,7 @@ 'prefix_indexes' => true, 'strict' => true, 'engine' => null, - 'options' => extension_loaded('pdo_mysql') ? array_filter([ - (PHP_VERSION_ID >= 80500 ? Mysql::ATTR_SSL_CA : PDO::MYSQL_ATTR_SSL_CA) => env('MYSQL_ATTR_SSL_CA'), - ]) : [], + 'options' => $mysqlOptions, ], 'mariadb' => [ @@ -82,9 +92,7 @@ 'prefix_indexes' => true, 'strict' => true, 'engine' => null, - 'options' => extension_loaded('pdo_mysql') ? array_filter([ - (PHP_VERSION_ID >= 80500 ? Mysql::ATTR_SSL_CA : PDO::MYSQL_ATTR_SSL_CA) => env('MYSQL_ATTR_SSL_CA'), - ]) : [], + 'options' => $mysqlOptions, ], 'pgsql' => [ @@ -102,9 +110,7 @@ 'sslmode' => 'prefer', // Keep native parameter binding without session-scoped statement // names, so transaction poolers can safely change backend sessions. - 'options' => extension_loaded('pdo_pgsql') ? [ - (PHP_VERSION_ID >= 80500 ? Pgsql::ATTR_DISABLE_PREPARES : PDO::PGSQL_ATTR_DISABLE_PREPARES) => true, - ] : [], + 'options' => $pgsqlOptions, ], 'sqlsrv' => [ diff --git a/tests/Feature/DatabaseStorageExhaustedTest.php b/tests/Feature/DatabaseStorageExhaustedTest.php index f17749ad..6b75c7bc 100644 --- a/tests/Feature/DatabaseStorageExhaustedTest.php +++ b/tests/Feature/DatabaseStorageExhaustedTest.php @@ -93,7 +93,7 @@ public function test_database_readiness_explains_that_write_capacity_is_not_veri $this->getJson('/api/ready') ->assertOk() ->assertJsonPath('checks.database.status', 'ok') - ->assertJsonPath('checks.database.check_scope', 'connection_only') + ->assertJsonPath('checks.database.check_scope', 'bound_read_query') ->assertJsonPath('checks.database.write_capacity_verified', false); } diff --git a/tests/Feature/HealthControllerTest.php b/tests/Feature/HealthControllerTest.php index d40ef79f..ac6c13c6 100644 --- a/tests/Feature/HealthControllerTest.php +++ b/tests/Feature/HealthControllerTest.php @@ -28,6 +28,7 @@ class HealthControllerTest extends TestCase public function test_readiness_refuses_a_connected_database_that_cannot_execute_bound_reads(): void { + $originalDefault = config('database.default'); $pdo = DB::connection()->getPdo(); DB::extend('broken_bound_reads', static fn (array $config): SQLiteConnection => new class($pdo, ':memory:', '', $config) extends SQLiteConnection { @@ -45,10 +46,17 @@ public function select($query, $bindings = [], $useReadPdo = true, array $fetchU 'database.default' => 'broken_bound_reads', ]); - $this->getJson('/api/ready') - ->assertStatus(503) - ->assertJsonPath('checks.database.status', 'unavailable') - ->assertJsonPath('checks.database.reason', 'database_query_failed'); + try { + $this->getJson('/api/ready') + ->assertStatus(503) + ->assertJsonPath('checks.database.status', 'unavailable') + ->assertJsonPath('checks.database.reason', 'database_query_failed'); + } finally { + // RefreshDatabase must roll back its original connection, not + // this wrapper around the same PDO. In-memory suites reuse it. + config(['database.default' => $originalDefault]); + DB::purge('broken_bound_reads'); + } } public function test_health_check_returns_serving_when_database_is_available(): void From 3c1cc665678b262b2e59b615b233b28984b49998 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 11:42:46 +0000 Subject: [PATCH 28/42] Take over a stopped PHP MySQL or MariaDB database in recorded steps MySQL and MariaDB commit DDL implicitly, and LOCK TABLES refuses to create or drop tables it does not hold, so their takeover cannot be one transaction. Under native's bootstrap lock, upgrade-takeover now: - refuses while any other session uses the database (which needs the PROCESS privilege to see), then verifies the inventory and fingerprint in one consistent snapshot against the backup digest; - writes native's marker with a rust-takeover: row holding the digest. PHP refuses the database from then on, and native startup refuses it with native_takeover_incomplete; - checks the sessions and the fingerprint again with the marker set aside, then runs each conversion step, recording its completion in the row. Row changes commit with their step; DDL steps check what is left to do, so repeating an interrupted one is harmless; - commits the migration ledger with native's final marker. A rerun with the same digest resumes after the last recorded step; a different digest is refused. Laravel's mariadb driver layout, whose backup hold owner is a uuid, is converted too. The upgrade source cohort now takes over a restored copy on every backend, and its steps are named for what they do. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/server-parity.yml | 12 +- rust/README.md | 43 +- rust/src/main.rs | 4 +- rust/src/runtime/mysql.rs | 432 +++++++++++++++++- rust/src/runtime/upgrade.rs | 94 ++-- rust/src/runtime/upgrade/fingerprint.rs | 78 ++-- rust/src/runtime/upgrade/takeover.rs | 139 +++--- .../runtime/upgrade/takeover/mysql_journal.rs | 397 ++++++++++++++++ .../server-parity/upgrade-source-cleanup.sh | 19 +- .../server-parity/upgrade-source-profile.sh | 20 +- 10 files changed, 1077 insertions(+), 161 deletions(-) create mode 100644 rust/src/runtime/upgrade/takeover/mysql_journal.rs diff --git a/.github/workflows/server-parity.yml b/.github/workflows/server-parity.yml index 90a4dbe4..aee48b1b 100644 --- a/.github/workflows/server-parity.yml +++ b/.github/workflows/server-parity.yml @@ -663,14 +663,14 @@ jobs: cmp parity-evidence/catalog-before.json parity-evidence/catalog-after.json - name: Compare separate configured role-token cohort run: bash scripts/conformance/server-parity/role-profile.sh mysql - - name: Inspect separate published PHP upgrade source without takeover + - name: Back up, take over and resume a separate published PHP upgrade source timeout-minutes: 8 run: bash scripts/conformance/server-parity/upgrade-source-profile.sh mysql - name: Retire upgrade source cohort if: always() timeout-minutes: 3 run: bash scripts/conformance/server-parity/upgrade-source-cleanup.sh mysql - - name: Inspect published PHP 2.9.7 upgrade source without takeover + - name: Back up, take over and resume a published PHP 2.9.7 upgrade source timeout-minutes: 8 env: DW_PARITY_UPGRADE_LABEL: php297 @@ -1157,14 +1157,14 @@ jobs: env: POSTGRES_IMAGE: ${{ matrix.image }} run: bash scripts/conformance/server-parity/role-profile.sh pgsql - - name: Inspect separate published PHP upgrade source without takeover + - name: Back up, take over and resume a separate published PHP upgrade source timeout-minutes: 8 run: bash scripts/conformance/server-parity/upgrade-source-profile.sh pgsql - name: Retire upgrade source cohort if: always() timeout-minutes: 3 run: bash scripts/conformance/server-parity/upgrade-source-cleanup.sh pgsql - - name: Inspect published PHP 2.9.7 upgrade source without takeover + - name: Back up, take over and resume a published PHP 2.9.7 upgrade source timeout-minutes: 8 env: DW_PARITY_UPGRADE_LABEL: php297 @@ -1642,7 +1642,7 @@ jobs: env: RESOURCE_SCOPE: dw-parity-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }} run: bash scripts/conformance/server-parity/role-profile.sh sqlite - - name: Inspect separate published PHP upgrade source without takeover + - name: Back up, take over and resume a separate published PHP upgrade source timeout-minutes: 8 env: RESOURCE_SCOPE: dw-parity-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }} @@ -1653,7 +1653,7 @@ jobs: env: RESOURCE_SCOPE: dw-parity-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }} run: bash scripts/conformance/server-parity/upgrade-source-cleanup.sh sqlite - - name: Inspect published PHP 2.9.7 upgrade source without takeover + - name: Back up, take over and resume a published PHP 2.9.7 upgrade source timeout-minutes: 8 env: DW_PARITY_UPGRADE_LABEL: php297 diff --git a/rust/README.md b/rust/README.md index 0876ce23..36b9eee2 100644 --- a/rust/README.md +++ b/rust/README.md @@ -412,38 +412,51 @@ version-1/version-2/version-3 development schemas are refused without conversion needed old development data separately; this unpublished slice supplies no older-development converter. Use a new isolated file for version 4. -This refusal is not the upgrade mechanism. A stopped PHP SQLite or PostgreSQL -database can be taken over with the development `upgrade-takeover` command, -following the [takeover design](../docs/rust-server-upgrade-takeover.md): +This refusal is not the upgrade mechanism. A stopped PHP database can be taken +over with the development `upgrade-takeover` command, following the +[takeover design](../docs/rust-server-upgrade-takeover.md): 1. Run PHP Server 2.5.14 or later, which refuses a Rust-owned database, and stop every PHP role. 2. Run `upgrade-fingerprint` on the source: a SQLite file mounted read-only, - or PostgreSQL through a read-only role. Back the database up (a file copy, - or `pg_dump`), restore the backup separately, and fingerprint that too. - The digests must be equal. + or the database through a read-only account. Back the database up (a file + copy, `pg_dump` or `mysqldump --single-transaction`), restore the backup + separately, and fingerprint that too. The digests must be equal. 3. Run `upgrade-takeover` with `DW_UPGRADE_BACKUP_FINGERPRINT` set to that digest and the original queue settings (`DW_RUST_UPGRADE_QUEUE_*`, as for `upgrade-queue-inspect`). PostgreSQL uses the owner's `DB_*` settings. + MySQL and MariaDB use an account that owns the database and has the + `PROCESS` privilege, so the takeover can see every other session. -Inspection, the fingerprint check, the conversion of the recorded PHP release -layout, the native relations, the migration ledger and native's own startup -verification all happen in one transaction. SQLite holds the file +On SQLite and PostgreSQL, inspection, the fingerprint check, the conversion +of the recorded PHP release layout, the native relations, the migration ledger +and native's own startup verification all happen in one transaction. SQLite +holds the file exclusively; PostgreSQL takes native's bootstrap lock and locks every table before reading, and refuses with `upgrade_source_in_use` if a session still holds one after five seconds. A refusal leaves the PHP database unchanged. An interrupted takeover commits nothing: rerun it with the same digest. On SQLite, the rerun lets SQLite roll back the journal the interrupted run left, -and the fingerprint check then proves the file is the backed-up one. The -command refuses: +and the fingerprint check then proves the file is the backed-up one. + +MySQL and MariaDB commit DDL implicitly, so their takeover runs in steps under +native's bootstrap lock. It refuses with `upgrade_source_in_use` while any +other session uses the database. Once the source is verified, it first writes +native's marker with a `rust-takeover:` row: PHP refuses the database +from then on, and native refuses it with `native_takeover_incomplete` until +the takeover finishes. It then checks the sessions and the fingerprint again, +and runs each conversion step, recording each in that row. An interrupted +takeover is rerun with the same digest and continues after the last recorded +step; a different digest is refused. + +The command refuses: - a digest that differs, so any write after the backup stops it; - unrecognized queued jobs, external payloads and service calls; -- a layout other than the frozen one, PHP 2.8.0 or PHP 2.9.7, or a database - native already owns. +- a layout other than the frozen one, PHP 2.8.0 or PHP 2.9.7 (with either + Laravel MariaDB driver), or a database native already owns. -MySQL/MariaDB takeover, nested payload columns and multi-node qualification -remain open. **Never connect PHP to a Rust development database.** +Nested payload columns and multi-node qualification remain open. **Never connect PHP to a Rust development database.** Named namespace isolation and configured worker/operator/admin roles have representative shared qualification. Principal/custom-provider credentials and the full authorization contract remain open. Broader timer/signal semantics, failure/retry policy, cancellation and the rest of the API also diff --git a/rust/src/main.rs b/rust/src/main.rs index 22c7793f..56a8fe46 100644 --- a/rust/src/main.rs +++ b/rust/src/main.rs @@ -143,7 +143,7 @@ async fn main() -> Result<(), Box> { use durable_workflow_server::runtime::upgrade::{ queue_binding::QueueSourceConfig, takeover, }; - if backend != "sqlite" && backend != "pgsql" { + if !matches!(backend.as_str(), "sqlite" | "pgsql" | "mysql" | "mariadb") { eprintln!("upgrade_takeover_backend_unqualified"); return Err("upgrade takeover refused".into()); } @@ -165,6 +165,8 @@ async fn main() -> Result<(), Box> { }; let result = if backend == "pgsql" { takeover::take_over_postgres(postgres_options()?, &fingerprint, &queues).await + } else if backend != "sqlite" { + takeover::take_over_mysql(mysql_options()?, &fingerprint, &queues).await } else { let path = env::var("DB_DATABASE").map_err(|_| "DB_DATABASE is required")?; takeover::take_over_sqlite(&path, &fingerprint, &queues).await diff --git a/rust/src/runtime/mysql.rs b/rust/src/runtime/mysql.rs index 35e68ac0..a4a76dbf 100644 --- a/rust/src/runtime/mysql.rs +++ b/rust/src/runtime/mysql.rs @@ -13,15 +13,19 @@ pub const VERSION: i64 = 4; type Catalog = Vec<(String, String)>; pub const MARKER_SQL: &str = "CREATE TABLE dw_server_schema (engine VARCHAR(64) NOT NULL PRIMARY KEY,version BIGINT NOT NULL,bootstrap_checksum VARBINARY(48) NOT NULL) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin"; -pub const LEDGER_SQL: &str = "CREATE TABLE IF NOT EXISTS _sqlx_migrations (version BIGINT PRIMARY KEY,description TEXT NOT NULL,installed_on TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,success BOOLEAN NOT NULL,checksum BLOB NOT NULL,execution_time BIGINT NOT NULL) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci"; +/// The marker row of a takeover in progress: `rust-takeover:`, with +/// the completed step count as its version and the backup fingerprint's +/// digest as its checksum. Native startup refuses it and PHP sees the marker. +pub(super) const TAKEOVER_ENGINE: &str = "rust-takeover:"; +pub const LEDGER_SQL: &str ="CREATE TABLE IF NOT EXISTS _sqlx_migrations (version BIGINT PRIMARY KEY,description TEXT NOT NULL,installed_on TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,success BOOLEAN NOT NULL,checksum BLOB NOT NULL,execution_time BIGINT NOT NULL) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci"; #[derive(Clone, Copy)] -enum Flavor { +pub(super) enum Flavor { MySql, MariaDb, } impl Flavor { - async fn detect(connection: &mut MySqlConnection) -> Result { + pub(super) async fn detect(connection: &mut MySqlConnection) -> Result { let version: String = sqlx::query_scalar("SELECT VERSION()") .fetch_one(connection) .await?; @@ -72,7 +76,35 @@ impl Flavor { Self::MariaDb => include_str!("../../migrations/mariadb/0004_catalog.json"), }))?) } - fn checksum(self) -> &'static [u8] { + /// The single native migration, which a takeover records as applied. + pub(super) fn migration(self) -> &'static Migration { + self.migrator().iter().next().expect("one native MySQL migration") + } + /// The native relations a fresh bootstrap adds after the frozen PHP + /// layout, each created only if missing so a resumed takeover can repeat + /// them. The poll index on `workflow_tasks` is added separately. + pub(super) fn native_relations_sql(self) -> Result { + let (frozen, incarnations) = match self { + Self::MySql => ( + include_str!("../../migrations/mysql/0003_full_schema.sql"), + include_str!("../../migrations/mysql/0004_worker_incarnations.sql"), + ), + Self::MariaDb => ( + include_str!("../../migrations/mariadb/0003_full_schema.sql"), + include_str!("../../migrations/mariadb/0004_worker_incarnations.sql"), + ), + }; + Ok(super::upgrade::native_relations( + frozen, + "-- Native receipts are separate", + incarnations, + )? + .replace( + "CREATE TABLE dw_worker_registration_incarnations", + "CREATE TABLE IF NOT EXISTS dw_worker_registration_incarnations", + )) + } + pub(super) fn checksum(self) -> &'static [u8] { self.migrator().iter().next().unwrap().checksum.as_ref() } } @@ -83,7 +115,7 @@ fn options(options: MySqlConnectOptions) -> MySqlConnectOptions { .collation("utf8mb4_unicode_ci") .timezone(Some("+00:00".into())) } -async fn configure(connection: &mut MySqlConnection) -> Result<()> { +pub(super) async fn configure(connection: &mut MySqlConnection) -> Result<()> { sqlx::query("SET sql_quote_show_create=1,lock_wait_timeout=5,innodb_lock_wait_timeout=5") .execute(connection) .await?; @@ -205,13 +237,44 @@ pub(super) async fn php_source_catalog(connection: &mut MySqlConnection) -> Resu )); } } + let actual = catalog(connection).await?; + source_layout(connection, actual).await +} + +/// The PHP source catalog of a database whose takeover has written its +/// marker and nothing else yet: native's exact marker table, set aside. +pub(super) async fn fenced_source_catalog(connection: &mut MySqlConnection) -> Result<(Catalog, &'static str)> { + let flavor = Flavor::detect(connection).await?; + let marker = flavor + .expected()? + .into_iter() + .find(|(table, _)| table == "dw_server_schema"); + let mut actual = catalog(connection).await?; + let position = actual.iter().position(|(table, _)| table == "dw_server_schema"); + match position { + Some(position) if Some(&actual[position]) == marker.as_ref() => { + actual.remove(position); + } + _ => { + return Err(refuse( + StatusCode::CONFLICT, + "upgrade_takeover_marker_unqualified", + )); + } + } + source_layout(connection, actual).await +} + +async fn source_layout( + connection: &mut MySqlConnection, + mut actual: Catalog, +) -> Result<(Catalog, &'static str)> { let flavor = Flavor::detect(connection).await?; let expected: Catalog = flavor.expected()?.into_iter() .filter(|(table, _)| !super::upgrade::native_relation(table)) .map(|(table, ddl)| (table, ddl.replace( ",\n KEY `dw_workflow_tasks_poll` (`namespace`(64),`queue`(64),`task_type`(16),`status`(16),`available_at`)", ""))) .collect(); - let mut actual = catalog(connection).await?; // MariaDB's information_schema collation ignores some underscores. Only // canonicalize catalog presentation; preserve every full definition. actual.sort_by(|left, right| left.0.cmp(&right.0)); @@ -322,7 +385,7 @@ pub(super) async fn verify_history(connection: &mut MySqlConnection) -> Result<( Ok(()) } -async fn verify(connection: &mut MySqlConnection, flavor: Flavor) -> Result<()> { +pub(super) async fn verify(connection: &mut MySqlConnection, flavor: Flavor) -> Result<()> { history(connection, flavor, false).await?; if catalog(connection).await? != flavor.expected()? { return Err(refuse( @@ -388,6 +451,9 @@ async fn preflight(connection: &mut MySqlConnection, flavor: Flavor) -> Result<( if engines == ["rust-initializing"] { verify_initializing(connection, flavor).await?; Ok(()) + } else if engines.iter().any(|engine| engine.starts_with(TAKEOVER_ENGINE)) { + // An interrupted takeover resumes only through upgrade-takeover. + Err(refuse(StatusCode::CONFLICT, "native_takeover_incomplete")) } else { verify(connection, flavor).await } @@ -1038,4 +1104,356 @@ mod tests { db.remove().await; } } + + use crate::runtime::upgrade::{fingerprint, queue_binding::QueueSourceConfig, takeover}; + + const TAKEOVER_QUEUES: QueueSourceConfig<'static> = QueueSourceConfig { + connection: "database", + default_connection: "database", + default_queue: "default", + }; + + /// The frozen PHP layout, or a release layout rebuilt from the recorded + /// deltas (each changed table exactly as that release's PHP created it). + async fn php_layout(db: &Database, release: Option<&str>) -> Flavor { + let mut connection = db.connection().await; + let flavor = Flavor::detect(&mut connection).await.unwrap(); + let (frozen, deltas) = match flavor { + Flavor::MySql => ( + include_str!("../../migrations/mysql/0003_full_schema.sql"), + [ + include_str!("../../upgrade/php-2.8.0/mysql.json"), + include_str!("../../upgrade/php-2.9.7/mysql.json"), + ], + ), + Flavor::MariaDb => ( + include_str!("../../migrations/mariadb/0003_full_schema.sql"), + [ + include_str!("../../upgrade/php-2.8.0/mariadb.json"), + include_str!("../../upgrade/php-2.9.7/mariadb.json"), + ], + ), + }; + let php = &frozen[..frozen.find("-- Native receipts are separate").unwrap()]; + connection.execute(php).await.unwrap(); + connection + .execute("SET FOREIGN_KEY_CHECKS=1; ALTER TABLE workflow_tasks DROP KEY dw_workflow_tasks_poll") + .await + .unwrap(); + let releases = match release { + None => 0, + Some("php-2.8.0") => 1, + Some(_) => 2, + }; + for delta in &deltas[..releases] { + let delta: Value = serde_json::from_str(delta).unwrap(); + for added in delta["add"].as_array().unwrap() { + let (table, ddl) = (added[0].as_str().unwrap(), added[1].as_str().unwrap()); + connection + .execute(table_sql("DROP TABLE IF EXISTS ", table, "").unwrap()) + .await + .unwrap(); + connection.execute(AssertSqlSafe(ddl.to_owned())).await.unwrap(); + } + } + connection.close().await.unwrap(); + flavor + } + + async fn backup_fingerprint(db: &Database) -> String { + fingerprint::fingerprint_mysql(db.options.clone()) + .await + .unwrap()["fingerprint"] + .as_str() + .unwrap() + .to_owned() + } + + /// The physical schema and the fingerprint of every PHP row. + async fn source_state(db: &Database) -> (Catalog, String) { + let mut connection = db.connection().await; + let schema = catalog(&mut connection).await.unwrap(); + connection.close().await.unwrap(); + (schema, backup_fingerprint(db).await) + } + + async fn sessions_ended(db: &Database) { + let deadline = tokio::time::Instant::now() + Duration::from_secs(15); + loop { + let mut connection = db.connection().await; + let others: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM information_schema.PROCESSLIST WHERE DB=DATABASE() AND ID<>CONNECTION_ID()") + .fetch_one(&mut connection) + .await + .unwrap(); + connection.close().await.unwrap(); + if others == 0 { + return; + } + assert!(tokio::time::Instant::now() < deadline, "a takeover session survived"); + tokio::time::sleep(Duration::from_millis(50)).await; + } + } + + fn takeover_refused(result: Result, expected: &str) { + match result { + Err(super::super::RuntimeError::Refused { reason, .. }) => assert_eq!(reason, expected), + Err(error) => panic!("unexpected takeover error: {error}"), + Ok(report) => panic!("takeover was accepted: {report}"), + } + } + + async fn registrations(db: &Database) { + let mut connection = db.connection().await; + connection.execute( + "INSERT INTO workflow_worker_registrations(worker_id,namespace,task_queue,runtime,registration_token,created_at,updated_at) VALUES ('current','default','q','php','0123456789abcdef0123456789abcdef','2026-10-20 09:00:00','2026-10-20 09:00:00'); + INSERT INTO workflow_worker_registrations(worker_id,namespace,task_queue,runtime,created_at,updated_at) VALUES ('older','default','q','php','2026-10-20 09:00:00','2026-10-20 09:00:00'); + INSERT INTO workflow_worker_registration_incarnations VALUES ('0123456789abcdef0123456789abcdef','default','current','active',0,NULL,'2026-10-20 09:00:00','2026-10-20 09:00:00'); + INSERT INTO workflow_worker_registration_incarnations VALUES ('11111111111111111111111111111111','default','current','superseded',0,'2026-10-20 08:00:00','2026-10-20 07:00:00','2026-10-20 08:00:00'); + INSERT INTO workflow_worker_registration_incarnations VALUES ('22222222222222222222222222222222','default','gone','deregistered',3,'2026-10-20 08:30:00','2026-10-20 07:00:00','2026-10-20 08:30:00');", + ) + .await + .unwrap(); + connection.close().await.unwrap(); + } + + /// Native opens the database and holds the converted incarnations. + async fn converted(db: &Database) { + let storage = MySqlStorage::open(db.options.clone()).await.unwrap(); + storage.close().await; + MySqlStorage::check(db.options.clone()).await.unwrap(); + let mut connection = db.connection().await; + let incarnations: Vec<(String, String, String, Option, bool)> = sqlx::query_as( + "SELECT token,worker_id,status,recovered_workflow_task_count,finished_at IS NOT NULL FROM dw_worker_registration_incarnations ORDER BY worker_id,status", + ) + .fetch_all(&mut connection) + .await + .unwrap(); + connection.close().await.unwrap(); + assert_eq!(incarnations.len(), 4, "{incarnations:?}"); + assert!(incarnations.contains(&( + "0123456789abcdef0123456789abcdef".into(), + "current".into(), + "active".into(), + None, + false + ))); + assert!(incarnations.contains(&( + "22222222222222222222222222222222".into(), + "gone".into(), + "deregistered".into(), + Some(3), + true + ))); + assert!(incarnations.iter().any(|row| row.1 == "older" && row.2 == "active" && row.0.len() == 32)); + } + + #[tokio::test] + #[ignore = "requires an isolated MySQL/MariaDB create-database role"] + async fn qualification_takeover_converts_php_297_and_native_opens_it() { + let db = Database::new().await; + php_layout(&db, Some("php-2.9.7")).await; + registrations(&db).await; + let fingerprint = backup_fingerprint(&db).await; + let report = takeover::take_over_mysql(db.options.clone(), &fingerprint, &TAKEOVER_QUEUES) + .await + .unwrap(); + assert_eq!(report["status"], "taken_over", "{report}"); + assert_eq!(report["source_layout"], "php-2.9.7"); + assert_eq!(report["converted"]["incarnations_copied"], 3, "{report}"); + assert_eq!(report["converted"]["incarnations_created"], 1, "{report}"); + converted(&db).await; + takeover_refused( + takeover::take_over_mysql(db.options.clone(), &fingerprint, &TAKEOVER_QUEUES).await, + "upgrade_source_already_owned", + ); + db.remove().await; + } + + #[tokio::test] + #[ignore = "requires an isolated MySQL/MariaDB create-database role"] + async fn qualification_takeover_converts_the_mariadb_driver_layout() { + let db = Database::new().await; + if let Flavor::MySql = php_layout(&db, Some("php-2.9.7")).await { + db.remove().await; + return; + } + // Laravel's mariadb connection stores the backup hold owner as uuid. + let mut connection = db.connection().await; + connection + .execute("ALTER TABLE runtime_external_payload_backup_hold MODIFY owner uuid DEFAULT NULL") + .await + .unwrap(); + connection.close().await.unwrap(); + let fingerprint = backup_fingerprint(&db).await; + let report = takeover::take_over_mysql(db.options.clone(), &fingerprint, &TAKEOVER_QUEUES) + .await + .unwrap(); + assert_eq!(report["source_layout"], "php-2.9.7+mariadb-driver", "{report}"); + MySqlStorage::check(db.options.clone()).await.unwrap(); + db.remove().await; + } + + #[tokio::test] + #[ignore = "requires an isolated MySQL/MariaDB create-database role"] + async fn qualification_takeover_of_the_frozen_layout_adds_only_native_relations() { + let db = Database::new().await; + php_layout(&db, None).await; + let fingerprint = backup_fingerprint(&db).await; + let report = takeover::take_over_mysql(db.options.clone(), &fingerprint, &TAKEOVER_QUEUES) + .await + .unwrap(); + assert_eq!(report["source_layout"], "frozen", "{report}"); + MySqlStorage::check(db.options.clone()).await.unwrap(); + db.remove().await; + } + + #[tokio::test] + #[ignore = "requires an isolated MySQL/MariaDB create-database role"] + async fn qualification_takeover_refusals_before_the_marker_leave_php_unchanged() { + let db = Database::new().await; + php_layout(&db, Some("php-2.9.7")).await; + let fingerprint = backup_fingerprint(&db).await; + let original = source_state(&db).await; + takeover_refused( + takeover::take_over_mysql(db.options.clone(), &format!("sha256:{}", "0".repeat(64)), &TAKEOVER_QUEUES).await, + "upgrade_backup_fingerprint_mismatch", + ); + takeover_refused( + takeover::take_over_mysql(db.options.clone(), &format!("sha256:{}", "z".repeat(64)), &TAKEOVER_QUEUES).await, + "upgrade_backup_fingerprint_required", + ); + // Any other session on the database refuses the takeover. + let php = db.connection().await; + takeover_refused( + takeover::take_over_mysql(db.options.clone(), &fingerprint, &TAKEOVER_QUEUES).await, + "upgrade_source_in_use", + ); + php.close().await.unwrap(); + assert_eq!(source_state(&db).await, original); + // A write after the backup changes the digest, so the old one is refused. + let mut php = db.connection().await; + php.execute("INSERT INTO migrations(migration,batch) VALUES ('after-backup',9)") + .await + .unwrap(); + php.close().await.unwrap(); + let written = source_state(&db).await; + takeover_refused( + takeover::take_over_mysql(db.options.clone(), &fingerprint, &TAKEOVER_QUEUES).await, + "upgrade_backup_fingerprint_mismatch", + ); + assert_eq!(source_state(&db).await, written); + db.remove().await; + for (statement, expected) in [ + ( + "INSERT INTO workflow_service_calls(id,endpoint_name,service_name,operation_name,status,operation_mode) VALUES ('call','e','s','o','accepted','sync')", + "upgrade_source_service_calls_unsupported", + ), + ( + "INSERT INTO runtime_external_payloads(id,namespace,storage_uri,storage_uri_sha256,codec,sha256,size_bytes) VALUES ('p','default','s3://b/k','x','avro','y',1)", + "upgrade_source_external_payloads_unsupported", + ), + ] { + let db = Database::new().await; + php_layout(&db, Some("php-2.9.7")).await; + let mut connection = db.connection().await; + connection.execute(statement).await.unwrap(); + connection.close().await.unwrap(); + let fingerprint = backup_fingerprint(&db).await; + let before = source_state(&db).await; + takeover_refused( + takeover::take_over_mysql(db.options.clone(), &fingerprint, &TAKEOVER_QUEUES).await, + expected, + ); + assert_eq!(source_state(&db).await, before, "{expected}"); + db.remove().await; + } + } + + // Named under bootstrap_child so the qualification run's existing skip of + // that subprocess helper also skips this one. + #[tokio::test] + #[ignore = "invoked only as a subprocess by the takeover interruption qualification"] + async fn bootstrap_child_takeover() { + let url = std::env::var("DW_TAKEOVER_TEST_DATABASE").unwrap(); + let fingerprint = std::env::var("DW_TAKEOVER_TEST_FINGERPRINT").unwrap(); + // Parks at DW_TAKEOVER_TEST_BOUNDARY, where the parent kills it. + let report = takeover::take_over_mysql( + MySqlConnectOptions::from_str(&url).unwrap(), + &fingerprint, + &TAKEOVER_QUEUES, + ) + .await; + panic!("takeover returned instead of parking: {report:?}"); + } + + #[cfg(unix)] + #[tokio::test] + #[ignore = "requires an isolated MySQL/MariaDB create-database role"] + async fn qualification_takeover_killed_at_each_step_resumes_with_the_same_fingerprint() { + use std::os::unix::process::ExitStatusExt; + for boundary in [ + "before-marker-row", + "after-step-1", + "after-step-3", + "after-step-8", + ] { + let db = Database::new().await; + php_layout(&db, Some("php-2.9.7")).await; + registrations(&db).await; + let fingerprint = backup_fingerprint(&db).await; + let directory = tempfile::tempdir().unwrap(); + let receipt = directory.path().join("boundary"); + let mut child = Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "runtime::mysql::tests::bootstrap_child_takeover", + "--ignored", + "--nocapture", + ]) + .env("DW_TAKEOVER_TEST_DATABASE", &db.url) + .env("DW_TAKEOVER_TEST_FINGERPRINT", &fingerprint) + .env("DW_TAKEOVER_TEST_BOUNDARY", boundary) + .env("DW_TAKEOVER_TEST_RECEIPT", &receipt) + .spawn() + .unwrap(); + let deadline = tokio::time::Instant::now() + Duration::from_secs(90); + while !receipt.exists() && tokio::time::Instant::now() < deadline { + if let Some(status) = child.try_wait().unwrap() { + panic!("child exited before {boundary}: {status}"); + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + if !receipt.exists() { + child.kill().unwrap(); + child.wait().unwrap(); + panic!("child did not reach {boundary}"); + } + child.kill().unwrap(); + assert_eq!(child.wait().unwrap().signal(), Some(9)); + sessions_ended(&db).await; + // The marker already fences PHP, and native will not open the + // database until the takeover finishes. + let mut connection = db.connection().await; + let marked: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM information_schema.TABLES WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME='dw_server_schema'") + .fetch_one(&mut connection) + .await + .unwrap(); + connection.close().await.unwrap(); + assert_eq!(marked, 1, "{boundary}"); + assert!(MySqlStorage::open(db.options.clone()).await.is_err(), "{boundary}"); + if boundary == "after-step-3" { + takeover_refused( + takeover::take_over_mysql(db.options.clone(), &format!("sha256:{}", "0".repeat(64)), &TAKEOVER_QUEUES).await, + "upgrade_backup_fingerprint_mismatch", + ); + } + let report = takeover::take_over_mysql(db.options.clone(), &fingerprint, &TAKEOVER_QUEUES) + .await + .unwrap(); + assert_eq!(report["status"], "taken_over", "{boundary}: {report}"); + assert_eq!(report["source_layout"], "php-2.9.7", "{boundary}"); + converted(&db).await; + db.remove().await; + } + } } diff --git a/rust/src/runtime/upgrade.rs b/rust/src/runtime/upgrade.rs index d9029d1c..ef6baf64 100644 --- a/rust/src/runtime/upgrade.rs +++ b/rust/src/runtime/upgrade.rs @@ -511,46 +511,68 @@ async fn inspect_mysql_inner( ) -> Result { queue_config(queues)?; let deadline = Instant::now() + BUDGET; - let (mut connection, backend) = mysql_source(options, deadline, false, 30).await?; - let result = timeout_at(deadline, async { + let (connection, backend) = mysql_source(options, deadline, false, 30).await?; + bounded(connection, deadline, async |connection| { let mut tx = connection.begin_with("START TRANSACTION READ ONLY").await?; - let (catalog, layout) = mysql::php_source_catalog(&mut tx).await?; - let tables: Vec = catalog.iter().map(|(name, _)| name.clone()).collect(); - let counts = count_tables!(tx, tables, '`'); - let queue_inspection = match queues { - Some(config) => { - Some(queue_scan::inspect::(&mut *tx, config, counts["jobs"]).await?) - } - None => None, - }; - let payload_inspection = if payloads { - Some(payload_scan::inspect::(&mut *tx, &counts).await?) - } else { - None - }; - // MySQL's metadata is not an MVCC snapshot. Refuse a changed visible - // catalog rather than joining counts to a different schema. - if mysql::php_source_catalog(&mut tx).await?.0 != catalog { - return Err(refuse( - StatusCode::CONFLICT, - "upgrade_source_catalog_changed", - )); - } + let report = mysql_inventory(&mut tx, backend, queues, payloads, false).await; tx.rollback().await?; - report( - backend, - &catalog, - layout, - counts, - queue_inspection, - payload_inspection, - ) + report }) .await - .map_err(|_| expired()) - .and_then(|result| result); - close(connection.close()).await?; - result +} + +/// The PHP source catalog, or with `fenced` the catalog of a source whose +/// takeover marker is all that has been written, with that marker set aside. +pub(super) async fn mysql_source_catalog( + connection: &mut MySqlConnection, + fenced: bool, +) -> Result<(Vec<(String, String)>, &'static str)> { + if fenced { + mysql::fenced_source_catalog(connection).await + } else { + mysql::php_source_catalog(connection).await + } +} + +/// The MySQL or MariaDB inspection report, read inside the caller's +/// transaction. +pub(super) async fn mysql_inventory( + tx: &mut MySqlConnection, + backend: &str, + queues: Option<&QueueSourceConfig<'_>>, + payloads: bool, + fenced: bool, +) -> Result { + let (catalog, layout) = mysql_source_catalog(&mut *tx, fenced).await?; + let tables: Vec = catalog.iter().map(|(name, _)| name.clone()).collect(); + let counts = count_tables!(tx, tables, '`'); + let queue_inspection = match queues { + Some(config) => { + Some(queue_scan::inspect::(&mut *tx, config, counts["jobs"]).await?) + } + None => None, + }; + let payload_inspection = if payloads { + Some(payload_scan::inspect::(&mut *tx, &counts).await?) + } else { + None + }; + // MySQL's metadata is not an MVCC snapshot. Refuse a changed visible + // catalog rather than joining counts to a different schema. + if mysql_source_catalog(&mut *tx, fenced).await?.0 != catalog { + return Err(refuse( + StatusCode::CONFLICT, + "upgrade_source_catalog_changed", + )); + } + report( + backend, + &catalog, + layout, + counts, + queue_inspection, + payload_inspection, + ) } pub async fn inspect_postgres(options: PgConnectOptions) -> Result { diff --git a/rust/src/runtime/upgrade/fingerprint.rs b/rust/src/runtime/upgrade/fingerprint.rs index 384995ff..df82bb3b 100644 --- a/rust/src/runtime/upgrade/fingerprint.rs +++ b/rust/src/runtime/upgrade/fingerprint.rs @@ -8,8 +8,8 @@ //! per-table row counts leave the database. The fingerprint grants nothing: //! every compatibility, backup, quiescence and takeover flag stays false. use super::{ - Result, bounded, expired, mysql, mysql_source, open_sqlite_source, postgres, readonly_source, - refuse, schema, + Result, bounded, expired, mysql_source, mysql_source_catalog, open_sqlite_source, postgres, + readonly_source, refuse, schema, }; use axum::http::StatusCode; use futures_util::TryStreamExt; @@ -17,8 +17,8 @@ use serde::Serialize; use serde_json::{Value, json}; use sha2::{Digest as _, Sha256}; use sqlx::{ - AssertSqlSafe, Connection, PgConnection, Row, SqliteConnection, mysql::MySqlConnectOptions, - postgres::PgConnectOptions, + AssertSqlSafe, Connection, MySqlConnection, PgConnection, Row, SqliteConnection, + mysql::MySqlConnectOptions, postgres::PgConnectOptions, }; use std::{collections::BTreeMap, time::Duration}; use tokio::time::{Instant, timeout, timeout_at}; @@ -249,38 +249,46 @@ pub async fn fingerprint_mysql(options: MySqlConnectOptions) -> Result { let mut tx = connection .begin_with("START TRANSACTION WITH CONSISTENT SNAPSHOT, READ ONLY") .await?; - let (catalog, layout) = mysql::php_source_catalog(&mut tx).await?; - let mut digest = Digest::new(backend, &catalog)?; - for (table, _) in &catalog { - let table = identifier(table)?; - let columns: Vec<(String, String)> = sqlx::query_as("SELECT CAST(COLUMN_NAME AS CHAR),CAST(DATA_TYPE AS CHAR) FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? ORDER BY ORDINAL_POSITION") - .bind(table).fetch_all(&mut *tx).await?; - let key: Vec = sqlx::query_scalar("SELECT CAST(COLUMN_NAME AS CHAR) FROM information_schema.KEY_COLUMN_USAGE WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? AND CONSTRAINT_NAME='PRIMARY' ORDER BY ORDINAL_POSITION") - .bind(table).fetch_all(&mut *tx).await?; - let (names, kinds): (Vec, Vec) = columns.into_iter().unzip(); - digest.table(table, &names); - // Binary values render as hex literals; everything else as its - // quoted string form. - let sql = row_select(&format!("`{table}`"), &names, &key, '`', |c, index| { - match kinds[index].to_ascii_lowercase().as_str() { - "binary" | "varbinary" | "tinyblob" | "blob" | "mediumblob" | "longblob" | "bit" => format!( - "CAST(IF(`{c}` IS NULL,'NULL',CONCAT('X''',HEX(`{c}`),'''')) AS CHAR CHARACTER SET utf8mb4)" - ), - _ => format!("CAST(QUOTE(`{c}`) AS CHAR CHARACTER SET utf8mb4)"), - } - })?; - hash_rows!(tx, digest, table, sql, names.len()); - } - // MySQL's metadata is not an MVCC snapshot. Refuse a catalog that - // changed while the rows were read. - if mysql::php_source_catalog(&mut tx).await?.0 != catalog { - return Err(refuse( - StatusCode::CONFLICT, - "upgrade_source_catalog_changed", - )); - } + let report = mysql_digest(&mut tx, backend, false).await; tx.rollback().await?; - Ok(digest.report(backend, layout)) + report }) .await } + +/// The MySQL or MariaDB fingerprint report, read inside the caller's +/// consistent snapshot. With `fenced`, a takeover's own marker is set aside, +/// so the digest is the one the backup had. +pub(super) async fn mysql_digest(tx: &mut MySqlConnection, backend: &str, fenced: bool) -> Result { + let (catalog, layout) = mysql_source_catalog(&mut *tx, fenced).await?; + let mut digest = Digest::new(backend, &catalog)?; + for (table, _) in &catalog { + let table = identifier(table)?; + let columns: Vec<(String, String)> = sqlx::query_as("SELECT CAST(COLUMN_NAME AS CHAR),CAST(DATA_TYPE AS CHAR) FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? ORDER BY ORDINAL_POSITION") + .bind(table).fetch_all(&mut *tx).await?; + let key: Vec = sqlx::query_scalar("SELECT CAST(COLUMN_NAME AS CHAR) FROM information_schema.KEY_COLUMN_USAGE WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? AND CONSTRAINT_NAME='PRIMARY' ORDER BY ORDINAL_POSITION") + .bind(table).fetch_all(&mut *tx).await?; + let (names, kinds): (Vec, Vec) = columns.into_iter().unzip(); + digest.table(table, &names); + // Binary values render as hex literals; everything else as its + // quoted string form. + let sql = row_select(&format!("`{table}`"), &names, &key, '`', |c, index| { + match kinds[index].to_ascii_lowercase().as_str() { + "binary" | "varbinary" | "tinyblob" | "blob" | "mediumblob" | "longblob" | "bit" => format!( + "CAST(IF(`{c}` IS NULL,'NULL',CONCAT('X''',HEX(`{c}`),'''')) AS CHAR CHARACTER SET utf8mb4)" + ), + _ => format!("CAST(QUOTE(`{c}`) AS CHAR CHARACTER SET utf8mb4)"), + } + })?; + hash_rows!(tx, digest, table, sql, names.len()); + } + // MySQL's metadata is not an MVCC snapshot. Refuse a catalog that + // changed while the rows were read. + if mysql_source_catalog(&mut *tx, fenced).await?.0 != catalog { + return Err(refuse( + StatusCode::CONFLICT, + "upgrade_source_catalog_changed", + )); + } + Ok(digest.report(backend, layout)) +} diff --git a/rust/src/runtime/upgrade/takeover.rs b/rust/src/runtime/upgrade/takeover.rs index b7dbcb1c..2eb10c25 100644 --- a/rust/src/runtime/upgrade/takeover.rs +++ b/rust/src/runtime/upgrade/takeover.rs @@ -24,6 +24,9 @@ use sqlx::{ use std::time::Duration; use tokio::time::{Instant, timeout_at}; +mod mysql_journal; +pub use mysql_journal::take_over_mysql; + /// Takeover reads every row twice (inspection and fingerprint). const BUDGET: Duration = Duration::from_secs(3600); /// Native's PostgreSQL bootstrap key, so a native node cannot open the @@ -116,7 +119,7 @@ async fn take_over_sqlite_in( // Exactly the check native startup makes: marker, ledger and catalog. schema::verify(&mut *tx).await?; #[cfg(test)] - interruption_point(); + interruption_point("before-commit"); Ok(report("sqlite", layout, &digest, &inventory, converted)) } @@ -172,16 +175,20 @@ async fn take_over_postgres_in( record_migration::(&mut *tx, postgres::migration()).await?; postgres::verify(&mut *tx).await?; #[cfg(test)] - interruption_point(); + interruption_point("before-commit"); Ok(report("pgsql", layout, &digest, &inventory, converted)) } -/// Lets the interruption qualification kill a real takeover process after -/// every step has run, just before the commit. +/// Lets the interruption qualification kill a real takeover process at a +/// named boundary: `before-commit` unless `DW_TAKEOVER_TEST_BOUNDARY` names +/// another. #[cfg(test)] -fn interruption_point() { - if let Ok(receipt) = std::env::var("DW_TAKEOVER_TEST_RECEIPT") { - std::fs::write(format!("{receipt}.partial"), "before-commit").unwrap(); +fn interruption_point(boundary: &str) { + let wanted = std::env::var("DW_TAKEOVER_TEST_BOUNDARY"); + if let Ok(receipt) = std::env::var("DW_TAKEOVER_TEST_RECEIPT") + && wanted.as_deref().unwrap_or("before-commit") == boundary + { + std::fs::write(format!("{receipt}.partial"), boundary).unwrap(); std::fs::rename(format!("{receipt}.partial"), &receipt).unwrap(); loop { std::thread::park(); @@ -283,29 +290,87 @@ where Ok(()) } -/// Converts a recorded PHP release layout to native's (the design's recipes). -async fn convert(tx: &mut DB::Connection, layout: &str) -> Result +/// Whether a recorded PHP release layout has PHP's own registration +/// incarnations. Layouts without a recipe are refused. +fn php_incarnations(layout: &str) -> Result { + match layout.trim_end_matches("+mariadb-driver") { + "frozen" => Ok(false), + "php-2.8.0" | "php-2.9.7" => Ok(true), + _ => Err(refuse( + StatusCode::CONFLICT, + "upgrade_source_layout_unconverted", + )), + } +} + +/// Service calls are not part of native yet. +async fn service_calls_absent(tx: &mut DB::Connection) -> Result<()> where for<'c> &'c mut DB::Connection: Executor<'c, Database = DB>, DB::Arguments: IntoArguments, - for<'q> String: Encode<'q, DB> + Type, { - let php_incarnations = match layout { - "frozen" => false, - "php-2.8.0" | "php-2.9.7" => true, - _ => { - return Err(refuse( - StatusCode::CONFLICT, - "upgrade_source_layout_unconverted", - )); - } - }; if count::(tx, "SELECT COUNT(*) AS n FROM workflow_service_calls").await? != 0 { return Err(refuse( StatusCode::CONFLICT, "upgrade_source_service_calls_unsupported", )); } + Ok(()) +} + +/// Converts a recorded PHP release layout to native's (the design's recipes). +async fn convert(tx: &mut DB::Connection, layout: &str) -> Result +where + for<'c> &'c mut DB::Connection: Executor<'c, Database = DB>, + DB::Arguments: IntoArguments, + for<'q> String: Encode<'q, DB> + Type, +{ + let php_incarnations = php_incarnations(layout)?; + service_calls_absent::(tx).await?; + let (copied, created) = adopt_incarnations::(tx, php_incarnations).await?; + if php_incarnations { + sqlx::raw_sql( + "DROP TABLE workflow_worker_registration_incarnations; \ + ALTER TABLE workflow_worker_registrations DROP COLUMN registration_token; \ + DROP INDEX wf_service_calls_deadline_idx; \ + ALTER TABLE workflow_service_calls DROP COLUMN call_deadline_at;", + ) + .execute(&mut *tx) + .await?; + } + let delivery_references = if layout == "php-2.9.7" { + // PHP's own queue delivery lookup metadata; native does not use PHP's + // queue, and the queue inspection bound every job to its task row. + let references = count::(tx, DELIVERY_REFERENCES).await?; + sqlx::raw_sql("ALTER TABLE workflow_tasks DROP COLUMN queue_delivery_reference") + .execute(&mut *tx) + .await?; + references + } else { + 0 + }; + Ok(json!({ + "incarnations_copied":copied, "incarnations_created":created, + "queue_delivery_references_removed":delivery_references + })) +} + +const DELIVERY_REFERENCES: &str = + "SELECT COUNT(*) AS n FROM workflow_tasks WHERE queue_delivery_reference IS NOT NULL"; + +/// Gives every registration the incarnation native expects. PHP's own +/// incarnations and tokens are copied, so a running SDK worker's fenced +/// deregistration still matches; an active registration without one gets a +/// new incarnation. Returns how many were copied and created. +async fn adopt_incarnations( + tx: &mut DB::Connection, + php_incarnations: bool, +) -> Result<(u64, usize)> +where + for<'c> &'c mut DB::Connection: Executor<'c, Database = DB>, + DB::Arguments: IntoArguments, + for<'q> String: Encode<'q, DB> + Type, +{ let now = super::super::store::now(); let mut copied = 0; if php_incarnations { @@ -331,8 +396,8 @@ where )); } } - // Every active registration gets the authority native expects. A PHP - // 2.8.0+ worker keeps the token it was given; an older one gets a new token. + // A PHP 2.8.0+ worker keeps the token it was given; an older one gets a + // new token. let missing = query::(if php_incarnations { "SELECT r.namespace,r.worker_id,r.registration_token AS token FROM workflow_worker_registrations r WHERE r.status='active' AND NOT EXISTS (SELECT 1 FROM dw_worker_registration_incarnations i WHERE i.namespace=r.namespace AND i.worker_id=r.worker_id AND i.status='active') ORDER BY r.namespace,r.worker_id" } else { @@ -354,35 +419,7 @@ where .execute(&mut *tx) .await?; } - if php_incarnations { - sqlx::raw_sql( - "DROP TABLE workflow_worker_registration_incarnations; \ - ALTER TABLE workflow_worker_registrations DROP COLUMN registration_token; \ - DROP INDEX wf_service_calls_deadline_idx; \ - ALTER TABLE workflow_service_calls DROP COLUMN call_deadline_at;", - ) - .execute(&mut *tx) - .await?; - } - let delivery_references = if layout == "php-2.9.7" { - // PHP's own queue delivery lookup metadata; native does not use PHP's - // queue, and the queue inspection bound every job to its task row. - let references = count::( - tx, - "SELECT COUNT(*) AS n FROM workflow_tasks WHERE queue_delivery_reference IS NOT NULL", - ) - .await?; - sqlx::raw_sql("ALTER TABLE workflow_tasks DROP COLUMN queue_delivery_reference") - .execute(&mut *tx) - .await?; - references - } else { - 0 - }; - Ok(json!({ - "incarnations_copied":copied, "incarnations_created":missing.len(), - "queue_delivery_references_removed":delivery_references - })) + Ok((copied, missing.len())) } async fn count(tx: &mut DB::Connection, sql: &'static str) -> Result diff --git a/rust/src/runtime/upgrade/takeover/mysql_journal.rs b/rust/src/runtime/upgrade/takeover/mysql_journal.rs new file mode 100644 index 00000000..15b3d8ce --- /dev/null +++ b/rust/src/runtime/upgrade/takeover/mysql_journal.rs @@ -0,0 +1,397 @@ +//! MySQL and MariaDB takeover. Their DDL commits implicitly, and `LOCK +//! TABLES` refuses to create or drop the tables it does not hold, so the +//! takeover cannot be one transaction. It writes native's marker first, with +//! a `rust-takeover:` row whose version counts the completed steps +//! and whose checksum holds the backup fingerprint's digest. From then on PHP +//! refuses the database and native refuses it until the last step. Every step +//! is idempotent, so a rerun with the same fingerprint resumes after the last +//! recorded one. +use super::{ + BUDGET, DELIVERY_REFERENCES, Result, adopt_incarnations, count, matches_backup, + php_incarnations, preconditions, qualified, report, service_calls_absent, +}; +use crate::runtime::{ + mysql::{self, Flavor, LEDGER_SQL, MARKER_SQL, TAKEOVER_ENGINE}, + upgrade::{ + bounded, fingerprint, mysql_inventory, mysql_source, queue_binding::QueueSourceConfig, + refuse, + }, +}; +use axum::http::StatusCode; +use serde_json::{Value, json}; +use sqlx::{Connection, Executor, MySql, MySqlConnection, mysql::MySqlConnectOptions}; +use tokio::time::Instant; + +/// Completed-step counts recorded in the marker row. +const MARKED: i64 = 0; +/// Sessions rechecked and the source fingerprinted again after the marker: +/// nothing PHP wrote can be missing from the backup. +const FENCED: i64 = 1; +const LAST: i64 = 8; + +/// One takeover run: what its steps need and what its report says. +struct Run<'a> { + backend: &'static str, + flavor: Flavor, + digest: Vec, + layout: String, + fingerprint: &'a str, + queues: &'a QueueSourceConfig<'a>, + inventory: Value, + converted: serde_json::Map, +} + +enum Journal { + /// No marker: a PHP database. + Absent, + /// The marker table without its row: a takeover killed between the two. + Empty, + /// A takeover in progress, with its layout, step count and digest. + Takeover(String, i64, Vec), + /// Native's own database, or one native is still initializing. + Owned, +} + +pub async fn take_over_mysql( + options: MySqlConnectOptions, + backup_fingerprint: &str, + queues: &QueueSourceConfig<'_>, +) -> Result { + preconditions(backup_fingerprint, queues)?; + let digest = fingerprint_digest(backup_fingerprint)?; + let deadline = Instant::now() + BUDGET; + // TIMESTAMP values render in UTC for the fingerprint, and lock waits stay + // short; the client deadline bounds whole-table work. + let (connection, backend) = mysql_source(options, deadline, true, 0).await?; + bounded(connection, deadline, async |connection| { + // The same session lock as native bootstrap, so neither a native node + // nor a second takeover can run on this database meanwhile. + let locked: Option = sqlx::query_scalar( + "SELECT GET_LOCK(CONCAT('dw-native-bootstrap:',SHA1(DATABASE())),5)", + ) + .fetch_one(&mut *connection) + .await?; + if locked != Some(1) { + return Err(refuse( + StatusCode::SERVICE_UNAVAILABLE, + "native_initialization_busy", + )); + } + let mut run = Run { + backend, + flavor: Flavor::detect(connection).await?, + digest, + layout: String::new(), + fingerprint: backup_fingerprint, + queues, + inventory: Value::Null, + converted: serde_json::Map::new(), + }; + let completed = match journal(connection).await? { + Journal::Absent => { + mark(connection, &mut run).await?; + MARKED + } + Journal::Empty => { + // Only a takeover adds the marker to a PHP database. Its layout + // is read again with the marker set aside. + let mut tx = connection + .begin_with("START TRANSACTION WITH CONSISTENT SNAPSHOT, READ ONLY") + .await?; + let layout = mysql_inventory(&mut tx, backend, Some(queues), true, true).await; + tx.rollback().await?; + run.layout = qualified(&layout?)?.to_owned(); + record(connection, &run).await?; + MARKED + } + Journal::Takeover(layout, completed, digest) => { + if digest != run.digest { + return Err(refuse( + StatusCode::CONFLICT, + "upgrade_backup_fingerprint_mismatch", + )); + } + php_incarnations(&layout)?; + run.layout = layout; + run.converted + .insert("resumed_after_step".into(), json!(completed)); + completed + } + Journal::Owned => { + return Err(refuse(StatusCode::CONFLICT, "upgrade_source_already_owned")); + } + }; + for step in completed + 1..=LAST { + perform(connection, &mut run, step).await?; + #[cfg(test)] + super::interruption_point(&format!("after-step-{step}")); + } + finish(connection, &run).await?; + mysql::verify(connection, run.flavor).await?; + let digest = json!({"fingerprint": run.fingerprint}); + Ok(report( + run.backend, + &run.layout, + &digest, + &run.inventory, + Value::Object(run.converted), + )) + }) + .await +} + +/// `sha256:` as the 32 bytes the marker row keeps. +fn fingerprint_digest(fingerprint: &str) -> Result> { + fingerprint["sha256:".len()..] + .as_bytes() + .chunks(2) + .map(|pair| { + std::str::from_utf8(pair) + .ok() + .and_then(|pair| u8::from_str_radix(pair, 16).ok()) + }) + .collect::>>() + .ok_or_else(|| { + refuse( + StatusCode::BAD_REQUEST, + "upgrade_backup_fingerprint_required", + ) + }) +} + +async fn journal(connection: &mut MySqlConnection) -> Result { + let marked: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM information_schema.TABLES WHERE TABLE_SCHEMA=DATABASE() AND LOWER(TABLE_NAME)='dw_server_schema'") + .fetch_one(&mut *connection) + .await?; + if marked == 0 { + return Ok(Journal::Absent); + } + let mut rows: Vec<(String, i64, Vec)> = + sqlx::query_as("SELECT engine,version,bootstrap_checksum FROM dw_server_schema") + .fetch_all(&mut *connection) + .await?; + Ok(match rows.pop() { + None => Journal::Empty, + Some((engine, completed, digest)) if rows.is_empty() => { + match engine.strip_prefix(TAKEOVER_ENGINE) { + Some(layout) => Journal::Takeover(layout.to_owned(), completed, digest), + None => Journal::Owned, + } + } + Some(_) => Journal::Owned, + }) +} + +/// No other session may use the database. That needs the PROCESS privilege, +/// since without it MySQL shows only the account's own sessions. +async fn sessions_clear(connection: &mut MySqlConnection) -> Result<()> { + let visible: bool = sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM information_schema.USER_PRIVILEGES WHERE PRIVILEGE_TYPE='PROCESS')", + ) + .fetch_one(&mut *connection) + .await?; + if !visible { + return Err(refuse( + StatusCode::CONFLICT, + "upgrade_source_sessions_unverifiable", + )); + } + let others: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM information_schema.PROCESSLIST WHERE DB=DATABASE() AND ID<>CONNECTION_ID()") + .fetch_one(&mut *connection) + .await?; + if others != 0 { + return Err(refuse(StatusCode::CONFLICT, "upgrade_source_in_use")); + } + Ok(()) +} + +/// The inventory and fingerprint from one consistent snapshot, refused +/// unless the source is qualified and matches the backup. +async fn verified_snapshot( + connection: &mut MySqlConnection, + run: &Run<'_>, + fenced: bool, +) -> Result<(Value, String)> { + sessions_clear(connection).await?; + let mut tx = connection + .begin_with("START TRANSACTION WITH CONSISTENT SNAPSHOT, READ ONLY") + .await?; + let read = async { + let inventory = + mysql_inventory(&mut tx, run.backend, Some(run.queues), true, fenced).await?; + let digest = fingerprint::mysql_digest(&mut tx, run.backend, fenced).await?; + Ok::<_, crate::runtime::RuntimeError>((inventory, digest)) + } + .await; + tx.rollback().await?; + let (inventory, digest) = read?; + let layout = qualified(&inventory)?.to_owned(); + matches_backup(&digest, run.fingerprint)?; + php_incarnations(&layout)?; + service_calls_absent::(connection).await?; + Ok((inventory, layout)) +} + +/// The first write: native's marker with the takeover row, once the source +/// is verified. PHP refuses the database from here on. +async fn mark(connection: &mut MySqlConnection, run: &mut Run<'_>) -> Result<()> { + let (inventory, layout) = verified_snapshot(connection, run, false).await?; + run.inventory = inventory; + run.layout = layout; + connection.execute(MARKER_SQL).await?; + #[cfg(test)] + super::interruption_point("before-marker-row"); + record(connection, run).await +} + +async fn record(connection: &mut MySqlConnection, run: &Run<'_>) -> Result<()> { + sqlx::query("INSERT INTO dw_server_schema(engine,version,bootstrap_checksum) VALUES (?,?,?)") + .bind(format!("{TAKEOVER_ENGINE}{}", run.layout)) + .bind(MARKED) + .bind(&run.digest[..]) + .execute(&mut *connection) + .await?; + Ok(()) +} + +/// Records one more completed step, in the caller's transaction or alone. +async fn advance<'c, E>(executor: E, run: &Run<'_>, step: i64) -> Result<()> +where + E: Executor<'c, Database = MySql>, +{ + let advanced = sqlx::query("UPDATE dw_server_schema SET version=? WHERE engine=? AND version=?") + .bind(step) + .bind(format!("{TAKEOVER_ENGINE}{}", run.layout)) + .bind(step - 1) + .execute(executor) + .await? + .rows_affected(); + if advanced != 1 { + return Err(refuse( + StatusCode::CONFLICT, + "upgrade_takeover_journal_changed", + )); + } + Ok(()) +} + +async fn column_exists(connection: &mut MySqlConnection, table: &str, column: &str) -> Result { + let found: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? AND COLUMN_NAME=?") + .bind(table) + .bind(column) + .fetch_one(&mut *connection) + .await?; + Ok(found != 0) +} + +/// Each step checks what it has to do, so repeating an interrupted one is +/// harmless, and then records itself. +async fn perform(connection: &mut MySqlConnection, run: &mut Run<'_>, step: i64) -> Result<()> { + let php_incarnations = php_incarnations(&run.layout)?; + match step { + FENCED => { + // A session that was still open when the marker appeared could + // have written; none is left, and the source is still the backup. + let (inventory, layout) = verified_snapshot(connection, run, true).await?; + if layout != run.layout { + return Err(refuse( + StatusCode::CONFLICT, + "upgrade_source_catalog_unqualified", + )); + } + run.inventory = inventory; + } + 2 => { + sqlx::raw_sql(sqlx::AssertSqlSafe(run.flavor.native_relations_sql()?)) + .execute(&mut *connection) + .await?; + let indexed: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME='workflow_tasks' AND INDEX_NAME='dw_workflow_tasks_poll'") + .fetch_one(&mut *connection) + .await?; + if indexed == 0 { + connection.execute("ALTER TABLE workflow_tasks ADD KEY dw_workflow_tasks_poll (namespace(64),queue(64),task_type(16),status(16),available_at)").await?; + } + } + 3 => { + // Rows, unlike DDL, commit with the step that records them. + let mut tx = connection.begin().await?; + let (copied, created) = adopt_incarnations::(&mut tx, php_incarnations).await?; + advance(&mut *tx, run, step).await?; + tx.commit().await?; + run.converted + .insert("incarnations_copied".into(), json!(copied)); + run.converted + .insert("incarnations_created".into(), json!(created)); + return Ok(()); + } + 4 => { + if php_incarnations { + connection + .execute("DROP TABLE IF EXISTS workflow_worker_registration_incarnations") + .await?; + } + } + 5 => { + if column_exists(connection, "workflow_worker_registrations", "registration_token").await? { + connection + .execute("ALTER TABLE workflow_worker_registrations DROP COLUMN registration_token") + .await?; + } + } + 6 => { + if column_exists(connection, "workflow_service_calls", "call_deadline_at").await? { + // Dropping only the column would leave its index on status alone. + connection.execute("ALTER TABLE workflow_service_calls DROP INDEX wf_service_calls_deadline_idx, DROP COLUMN call_deadline_at").await?; + } + } + 7 => { + if column_exists(connection, "workflow_tasks", "queue_delivery_reference").await? { + let references = count::(connection, DELIVERY_REFERENCES).await?; + run.converted + .insert("queue_delivery_references_removed".into(), json!(references)); + connection + .execute("ALTER TABLE workflow_tasks DROP COLUMN queue_delivery_reference") + .await?; + } + } + 8 => { + if run.layout.ends_with("+mariadb-driver") { + // Laravel's mariadb connection made the backup hold owner a uuid. + connection + .execute("ALTER TABLE runtime_external_payload_backup_hold MODIFY owner char(36) DEFAULT NULL") + .await?; + } + } + _ => {} + } + advance(&mut *connection, run, step).await +} + +/// The ledger row and native's final marker commit together. +async fn finish(connection: &mut MySqlConnection, run: &Run<'_>) -> Result<()> { + connection.execute(LEDGER_SQL).await?; + let migration = run.flavor.migration(); + let mut tx = connection.begin().await?; + sqlx::query("INSERT INTO _sqlx_migrations(version,description,success,checksum,execution_time) VALUES (?,?,TRUE,?,0)") + .bind(migration.version) + .bind(migration.description.as_ref()) + .bind(migration.checksum.as_ref()) + .execute(&mut *tx) + .await?; + let marked = sqlx::query("UPDATE dw_server_schema SET engine='rust-development',version=?,bootstrap_checksum=? WHERE engine=? AND version=?") + .bind(mysql::VERSION) + .bind(run.flavor.checksum()) + .bind(format!("{TAKEOVER_ENGINE}{}", run.layout)) + .bind(LAST) + .execute(&mut *tx) + .await? + .rows_affected(); + if marked != 1 { + return Err(refuse( + StatusCode::CONFLICT, + "upgrade_takeover_journal_changed", + )); + } + tx.commit().await?; + Ok(()) +} diff --git a/scripts/conformance/server-parity/upgrade-source-cleanup.sh b/scripts/conformance/server-parity/upgrade-source-cleanup.sh index 418448a3..e5a1ac52 100755 --- a/scripts/conformance/server-parity/upgrade-source-cleanup.sh +++ b/scripts/conformance/server-parity/upgrade-source-cleanup.sh @@ -53,11 +53,20 @@ case "$backend" in *) ;; esac fi -if [[ $backend == pgsql && -f "$evidence/takeover-owned.txt" && "$(cat "$evidence/takeover-owned.txt")" == "$RESOURCE_SCOPE" ]]; then - docker run --rm --user=1000:1000 --network "$RESOURCE_SCOPE" -e PGPASSWORD="$password" \ - --entrypoint psql "$POSTGRES_IMAGE" -h "${DW_PARITY_UPGRADE_DB_HOST:-parity-pg}" -U parity -d postgres -v ON_ERROR_STOP=1 \ - -c "DROP DATABASE IF EXISTS upgrade_takeover_ref${label:+_$label}" \ - >> "$evidence/database-cleanup.txt" 2>&1 +if [[ -f "$evidence/takeover-owned.txt" && "$(cat "$evidence/takeover-owned.txt")" == "$RESOURCE_SCOPE" ]]; then +case "$backend" in + mysql) + docker run --rm --user=1000:1000 --network "$RESOURCE_SCOPE" --entrypoint mysql "$MYSQL_IMAGE" \ + -h "${DW_PARITY_UPGRADE_DB_HOST:-parity-mysql}" -uroot -p"$password" \ + -e "DROP DATABASE IF EXISTS upgrade_takeover_ref${label:+_$label}" \ + >> "$evidence/database-cleanup.txt" 2>&1 ;; + pgsql) + docker run --rm --user=1000:1000 --network "$RESOURCE_SCOPE" -e PGPASSWORD="$password" \ + --entrypoint psql "$POSTGRES_IMAGE" -h "${DW_PARITY_UPGRADE_DB_HOST:-parity-pg}" -U parity -d postgres -v ON_ERROR_STOP=1 \ + -c "DROP DATABASE IF EXISTS upgrade_takeover_ref${label:+_$label}" \ + >> "$evidence/database-cleanup.txt" 2>&1 ;; + *) ;; +esac fi if test -f "$evidence/roles-owned.txt" && test "$(cat "$evidence/roles-owned.txt")" = "$RESOURCE_SCOPE"; then if test "$backend" = mysql; then diff --git a/scripts/conformance/server-parity/upgrade-source-profile.sh b/scripts/conformance/server-parity/upgrade-source-profile.sh index b50f7c58..e41b52c0 100755 --- a/scripts/conformance/server-parity/upgrade-source-profile.sh +++ b/scripts/conformance/server-parity/upgrade-source-profile.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash -# A separate published PHP cohort; inventory is not migration or SDK takeover. +# A separate published PHP cohort: inventory and refusals, a verified backup, +# and the takeover of a restored copy that native then resumes. set -euo pipefail test "$#" = 1 || exit 2 backend=$1 @@ -52,10 +53,9 @@ else chmod 0666 "$state/database.sqlite" fi printf '%s\n' "$RESOURCE_SCOPE" > "$evidence/database-owned.txt" -# SQLite and PostgreSQL also take over a copy and resume the originals, so -# their timer is short. -timer_seconds=600 -if [[ $backend == sqlite || $backend == pgsql ]]; then timer_seconds=60; fi +# Every backend also takes over a copy and resumes the originals, so the +# timer is short. +timer_seconds=60 configuration=("${admin[@]}" -e DB_DATABASE="$database" -e DW_AUTH_TOKEN=upgrade-fixture-token -e CACHE_STORE=file -e QUEUE_CONNECTION=database -e DB_SQLITE_JOURNAL_MODE=DELETE) php_run() { @@ -224,6 +224,16 @@ case "$backend" in target=(-e DB_CONNECTION=pgsql -e DB_HOST="$host" -e DB_DATABASE="$takeover_database" -e DB_USERNAME=parity -e DB_PASSWORD="$password") target_network=$RESOURCE_SCOPE ;; + mysql) + # The same backup restored once more, taken over as root, which can see + # every session (the PROCESS privilege). + takeover_database=upgrade_takeover_ref${label:+_$label} + printf '%s\n' "$RESOURCE_SCOPE" > "$evidence/takeover-owned.txt" + mysql_admin mysql -e "CREATE DATABASE $takeover_database CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci" 2>> "$evidence/backup.stderr" + mysql_admin mysql "$takeover_database" < "$state/backup.sql" 2>> "$evidence/backup.stderr" + target=(-e DB_CONNECTION=mysql -e DB_HOST="$host" -e DB_DATABASE="$takeover_database" + -e DB_USERNAME=root -e DB_PASSWORD="$password") + target_network=$RESOURCE_SCOPE ;; *) ;; esac if (( ${#target[@]} )); then From fe2b26e6f64e2b4b5ad7d4e14f6ee4b0a9dceaf2 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 11:45:56 +0000 Subject: [PATCH 29/42] Retry lock-pressure refusals inside the probes' recorders Guzzle wraps a handler stack in reverse, so HandlerStack::unshift put the retry outermost and each recorder logged the refused attempt as well as the retried reply. The patch deployment contract then rejected a published PHP activity-poll refusal it did not expect. The retry is now pushed after each recorder, which makes it innermost, so recorders see only the final reply, as the middleware intended. Co-Authored-By: Claude Opus 5.5 --- .../conformance/server-parity/cooperative-probe.php | 2 +- .../server-parity/lock-pressure-middleware.php | 10 ++++++---- scripts/conformance/server-parity/namespace-probe.php | 2 +- .../server-parity/patch-deployment-probe.php | 2 +- scripts/conformance/server-parity/probe.php | 2 +- 5 files changed, 10 insertions(+), 8 deletions(-) diff --git a/scripts/conformance/server-parity/cooperative-probe.php b/scripts/conformance/server-parity/cooperative-probe.php index b856b308..3a52e3ee 100644 --- a/scripts/conformance/server-parity/cooperative-probe.php +++ b/scripts/conformance/server-parity/cooperative-probe.php @@ -26,7 +26,6 @@ function cooperativeObservedTransport(array &$traffic): Psr18Transport { $stack = HandlerStack::create(); - $stack->unshift(LockPressurePollRetries::middleware(), 'lock_pressure_poll_retry'); $stack->push(static function (callable $handler) use (&$traffic): callable { return static function (RequestInterface $request, array $options) use ($handler, &$traffic) { return $handler($request, $options)->then(static function (ResponseInterface $response) use ($request, &$traffic): ResponseInterface { @@ -44,6 +43,7 @@ function cooperativeObservedTransport(array &$traffic): Psr18Transport }); }; }); + $stack->push(LockPressurePollRetries::middleware(), 'lock_pressure_poll_retry'); return new Psr18Transport(client: new GuzzleClient(['http_errors' => false, 'handler' => $stack])); } diff --git a/scripts/conformance/server-parity/lock-pressure-middleware.php b/scripts/conformance/server-parity/lock-pressure-middleware.php index 11cf1d5e..1ced25c8 100644 --- a/scripts/conformance/server-parity/lock-pressure-middleware.php +++ b/scripts/conformance/server-parity/lock-pressure-middleware.php @@ -19,9 +19,11 @@ * (see lockPressureOperationRetryDelay and lockPressurePollRetryDelay for the * exact conditions). This Guzzle middleware retries only that refusal, below the SDK transport, so * the transport keeps every capability the SDK checks (bounded requests, - * payload transfer). It stops within the request's own timeout. Push it - * innermost (HandlerStack::unshift) so recording middleware sees only the final - * reply and observations stay comparable across targets. Retries are reported + * payload transfer). It stops within the request's own timeout. Push it after + * any recording middleware (HandlerStack::push), which makes it innermost: + * Guzzle wraps the stack in reverse, so unshift would put it outermost and let + * recorders see each refused attempt. Recorders then see only the final reply + * and observations stay comparable across targets. Retries are reported * once on stderr when the process exits. */ final class LockPressurePollRetries @@ -80,7 +82,7 @@ public static function middleware(): callable public static function transport(): Psr18Transport { $stack = HandlerStack::create(); - $stack->unshift(self::middleware(), 'lock_pressure_poll_retry'); + $stack->push(self::middleware(), 'lock_pressure_poll_retry'); return new Psr18Transport(client: new GuzzleClient(['http_errors' => false, 'handler' => $stack])); } diff --git a/scripts/conformance/server-parity/namespace-probe.php b/scripts/conformance/server-parity/namespace-probe.php index 4f756671..f6d1e0fa 100644 --- a/scripts/conformance/server-parity/namespace-probe.php +++ b/scripts/conformance/server-parity/namespace-probe.php @@ -46,7 +46,6 @@ function namespaceHttpRun(Client $client, string $workflowId, string $runId): ar function namespaceTransport(array &$receipts, ?Closure $beforeCompletion = null): Psr18Transport { $stack = HandlerStack::create(); - $stack->unshift(LockPressurePollRetries::middleware(), 'lock_pressure_poll_retry'); $stack->push(static function (callable $handler) use (&$receipts, $beforeCompletion): callable { return static function (RequestInterface $request, array $options) use ($handler, &$receipts, $beforeCompletion) { $path = $request->getUri()->getPath(); @@ -68,6 +67,7 @@ function namespaceTransport(array &$receipts, ?Closure $beforeCompletion = null) }); }; }); + $stack->push(LockPressurePollRetries::middleware(), 'lock_pressure_poll_retry'); return new Psr18Transport(client: new NamespaceHttp(['http_errors' => false, 'handler' => $stack, 'connect_timeout' => 1, 'timeout' => 5])); diff --git a/scripts/conformance/server-parity/patch-deployment-probe.php b/scripts/conformance/server-parity/patch-deployment-probe.php index 204b073c..030b3477 100644 --- a/scripts/conformance/server-parity/patch-deployment-probe.php +++ b/scripts/conformance/server-parity/patch-deployment-probe.php @@ -101,7 +101,6 @@ function patchHttpPhase(array $fixture, array $options): array $worker = $handle = null; $requests = $diagnostics = []; $stack = HandlerStack::create(); - $stack->unshift(LockPressurePollRetries::middleware(), 'lock_pressure_poll_retry'); $stack->push(static function (callable $handler) use (&$worker, &$requests, $phase, $fixture): callable { return static function (RequestInterface $request, array $transportOptions) use ($handler, &$worker, &$requests, $phase, $fixture) { return $handler($request, $transportOptions)->then(static function (ResponseInterface $response) use ($request, &$worker, &$requests, $phase, $fixture): ResponseInterface { @@ -128,6 +127,7 @@ function patchHttpPhase(array $fixture, array $options): array }); }; }); + $stack->push(LockPressurePollRetries::middleware(), 'lock_pressure_poll_retry'); $client = parityClient($options['url'], namespace: 'default', token: getenv('DW_PARITY_TOKEN') ?: null, transport: new Psr18Transport(client: new GuzzleClient(['http_errors' => false, 'handler' => $stack]))); if ($phase === 'replacement') { diff --git a/scripts/conformance/server-parity/probe.php b/scripts/conformance/server-parity/probe.php index 9c078e33..060d2674 100644 --- a/scripts/conformance/server-parity/probe.php +++ b/scripts/conformance/server-parity/probe.php @@ -71,7 +71,6 @@ function observedChildTransport(array &$polls, array &$completions, array &$acti // Observe real published transport I/O. No response or request is changed, // and authentication headers are never retained. $stack = HandlerStack::create(); - $stack->unshift(LockPressurePollRetries::middleware(), 'lock_pressure_poll_retry'); $stack->push(static function (callable $handler) use (&$polls, &$completions, &$activityPolls, &$activityOutcomes, $afterCompletion, &$scheduleReceipts): callable { return static function (RequestInterface $request, array $options) use ($handler, &$polls, &$completions, &$activityPolls, &$activityOutcomes, $afterCompletion, &$scheduleReceipts) { return $handler($request, $options)->then(static function (ResponseInterface $response) use ($request, &$polls, &$completions, &$activityPolls, &$activityOutcomes, $afterCompletion, &$scheduleReceipts): ResponseInterface { @@ -108,6 +107,7 @@ function observedChildTransport(array &$polls, array &$completions, array &$acti }); }; }); + $stack->push(LockPressurePollRetries::middleware(), 'lock_pressure_poll_retry'); return new Psr18Transport(client: new GuzzleClient(['http_errors' => false, 'handler' => $stack])); } From 48ced5bb1ff31368bbb69232ca1fedd3bf2f9702 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 11:56:32 +0000 Subject: [PATCH 30/42] Accept MariaDB's backend name in the cohort's takeover check The cohort's mysql backend runs on MariaDB too, where native reports the takeover backend as mariadb. Co-Authored-By: Claude Opus 5.5 --- scripts/conformance/server-parity/upgrade-source-profile.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/conformance/server-parity/upgrade-source-profile.sh b/scripts/conformance/server-parity/upgrade-source-profile.sh index df69ac0e..23c3b2f0 100755 --- a/scripts/conformance/server-parity/upgrade-source-profile.sh +++ b/scripts/conformance/server-parity/upgrade-source-profile.sh @@ -248,7 +248,8 @@ if (( ${#target[@]} )); then --entrypoint /target/debug/durable-workflow-server "$rust_image" upgrade-takeover \ > "$evidence/takeover.json" 2> "$evidence/takeover.stderr" jq -e --arg backend "$backend" --arg layout "$layout" --arg fingerprint "$source_fingerprint" '.status=="taken_over" - and .backend==$backend and .source_layout==$layout and .fingerprint==$fingerprint + and (.backend==$backend or ($backend=="mysql" and .backend=="mariadb")) + and .source_layout==$layout and .fingerprint==$fingerprint and .backup_fingerprint_matched==true and .converted.incarnations_copied==(if $layout=="frozen" then 0 else 1 end) and .stored_representation_compatibility_verified==false' "$evidence/takeover.json" >/dev/null From 4d8a28bd77f4e1a91e42c7ff804321afc7df4032 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 12:16:02 +0000 Subject: [PATCH 31/42] chore: prepare Server 2.9.8 pooler compatibility release --- composer.json | 2 +- composer.lock | 2 +- docker-compose.dedicated-matching.yml | 4 ++-- docker-compose.memo-rolling.yml | 4 ++-- docker-compose.published.yml | 4 ++-- docker-compose.small-cluster.yml | 2 +- docker-compose.yml | 8 ++++---- k8s/README.md | 10 +++++----- k8s/helm/durable-workflow/Chart.yaml | 8 ++++---- k8s/helm/durable-workflow/README.md | 2 +- .../durable-workflow/ci/existing-secrets-values.yaml | 2 +- .../durable-workflow/ci/ingress-and-hpa-values.yaml | 2 +- .../durable-workflow/ci/inline-secrets-values.yaml | 2 +- k8s/helm/durable-workflow/templates/_helpers.tpl | 2 +- k8s/helm/durable-workflow/values.yaml | 2 +- k8s/helm/examples/values-dev.yaml | 2 +- .../examples/values-external-secrets-operator.yaml | 2 +- .../examples/values-production-existing-secrets.yaml | 2 +- k8s/migration-job.yaml | 2 +- k8s/scheduler-cronjob.yaml | 2 +- k8s/secret.yaml | 2 +- k8s/server-deployment.yaml | 2 +- k8s/worker-deployment.yaml | 2 +- resources/release/source-release.json | 4 ++-- scripts/k8s-kind-smoke.sh | 2 +- 25 files changed, 39 insertions(+), 39 deletions(-) diff --git a/composer.json b/composer.json index 00e3ccfb..a31e9251 100644 --- a/composer.json +++ b/composer.json @@ -48,7 +48,7 @@ }, "extra": { "durable-workflow": { - "product-train": "2.9.7" + "product-train": "2.9.8" }, "laravel": { "dont-discover": [] diff --git a/composer.lock b/composer.lock index 4f4553fc..2e274b22 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "aa78b1f467750bfc7db005c71eb290e7", + "content-hash": "f3629921342639b58b34196e3ede3051", "packages": [ { "name": "apache/avro", diff --git a/docker-compose.dedicated-matching.yml b/docker-compose.dedicated-matching.yml index 07182204..099c8775 100644 --- a/docker-compose.dedicated-matching.yml +++ b/docker-compose.dedicated-matching.yml @@ -32,13 +32,13 @@ name: durable-workflow-server # daemon reports `shape: dedicated`. # Generated by scripts/ci/sync-source-release.mjs. Do not edit the fallback. -x-server-image: &server-image ${DW_SERVER_IMAGE:-durableworkflow/server:${DW_SERVER_TAG:-2.9.7}} +x-server-image: &server-image ${DW_SERVER_IMAGE:-durableworkflow/server:${DW_SERVER_TAG:-2.9.8}} x-server-environment: &server-environment APP_NAME: "Durable Workflow Server" APP_ENV: ${APP_ENV:-local} DW_SERVER_KEY: ${DW_SERVER_KEY:-} - APP_VERSION: ${APP_VERSION:-${DW_SERVER_TAG:-2.9.7}} + APP_VERSION: ${APP_VERSION:-${DW_SERVER_TAG:-2.9.8}} APP_DEBUG: ${APP_DEBUG:-false} DB_CONNECTION: mysql DB_HOST: mysql diff --git a/docker-compose.memo-rolling.yml b/docker-compose.memo-rolling.yml index 0fc31e22..6bb030f7 100644 --- a/docker-compose.memo-rolling.yml +++ b/docker-compose.memo-rolling.yml @@ -49,14 +49,14 @@ services: command: ["server-bootstrap"] environment: <<: *runtime-environment - APP_VERSION: ${APP_VERSION:-2.9.7} + APP_VERSION: ${APP_VERSION:-2.9.8} successor: image: ${DW_MEMO_SUCCESSOR_IMAGE:-durable-workflow/server-memo-rolling:local} ports: !override [] environment: <<: *runtime-environment - APP_VERSION: ${APP_VERSION:-2.9.7} + APP_VERSION: ${APP_VERSION:-2.9.8} DW_SERVER_ID: memo-successor DW_SERVER_TOPOLOGY_SHAPE: standalone_server DW_SERVER_PROCESS_CLASS: server_http_node diff --git a/docker-compose.published.yml b/docker-compose.published.yml index e6fed642..e7acdad0 100644 --- a/docker-compose.published.yml +++ b/docker-compose.published.yml @@ -1,13 +1,13 @@ name: durable-workflow-server # Generated by scripts/ci/sync-source-release.mjs. Do not edit the fallback. -x-server-image: &server-image ${DW_SERVER_IMAGE:-durableworkflow/server:${DW_SERVER_TAG:-2.9.7}} +x-server-image: &server-image ${DW_SERVER_IMAGE:-durableworkflow/server:${DW_SERVER_TAG:-2.9.8}} x-server-environment: &server-environment APP_NAME: "Durable Workflow Server" APP_ENV: ${APP_ENV:-local} DW_SERVER_KEY: ${DW_SERVER_KEY:-} - APP_VERSION: ${APP_VERSION:-${DW_SERVER_TAG:-2.9.7}} + APP_VERSION: ${APP_VERSION:-${DW_SERVER_TAG:-2.9.8}} APP_DEBUG: ${APP_DEBUG:-false} LOG_CHANNEL: ${LOG_CHANNEL:-stderr} LOG_LEVEL: ${LOG_LEVEL:-info} diff --git a/docker-compose.small-cluster.yml b/docker-compose.small-cluster.yml index b5a994c3..3f3715b0 100644 --- a/docker-compose.small-cluster.yml +++ b/docker-compose.small-cluster.yml @@ -12,7 +12,7 @@ x-server-build: &server-build x-server-environment: &server-environment APP_NAME: "Durable Workflow Server" APP_ENV: testing - APP_VERSION: ${APP_VERSION:-2.9.7} + APP_VERSION: ${APP_VERSION:-2.9.8} APP_DEBUG: "false" DW_SERVER_KEY: ${DW_SERVER_KEY:-base64:5Zt4nUhlCm3DD0nLXZJQdHiwPfb56yGo9gNV/g3jYbY=} DB_CONNECTION: ${DW_SMALL_CLUSTER_DB:-mysql} diff --git a/docker-compose.yml b/docker-compose.yml index 5d587e3b..6f18015e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,7 +15,7 @@ services: DW_SERVER_KEY: "${DW_SERVER_KEY:-}" DW_SERVER_TOPOLOGY_SHAPE: standalone_server DW_SERVER_PROCESS_CLASS: server_http_node - APP_VERSION: "${APP_VERSION:-2.9.7}" + APP_VERSION: "${APP_VERSION:-2.9.8}" APP_DEBUG: "false" DB_CONNECTION: mysql DB_HOST: mysql @@ -62,7 +62,7 @@ services: APP_NAME: "Durable Workflow Server" APP_ENV: local DW_SERVER_KEY: "${DW_SERVER_KEY:-}" - APP_VERSION: "${APP_VERSION:-2.9.7}" + APP_VERSION: "${APP_VERSION:-2.9.8}" APP_DEBUG: "false" DB_CONNECTION: mysql DB_HOST: mysql @@ -124,7 +124,7 @@ services: DW_SERVER_KEY: "${DW_SERVER_KEY:-}" DW_SERVER_TOPOLOGY_SHAPE: standalone_server DW_SERVER_PROCESS_CLASS: worker_node - APP_VERSION: "${APP_VERSION:-2.9.7}" + APP_VERSION: "${APP_VERSION:-2.9.8}" DB_CONNECTION: mysql DB_HOST: mysql DB_PORT: 3306 @@ -178,7 +178,7 @@ services: DW_SERVER_KEY: "${DW_SERVER_KEY:-}" DW_SERVER_TOPOLOGY_SHAPE: standalone_server DW_SERVER_PROCESS_CLASS: scheduler_node - APP_VERSION: "${APP_VERSION:-2.9.7}" + APP_VERSION: "${APP_VERSION:-2.9.8}" DB_CONNECTION: mysql DB_HOST: mysql DB_PORT: 3306 diff --git a/k8s/README.md b/k8s/README.md index cf7c9a54..2375dfd6 100644 --- a/k8s/README.md +++ b/k8s/README.md @@ -13,7 +13,7 @@ The checked-in manifests are synchronized with the repository's stable source release and pin its Docker Hub tag: ```text -durableworkflow/server:2.9.7 +durableworkflow/server:2.9.8 ``` Before production use, patch every workload image to the exact published tag or @@ -21,15 +21,15 @@ digest you intend to run: ```bash kubectl set image -n durable-workflow deploy/durable-workflow-server \ - server=durableworkflow/server:2.9.7 + server=durableworkflow/server:2.9.8 kubectl set image -n durable-workflow deploy/durable-workflow-worker \ - worker=durableworkflow/server:2.9.7 + worker=durableworkflow/server:2.9.8 kubectl set image -n durable-workflow cronjob/durable-workflow-scheduler \ - scheduler=durableworkflow/server:2.9.7 + scheduler=durableworkflow/server:2.9.8 ``` GitHub Container Registry publishes the same release line at -`ghcr.io/durable-workflow/server:2.9.7`. Digest pinning is preferred for strict +`ghcr.io/durable-workflow/server:2.9.8`. Digest pinning is preferred for strict change control. The manifests expect you to provide: diff --git a/k8s/helm/durable-workflow/Chart.yaml b/k8s/helm/durable-workflow/Chart.yaml index 3c188469..5801c1bc 100644 --- a/k8s/helm/durable-workflow/Chart.yaml +++ b/k8s/helm/durable-workflow/Chart.yaml @@ -5,11 +5,11 @@ type: application # The chart's own semver version. Bumped on every chart release; treated as # independent of the server image version (appVersion). Breaking-change rules # for this version live in docs/helm-upgrading.md alongside the chart. -version: 0.1.183 +version: 0.1.184 # The immutable Durable Workflow Server identity this chart release packages. # The onboarding default in values.yaml and appVersion are generated from the # checked-in source release record. -appVersion: "2.9.7" +appVersion: "2.9.8" kubeVersion: ">=1.27.0-0" home: https://durable-workflow.github.io/docs/2.0/deployment sources: @@ -30,10 +30,10 @@ annotations: # exact commit that most recently changed the packaged chart. org.opencontainers.image.source: https://github.com/durable-workflow/server dev.durable-workflow.source-revision: "unreleased" - dev.durable-workflow.image-reference: "docker.io/durableworkflow/server:2.9.7" + dev.durable-workflow.image-reference: "docker.io/durableworkflow/server:2.9.8" artifacthub.io/license: MIT artifacthub.io/category: integration-delivery # Free-form changelog for the current chart release shown by Artifact Hub. artifacthub.io/changes: | - kind: fixed - description: Preserve accepted signals at terminal workflow boundaries and exclude intentionally pruned run summaries from projection drift. + description: Use unnamed bound PostgreSQL statements with transaction poolers and verify a bound database read during readiness checks. diff --git a/k8s/helm/durable-workflow/README.md b/k8s/helm/durable-workflow/README.md index e286e2c2..69d84284 100644 --- a/k8s/helm/durable-workflow/README.md +++ b/k8s/helm/durable-workflow/README.md @@ -63,7 +63,7 @@ helm install durable-workflow ./k8s/helm/durable-workflow \ ```yaml image: - tag: "2.9.7" + tag: "2.9.8" # Pin a digest in production: # digest: "sha256:abc123..." # memoPayloadStorage: "raw-json-v1" # Required for a digest or custom image. diff --git a/k8s/helm/durable-workflow/ci/existing-secrets-values.yaml b/k8s/helm/durable-workflow/ci/existing-secrets-values.yaml index 2aa2fb07..03daee1a 100644 --- a/k8s/helm/durable-workflow/ci/existing-secrets-values.yaml +++ b/k8s/helm/durable-workflow/ci/existing-secrets-values.yaml @@ -1,7 +1,7 @@ # CI fixture: GitOps / externally-managed-secret path. The chart consumes # existing Secrets and renders no Secret resources of its own. image: - tag: "2.9.7" + tag: "2.9.8" externalDatabase: connection: pgsql diff --git a/k8s/helm/durable-workflow/ci/ingress-and-hpa-values.yaml b/k8s/helm/durable-workflow/ci/ingress-and-hpa-values.yaml index bfc1fa4a..298204fb 100644 --- a/k8s/helm/durable-workflow/ci/ingress-and-hpa-values.yaml +++ b/k8s/helm/durable-workflow/ci/ingress-and-hpa-values.yaml @@ -1,6 +1,6 @@ # CI fixture: ingress + autoscaling enabled. Exercises optional templates. image: - tag: "2.9.7" + tag: "2.9.8" externalDatabase: connection: mysql diff --git a/k8s/helm/durable-workflow/ci/inline-secrets-values.yaml b/k8s/helm/durable-workflow/ci/inline-secrets-values.yaml index ab0467cf..2e88b700 100644 --- a/k8s/helm/durable-workflow/ci/inline-secrets-values.yaml +++ b/k8s/helm/durable-workflow/ci/inline-secrets-values.yaml @@ -2,7 +2,7 @@ # chart's render path is exercised end-to-end. Real deployments should use # existingSecret instead. image: - tag: "2.9.7" + tag: "2.9.8" externalDatabase: connection: mysql diff --git a/k8s/helm/durable-workflow/templates/_helpers.tpl b/k8s/helm/durable-workflow/templates/_helpers.tpl index d09b528d..ef7394b2 100644 --- a/k8s/helm/durable-workflow/templates/_helpers.tpl +++ b/k8s/helm/durable-workflow/templates/_helpers.tpl @@ -88,7 +88,7 @@ resolved by an explicit capability declaration or an existing workload marker. {{- define "durable-workflow.memoPayloadStorageForImage" -}} {{- $image := toString . -}} {{- $normalized := regexReplaceAll "^index\\.docker\\.io/" $image "docker.io/" -}} -{{- if eq $normalized "docker.io/durableworkflow/server:2.9.7" -}} +{{- if eq $normalized "docker.io/durableworkflow/server:2.9.8" -}} dual-v1 {{- else if regexMatch "^docker\\.io/durableworkflow/server:2\\.0\\.0-rc\\.[0-9]+$" $normalized -}} {{- $releaseCandidate := atoi (regexFind "[0-9]+$" $normalized) -}} diff --git a/k8s/helm/durable-workflow/values.yaml b/k8s/helm/durable-workflow/values.yaml index 33e0ae40..e4f51aa9 100644 --- a/k8s/helm/durable-workflow/values.yaml +++ b/k8s/helm/durable-workflow/values.yaml @@ -21,7 +21,7 @@ image: registry: docker.io repository: durableworkflow/server # Generated by scripts/ci/sync-source-release.mjs. Do not edit this default. - tag: "2.9.7" + tag: "2.9.8" # Optional digest pin. When set, takes precedence over tag for change control. # Example: "sha256:abc123..." digest: "" diff --git a/k8s/helm/examples/values-dev.yaml b/k8s/helm/examples/values-dev.yaml index 3a413fe4..b20fdfdf 100644 --- a/k8s/helm/examples/values-dev.yaml +++ b/k8s/helm/examples/values-dev.yaml @@ -3,7 +3,7 @@ # shape in production. image: - tag: "2.9.7" + tag: "2.9.8" externalDatabase: connection: mysql diff --git a/k8s/helm/examples/values-external-secrets-operator.yaml b/k8s/helm/examples/values-external-secrets-operator.yaml index 88144260..b251debb 100644 --- a/k8s/helm/examples/values-external-secrets-operator.yaml +++ b/k8s/helm/examples/values-external-secrets-operator.yaml @@ -5,7 +5,7 @@ # concern. image: - tag: "2.9.7" + tag: "2.9.8" externalDatabase: connection: pgsql diff --git a/k8s/helm/examples/values-production-existing-secrets.yaml b/k8s/helm/examples/values-production-existing-secrets.yaml index 248b16bb..0f3cb833 100644 --- a/k8s/helm/examples/values-production-existing-secrets.yaml +++ b/k8s/helm/examples/values-production-existing-secrets.yaml @@ -10,7 +10,7 @@ image: repository: durable-workflow/server # Pin a digest in production for change-control auditability. digest: "" # e.g. "sha256:abc123..." - tag: "2.9.7" + tag: "2.9.8" externalDatabase: connection: pgsql diff --git a/k8s/migration-job.yaml b/k8s/migration-job.yaml index 6d6e2ed7..e5e21c00 100644 --- a/k8s/migration-job.yaml +++ b/k8s/migration-job.yaml @@ -13,7 +13,7 @@ spec: restartPolicy: OnFailure containers: - name: migrate - image: durableworkflow/server:2.9.7 + image: durableworkflow/server:2.9.8 command: ["server-entrypoint"] args: ["server-bootstrap"] envFrom: diff --git a/k8s/scheduler-cronjob.yaml b/k8s/scheduler-cronjob.yaml index fd8f44b8..18aaba0d 100644 --- a/k8s/scheduler-cronjob.yaml +++ b/k8s/scheduler-cronjob.yaml @@ -24,7 +24,7 @@ spec: restartPolicy: Never containers: - name: scheduler - image: durableworkflow/server:2.9.7 + image: durableworkflow/server:2.9.8 command: ["server-entrypoint"] args: - sh diff --git a/k8s/secret.yaml b/k8s/secret.yaml index 303e2bf6..c0ac4bfe 100644 --- a/k8s/secret.yaml +++ b/k8s/secret.yaml @@ -12,7 +12,7 @@ metadata: app.kubernetes.io/name: durable-workflow data: APP_NAME: "Durable Workflow Server" - APP_VERSION: "2.9.7" + APP_VERSION: "2.9.8" APP_ENV: production APP_DEBUG: "false" DB_CONNECTION: mysql diff --git a/k8s/server-deployment.yaml b/k8s/server-deployment.yaml index 5b7a8b68..e486ec44 100644 --- a/k8s/server-deployment.yaml +++ b/k8s/server-deployment.yaml @@ -23,7 +23,7 @@ spec: spec: containers: - name: server - image: durableworkflow/server:2.9.7 + image: durableworkflow/server:2.9.8 ports: - containerPort: 8080 name: http diff --git a/k8s/worker-deployment.yaml b/k8s/worker-deployment.yaml index 0e933e8b..2137c6c9 100644 --- a/k8s/worker-deployment.yaml +++ b/k8s/worker-deployment.yaml @@ -19,7 +19,7 @@ spec: spec: containers: - name: worker - image: durableworkflow/server:2.9.7 + image: durableworkflow/server:2.9.8 command: ["server-entrypoint"] args: ["php", "artisan", "queue:work", "--sleep=1", "--tries=3", "--max-time=3600"] envFrom: diff --git a/resources/release/source-release.json b/resources/release/source-release.json index 6397bb46..ffb24cee 100644 --- a/resources/release/source-release.json +++ b/resources/release/source-release.json @@ -1,9 +1,9 @@ { "schema": "durable-workflow.server.source-release/v1", "server": { - "version": "2.9.7" + "version": "2.9.8" }, "helm_chart": { - "version": "0.1.183" + "version": "0.1.184" } } diff --git a/scripts/k8s-kind-smoke.sh b/scripts/k8s-kind-smoke.sh index b936d210..9ce085d8 100755 --- a/scripts/k8s-kind-smoke.sh +++ b/scripts/k8s-kind-smoke.sh @@ -7,7 +7,7 @@ cluster="${K8S_SMOKE_CLUSTER:-durable-workflow-server-smoke}" image="${K8S_SMOKE_IMAGE:-durableworkflow/server:k8s-smoke}" # Generated by scripts/ci/sync-source-release.mjs so the smoke replaces the # same default shipped by the public manifests. -manifest_image="durableworkflow/server:2.9.7" +manifest_image="durableworkflow/server:2.9.8" kind_node_image="${K8S_SMOKE_KIND_NODE_IMAGE:-kindest/node:v1.29.4}" artifact_dir="${K8S_SMOKE_ARTIFACT_DIR:-/tmp/durable-workflow-k8s-kind-smoke-artifacts}" rendered_dir="${artifact_dir}/rendered-manifests" From 132755a476ee2b257f8fea98a3a3425af452d1ab Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 12:33:35 +0000 Subject: [PATCH 32/42] Prepare a PHP SQLite source in WAL mode for the upgrade commands PHP Server keeps SQLite in WAL mode, and inspection, fingerprint and takeover need the rollback journal, so a default installation could not be upgraded. upgrade-prepare-sqlite, run once PHP is stopped, opens the file as its only connection and switches it to the rollback journal: SQLite checkpoints every committed WAL frame into the file first, so the stored rows are unchanged. It refuses with upgrade_source_in_use while another process has the file open, and confirms the header and that no WAL, shared-memory or journal file remains. A journal left by a crashed writer is rolled back the same way, and a source already in rollback mode is left byte-identical. The upgrade source cohort now runs its SQLite source in PHP's default WAL mode and prepares it after PHP stops, as an operator does. Co-Authored-By: Claude Opus 5.5 --- rust/README.md | 7 + rust/src/main.rs | 22 +++ rust/src/runtime/upgrade.rs | 146 +++++++++++++++--- rust/src/runtime/upgrade_tests.rs | 89 +++++++++++ .../server-parity/upgrade-source-cleanup.sh | 2 +- .../server-parity/upgrade-source-profile.sh | 16 +- 6 files changed, 258 insertions(+), 24 deletions(-) diff --git a/rust/README.md b/rust/README.md index 36b9eee2..679d895d 100644 --- a/rust/README.md +++ b/rust/README.md @@ -418,6 +418,13 @@ over with the development `upgrade-takeover` command, following the 1. Run PHP Server 2.5.14 or later, which refuses a Rust-owned database, and stop every PHP role. + PHP Server keeps SQLite in WAL mode, which the upgrade commands refuse. With + PHP stopped, run `upgrade-prepare-sqlite` on the source file (writable). + It has SQLite checkpoint every committed WAL frame into the file and switch + it to the rollback journal, refusing with `upgrade_source_in_use` while any + other process has the file open. The rows are unchanged; the backup below + is taken afterwards. A journal left by a crashed writer is rolled back the + same way. 2. Run `upgrade-fingerprint` on the source: a SQLite file mounted read-only, or the database through a read-only account. Back the database up (a file copy, `pg_dump` or `mysqldump --single-transaction`), restore the backup diff --git a/rust/src/main.rs b/rust/src/main.rs index 56a8fe46..ae8efabc 100644 --- a/rust/src/main.rs +++ b/rust/src/main.rs @@ -133,12 +133,34 @@ async fn main() -> Result<(), Box> { | "upgrade-queue-inspect" | "upgrade-payload-inspect" | "upgrade-fingerprint" + | "upgrade-prepare-sqlite" | "upgrade-takeover" ) { return Err("unknown development command".into()); } let backend = env::var("DB_CONNECTION").unwrap_or_default(); + if command == "upgrade-prepare-sqlite" { + if backend != "sqlite" { + return Err("upgrade-prepare-sqlite requires DB_CONNECTION=sqlite".into()); + } + let path = env::var("DB_DATABASE").map_err(|_| "DB_DATABASE is required")?; + return match durable_workflow_server::runtime::upgrade::prepare_sqlite_source(&path).await + { + Ok(report) => { + println!("{report}"); + Ok(()) + } + Err(error) => { + let reason = match error { + RuntimeError::Refused { reason, .. } => reason, + _ => "upgrade_prepare_unavailable", + }; + eprintln!("{reason}"); + Err("SQLite source preparation refused".into()) + } + }; + } if command == "upgrade-takeover" { use durable_workflow_server::runtime::upgrade::{ queue_binding::QueueSourceConfig, takeover, diff --git a/rust/src/runtime/upgrade.rs b/rust/src/runtime/upgrade.rs index ef6baf64..b5ac2421 100644 --- a/rust/src/runtime/upgrade.rs +++ b/rust/src/runtime/upgrade.rs @@ -407,6 +407,128 @@ pub(super) async fn open_sqlite_source(path: &str, deadline: Instant) -> Result< open_sqlite_file(path, deadline, false).await } +/// The file format version bytes of a SQLite header: rollback journal or WAL. +const ROLLBACK_JOURNAL: [u8; 2] = [1, 1]; +const WRITE_AHEAD_LOG: [u8; 2] = [2, 2]; + +/// The journal format bytes of a regular SQLite file, read without SQLite. +async fn journal_format(path: &str) -> Result<[u8; 2]> { + let regular_file_required = || { + refuse( + StatusCode::BAD_REQUEST, + "upgrade_source_regular_file_required", + ) + }; + if !tokio::fs::metadata(path).await?.is_file() { + return Err(regular_file_required()); + } + let mut options = tokio::fs::OpenOptions::new(); + options.read(true); + // Tokio filesystem opens use blocking workers. A replaced FIFO must + // not strand one beyond the observation budget or prevent CLI exit. + #[cfg(unix)] + options.custom_flags(libc::O_NONBLOCK); + let mut file = options.open(path).await?; + if !file.metadata().await?.is_file() { + return Err(regular_file_required()); + } + let mut header = [0_u8; 20]; + file.read_exact(&mut header).await?; + if &header[..16] != b"SQLite format 3\0" { + return Err(refuse( + StatusCode::CONFLICT, + "upgrade_source_sqlite_journal_unqualified", + )); + } + Ok([header[18], header[19]]) +} + +/// Whether a SQLite file has a WAL, shared-memory or rollback journal beside it. +async fn sidecars(path: &str) -> Result { + for suffix in ["-wal", "-shm", "-journal"] { + if tokio::fs::try_exists(format!("{path}{suffix}")).await? { + return Ok(true); + } + } + Ok(false) +} + +/// Prepares a stopped PHP SQLite source for the upgrade commands, which need +/// the rollback journal. PHP Server's default is WAL: switching it makes +/// SQLite checkpoint every committed WAL frame into the file first, so the +/// stored rows are unchanged. A journal left by a crashed writer is rolled +/// back the same way. Refuses while another process has the file open. +pub async fn prepare_sqlite_source(path: &str) -> Result { + if path.is_empty() || path.contains(['?', ':']) { + return Err(refuse( + StatusCode::BAD_REQUEST, + "upgrade_source_file_required", + )); + } + let deadline = Instant::now() + BUDGET; + let previous = match timeout_at(deadline, journal_format(path)) + .await + .map_err(|_| expired())?? + { + WRITE_AHEAD_LOG => "wal", + ROLLBACK_JOURNAL => "delete", + _ => { + return Err(refuse( + StatusCode::CONFLICT, + "upgrade_source_sqlite_journal_unqualified", + )); + } + }; + let switched = previous == "wal" || sidecars(path).await?; + if switched { + let in_use = || refuse(StatusCode::CONFLICT, "upgrade_source_in_use"); + let opened = timeout_at( + deadline, + SqliteConnection::connect_with( + &SqliteConnectOptions::new() + .filename(path) + .create_if_missing(false) + .foreign_keys(false) + .busy_timeout(Duration::ZERO) + .journal_mode(sqlx::sqlite::SqliteJournalMode::Delete), + ), + ) + .await + .map_err(|_| expired())?; + // Leaving WAL needs the only connection; SQLite answers busy otherwise. + let mut connection = match opened { + Ok(connection) => connection, + Err(sqlx::Error::Database(error)) + if error + .code() + .and_then(|code| code.parse::().ok()) + .is_some_and(|code| code & 0xff == 5) => + { + return Err(in_use()); + } + Err(error) => return Err(error.into()), + }; + let mode: String = sqlx::query_scalar("PRAGMA journal_mode") + .fetch_one(&mut connection) + .await?; + close(connection.close()).await?; + if mode != "delete" { + return Err(in_use()); + } + } + if journal_format(path).await? != ROLLBACK_JOURNAL || sidecars(path).await? { + return Err(refuse( + StatusCode::CONFLICT, + "upgrade_source_sqlite_journal_unqualified", + )); + } + Ok(json!({ + "schema":"durable-workflow.rust-upgrade-sqlite-prepare.v1", + "status":"rollback_journal", "previous_journal_mode":previous, + "source_mutated":switched, "takeover_allowed":false, "backup_verified":false + })) +} + /// Opens a qualified rollback-journal source file, writable only for takeover. /// It keeps the rollback journal: native enables WAL itself after takeover. pub(super) async fn open_sqlite_file( @@ -424,29 +546,9 @@ pub(super) async fn open_sqlite_file( // not open an unqualified WAL source or attempt journal recovery. Unlike // immutable=1 this preserves normal SQLite locking/change detection for // the supported rollback-journal source, rather than ignoring live WAL. + // A WAL source is first switched with upgrade-prepare-sqlite. timeout_at(deadline, async { - let regular_file_required = || { - refuse( - StatusCode::BAD_REQUEST, - "upgrade_source_regular_file_required", - ) - }; - if !tokio::fs::metadata(path).await?.is_file() { - return Err(regular_file_required()); - } - let mut options = tokio::fs::OpenOptions::new(); - options.read(true); - // Tokio filesystem opens use blocking workers. A replaced FIFO must - // not strand one beyond the observation budget or prevent CLI exit. - #[cfg(unix)] - options.custom_flags(libc::O_NONBLOCK); - let mut file = options.open(path).await?; - if !file.metadata().await?.is_file() { - return Err(regular_file_required()); - } - let mut header = [0_u8; 20]; - file.read_exact(&mut header).await?; - if &header[..16] != b"SQLite format 3\0" || header[18..20] != [1, 1] { + if journal_format(path).await? != ROLLBACK_JOURNAL { return Err(refuse( StatusCode::CONFLICT, "upgrade_source_sqlite_journal_unqualified", diff --git a/rust/src/runtime/upgrade_tests.rs b/rust/src/runtime/upgrade_tests.rs index a140ec2b..7dee1890 100644 --- a/rust/src/runtime/upgrade_tests.rs +++ b/rust/src/runtime/upgrade_tests.rs @@ -1054,3 +1054,92 @@ async fn takeover_refusals_leave_the_php_database_unchanged() { assert_eq!(std::fs::read(&path).unwrap(), before, "{expected}"); } } + +/// The frozen layout in PHP Server's default WAL mode, copied while its +/// writer was still open: one committed row is only in the WAL, as a crashed +/// PHP would leave it. +async fn wal_source() -> (tempfile::TempDir, String) { + let (directory, path) = php_layout().await; + let mut writer = SqliteConnection::connect_with( + &SqliteConnectOptions::new() + .filename(&path) + .journal_mode(SqliteJournalMode::Wal), + ) + .await + .unwrap(); + writer.execute("PRAGMA wal_autocheckpoint=0").await.unwrap(); + writer + .execute("INSERT INTO migrations(migration,batch) VALUES ('only-in-wal',7)") + .await + .unwrap(); + let copy = directory.path().join("wal.sqlite").to_str().unwrap().to_owned(); + for suffix in ["", "-wal", "-shm"] { + std::fs::copy(format!("{path}{suffix}"), format!("{copy}{suffix}")).unwrap(); + } + writer.close().await.unwrap(); + (directory, copy) +} + +#[tokio::test] +async fn prepare_switches_a_wal_source_to_the_rollback_journal_and_keeps_its_rows() { + let (_directory, path) = wal_source().await; + assert!(Path::new(&format!("{path}-wal")).exists()); + reason( + inspect_sqlite(&path).await, + "upgrade_source_sqlite_journal_unqualified", + ); + let report = prepare_sqlite_source(&path).await.unwrap(); + assert_eq!(report["status"], "rollback_journal", "{report}"); + assert_eq!(report["previous_journal_mode"], "wal"); + assert_eq!(report["source_mutated"], true); + for suffix in ["-wal", "-shm", "-journal"] { + assert!(!Path::new(&format!("{path}{suffix}")).exists(), "{suffix}"); + } + assert_eq!(journal_format(&path).await.unwrap(), ROLLBACK_JOURNAL); + let inventory = inspect_sqlite(&path).await.unwrap(); + assert_eq!(inventory["table_rows"]["migrations"], 1, "{inventory}"); + let mut connection = SqliteConnection::connect(&format!("sqlite://{path}?mode=ro")) + .await + .unwrap(); + let batch: i64 = sqlx::query_scalar("SELECT batch FROM migrations WHERE migration='only-in-wal'") + .fetch_one(&mut connection) + .await + .unwrap(); + assert_eq!(batch, 7); + connection.close().await.unwrap(); + // Preparing again changes nothing. + let original = std::fs::read(&path).unwrap(); + assert_eq!(prepare_sqlite_source(&path).await.unwrap()["source_mutated"], false); + assert_eq!(std::fs::read(&path).unwrap(), original); +} + +#[tokio::test] +async fn prepare_refuses_while_another_connection_holds_the_source() { + let (_directory, path) = php_layout().await; + let mut writer = SqliteConnection::connect_with( + &SqliteConnectOptions::new() + .filename(&path) + .journal_mode(SqliteJournalMode::Wal), + ) + .await + .unwrap(); + writer + .execute("INSERT INTO migrations(migration,batch) VALUES ('open',8)") + .await + .unwrap(); + reason(prepare_sqlite_source(&path).await, "upgrade_source_in_use"); + assert_eq!(journal_format(&path).await.unwrap(), WRITE_AHEAD_LOG); + writer.close().await.unwrap(); + let report = prepare_sqlite_source(&path).await.unwrap(); + assert_eq!(report["previous_journal_mode"], "wal", "{report}"); +} + +#[tokio::test] +async fn prepare_leaves_a_rollback_journal_source_byte_identical() { + let (_directory, path) = php_layout().await; + let original = std::fs::read(&path).unwrap(); + let report = prepare_sqlite_source(&path).await.unwrap(); + assert_eq!(report["previous_journal_mode"], "delete", "{report}"); + assert_eq!(report["source_mutated"], false); + assert_eq!(std::fs::read(&path).unwrap(), original); +} diff --git a/scripts/conformance/server-parity/upgrade-source-cleanup.sh b/scripts/conformance/server-parity/upgrade-source-cleanup.sh index e5a1ac52..bd2b6714 100755 --- a/scripts/conformance/server-parity/upgrade-source-cleanup.sh +++ b/scripts/conformance/server-parity/upgrade-source-cleanup.sh @@ -15,7 +15,7 @@ source=upgrade_source_ref${label:+_$label} restored=upgrade_restore_ref${label:+_$label} image=${DW_PARITY_UPGRADE_PHP_IMAGE:-ghcr.io/durable-workflow/server@sha256:cba7c4c6732529293ef6c2625a64785dcb21b4f7fe0013fdc74f1af466f534ac} [[ "$image" =~ ^ghcr.io/durable-workflow/server@sha256:[0-9a-f]{64}$ ]] || exit 2 -for role in author control inspect php takeover native resume; do +for role in author control inspect php prepare takeover native resume; do if docker inspect "$resource-$role" >/dev/null 2>&1; then docker stop --signal SIGTERM --time 5 "$resource-$role" >/dev/null docker rm "$resource-$role" >/dev/null diff --git a/scripts/conformance/server-parity/upgrade-source-profile.sh b/scripts/conformance/server-parity/upgrade-source-profile.sh index 23c3b2f0..3613cf5b 100755 --- a/scripts/conformance/server-parity/upgrade-source-profile.sh +++ b/scripts/conformance/server-parity/upgrade-source-profile.sh @@ -57,7 +57,7 @@ printf '%s\n' "$RESOURCE_SCOPE" > "$evidence/database-owned.txt" # timer is short. timer_seconds=60 configuration=("${admin[@]}" -e DB_DATABASE="$database" -e DW_AUTH_TOKEN=upgrade-fixture-token - -e CACHE_STORE=file -e QUEUE_CONNECTION=database -e DB_SQLITE_JOURNAL_MODE=DELETE) + -e CACHE_STORE=file -e QUEUE_CONNECTION=database) php_run() { docker run --rm --name "$resource-control" --user=1000:1000 --cpus=1 --memory=512m --memory-swap=512m \ --network "$RESOURCE_SCOPE" "${configuration[@]}" -v "$PWD/$state:/state" -v "$PWD:/source:ro" \ @@ -81,6 +81,20 @@ docker run --rm --name "$resource-author" --user=1000:1000 --cpus=1 --memory=512 docker stop --signal SIGTERM --time 10 "$resource-php" >/dev/null docker inspect --format '{{json .State}}' "$resource-php" > "$evidence/stopped-state.json" jq -e '.Running==false and .ExitCode==0 and .OOMKilled==false' "$evidence/stopped-state.json" >/dev/null +if test "$backend" = sqlite; then + # PHP Server keeps SQLite in WAL mode. Once PHP is stopped, the source is + # switched to the rollback journal the upgrade commands need, as an operator + # does first; its rows are unchanged. + [[ "$(od -An -tu1 -j18 -N2 "$state/database.sqlite" | tr -s ' ')" == " 2 2" ]] + docker run --rm --name "$resource-prepare" --user=1000:1000 --cpus=1 --memory=256m --memory-swap=256m \ + --network none -e DB_CONNECTION=sqlite -e DB_DATABASE=/state/database.sqlite -e DW_RUST_EXPERIMENTAL=1 \ + -v "$PWD/$state:/state" -v "$PWD/rust-target:/target:ro" \ + --entrypoint /target/debug/durable-workflow-server "$rust_image" upgrade-prepare-sqlite \ + > "$evidence/prepare.json" 2> "$evidence/prepare.stderr" + jq -e '.status=="rollback_journal" and .previous_journal_mode=="wal" and .source_mutated==true' \ + "$evidence/prepare.json" >/dev/null + [[ "$(od -An -tu1 -j18 -N2 "$state/database.sqlite" | tr -s ' ')" == " 1 1" ]] +fi if test "$backend" = sqlite; then docker run --rm --name "$resource-control" --user=1000:1000 --cpus=1 --memory=512m --memory-swap=512m \ --network=none -v "$PWD/$state:/state:ro" -v "$PWD:/source:ro" --entrypoint php "$image" \ From 924e45f4280f7287712df4caf9faa350a5b0e2a6 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 12:42:19 +0000 Subject: [PATCH 33/42] Describe the MySQL journal and the SQLite WAL preparation as built MySQL and MariaDB use native's bootstrap lock and a session check rather than LOCK TABLES, which refuses to create or drop tables it does not hold. Their journal is a rust-takeover row in native's marker table. A SQLite source in PHP's default WAL mode is first switched to the rollback journal with upgrade-prepare-sqlite. Co-Authored-By: Claude Opus 5.5 --- docs/rust-server-upgrade-takeover.md | 32 +++++++++++++++++++--------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/docs/rust-server-upgrade-takeover.md b/docs/rust-server-upgrade-takeover.md index 0aa1f821..dfa5d700 100644 --- a/docs/rust-server-upgrade-takeover.md +++ b/docs/rust-server-upgrade-takeover.md @@ -67,7 +67,7 @@ with any other writer. | --- | --- | --- | | SQLite | `BEGIN EXCLUSIVE` on the database file | Fingerprint, marker, native relations and conversions commit in one transaction. | | PostgreSQL | Native's bootstrap advisory key, then `LOCK TABLE … IN ACCESS EXCLUSIVE MODE` on every table, with a short lock wait. The later DDL needs that mode anyway, so no lock is upgraded while another session waits. | DDL is transactional, so everything commits in one transaction. | -| MySQL and MariaDB | To be qualified: `LOCK TABLES … WRITE` on every relation, or a check that no other session uses the database, repeated after the marker is written | DDL commits implicitly, so the marker records a step journal (below). | +| MySQL and MariaDB | Native's bootstrap lock and a check that no other session uses the database (which needs the `PROCESS` privilege), repeated after the marker is written. `LOCK TABLES` cannot be used: under it MySQL refuses to create or drop any table it does not hold. | DDL commits implicitly, so the marker records a step journal (below). | On SQLite and PostgreSQL, a PHP connection that slipped past the stop blocks on these locks, and a write it committed earlier changes the fingerprint. Neither @@ -81,13 +81,27 @@ The operator reruns `upgrade-takeover` with the same fingerprint. A killed SQLite takeover leaves a rollback journal; read-only inspection refuses it, and the rerun lets SQLite roll it back before the fingerprint check. -MySQL and MariaDB commit DDL implicitly, so there the marker is written first, -with state `takeover`, the fingerprint and the step list. Native startup -refuses state `takeover`. Each step is idempotent and records its completion -in the marker. Rerunning `upgrade-takeover` with the same fingerprint resumes -after the last recorded step; a different fingerprint is refused. The final -step sets state `ready` and the native schema version. The PHP fence refuses -the database from the first step, because the marker exists. +MySQL and MariaDB commit DDL implicitly, so there native's marker table is +written first, with one row `('rust-takeover:', , +)`. Native startup refuses it with +`native_takeover_incomplete`. The sessions and the fingerprint are checked +again with the marker set aside, then each step runs and records its +completion in that row; row changes commit with their step, and DDL steps +check what is left to do. Rerunning `upgrade-takeover` with the same +fingerprint resumes after the last recorded step; a different fingerprint is +refused. The ledger and native's final marker `('rust-development', 4, +)` commit together. The PHP fence refuses the database from the +first step, because the marker table exists. + +## SQLite sources in WAL mode + +PHP Server keeps SQLite in WAL mode, and a read-only SQLite open can still +create WAL and shared-memory sidecars, so inspection, fingerprint and takeover +require the rollback journal. With PHP stopped, `upgrade-prepare-sqlite` opens +the file as its only connection and switches it to the rollback journal. +SQLite first checkpoints every committed WAL frame into the file, so the stored +rows are unchanged. The command refuses while another process holds the file, +and runs before the fingerprint and the backup. ## Steps @@ -142,8 +156,6 @@ acknowledged signal and an expired lease. The tests check: ## Open questions -- Whether MySQL and MariaDB can create the native relations while - `LOCK TABLES` holds every existing one, or need the session check instead. - Whether the declared column types print identically after a restore into another minor server version. Until that is shown, the restore check uses the same server version as the source. From a25fa493542f2441e0f96fc6ffdd45d379394fd0 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 13:33:36 +0000 Subject: [PATCH 34/42] Keep the activity deadline probe's own poll retry accounting This probe retries lock-pressure poll refusals itself and records each retry, while a recording proxy outside the client logs every refused poll. Retrying below the SDK hid those retries from the probe, so the contract found a refused poll without its recorded retry. The probe keeps its plain client. Co-Authored-By: Claude Opus 5.5 --- scripts/conformance/server-parity/activity-deadline-probe.php | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/conformance/server-parity/activity-deadline-probe.php b/scripts/conformance/server-parity/activity-deadline-probe.php index 4728f626..b4bd61df 100644 --- a/scripts/conformance/server-parity/activity-deadline-probe.php +++ b/scripts/conformance/server-parity/activity-deadline-probe.php @@ -3,7 +3,6 @@ declare(strict_types=1); use DurableWorkflow\Client; -require_once __DIR__.'/lock-pressure-middleware.php'; require_once __DIR__.'/namespace-probe.php'; require_once __DIR__.'/activity-deadline-poll.php'; @@ -35,7 +34,7 @@ function activityDeadlineHttpObservation(array $fixture, array $options): array if (!preg_match('#^http://127\.0\.0\.1:[0-9]+$#', $url)) { throw new RuntimeException('Activity deadline recorder did not bind its local port'); } - $client = parityClient($url, namespace:'default', token:getenv('DW_PARITY_TOKEN') ?: null); + $client = new Client($url, namespace:'default', token:getenv('DW_PARITY_TOKEN') ?: null); $checkBudget = static function () use ($deadline): void { if (microtime(true) >= $deadline) { throw new RuntimeException('Original activity deadline observation expired'); From d3c3c34ebd961c6836786f36cc706b59928ca75b Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 13:44:46 +0000 Subject: [PATCH 35/42] Let SQLite source preparation wait briefly for a closing writer With no lock wait, a writer that had just closed could still hold its lock when the preparation opened the file, and the preparation refused it as in use. A two-second wait covers that; a file another process keeps open is still refused. Co-Authored-By: Claude Opus 5.5 --- rust/src/runtime/upgrade.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/rust/src/runtime/upgrade.rs b/rust/src/runtime/upgrade.rs index b5ac2421..e3f3f7b4 100644 --- a/rust/src/runtime/upgrade.rs +++ b/rust/src/runtime/upgrade.rs @@ -489,13 +489,15 @@ pub async fn prepare_sqlite_source(path: &str) -> Result { .filename(path) .create_if_missing(false) .foreign_keys(false) - .busy_timeout(Duration::ZERO) + .busy_timeout(Duration::from_secs(2)) .journal_mode(sqlx::sqlite::SqliteJournalMode::Delete), ), ) .await .map_err(|_| expired())?; - // Leaving WAL needs the only connection; SQLite answers busy otherwise. + // Leaving WAL needs the only connection. SQLite answers busy when another + // still has the file after a short wait, which also covers a writer + // that is just closing. let mut connection = match opened { Ok(connection) => connection, Err(sqlx::Error::Database(error)) From fa7695fd38810f7329fa6b94a907c6deb4ce6635 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 13:45:02 +0000 Subject: [PATCH 36/42] Resume a PHP-accepted signal after takeover and refuse unsupported features The upgrade source author now also writes a third original: a workflow that opens a signal wait, then receives its declared signal under published PHP, whose resume task stays unclaimed. After the takeover, native must deliver that signal from PHP's rows: the resumed task keeps the original history and the signal's PHP-written arguments, and the workflow completes. The resume accepts the three originals in any order. The payload inspection now also checks the codec-tagged payloads of workflow commands, signal records and updates. Takeover refuses a source holding rows of a published PHP feature native does not implement yet (runtime credentials, message streams, service definitions, update validation tasks, worker sessions, build ID rollouts and search attribute definitions), rather than leaving them unread. Co-Authored-By: Claude Opus 5.5 --- rust/src/runtime/upgrade/payload_scan.rs | 11 +++-- rust/src/runtime/upgrade/takeover.rs | 40 ++++++++++++++++++- rust/src/runtime/upgrade_tests.rs | 15 ++++++- .../server-parity/upgrade-source-author.php | 36 ++++++++++++----- .../server-parity/upgrade-source-profile.sh | 14 +++---- .../upgrade-source-sqlite-representations.php | 6 +-- .../server-parity/upgrade-takeover-resume.php | 26 +++++++++--- 7 files changed, 118 insertions(+), 30 deletions(-) diff --git a/rust/src/runtime/upgrade/payload_scan.rs b/rust/src/runtime/upgrade/payload_scan.rs index 555eea3b..412c2b91 100644 --- a/rust/src/runtime/upgrade/payload_scan.rs +++ b/rust/src/runtime/upgrade/payload_scan.rs @@ -72,11 +72,15 @@ macro_rules! column { }; } -const COLUMNS: [Column; 4] = [ +const COLUMNS: [Column; 8] = [ column!("workflow_runs", "arguments", "payload_codec"), column!("workflow_runs", "output", "output_payload_codec"), column!("activity_executions", "arguments", "payload_codec"), column!("activity_executions", "result", "payload_codec"), + column!("workflow_commands", "payload", "payload_codec"), + column!("workflow_signal_records", "arguments", "payload_codec"), + column!("workflow_updates", "arguments", "payload_codec"), + column!("workflow_updates", "result", "payload_codec"), ]; #[derive(Default, Serialize)] @@ -218,9 +222,10 @@ where "references_metadata_supported":references_supported, "diagnostics":diagnostics, "diagnostics_omitted":omitted, "inline_values_verified":inline_supported, "nested_payloads_verified":false, - // Published migrations also give these tables codec-tagged payloads. + // These tables also hold codec-tagged payloads; takeover refuses a + // source with any of their rows, since native lacks the features. "other_root_payload_columns_verified":false, - "unscanned_payload_codec_tables":["workflow_commands","workflow_signal_records","workflow_updates"], + "unscanned_payload_codec_tables":["workflow_durable_stream_items","workflow_inbound_stream_items","workflow_update_validation_tasks"], "inline_validation_limits":{"encoded_frame_bytes":65536,"traversal_depth":64, "traversal_visits":16384,"scalar_allocation_bytes":apache_avro::util::max_allocation_bytes(apache_avro::util::DEFAULT_MAX_ALLOCATION_BYTES)}, "provider_bytes_verified":false, "registry_binding_verified":false, diff --git a/rust/src/runtime/upgrade/takeover.rs b/rust/src/runtime/upgrade/takeover.rs index 2eb10c25..0d9188b8 100644 --- a/rust/src/runtime/upgrade/takeover.rs +++ b/rust/src/runtime/upgrade/takeover.rs @@ -218,10 +218,46 @@ fn report(backend: &str, layout: &str, digest: &Value, inventory: &Value, conver }) } +/// Published PHP features native does not implement yet. A source holding any +/// of their rows is refused, rather than taken over without them. Derived +/// projections and PHP's own queue and poll bookkeeping are left in place. +const UNSUPPORTED_FEATURES: [(&str, &[&str]); 7] = [ + ("upgrade_source_runtime_credentials_unsupported", &["runtime_credentials"]), + ( + "upgrade_source_message_streams_unsupported", + &[ + "workflow_messages", + "workflow_durable_streams", + "workflow_durable_stream_items", + "workflow_inbound_streams", + "workflow_inbound_stream_items", + ], + ), + ( + "upgrade_source_services_unsupported", + &["workflow_services", "workflow_service_endpoints", "workflow_service_operations"], + ), + ("upgrade_source_update_validation_unsupported", &["workflow_update_validation_tasks"]), + ("upgrade_source_worker_sessions_unsupported", &["workflow_worker_sessions"]), + ("upgrade_source_build_id_rollouts_unsupported", &["workflow_worker_build_id_rollouts"]), + ( + "upgrade_source_search_attribute_definitions_unsupported", + &["search_attribute_definitions"], + ), +]; + /// Takeover needs every inspection result it relies on, and refuses what -/// native cannot represent yet: external payloads and service calls. -/// Returns the qualified source layout. +/// native cannot represent yet: external payloads, service calls and the +/// other unsupported features above. Returns the qualified source layout. fn qualified(inventory: &Value) -> Result<&str> { + for (reason, tables) in UNSUPPORTED_FEATURES { + if tables + .iter() + .any(|table| inventory["table_rows"][*table].as_i64().unwrap_or(0) != 0) + { + return Err(refuse(StatusCode::CONFLICT, reason)); + } + } if inventory["queue_inspection"]["status"] != "known_subset_bound" { return Err(refuse( StatusCode::CONFLICT, diff --git a/rust/src/runtime/upgrade_tests.rs b/rust/src/runtime/upgrade_tests.rs index 7dee1890..35e563a5 100644 --- a/rust/src/runtime/upgrade_tests.rs +++ b/rust/src/runtime/upgrade_tests.rs @@ -406,7 +406,11 @@ async fn payload_scan_validates_inline_pages_and_reports_late_bad_values_without assert_eq!(report["payload_inspection"]["other_root_payload_columns_verified"], false); assert_eq!( report["payload_inspection"]["unscanned_payload_codec_tables"], - json!(["workflow_commands", "workflow_signal_records", "workflow_updates"]) + json!([ + "workflow_durable_stream_items", + "workflow_inbound_stream_items", + "workflow_update_validation_tasks" + ]) ); assert_eq!(report["payload_inspection"]["diagnostics"], json!([])); for (table, column, count) in [ @@ -1042,6 +1046,15 @@ async fn takeover_refusals_leave_the_php_database_unchanged() { "INSERT INTO runtime_external_payloads(id,namespace,storage_uri,storage_uri_sha256,codec,sha256,size_bytes) VALUES ('p','default','s3://b/k','x','avro','y',1)", "upgrade_source_external_payloads_unsupported", ), + // Features native lacks: their rows would be left behind unread. + ( + "INSERT INTO runtime_credentials(id,subject,roles,tenant,token_prefix,token_hash) VALUES ('c','worker','[]','default','dw_','h')", + "upgrade_source_runtime_credentials_unsupported", + ), + ( + "INSERT INTO workflow_messages(workflow_instance_id,direction,channel,stream_key,sequence) VALUES ('w','inbound','c','k',1)", + "upgrade_source_message_streams_unsupported", + ), ] { let (_directory, path) = php297_layout().await; mutate(&path, statement).await; diff --git a/scripts/conformance/server-parity/upgrade-source-author.php b/scripts/conformance/server-parity/upgrade-source-author.php index cae370f9..c1623770 100644 --- a/scripts/conformance/server-parity/upgrade-source-author.php +++ b/scripts/conformance/server-parity/upgrade-source-author.php @@ -42,7 +42,12 @@ $worker = $prefix.'-worker'; $expect($request('POST', '/api/worker/register', ['worker_id' => $worker, 'task_queue' => $prefix, 'runtime' => 'php', 'sdk_version' => 'raw-upgrade-inspection-author', - 'supported_workflow_types' => ['parity.upgrade_activity', 'parity.upgrade_timer'], + 'supported_workflow_types' => ['parity.upgrade_activity', 'parity.upgrade_timer', 'parity.upgrade_signal'], + // The signal original declares its one signal, as an SDK worker would. + 'workflow_command_contracts' => ['parity.upgrade_signal' => ['signals' => ['go'], 'signal_contracts' => [[ + 'name' => 'go', 'parameters' => [['name' => 'value', 'position' => 0, 'type' => 'array', 'required' => true, + 'variadic' => false, 'allows_null' => false, 'default_available' => false]], + ]]]], 'supported_activity_types' => ['parity.upgrade_activity'], 'capabilities' => [], 'capability_manifest' => array_fill_keys(['local_activities', 'worker_sessions', 'sticky_execution'], ['supported' => false, 'minimum_protocol_version' => '1.18', 'reason' => 'raw_protocol_author']), @@ -54,7 +59,7 @@ $timerSeconds = (int) (getenv('DW_PARITY_UPGRADE_TIMER_SECONDS') ?: 600); $check($timerSeconds >= 30 && $timerSeconds <= 600, 'Unsupported original timer delay'); $originals = []; -foreach (['activity', 'timer'] as $kind) { +foreach (['activity', 'timer', 'signal'] as $kind) { $id = $prefix.'-'.$kind; $run = $expect($request('POST', '/api/workflows', ['workflow_id' => $id, 'workflow_type' => 'parity.upgrade_'.$kind, 'task_queue' => $prefix, 'input' => $envelope, @@ -63,11 +68,13 @@ 'started_at' => $started, 'original_deadline' => $deadline], JSON_THROW_ON_ERROR)."\n"); $task = $expect($request('POST', '/api/worker/workflow-tasks/poll', $poll), 200, $kind.' claim')['task']; $check($task['run_id'] === $run['run_id'], 'Original workflow claim changed'); - $command = $kind === 'activity' - ? ['type' => 'schedule_activity', 'activity_type' => 'parity.upgrade_activity', 'arguments' => $envelope, + $command = match ($kind) { + 'activity' => ['type' => 'schedule_activity', 'activity_type' => 'parity.upgrade_activity', 'arguments' => $envelope, 'retry_policy' => ['max_attempts' => 1, 'backoff_seconds' => []], - 'start_to_close_timeout' => 600, 'schedule_to_close_timeout' => 900] - : ['type' => 'start_timer', 'delay_seconds' => $timerSeconds]; + 'start_to_close_timeout' => 600, 'schedule_to_close_timeout' => 900], + 'timer' => ['type' => 'start_timer', 'delay_seconds' => $timerSeconds], + 'signal' => ['type' => 'open_signal_wait', 'signal_name' => 'go'], + }; $expect($request('POST', '/api/worker/workflow-tasks/'.$task['task_id'].'/complete', [ 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], 'commands' => [$command], @@ -77,10 +84,21 @@ $activity = $expect($request('POST', '/api/worker/activity-tasks/poll', $poll), 200, 'activity claim')['task']; $check($activity['run_id'] === $run['run_id'] && $activity['attempt_number'] === 1, 'Original activity changed'); } + if ($kind === 'signal') { + // PHP accepts the signal and makes its resume task, which stays + // unclaimed until native resumes it after the takeover. + $signalled = $request('POST', '/api/workflows/'.$id.'/signal/go', ['input' => $envelope]); + $check(in_array($signalled[0], [200, 202], true), 'signal: '.json_encode(['status' => $signalled[0], + 'reason' => $signalled[1]['reason'] ?? null, 'errors' => $signalled[1]['errors'] ?? null])); + } $history = $expect($request('GET', '/api/workflows/'.$id.'/runs/'.$run['run_id'].'/history'), 200, 'history')['events']; - $check(array_column($history, 'event_type') === ($kind === 'activity' - ? ['StartAccepted', 'WorkflowStarted', 'ActivityScheduled', 'ActivityStarted'] - : ['StartAccepted', 'WorkflowStarted', 'TimerScheduled']), 'Original pending history differs'); + fwrite(STDERR, json_encode(['phase' => 'history', 'kind' => $kind, 'events' => array_column($history, 'event_type')], + JSON_THROW_ON_ERROR)."\n"); + $check(array_column($history, 'event_type') === match ($kind) { + 'activity' => ['StartAccepted', 'WorkflowStarted', 'ActivityScheduled', 'ActivityStarted'], + 'timer' => ['StartAccepted', 'WorkflowStarted', 'TimerScheduled'], + 'signal' => ['StartAccepted', 'WorkflowStarted', 'SignalWaitOpened', 'SignalReceived'], + }, 'Original pending history differs'); $originals[$kind] = ['workflow_id' => $id, 'run_id' => $run['run_id'], 'workflow_task' => $task, 'activity_task' => $activity, 'history' => $history]; } diff --git a/scripts/conformance/server-parity/upgrade-source-profile.sh b/scripts/conformance/server-parity/upgrade-source-profile.sh index 3613cf5b..e7a504c9 100755 --- a/scripts/conformance/server-parity/upgrade-source-profile.sh +++ b/scripts/conformance/server-parity/upgrade-source-profile.sh @@ -101,8 +101,8 @@ if test "$backend" = sqlite; then /source/scripts/conformance/server-parity/upgrade-source-sqlite-representations.php upgrade-source \ > "$evidence/representations-comparison.json" jq -e '.outcome=="pass" and .source_mutated==false and .php_objects_executed==false - and (.runs|length)==2 and (.tasks|length)==4 and (.activities|length)==1 - and (.history|length)==7 and (.queue_jobs|length)==1 + and (.runs|length)==3 and (.tasks|length)==6 and (.activities|length)==1 + and (.history|length)==11 and (.queue_jobs|length)==1 and .queue_jobs[0].canonical_timer_task_match==true and all(.runs[]; .typed_arguments_match==true) and .backup_verified==false and .stored_representation_compatibility_verified==false @@ -134,7 +134,7 @@ snapshot() { } snapshot > "$evidence/before.json" inspect_source > "$evidence/inspection.json" 2> "$evidence/inspection.stderr" -jq -e --arg layout "$layout" '.status=="inspected" and .source_layout==$layout and (.table_rows|length)==50 and .table_rows.workflow_runs==2 +jq -e --arg layout "$layout" '.status=="inspected" and .source_layout==$layout and (.table_rows|length)==50 and .table_rows.workflow_runs==3 and .table_rows.activity_executions==1 and .catalog_visibility=="configured_role" and .source_mutated==false and .takeover_allowed==false and .backup_verified==false and .php_quiescence_verified==false and .stored_representation_compatibility_verified==false' "$evidence/inspection.json" >/dev/null @@ -145,10 +145,10 @@ jq -e '.status=="inspected" and .queue_inspection.status=="known_subset_bound" and .source_mutated==false and .takeover_allowed==false and .backup_verified==false and .php_quiescence_verified==false and .stored_representation_compatibility_verified==false' "$evidence/queue-inspection.json" >/dev/null inspect_source upgrade-payload-inspect > "$evidence/payload-inspection.json" 2> "$evidence/payload-inspection.stderr" -jq -e '.status=="inspected" and (.payload_inspection.columns|length)==4 +jq -e '.status=="inspected" and (.payload_inspection.columns|length)==8 and .payload_inspection.references_metadata_supported==true and ([.payload_inspection.columns[].counts.php_v1_references]|add)==0 - and ([.payload_inspection.columns[].counts.avro_inline_values]|add)==3 + and ([.payload_inspection.columns[].counts.avro_inline_values]|add)==9 and ([.payload_inspection.columns[].counts.other_unvalidated]|add)==0 and .payload_inspection.inline_values_verified==true and .payload_inspection.inline_validation_limits=={encoded_frame_bytes:65536,traversal_depth:64,traversal_visits:16384,scalar_allocation_bytes:65536} @@ -166,7 +166,7 @@ fingerprint() { } fingerprint "$database" > "$evidence/fingerprint.json" 2> "$evidence/fingerprint.stderr" jq -e --arg layout "$layout" '.status=="fingerprinted" and .source_layout==$layout and (.fingerprint|test("^sha256:[0-9a-f]{64}$")) - and (.table_rows|length)==50 and .table_rows.workflow_runs==2 and .table_rows.jobs==1 + and (.table_rows|length)==50 and .table_rows.workflow_runs==3 and .table_rows.jobs==1 and .source_mutated==false and .takeover_allowed==false and .backup_verified==false and .php_quiescence_verified==false and .stored_representation_compatibility_verified==false' "$evidence/fingerprint.json" >/dev/null source_fingerprint=$(jq -r .fingerprint "$evidence/fingerprint.json") @@ -285,7 +285,7 @@ if (( ${#target[@]} )); then "/source/$evidence/originals.json" 150 > "$evidence/resume.json" 2> "$evidence/resume.stderr" jq -e '.outcome=="pass" and .activity.stale_attempt_status==409 and .activity.events[-1]=="WorkflowCompleted" and .timer.fired_after_seconds>=0 - and .timer.events[-1]=="WorkflowCompleted"' "$evidence/resume.json" >/dev/null + and .timer.events[-1]=="WorkflowCompleted" and .signal.events[-1]=="WorkflowCompleted"' "$evidence/resume.json" >/dev/null docker stop --signal SIGTERM --time 10 "$resource-native" >/dev/null if docker run --rm --name "$resource-control" --user=1000:1000 --cpus=1 --memory=512m --memory-swap=512m \ --network "$target_network" "${target[@]}" -e DW_AUTH_TOKEN=upgrade-fixture-token \ diff --git a/scripts/conformance/server-parity/upgrade-source-sqlite-representations.php b/scripts/conformance/server-parity/upgrade-source-sqlite-representations.php index da17fe17..b36f69b1 100644 --- a/scripts/conformance/server-parity/upgrade-source-sqlite-representations.php +++ b/scripts/conformance/server-parity/upgrade-source-sqlite-representations.php @@ -20,11 +20,11 @@ if ($pdo->query('PRAGMA journal_mode')->fetchColumn() !== 'delete') { throw new RuntimeException('Audit requires the already stopped DELETE-journal fixture'); } -$runs = $pdo->query('SELECT * FROM workflow_runs ORDER BY id LIMIT 3')->fetchAll(PDO::FETCH_ASSOC); +$runs = $pdo->query('SELECT * FROM workflow_runs ORDER BY id LIMIT 4')->fetchAll(PDO::FETCH_ASSOC); $workflowIds = array_column($runs, 'workflow_instance_id'); sort($workflowIds); -if ($workflowIds !== [$prefix.'-activity', $prefix.'-timer']) { - throw new RuntimeException('This reader accepts only the two isolated source-author originals'); +if ($workflowIds !== [$prefix.'-activity', $prefix.'-signal', $prefix.'-timer']) { + throw new RuntimeException('This reader accepts only the three isolated source-author originals'); } $report = ['scope' => 'published_php_stored_representation_observation', 'runs' => [], 'table_rows' => [], 'tasks' => [], 'activities' => [], 'history' => [], 'queue_jobs' => [], 'cache_files' => []]; diff --git a/scripts/conformance/server-parity/upgrade-takeover-resume.php b/scripts/conformance/server-parity/upgrade-takeover-resume.php index bddd3db3..2b695dc8 100644 --- a/scripts/conformance/server-parity/upgrade-takeover-resume.php +++ b/scripts/conformance/server-parity/upgrade-takeover-resume.php @@ -56,6 +56,7 @@ $outcome = []; $activity = $originals['activity']; $timer = $originals['timer']; +$signal = $originals['signal']; $complete = static function (array $claimed, string $operation) use ($expect, $request, $result): void { $expect($request('POST', '/api/worker/workflow-tasks/'.$claimed['task_id'].'/complete', ['lease_owner' => $claimed['lease_owner'], 'workflow_task_attempt' => $claimed['workflow_task_attempt'], @@ -77,20 +78,20 @@ $replayed = $expect($request('POST', '/api/worker/activity-tasks/'.$task['task_id'].'/complete', $completion), 200, 'activity completion replay'); $check(($replayed['recorded'] ?? null) === false, 'A replayed activity completion was recorded twice'); -// Each original workflow then gets one workflow task: the activity's at -// once and the timer's at its deadline. On a slow host the timer is already -// due, so either may be claimed first. +// Each original workflow then gets one workflow task: the activity's and the +// signal's at once, the timer's at its deadline. On a slow host the timer is +// already due, so they may be claimed in any order. $original = $activity['history']; $claims = []; $until = max($deadline, microtime(true)) + 30; -while (count($claims) < 2 && microtime(true) < $until) { +while (count($claims) < 3 && microtime(true) < $until) { $claimed = $expect($request('POST', '/api/worker/workflow-tasks/poll', ['worker_id' => $worker, 'task_queue' => $queue, 'timeout_seconds' => 5]), 200, 'workflow task claim')['task'] ?? null; if (! is_array($claimed)) { continue; } $run = $claimed['run_id']; - $check(in_array($run, [$activity['run_id'], $timer['run_id']], true) && ! isset($claims[$run]), + $check(in_array($run, [$activity['run_id'], $timer['run_id'], $signal['run_id']], true) && ! isset($claims[$run]), 'An unexpected workflow task was claimed'); $claims[$run] = true; if ($run === $activity['run_id']) { @@ -98,12 +99,23 @@ 'Original activity history changed in the resumed task'); $check(end($claimed['history_events'])['event_type'] === 'ActivityCompleted', 'Activity completion is missing from history'); $complete($claimed, 'resumed workflow completion'); + } elseif ($run === $signal['run_id']) { + // The signal PHP accepted is delivered from PHP's rows, with its arguments. + $check(array_slice($types($claimed['history_events']), 0, count($signal['history'])) === $types($signal['history']), + 'Original signal history changed in the resumed task'); + $received = array_values(array_filter($claimed['history_events'], + static fn (array $event): bool => $event['event_type'] === 'SignalReceived')); + $check(count($received) === 1 && json_encode($sorted($received[0]['payload']['arguments']), JSON_THROW_ON_ERROR) + === json_encode($sorted(end($signal['history'])['payload']['arguments']), JSON_THROW_ON_ERROR), + 'The PHP-accepted signal arguments differ under native'); + $complete($claimed, 'signal workflow completion'); } else { $check(end($claimed['history_events'])['event_type'] === 'TimerFired', 'Timer firing is missing from history'); $complete($claimed, 'timer workflow completion'); } } $check(isset($claims[$activity['run_id']]), 'The resumed workflow task was not claimed'); +$check(isset($claims[$signal['run_id']]), 'The signal workflow task was not claimed'); $check(isset($claims[$timer['run_id']]), 'The original timer did not fire under native'); // The workflow task PHP already completed is a stale attempt. @@ -124,6 +136,10 @@ $check($firedAt >= $deadline, 'The original timer fired before its deadline'); $check($keep($timer['history'], $timerHistory) && end($timerHistory)['event_type'] === 'WorkflowCompleted', 'The timer workflow did not complete on its original history'); +$signalHistory = $history($signal); +$check($keep($signal['history'], $signalHistory) && end($signalHistory)['event_type'] === 'WorkflowCompleted', + 'The signal workflow did not complete on its original history'); +$outcome['signal'] = ['run_id' => $signal['run_id'], 'events' => array_column($signalHistory, 'event_type')]; $outcome['timer'] = ['run_id' => $timer['run_id'], 'fire_at' => $fireAt, 'fired_after_seconds' => round($firedAt - $deadline, 3), 'events' => array_column($timerHistory, 'event_type')]; echo json_encode(['outcome' => 'pass', 'scope' => 'native_resumes_published_php_originals_after_takeover'] + $outcome, From 6978d60b137a26c70e946a1ad33a4dafa28c3c32 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 14:43:53 +0000 Subject: [PATCH 37/42] Redeliver a PHP lease that expires after the takeover The upgrade source author adds a fourth original whose worker stops after claiming its workflow task. PHP's lease on that task expires after the takeover. Native must then redeliver the same task as its next attempt, refuse the expired PHP attempt's completion, and complete the workflow on its original history. Part of #325. Co-Authored-By: Claude Opus 5.5 --- .../server-parity/upgrade-source-author.php | 20 ++++++--- .../server-parity/upgrade-source-profile.sh | 14 +++--- .../upgrade-source-sqlite-representations.php | 6 +-- .../server-parity/upgrade-takeover-resume.php | 44 ++++++++++++++++--- 4 files changed, 63 insertions(+), 21 deletions(-) diff --git a/scripts/conformance/server-parity/upgrade-source-author.php b/scripts/conformance/server-parity/upgrade-source-author.php index c1623770..fa85db11 100644 --- a/scripts/conformance/server-parity/upgrade-source-author.php +++ b/scripts/conformance/server-parity/upgrade-source-author.php @@ -42,7 +42,8 @@ $worker = $prefix.'-worker'; $expect($request('POST', '/api/worker/register', ['worker_id' => $worker, 'task_queue' => $prefix, 'runtime' => 'php', 'sdk_version' => 'raw-upgrade-inspection-author', - 'supported_workflow_types' => ['parity.upgrade_activity', 'parity.upgrade_timer', 'parity.upgrade_signal'], + 'supported_workflow_types' => ['parity.upgrade_activity', 'parity.upgrade_timer', 'parity.upgrade_signal', + 'parity.upgrade_lease'], // The signal original declares its one signal, as an SDK worker would. 'workflow_command_contracts' => ['parity.upgrade_signal' => ['signals' => ['go'], 'signal_contracts' => [[ 'name' => 'go', 'parameters' => [['name' => 'value', 'position' => 0, 'type' => 'array', 'required' => true, @@ -59,7 +60,8 @@ $timerSeconds = (int) (getenv('DW_PARITY_UPGRADE_TIMER_SECONDS') ?: 600); $check($timerSeconds >= 30 && $timerSeconds <= 600, 'Unsupported original timer delay'); $originals = []; -foreach (['activity', 'timer', 'signal'] as $kind) { +// The lease original claims before the signal original makes its resume task. +foreach (['activity', 'timer', 'lease', 'signal'] as $kind) { $id = $prefix.'-'.$kind; $run = $expect($request('POST', '/api/workflows', ['workflow_id' => $id, 'workflow_type' => 'parity.upgrade_'.$kind, 'task_queue' => $prefix, 'input' => $envelope, @@ -74,11 +76,16 @@ 'start_to_close_timeout' => 600, 'schedule_to_close_timeout' => 900], 'timer' => ['type' => 'start_timer', 'delay_seconds' => $timerSeconds], 'signal' => ['type' => 'open_signal_wait', 'signal_name' => 'go'], + // The lease original's worker stops after its claim. PHP's workflow + // task lease expires after the takeover, and native redelivers it. + 'lease' => null, }; - $expect($request('POST', '/api/worker/workflow-tasks/'.$task['task_id'].'/complete', [ - 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], - 'commands' => [$command], - ]), 200, $kind.' command'); + if ($command !== null) { + $expect($request('POST', '/api/worker/workflow-tasks/'.$task['task_id'].'/complete', [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'commands' => [$command], + ]), 200, $kind.' command'); + } $activity = null; if ($kind === 'activity') { $activity = $expect($request('POST', '/api/worker/activity-tasks/poll', $poll), 200, 'activity claim')['task']; @@ -98,6 +105,7 @@ 'activity' => ['StartAccepted', 'WorkflowStarted', 'ActivityScheduled', 'ActivityStarted'], 'timer' => ['StartAccepted', 'WorkflowStarted', 'TimerScheduled'], 'signal' => ['StartAccepted', 'WorkflowStarted', 'SignalWaitOpened', 'SignalReceived'], + 'lease' => ['StartAccepted', 'WorkflowStarted'], }, 'Original pending history differs'); $originals[$kind] = ['workflow_id' => $id, 'run_id' => $run['run_id'], 'workflow_task' => $task, 'activity_task' => $activity, 'history' => $history]; diff --git a/scripts/conformance/server-parity/upgrade-source-profile.sh b/scripts/conformance/server-parity/upgrade-source-profile.sh index e7a504c9..b5f340ef 100755 --- a/scripts/conformance/server-parity/upgrade-source-profile.sh +++ b/scripts/conformance/server-parity/upgrade-source-profile.sh @@ -101,8 +101,8 @@ if test "$backend" = sqlite; then /source/scripts/conformance/server-parity/upgrade-source-sqlite-representations.php upgrade-source \ > "$evidence/representations-comparison.json" jq -e '.outcome=="pass" and .source_mutated==false and .php_objects_executed==false - and (.runs|length)==3 and (.tasks|length)==6 and (.activities|length)==1 - and (.history|length)==11 and (.queue_jobs|length)==1 + and (.runs|length)==4 and (.tasks|length)==7 and (.activities|length)==1 + and (.history|length)==13 and (.queue_jobs|length)==1 and .queue_jobs[0].canonical_timer_task_match==true and all(.runs[]; .typed_arguments_match==true) and .backup_verified==false and .stored_representation_compatibility_verified==false @@ -134,7 +134,7 @@ snapshot() { } snapshot > "$evidence/before.json" inspect_source > "$evidence/inspection.json" 2> "$evidence/inspection.stderr" -jq -e --arg layout "$layout" '.status=="inspected" and .source_layout==$layout and (.table_rows|length)==50 and .table_rows.workflow_runs==3 +jq -e --arg layout "$layout" '.status=="inspected" and .source_layout==$layout and (.table_rows|length)==50 and .table_rows.workflow_runs==4 and .table_rows.activity_executions==1 and .catalog_visibility=="configured_role" and .source_mutated==false and .takeover_allowed==false and .backup_verified==false and .php_quiescence_verified==false and .stored_representation_compatibility_verified==false' "$evidence/inspection.json" >/dev/null @@ -148,7 +148,7 @@ inspect_source upgrade-payload-inspect > "$evidence/payload-inspection.json" 2> jq -e '.status=="inspected" and (.payload_inspection.columns|length)==8 and .payload_inspection.references_metadata_supported==true and ([.payload_inspection.columns[].counts.php_v1_references]|add)==0 - and ([.payload_inspection.columns[].counts.avro_inline_values]|add)==9 + and ([.payload_inspection.columns[].counts.avro_inline_values]|add)==11 and ([.payload_inspection.columns[].counts.other_unvalidated]|add)==0 and .payload_inspection.inline_values_verified==true and .payload_inspection.inline_validation_limits=={encoded_frame_bytes:65536,traversal_depth:64,traversal_visits:16384,scalar_allocation_bytes:65536} @@ -166,7 +166,7 @@ fingerprint() { } fingerprint "$database" > "$evidence/fingerprint.json" 2> "$evidence/fingerprint.stderr" jq -e --arg layout "$layout" '.status=="fingerprinted" and .source_layout==$layout and (.fingerprint|test("^sha256:[0-9a-f]{64}$")) - and (.table_rows|length)==50 and .table_rows.workflow_runs==3 and .table_rows.jobs==1 + and (.table_rows|length)==50 and .table_rows.workflow_runs==4 and .table_rows.jobs==1 and .source_mutated==false and .takeover_allowed==false and .backup_verified==false and .php_quiescence_verified==false and .stored_representation_compatibility_verified==false' "$evidence/fingerprint.json" >/dev/null source_fingerprint=$(jq -r .fingerprint "$evidence/fingerprint.json") @@ -285,7 +285,9 @@ if (( ${#target[@]} )); then "/source/$evidence/originals.json" 150 > "$evidence/resume.json" 2> "$evidence/resume.stderr" jq -e '.outcome=="pass" and .activity.stale_attempt_status==409 and .activity.events[-1]=="WorkflowCompleted" and .timer.fired_after_seconds>=0 - and .timer.events[-1]=="WorkflowCompleted" and .signal.events[-1]=="WorkflowCompleted"' "$evidence/resume.json" >/dev/null + and .timer.events[-1]=="WorkflowCompleted" and .signal.events[-1]=="WorkflowCompleted" + and .lease.attempt==2 and .lease.redelivered_after_seconds>=0 and .lease.stale_attempt_status==409 + and .lease.events[-1]=="WorkflowCompleted"' "$evidence/resume.json" >/dev/null docker stop --signal SIGTERM --time 10 "$resource-native" >/dev/null if docker run --rm --name "$resource-control" --user=1000:1000 --cpus=1 --memory=512m --memory-swap=512m \ --network "$target_network" "${target[@]}" -e DW_AUTH_TOKEN=upgrade-fixture-token \ diff --git a/scripts/conformance/server-parity/upgrade-source-sqlite-representations.php b/scripts/conformance/server-parity/upgrade-source-sqlite-representations.php index b36f69b1..09da717d 100644 --- a/scripts/conformance/server-parity/upgrade-source-sqlite-representations.php +++ b/scripts/conformance/server-parity/upgrade-source-sqlite-representations.php @@ -20,11 +20,11 @@ if ($pdo->query('PRAGMA journal_mode')->fetchColumn() !== 'delete') { throw new RuntimeException('Audit requires the already stopped DELETE-journal fixture'); } -$runs = $pdo->query('SELECT * FROM workflow_runs ORDER BY id LIMIT 4')->fetchAll(PDO::FETCH_ASSOC); +$runs = $pdo->query('SELECT * FROM workflow_runs ORDER BY id LIMIT 5')->fetchAll(PDO::FETCH_ASSOC); $workflowIds = array_column($runs, 'workflow_instance_id'); sort($workflowIds); -if ($workflowIds !== [$prefix.'-activity', $prefix.'-signal', $prefix.'-timer']) { - throw new RuntimeException('This reader accepts only the three isolated source-author originals'); +if ($workflowIds !== [$prefix.'-activity', $prefix.'-lease', $prefix.'-signal', $prefix.'-timer']) { + throw new RuntimeException('This reader accepts only the four isolated source-author originals'); } $report = ['scope' => 'published_php_stored_representation_observation', 'runs' => [], 'table_rows' => [], 'tasks' => [], 'activities' => [], 'history' => [], 'queue_jobs' => [], 'cache_files' => []]; diff --git a/scripts/conformance/server-parity/upgrade-takeover-resume.php b/scripts/conformance/server-parity/upgrade-takeover-resume.php index 2b695dc8..41f0281d 100644 --- a/scripts/conformance/server-parity/upgrade-takeover-resume.php +++ b/scripts/conformance/server-parity/upgrade-takeover-resume.php @@ -57,7 +57,8 @@ $activity = $originals['activity']; $timer = $originals['timer']; $signal = $originals['signal']; -$complete = static function (array $claimed, string $operation) use ($expect, $request, $result): void { +$lease = $originals['lease']; +$complete =static function (array $claimed, string $operation) use ($expect, $request, $result): void { $expect($request('POST', '/api/worker/workflow-tasks/'.$claimed['task_id'].'/complete', ['lease_owner' => $claimed['lease_owner'], 'workflow_task_attempt' => $claimed['workflow_task_attempt'], 'commands' => [['type' => 'complete_workflow', 'result' => $result]]]), 200, $operation); @@ -70,6 +71,12 @@ $deadline = (float) (new DateTimeImmutable($fireAt))->format('U.u'); $check($deadline - microtime(true) <= (float) $timerBudget, 'The original timer deadline is outside the wait budget'); +// The workflow task PHP leased to a stopped worker keeps its PHP expiry. +$check(is_string($lease['workflow_task']['lease_expires_at'] ?? null), 'The original lease expiry is missing'); +$leaseExpiry = (float) (new DateTimeImmutable($lease['workflow_task']['lease_expires_at']))->format('U.u'); +$check($leaseExpiry - microtime(true) <= (float) $timerBudget, 'The original lease expiry is outside the wait budget'); +$check($keep($lease['history'], $history($lease)), 'Original lease history changed'); + // The activity leased under PHP completes with its original attempt. $task = $activity['activity_task']; $completion = ['activity_attempt_id' => $task['activity_attempt_id'], 'lease_owner' => $task['lease_owner'], 'result' => $result]; @@ -79,19 +86,21 @@ $check(($replayed['recorded'] ?? null) === false, 'A replayed activity completion was recorded twice'); // Each original workflow then gets one workflow task: the activity's and the -// signal's at once, the timer's at its deadline. On a slow host the timer is -// already due, so they may be claimed in any order. +// signal's at once, the timer's at its deadline and the lease's when PHP's +// lease expires. On a slow host some are already due, so they may be claimed +// in any order. $original = $activity['history']; $claims = []; -$until = max($deadline, microtime(true)) + 30; -while (count($claims) < 3 && microtime(true) < $until) { +$leaseOutcome = null; +$until = max($deadline, $leaseExpiry, microtime(true)) + 30; +while (count($claims) < 4 && microtime(true) < $until) { $claimed = $expect($request('POST', '/api/worker/workflow-tasks/poll', ['worker_id' => $worker, 'task_queue' => $queue, 'timeout_seconds' => 5]), 200, 'workflow task claim')['task'] ?? null; if (! is_array($claimed)) { continue; } $run = $claimed['run_id']; - $check(in_array($run, [$activity['run_id'], $timer['run_id'], $signal['run_id']], true) && ! isset($claims[$run]), + $check(in_array($run, [$activity['run_id'], $timer['run_id'], $signal['run_id'], $lease['run_id']], true) && ! isset($claims[$run]), 'An unexpected workflow task was claimed'); $claims[$run] = true; if ($run === $activity['run_id']) { @@ -109,6 +118,23 @@ === json_encode($sorted(end($signal['history'])['payload']['arguments']), JSON_THROW_ON_ERROR), 'The PHP-accepted signal arguments differ under native'); $complete($claimed, 'signal workflow completion'); + } elseif ($run === $lease['run_id']) { + // After PHP's lease expires, native redelivers the same task as its + // next attempt. The expired PHP attempt cannot complete over it. + $expired = $lease['workflow_task']; + $check($claimed['task_id'] === $expired['task_id'] + && $claimed['workflow_task_attempt'] === $expired['workflow_task_attempt'] + 1, + 'The expired PHP lease was not redelivered as its next attempt'); + $check(microtime(true) >= $leaseExpiry, 'Native redelivered a PHP lease before it expired'); + $check($types($claimed['history_events']) === $types($lease['history']), + 'Original lease history changed in the redelivered task'); + $stale = $request('POST', '/api/worker/workflow-tasks/'.$expired['task_id'].'/complete', ['lease_owner' => $expired['lease_owner'], + 'workflow_task_attempt' => $expired['workflow_task_attempt'], 'commands' => [['type' => 'complete_workflow', 'result' => $result]]]); + $check($stale[0] === 409, 'The expired PHP attempt completed over its redelivery'); + $leaseOutcome = ['attempt' => $claimed['workflow_task_attempt'], + 'redelivered_after_seconds' => round(microtime(true) - $leaseExpiry, 3), + 'stale_attempt_status' => $stale[0], 'stale_attempt_reason' => $stale[1]['reason'] ?? null]; + $complete($claimed, 'redelivered lease completion'); } else { $check(end($claimed['history_events'])['event_type'] === 'TimerFired', 'Timer firing is missing from history'); $complete($claimed, 'timer workflow completion'); @@ -117,6 +143,7 @@ $check(isset($claims[$activity['run_id']]), 'The resumed workflow task was not claimed'); $check(isset($claims[$signal['run_id']]), 'The signal workflow task was not claimed'); $check(isset($claims[$timer['run_id']]), 'The original timer did not fire under native'); +$check(isset($claims[$lease['run_id']]), 'The expired PHP lease was not redelivered'); // The workflow task PHP already completed is a stale attempt. $stale = $activity['workflow_task']; @@ -140,6 +167,11 @@ $check($keep($signal['history'], $signalHistory) && end($signalHistory)['event_type'] === 'WorkflowCompleted', 'The signal workflow did not complete on its original history'); $outcome['signal'] = ['run_id' => $signal['run_id'], 'events' => array_column($signalHistory, 'event_type')]; +$leaseHistory = $history($lease); +$check($keep($lease['history'], $leaseHistory) && end($leaseHistory)['event_type'] === 'WorkflowCompleted', + 'The redelivered lease workflow did not complete on its original history'); +$outcome['lease'] = ['run_id' => $lease['run_id'], 'lease_expires_at' => $lease['workflow_task']['lease_expires_at']] + + $leaseOutcome + ['events' => array_column($leaseHistory, 'event_type')]; $outcome['timer'] = ['run_id' => $timer['run_id'], 'fire_at' => $fireAt, 'fired_after_seconds' => round($firedAt - $deadline, 3), 'events' => array_column($timerHistory, 'event_type')]; echo json_encode(['outcome' => 'pass', 'scope' => 'native_resumes_published_php_originals_after_takeover'] + $outcome, From ebfcbff19970cd8324df0e6792bef6ed929925de Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 15:03:02 +0000 Subject: [PATCH 38/42] Fence entry-schema failures and require safe replacement workers --- app/Http/Controllers/Api/WorkerController.php | 1 + app/Support/WorkflowDefinitionRouting.php | 18 ++++- docs/server-reference.md | 7 +- .../WorkflowDefinitionEvolutionTest.php | 69 +++++++++++++++++++ 4 files changed, 91 insertions(+), 4 deletions(-) diff --git a/app/Http/Controllers/Api/WorkerController.php b/app/Http/Controllers/Api/WorkerController.php index 4da9a14e..b192dc0f 100644 --- a/app/Http/Controllers/Api/WorkerController.php +++ b/app/Http/Controllers/Api/WorkerController.php @@ -161,6 +161,7 @@ class WorkerController 'version_marker_kind_mismatch', 'version_marker_sequence_invalid', 'version_range_invalid', + 'workflow_entry_arguments_mismatch', 'workflow_nondeterministic', ]; diff --git a/app/Support/WorkflowDefinitionRouting.php b/app/Support/WorkflowDefinitionRouting.php index 2719ee01..3109f8ba 100644 --- a/app/Support/WorkflowDefinitionRouting.php +++ b/app/Support/WorkflowDefinitionRouting.php @@ -13,6 +13,8 @@ final class WorkflowDefinitionRouting { + public const ENTRY_VALIDATION_CAPABILITY = 'workflow_entry_arguments_validation'; + /** @return array */ public function forWorker( WorkflowRun $run, @@ -44,6 +46,10 @@ public function forWorker( } } + if (! in_array(self::ENTRY_VALIDATION_CAPABILITY, $worker->capabilities ?? [], true)) { + return [...$decision, 'allowed' => false, 'status' => 'entry_validation_unavailable']; + } + // Match the embedded resolver: prefer the recorded source if it is // available, otherwise let the current definition validate replay. // This never changes the run's original fingerprint or build pin. @@ -63,7 +69,7 @@ public function inspect(WorkflowRun $run, ?string $taskQueue = null, ?string $re $requiredCompatibility ??= TaskCompatibility::resolve($task, $run); } $context = $this->context($run, $taskQueue, $requiredCompatibility); - $matching = $changed = $missing = 0; + $matching = $changed = $missing = $unsafeReplacement = 0; $examples = []; foreach ($this->eligibleWorkers($run, (string) $context['task_queue'], $context['required_build_id']) as $worker) { @@ -74,6 +80,9 @@ public function inspect(WorkflowRun $run, ?string $taskQueue = null, ?string $re $matching++; } else { $changed++; + if (! in_array(self::ENTRY_VALIDATION_CAPABILITY, $worker->capabilities ?? [], true)) { + $unsafeReplacement++; + } } if (count($examples) < 10) { $examples[] = ['worker_id' => $worker->worker_id, 'fingerprint' => $fingerprint]; @@ -86,14 +95,16 @@ public function inspect(WorkflowRun $run, ?string $taskQueue = null, ?string $re ! $context['pinning_enabled'] => 'pinning_disabled', $context['recorded_fingerprint'] === null => 'recorded_definition_missing', $matching > 0 => 'matching_definition_available', - $changed > 0 => 'replay_required', + $changed > $unsafeReplacement => 'replay_required', + $unsafeReplacement > 0 => 'entry_validation_unavailable', $missing > 0 => 'worker_definition_missing', default => 'no_eligible_worker', }; $runnable = ! $run->status->isTerminal() && $task instanceof WorkflowTask; $blocked = $runnable && $context['worker_pinned'] && $context['pinning_enabled'] - && in_array($status, ['recorded_definition_missing', 'worker_definition_missing', 'no_eligible_worker'], true); + && in_array($status, ['recorded_definition_missing', 'worker_definition_missing', 'no_eligible_worker', 'entry_validation_unavailable'], true); $resolution = match (true) { + $blocked && $status === 'entry_validation_unavailable' => 'Upgrade the replacement worker SDK to an implementation advertising workflow_entry_arguments_validation, or restore the recorded definition. The current replacement cannot safely refuse incompatible entry input.', $blocked => 'Register a fresh worker for the original namespace, task queue, workflow type, build and replay capabilities with its definition fingerprint. Do not rewrite the run history.', $status === 'replay_required' => 'Current workers may replay the original history. Use versioned branches for command changes; incompatible replay still refuses.', default => 'Recorded build, namespace, type and replay-capability guards remain in force.', @@ -102,6 +113,7 @@ public function inspect(WorkflowRun $run, ?string $taskQueue = null, ?string $re return [...$context, 'status' => $status, 'blocked' => $blocked, 'runnable_task' => $runnable, 'eligible_worker_count' => $eligible, 'matching_worker_count' => $matching, 'changed_worker_count' => $changed, 'missing_fingerprint_count' => $missing, + 'entry_validation_unavailable_count' => $unsafeReplacement, 'workers' => $examples, 'workers_truncated' => $eligible > count($examples), 'resolution' => $resolution, ]; diff --git a/docs/server-reference.md b/docs/server-reference.md index 00af2598..2c3fe5e4 100644 --- a/docs/server-reference.md +++ b/docs/server-reference.md @@ -422,7 +422,12 @@ Existing runs retain their original definition fingerprint, build pin, history and deadlines. By default, the Server prefers a fresh eligible worker with the recorded definition. When that definition is no longer available in the run's namespace, queue, workflow type and build, a replacement definition can claim -the original task and must pass the SDK's strict replay checks. Use versioned +the original task only when it advertises `workflow_entry_arguments_validation`, +and must pass the SDK's strict replay checks. This worker capability declares +pre-entry refusal without failing the business run; older worker implementations +do not gain it from their runtime or protocol version. `entry_validation_unavailable` +in `definition_routing` identifies a replacement that needs an SDK upgrade. +Use versioned branches such as `get_version` for changes to durable commands. A different source fingerprint alone is not proof of incompatible replay. diff --git a/tests/Feature/WorkflowDefinitionEvolutionTest.php b/tests/Feature/WorkflowDefinitionEvolutionTest.php index f8398e18..52e97365 100644 --- a/tests/Feature/WorkflowDefinitionEvolutionTest.php +++ b/tests/Feature/WorkflowDefinitionEvolutionTest.php @@ -4,6 +4,7 @@ use App\Models\WorkerRegistration; use App\Support\WorkflowReplayEligibility; +use App\Support\WorkflowDefinitionRouting; use Illuminate\Foundation\Testing\RefreshDatabase; use Illuminate\Support\Facades\Queue; use Illuminate\Testing\TestResponse; @@ -64,6 +65,24 @@ public function test_explicit_opt_out_does_not_wait_for_a_matching_peer(): void ->assertJsonPath('task.task_id', $task->id); } + public function test_legacy_replacement_cannot_gain_unsafe_default_fallback(): void + { + [$run, $task] = $this->original(); + $this->deleteJson('/api/workers/original-worker', [], $this->apiHeaders())->assertOk(); + $this->worker('legacy-replacement', 'definition-v2', ['capabilities' => []]); + $this->poll('legacy-replacement')->assertOk()->assertJsonPath('task', null); + self::assertSame(0, $task->fresh()->attempt_count); + $this->getJson('/api/workflows/definition-original', $this->apiHeaders())->assertOk() + ->assertJsonPath('definition_routing.status', 'entry_validation_unavailable') + ->assertJsonPath('definition_routing.blocked', true) + ->assertJsonPath('definition_routing.entry_validation_unavailable_count', 1); + $this->worker('safe-replacement', 'definition-v2'); + $this->getJson('/api/workflows/definition-original', $this->apiHeaders())->assertOk() + ->assertJsonPath('definition_routing.status', 'replay_required') + ->assertJsonPath('definition_routing.blocked', false); + $this->poll('safe-replacement')->assertOk()->assertJsonPath('task.run_id', $run->id); + } + public function test_default_keeps_the_matching_registered_peer_preference(): void { [, $task] = $this->original(); @@ -252,6 +271,55 @@ public static function notRunnableTasks(): array return [['terminal'], ['leased'], ['future']]; } + #[DataProvider('entryFailureTypes')] + public function test_entry_binding_failure_preserves_the_original_until_supported_repair(string $failureType): void + { + [$run, $task, $started] = $this->original(); + $original = [$started->getRawOriginal('payload'), $run->getRawOriginal('run_deadline_at'), $run->getRawOriginal('execution_deadline_at')]; + $claim = $this->poll('original-worker')->assertOk(); + $failure = [ + 'lease_owner' => $claim->json('task.lease_owner'), + 'workflow_task_attempt' => $claim->json('task.workflow_task_attempt'), + 'failure' => [ + 'type' => $failureType, + 'reason' => 'workflow_entry_arguments_mismatch', + 'message' => 'The recorded input cannot bind to the deployed entry.', + ], + ]; + + $this->postJson('/api/worker/workflow-tasks/'.$task->id.'/fail', [ + ...$failure, 'lease_owner' => 'another-worker', + ], $this->workerHeaders())->assertConflict(); + self::assertSame(TaskStatus::Leased, $task->fresh()->status); + + $this->postJson('/api/worker/workflow-tasks/'.$task->id.'/fail', $failure, $this->workerHeaders()) + ->assertOk()->assertJsonPath('recorded', true)->assertJsonPath('next_task_id', null); + self::assertSame(TaskStatus::Failed, $task->fresh()->status); + self::assertTrue($task->fresh()->payload['replay_blocked'] ?? false); + self::assertSame('workflow_entry_arguments_mismatch', $task->fresh()->payload['failure_reason']); + self::assertSame(1, WorkflowTask::query()->where('workflow_run_id', $run->id)->count()); + self::assertNotContains($run->fresh()->status->value, ['failed', 'completed']); + $this->poll('original-worker')->assertOk()->assertJsonPath('task', null); + + $this->postJson('/api/workflows/definition-original/repair', [ + 'request_id' => 'entry-compatible-repair', + ], $this->apiHeaders())->assertOk(); + $this->poll('original-worker')->assertOk()->assertJsonPath('task.task_id', $task->id); + self::assertSame(1, $task->fresh()->repair_count); + self::assertSame($original, [ + $started->fresh()->getRawOriginal('payload'), $run->fresh()->getRawOriginal('run_deadline_at'), $run->fresh()->getRawOriginal('execution_deadline_at'), + ]); + } + + public static function entryFailureTypes(): array + { + return [ + 'PHP' => ['DurableWorkflow\\Exception\\WorkflowEntryArgumentsMismatch'], + 'Python' => ['WorkflowEntryArgumentsMismatchError'], + 'Rust' => ['WorkflowEntryArgumentsMismatch'], + ]; + } + private function original(): array { $this->worker('original-worker', 'definition-v1'); @@ -273,6 +341,7 @@ private function worker(string $id, string $fingerprint, array $overrides = []): { $this->postJson('/api/worker/register', [ 'worker_id' => $id, 'task_queue' => self::QUEUE, 'runtime' => 'python', 'sdk_version' => '2.6.1', + 'capabilities' => [WorkflowDefinitionRouting::ENTRY_VALIDATION_CAPABILITY], 'capability_manifest' => $this->portableWorkerAffinityRefusalManifest(), 'supported_workflow_types' => [self::TYPE], 'workflow_definition_fingerprints' => [self::TYPE => $fingerprint], From 85a81e67e2db0469a86831efe0171e8ce69dbdf4 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 15:24:26 +0000 Subject: [PATCH 39/42] Keep replacement safety decision in one final return --- app/Support/WorkflowDefinitionRouting.php | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/app/Support/WorkflowDefinitionRouting.php b/app/Support/WorkflowDefinitionRouting.php index 3109f8ba..0579f3ba 100644 --- a/app/Support/WorkflowDefinitionRouting.php +++ b/app/Support/WorkflowDefinitionRouting.php @@ -46,14 +46,13 @@ public function forWorker( } } - if (! in_array(self::ENTRY_VALIDATION_CAPABILITY, $worker->capabilities ?? [], true)) { - return [...$decision, 'allowed' => false, 'status' => 'entry_validation_unavailable']; - } - // Match the embedded resolver: prefer the recorded source if it is // available, otherwise let the current definition validate replay. // This never changes the run's original fingerprint or build pin. - return [...$decision, 'allowed' => true, 'status' => 'replay_required']; + $entryValidation = in_array(self::ENTRY_VALIDATION_CAPABILITY, $worker->capabilities ?? [], true); + + return [...$decision, 'allowed' => $entryValidation, + 'status' => $entryValidation ? 'replay_required' : 'entry_validation_unavailable']; } /** @return array */ From 3a07c5585b2be7a2610f15250ea4d4dccf3b5bfe Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 15:40:45 +0000 Subject: [PATCH 40/42] Prepare Server 2.9.9 entry-safety release --- composer.json | 2 +- composer.lock | 2 +- docker-compose.dedicated-matching.yml | 4 ++-- docker-compose.memo-rolling.yml | 4 ++-- docker-compose.published.yml | 4 ++-- docker-compose.small-cluster.yml | 2 +- docker-compose.yml | 8 ++++---- k8s/README.md | 10 +++++----- k8s/helm/durable-workflow/Chart.yaml | 8 ++++---- k8s/helm/durable-workflow/README.md | 2 +- .../durable-workflow/ci/existing-secrets-values.yaml | 2 +- .../durable-workflow/ci/ingress-and-hpa-values.yaml | 2 +- .../durable-workflow/ci/inline-secrets-values.yaml | 2 +- k8s/helm/durable-workflow/templates/_helpers.tpl | 2 +- k8s/helm/durable-workflow/values.yaml | 2 +- k8s/helm/examples/values-dev.yaml | 2 +- .../examples/values-external-secrets-operator.yaml | 2 +- .../examples/values-production-existing-secrets.yaml | 2 +- k8s/migration-job.yaml | 2 +- k8s/scheduler-cronjob.yaml | 2 +- k8s/secret.yaml | 2 +- k8s/server-deployment.yaml | 2 +- k8s/worker-deployment.yaml | 2 +- resources/release/source-release.json | 4 ++-- scripts/k8s-kind-smoke.sh | 2 +- 25 files changed, 39 insertions(+), 39 deletions(-) diff --git a/composer.json b/composer.json index a31e9251..2de1c971 100644 --- a/composer.json +++ b/composer.json @@ -48,7 +48,7 @@ }, "extra": { "durable-workflow": { - "product-train": "2.9.8" + "product-train": "2.9.9" }, "laravel": { "dont-discover": [] diff --git a/composer.lock b/composer.lock index 2e274b22..6ffbeab6 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "f3629921342639b58b34196e3ede3051", + "content-hash": "1230e17c71101ee7340e52d0c1fe2e35", "packages": [ { "name": "apache/avro", diff --git a/docker-compose.dedicated-matching.yml b/docker-compose.dedicated-matching.yml index 099c8775..48c69e0a 100644 --- a/docker-compose.dedicated-matching.yml +++ b/docker-compose.dedicated-matching.yml @@ -32,13 +32,13 @@ name: durable-workflow-server # daemon reports `shape: dedicated`. # Generated by scripts/ci/sync-source-release.mjs. Do not edit the fallback. -x-server-image: &server-image ${DW_SERVER_IMAGE:-durableworkflow/server:${DW_SERVER_TAG:-2.9.8}} +x-server-image: &server-image ${DW_SERVER_IMAGE:-durableworkflow/server:${DW_SERVER_TAG:-2.9.9}} x-server-environment: &server-environment APP_NAME: "Durable Workflow Server" APP_ENV: ${APP_ENV:-local} DW_SERVER_KEY: ${DW_SERVER_KEY:-} - APP_VERSION: ${APP_VERSION:-${DW_SERVER_TAG:-2.9.8}} + APP_VERSION: ${APP_VERSION:-${DW_SERVER_TAG:-2.9.9}} APP_DEBUG: ${APP_DEBUG:-false} DB_CONNECTION: mysql DB_HOST: mysql diff --git a/docker-compose.memo-rolling.yml b/docker-compose.memo-rolling.yml index 6bb030f7..4c77c894 100644 --- a/docker-compose.memo-rolling.yml +++ b/docker-compose.memo-rolling.yml @@ -49,14 +49,14 @@ services: command: ["server-bootstrap"] environment: <<: *runtime-environment - APP_VERSION: ${APP_VERSION:-2.9.8} + APP_VERSION: ${APP_VERSION:-2.9.9} successor: image: ${DW_MEMO_SUCCESSOR_IMAGE:-durable-workflow/server-memo-rolling:local} ports: !override [] environment: <<: *runtime-environment - APP_VERSION: ${APP_VERSION:-2.9.8} + APP_VERSION: ${APP_VERSION:-2.9.9} DW_SERVER_ID: memo-successor DW_SERVER_TOPOLOGY_SHAPE: standalone_server DW_SERVER_PROCESS_CLASS: server_http_node diff --git a/docker-compose.published.yml b/docker-compose.published.yml index e7acdad0..e5cc0b81 100644 --- a/docker-compose.published.yml +++ b/docker-compose.published.yml @@ -1,13 +1,13 @@ name: durable-workflow-server # Generated by scripts/ci/sync-source-release.mjs. Do not edit the fallback. -x-server-image: &server-image ${DW_SERVER_IMAGE:-durableworkflow/server:${DW_SERVER_TAG:-2.9.8}} +x-server-image: &server-image ${DW_SERVER_IMAGE:-durableworkflow/server:${DW_SERVER_TAG:-2.9.9}} x-server-environment: &server-environment APP_NAME: "Durable Workflow Server" APP_ENV: ${APP_ENV:-local} DW_SERVER_KEY: ${DW_SERVER_KEY:-} - APP_VERSION: ${APP_VERSION:-${DW_SERVER_TAG:-2.9.8}} + APP_VERSION: ${APP_VERSION:-${DW_SERVER_TAG:-2.9.9}} APP_DEBUG: ${APP_DEBUG:-false} LOG_CHANNEL: ${LOG_CHANNEL:-stderr} LOG_LEVEL: ${LOG_LEVEL:-info} diff --git a/docker-compose.small-cluster.yml b/docker-compose.small-cluster.yml index 3f3715b0..a6a24201 100644 --- a/docker-compose.small-cluster.yml +++ b/docker-compose.small-cluster.yml @@ -12,7 +12,7 @@ x-server-build: &server-build x-server-environment: &server-environment APP_NAME: "Durable Workflow Server" APP_ENV: testing - APP_VERSION: ${APP_VERSION:-2.9.8} + APP_VERSION: ${APP_VERSION:-2.9.9} APP_DEBUG: "false" DW_SERVER_KEY: ${DW_SERVER_KEY:-base64:5Zt4nUhlCm3DD0nLXZJQdHiwPfb56yGo9gNV/g3jYbY=} DB_CONNECTION: ${DW_SMALL_CLUSTER_DB:-mysql} diff --git a/docker-compose.yml b/docker-compose.yml index 6f18015e..95d538a3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,7 +15,7 @@ services: DW_SERVER_KEY: "${DW_SERVER_KEY:-}" DW_SERVER_TOPOLOGY_SHAPE: standalone_server DW_SERVER_PROCESS_CLASS: server_http_node - APP_VERSION: "${APP_VERSION:-2.9.8}" + APP_VERSION: "${APP_VERSION:-2.9.9}" APP_DEBUG: "false" DB_CONNECTION: mysql DB_HOST: mysql @@ -62,7 +62,7 @@ services: APP_NAME: "Durable Workflow Server" APP_ENV: local DW_SERVER_KEY: "${DW_SERVER_KEY:-}" - APP_VERSION: "${APP_VERSION:-2.9.8}" + APP_VERSION: "${APP_VERSION:-2.9.9}" APP_DEBUG: "false" DB_CONNECTION: mysql DB_HOST: mysql @@ -124,7 +124,7 @@ services: DW_SERVER_KEY: "${DW_SERVER_KEY:-}" DW_SERVER_TOPOLOGY_SHAPE: standalone_server DW_SERVER_PROCESS_CLASS: worker_node - APP_VERSION: "${APP_VERSION:-2.9.8}" + APP_VERSION: "${APP_VERSION:-2.9.9}" DB_CONNECTION: mysql DB_HOST: mysql DB_PORT: 3306 @@ -178,7 +178,7 @@ services: DW_SERVER_KEY: "${DW_SERVER_KEY:-}" DW_SERVER_TOPOLOGY_SHAPE: standalone_server DW_SERVER_PROCESS_CLASS: scheduler_node - APP_VERSION: "${APP_VERSION:-2.9.8}" + APP_VERSION: "${APP_VERSION:-2.9.9}" DB_CONNECTION: mysql DB_HOST: mysql DB_PORT: 3306 diff --git a/k8s/README.md b/k8s/README.md index 2375dfd6..e1a7dd00 100644 --- a/k8s/README.md +++ b/k8s/README.md @@ -13,7 +13,7 @@ The checked-in manifests are synchronized with the repository's stable source release and pin its Docker Hub tag: ```text -durableworkflow/server:2.9.8 +durableworkflow/server:2.9.9 ``` Before production use, patch every workload image to the exact published tag or @@ -21,15 +21,15 @@ digest you intend to run: ```bash kubectl set image -n durable-workflow deploy/durable-workflow-server \ - server=durableworkflow/server:2.9.8 + server=durableworkflow/server:2.9.9 kubectl set image -n durable-workflow deploy/durable-workflow-worker \ - worker=durableworkflow/server:2.9.8 + worker=durableworkflow/server:2.9.9 kubectl set image -n durable-workflow cronjob/durable-workflow-scheduler \ - scheduler=durableworkflow/server:2.9.8 + scheduler=durableworkflow/server:2.9.9 ``` GitHub Container Registry publishes the same release line at -`ghcr.io/durable-workflow/server:2.9.8`. Digest pinning is preferred for strict +`ghcr.io/durable-workflow/server:2.9.9`. Digest pinning is preferred for strict change control. The manifests expect you to provide: diff --git a/k8s/helm/durable-workflow/Chart.yaml b/k8s/helm/durable-workflow/Chart.yaml index 5801c1bc..321c99ce 100644 --- a/k8s/helm/durable-workflow/Chart.yaml +++ b/k8s/helm/durable-workflow/Chart.yaml @@ -5,11 +5,11 @@ type: application # The chart's own semver version. Bumped on every chart release; treated as # independent of the server image version (appVersion). Breaking-change rules # for this version live in docs/helm-upgrading.md alongside the chart. -version: 0.1.184 +version: 0.1.185 # The immutable Durable Workflow Server identity this chart release packages. # The onboarding default in values.yaml and appVersion are generated from the # checked-in source release record. -appVersion: "2.9.8" +appVersion: "2.9.9" kubeVersion: ">=1.27.0-0" home: https://durable-workflow.github.io/docs/2.0/deployment sources: @@ -30,10 +30,10 @@ annotations: # exact commit that most recently changed the packaged chart. org.opencontainers.image.source: https://github.com/durable-workflow/server dev.durable-workflow.source-revision: "unreleased" - dev.durable-workflow.image-reference: "docker.io/durableworkflow/server:2.9.8" + dev.durable-workflow.image-reference: "docker.io/durableworkflow/server:2.9.9" artifacthub.io/license: MIT artifacthub.io/category: integration-delivery # Free-form changelog for the current chart release shown by Artifact Hub. artifacthub.io/changes: | - kind: fixed - description: Use unnamed bound PostgreSQL statements with transaction poolers and verify a bound database read during readiness checks. + description: Allow compatible workflow-code replacement with validated workers and preserve incompatible entry failures for supported same-run repair. diff --git a/k8s/helm/durable-workflow/README.md b/k8s/helm/durable-workflow/README.md index 69d84284..42c39a15 100644 --- a/k8s/helm/durable-workflow/README.md +++ b/k8s/helm/durable-workflow/README.md @@ -63,7 +63,7 @@ helm install durable-workflow ./k8s/helm/durable-workflow \ ```yaml image: - tag: "2.9.8" + tag: "2.9.9" # Pin a digest in production: # digest: "sha256:abc123..." # memoPayloadStorage: "raw-json-v1" # Required for a digest or custom image. diff --git a/k8s/helm/durable-workflow/ci/existing-secrets-values.yaml b/k8s/helm/durable-workflow/ci/existing-secrets-values.yaml index 03daee1a..11f807c4 100644 --- a/k8s/helm/durable-workflow/ci/existing-secrets-values.yaml +++ b/k8s/helm/durable-workflow/ci/existing-secrets-values.yaml @@ -1,7 +1,7 @@ # CI fixture: GitOps / externally-managed-secret path. The chart consumes # existing Secrets and renders no Secret resources of its own. image: - tag: "2.9.8" + tag: "2.9.9" externalDatabase: connection: pgsql diff --git a/k8s/helm/durable-workflow/ci/ingress-and-hpa-values.yaml b/k8s/helm/durable-workflow/ci/ingress-and-hpa-values.yaml index 298204fb..70d9fa26 100644 --- a/k8s/helm/durable-workflow/ci/ingress-and-hpa-values.yaml +++ b/k8s/helm/durable-workflow/ci/ingress-and-hpa-values.yaml @@ -1,6 +1,6 @@ # CI fixture: ingress + autoscaling enabled. Exercises optional templates. image: - tag: "2.9.8" + tag: "2.9.9" externalDatabase: connection: mysql diff --git a/k8s/helm/durable-workflow/ci/inline-secrets-values.yaml b/k8s/helm/durable-workflow/ci/inline-secrets-values.yaml index 2e88b700..8160b1c3 100644 --- a/k8s/helm/durable-workflow/ci/inline-secrets-values.yaml +++ b/k8s/helm/durable-workflow/ci/inline-secrets-values.yaml @@ -2,7 +2,7 @@ # chart's render path is exercised end-to-end. Real deployments should use # existingSecret instead. image: - tag: "2.9.8" + tag: "2.9.9" externalDatabase: connection: mysql diff --git a/k8s/helm/durable-workflow/templates/_helpers.tpl b/k8s/helm/durable-workflow/templates/_helpers.tpl index ef7394b2..fb7c0577 100644 --- a/k8s/helm/durable-workflow/templates/_helpers.tpl +++ b/k8s/helm/durable-workflow/templates/_helpers.tpl @@ -88,7 +88,7 @@ resolved by an explicit capability declaration or an existing workload marker. {{- define "durable-workflow.memoPayloadStorageForImage" -}} {{- $image := toString . -}} {{- $normalized := regexReplaceAll "^index\\.docker\\.io/" $image "docker.io/" -}} -{{- if eq $normalized "docker.io/durableworkflow/server:2.9.8" -}} +{{- if eq $normalized "docker.io/durableworkflow/server:2.9.9" -}} dual-v1 {{- else if regexMatch "^docker\\.io/durableworkflow/server:2\\.0\\.0-rc\\.[0-9]+$" $normalized -}} {{- $releaseCandidate := atoi (regexFind "[0-9]+$" $normalized) -}} diff --git a/k8s/helm/durable-workflow/values.yaml b/k8s/helm/durable-workflow/values.yaml index e4f51aa9..3aac505f 100644 --- a/k8s/helm/durable-workflow/values.yaml +++ b/k8s/helm/durable-workflow/values.yaml @@ -21,7 +21,7 @@ image: registry: docker.io repository: durableworkflow/server # Generated by scripts/ci/sync-source-release.mjs. Do not edit this default. - tag: "2.9.8" + tag: "2.9.9" # Optional digest pin. When set, takes precedence over tag for change control. # Example: "sha256:abc123..." digest: "" diff --git a/k8s/helm/examples/values-dev.yaml b/k8s/helm/examples/values-dev.yaml index b20fdfdf..871391cb 100644 --- a/k8s/helm/examples/values-dev.yaml +++ b/k8s/helm/examples/values-dev.yaml @@ -3,7 +3,7 @@ # shape in production. image: - tag: "2.9.8" + tag: "2.9.9" externalDatabase: connection: mysql diff --git a/k8s/helm/examples/values-external-secrets-operator.yaml b/k8s/helm/examples/values-external-secrets-operator.yaml index b251debb..2bbfd25c 100644 --- a/k8s/helm/examples/values-external-secrets-operator.yaml +++ b/k8s/helm/examples/values-external-secrets-operator.yaml @@ -5,7 +5,7 @@ # concern. image: - tag: "2.9.8" + tag: "2.9.9" externalDatabase: connection: pgsql diff --git a/k8s/helm/examples/values-production-existing-secrets.yaml b/k8s/helm/examples/values-production-existing-secrets.yaml index 0f3cb833..e1faeeff 100644 --- a/k8s/helm/examples/values-production-existing-secrets.yaml +++ b/k8s/helm/examples/values-production-existing-secrets.yaml @@ -10,7 +10,7 @@ image: repository: durable-workflow/server # Pin a digest in production for change-control auditability. digest: "" # e.g. "sha256:abc123..." - tag: "2.9.8" + tag: "2.9.9" externalDatabase: connection: pgsql diff --git a/k8s/migration-job.yaml b/k8s/migration-job.yaml index e5e21c00..a514b2e4 100644 --- a/k8s/migration-job.yaml +++ b/k8s/migration-job.yaml @@ -13,7 +13,7 @@ spec: restartPolicy: OnFailure containers: - name: migrate - image: durableworkflow/server:2.9.8 + image: durableworkflow/server:2.9.9 command: ["server-entrypoint"] args: ["server-bootstrap"] envFrom: diff --git a/k8s/scheduler-cronjob.yaml b/k8s/scheduler-cronjob.yaml index 18aaba0d..6017ed7c 100644 --- a/k8s/scheduler-cronjob.yaml +++ b/k8s/scheduler-cronjob.yaml @@ -24,7 +24,7 @@ spec: restartPolicy: Never containers: - name: scheduler - image: durableworkflow/server:2.9.8 + image: durableworkflow/server:2.9.9 command: ["server-entrypoint"] args: - sh diff --git a/k8s/secret.yaml b/k8s/secret.yaml index c0ac4bfe..753b17c9 100644 --- a/k8s/secret.yaml +++ b/k8s/secret.yaml @@ -12,7 +12,7 @@ metadata: app.kubernetes.io/name: durable-workflow data: APP_NAME: "Durable Workflow Server" - APP_VERSION: "2.9.8" + APP_VERSION: "2.9.9" APP_ENV: production APP_DEBUG: "false" DB_CONNECTION: mysql diff --git a/k8s/server-deployment.yaml b/k8s/server-deployment.yaml index e486ec44..15c71b38 100644 --- a/k8s/server-deployment.yaml +++ b/k8s/server-deployment.yaml @@ -23,7 +23,7 @@ spec: spec: containers: - name: server - image: durableworkflow/server:2.9.8 + image: durableworkflow/server:2.9.9 ports: - containerPort: 8080 name: http diff --git a/k8s/worker-deployment.yaml b/k8s/worker-deployment.yaml index 2137c6c9..910009de 100644 --- a/k8s/worker-deployment.yaml +++ b/k8s/worker-deployment.yaml @@ -19,7 +19,7 @@ spec: spec: containers: - name: worker - image: durableworkflow/server:2.9.8 + image: durableworkflow/server:2.9.9 command: ["server-entrypoint"] args: ["php", "artisan", "queue:work", "--sleep=1", "--tries=3", "--max-time=3600"] envFrom: diff --git a/resources/release/source-release.json b/resources/release/source-release.json index ffb24cee..234add48 100644 --- a/resources/release/source-release.json +++ b/resources/release/source-release.json @@ -1,9 +1,9 @@ { "schema": "durable-workflow.server.source-release/v1", "server": { - "version": "2.9.8" + "version": "2.9.9" }, "helm_chart": { - "version": "0.1.184" + "version": "0.1.185" } } diff --git a/scripts/k8s-kind-smoke.sh b/scripts/k8s-kind-smoke.sh index 9ce085d8..9d49762d 100755 --- a/scripts/k8s-kind-smoke.sh +++ b/scripts/k8s-kind-smoke.sh @@ -7,7 +7,7 @@ cluster="${K8S_SMOKE_CLUSTER:-durable-workflow-server-smoke}" image="${K8S_SMOKE_IMAGE:-durableworkflow/server:k8s-smoke}" # Generated by scripts/ci/sync-source-release.mjs so the smoke replaces the # same default shipped by the public manifests. -manifest_image="durableworkflow/server:2.9.8" +manifest_image="durableworkflow/server:2.9.9" kind_node_image="${K8S_SMOKE_KIND_NODE_IMAGE:-kindest/node:v1.29.4}" artifact_dir="${K8S_SMOKE_ARTIFACT_DIR:-/tmp/durable-workflow-k8s-kind-smoke-artifacts}" rendered_dir="${artifact_dir}/rendered-manifests" From f4f78d0d97a336208c6744c5731bab5b3886c7bc Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 15:51:41 +0000 Subject: [PATCH 41/42] Record two repeated matched PHP/native pairs after #516 Two alternating PHP/native pairs on a second dedicated host repeat the morning's matched comparison on main after #516. Native meets or beats PHP in every window, with no drift within any run. PHP's own capacity at 1/s differs between the two hosts, which the observations record. Part of #325. Co-Authored-By: Claude Opus 5.5 --- benchmarks/server-port/v3/README.md | 4 +- .../server-port/v3/repeated-2026-10-20.md | 101 ++ .../summary-repeated-native-1-2026-10-20.json | 911 +++++++++++++ .../summary-repeated-native-2-2026-10-20.json | 911 +++++++++++++ .../v3/summary-repeated-php-1-2026-10-20.json | 1124 +++++++++++++++++ .../v3/summary-repeated-php-2-2026-10-20.json | 1124 +++++++++++++++++ 6 files changed, 4174 insertions(+), 1 deletion(-) create mode 100644 benchmarks/server-port/v3/repeated-2026-10-20.md create mode 100644 benchmarks/server-port/v3/summary-repeated-native-1-2026-10-20.json create mode 100644 benchmarks/server-port/v3/summary-repeated-native-2-2026-10-20.json create mode 100644 benchmarks/server-port/v3/summary-repeated-php-1-2026-10-20.json create mode 100644 benchmarks/server-port/v3/summary-repeated-php-2-2026-10-20.json diff --git a/benchmarks/server-port/v3/README.md b/benchmarks/server-port/v3/README.md index c9965658..f74ef55e 100644 --- a/benchmarks/server-port/v3/README.md +++ b/benchmarks/server-port/v3/README.md @@ -5,7 +5,9 @@ run's native source, optimized binary and machine facts filled in. The workload, artifacts, limits and probe hashes are unchanged. The [October 20 observations](observations-2026-10-20.md) are one PHP/native pair on a disposable VM, run with `scripts/perf/run-server-port-matched.sh` -and summarized with `scripts/perf/summarize-server-port-matched.py`. +and summarized with `scripts/perf/summarize-server-port-matched.py`. The +[repeated observations](repeated-2026-10-20.md) are two alternating pairs +on a second host, after #516. To repeat it on a fresh dedicated host with Docker and Compose: diff --git a/benchmarks/server-port/v3/repeated-2026-10-20.md b/benchmarks/server-port/v3/repeated-2026-10-20.md new file mode 100644 index 00000000..f5d1eebe --- /dev/null +++ b/benchmarks/server-port/v3/repeated-2026-10-20.md @@ -0,0 +1,101 @@ +# Repeated matched observations, October 20 + +Two alternating PHP/native pairs on one dedicated experiment host, after #516 +landed on main. They repeat the [morning pair](observations-2026-10-20.md) +with the same workload, artifacts, limits and commands. These are development +observations on one host and one database family, not a release claim. + +## Conditions + +- **Source.** Server main `8957ba27a` (#516 merged), native tree `55aec4d8`. + The optimized binary was built with `cargo build --release --locked --bin + durable-workflow-server`, SHA-256 `edc283f1…6e2cc`. The PHP references, + PHP SDK, MySQL and Redis artifacts are the profile's, unchanged. +- **Host.** A fresh Vultr `vhp-4c-8gb-amd` VM: 4 vCPU AMD EPYC-Rome, 7.7 GiB, + Ubuntu 24.04.5 with kernel 6.8, Docker 29.1.3 and Compose 2.40.3, no swap. + No other workload ran. It ran from 13:15 to 15:48 UTC, was deleted at the + end and its deletion was verified, at about $0.20. +- **Procedure.** A native smoke check, then PHP, native, PHP and native full + runs in that order. Each run uses its own Compose project and fresh + database. Each run is the morning procedure: warmup, three 120 s windows at + 0.25, 0.5 and 1 workflow/s, the 0.25/s windows repeated after as a drift + check, and mixed idle polls. The four runs' summaries are in + `summary-repeated-{php,native}-{1,2}-2026-10-20.json`. + +All four runs finished with status 0. Every window's workflows verified, with +nothing pending. No container restarted or was OOM-killed. + +## Offered work and latency + +Each cell gives the range of the three per-window values. + +| Run / rate | In-window completions | p50 s | p95 s | p99 s | +| --- | --- | --- | --- | --- | +| PHP 1 / 0.25 before | 30/30/30 | 1.129–1.188 | 1.459–1.620 | 1.515–1.652 | +| PHP 1 / 0.5 | 60/60/60 | 1.129–1.215 | 1.517–1.545 | 1.633–1.675 | +| PHP 1 / 1 | 69/68/66 of 120 | 44.782–48.343 | 64.771–67.389 | 66.026–69.027 | +| PHP 1 / 0.25 after | 30/30/30 | 1.195–1.226 | 1.568–1.732 | 1.686–1.818 | +| Native 1 / 0.25 before | 30/30/30 | 0.815–1.171 | 1.485–1.785 | 1.867–2.202 | +| Native 1 / 0.5 | 60/60/60 | 0.806–0.862 | 1.361–1.434 | 1.490–1.730 | +| Native 1 / 1 | 120/120/120 | 0.642–0.688 | 1.139–2.111 | 1.604–2.545 | +| Native 1 / 0.25 after | 30/30/30 | 1.061–1.139 | 1.691–1.887 | 1.916–2.042 | +| PHP 2 / 0.25 before | 30/30/30 | 1.236–1.306 | 1.649–1.680 | 1.697–1.818 | +| PHP 2 / 0.5 | 60/60/60 | 1.179–1.268 | 1.557–1.635 | 1.694–1.867 | +| PHP 2 / 1 | 62/67/65 of 120 | 48.434–55.744 | 65.491–72.870 | 66.325–74.189 | +| PHP 2 / 0.25 after | 30/30/30 | 1.164–1.258 | 1.441–1.643 | 1.531–1.731 | +| Native 2 / 0.25 before | 30/30/30 | 0.942–0.989 | 1.655–1.876 | 1.860–1.944 | +| Native 2 / 0.5 | 60/60/60 | 0.874–0.985 | 1.380–1.555 | 1.618–1.918 | +| Native 2 / 1 | 120/119/120 | 0.675–0.715 | 1.121–1.567 | 1.759–2.945 | +| Native 2 / 0.25 after | 30/30/30 | 0.951–1.049 | 1.503–1.822 | 1.716–1.958 | + +PHP still does not sustain 1/s. Its in-window completions at 1/s fell from +93–96 this morning to 62–69 here, and its 1/s median latency grew from 15–18 s +to 45–56 s. Its CPU per workflow also rose about a quarter on this host, as the +next table shows. Native completed all 120 in both pairs here and this +morning. The two PHP runs agree with each other, and so do the two native runs. +So the change is between hosts, not drift within this one. + +## CPU and memory + +Core-seconds per verified workflow, and the highest container memory sampled. + +| Run / rate | Runtime core-s/wf | MySQL core-s/wf | Runtime max MiB | +| --- | --- | --- | --- | +| PHP 1 / 0.25 before | 2.574–2.755 | 0.662–0.713 | 192.0 | +| PHP 1 / 1 | 1.599–1.638 | 0.554–0.566 | 201.2 | +| PHP 2 / 0.25 before | 2.737–2.829 | 0.710–0.748 | 203.0 | +| PHP 2 / 1 | 1.603–1.676 | 0.549–0.573 | 213.1 | +| Native 1 / 0.25 before | 0.239–0.247 | 0.489–0.506 | 6.3 | +| Native 1 / 1 | 0.102–0.104 | 0.241–0.245 | 7.0 | +| Native 2 / 0.25 before | 0.245–0.256 | 0.503–0.518 | 6.9 | +| Native 2 / 1 | 0.110–0.116 | 0.264–0.275 | 6.8 | + +At 1/s, native uses about a fifteenth of PHP's runtime CPU per workflow and +about half of its database CPU. Its peak memory is about 7 MiB against PHP's +190–215 MiB. + +## Idle mixed long polls + +Runtime core-seconds per idle window, three windows each. + +| Run | 6 polls | 12 polls | 24 polls | Health failures | +| --- | --- | --- | --- | --- | +| PHP 1 | 6.117–6.877 | 7.279–8.642 | 9.160–9.564 | 0, 1, 2 per window | +| Native 1 | 0.645–0.683 | 0.790–0.917 | 1.206–1.256 | none | +| PHP 2 | 5.699–6.651 | 7.803–8.198 | 9.408–9.695 | 0, 1, 2–3 per window | +| Native 2 | 0.684–0.755 | 0.822–0.922 | 1.189–1.241 | none | + +PHP answers part of its 12- and 24-poll windows with backpressure, and its +health checks fail in them. Native answers every poll empty, as expected. + +## What this shows and what it does not + +On this profile, native meets or beats PHP on throughput, CPU, memory and idle +long-poll cost in every window of two repeated pairs. The before and after +windows show no drift within any run. + +It does not show the result on PostgreSQL or SQLite, on other machine types, +across nodes, or under the CPU-heavy deadline profile. The host-to-host change +in PHP's capacity also says one host is not enough for a release claim. The +driver records host load and swap but not CPU steal. Sampling steal in +`host-state.tsv` is the next change to the driver. diff --git a/benchmarks/server-port/v3/summary-repeated-native-1-2026-10-20.json b/benchmarks/server-port/v3/summary-repeated-native-1-2026-10-20.json new file mode 100644 index 00000000..89b505fd --- /dev/null +++ b/benchmarks/server-port/v3/summary-repeated-native-1-2026-10-20.json @@ -0,0 +1,911 @@ +{ + "identity": { + "implementation": "native", + "compose_project": "dw-port-native-20261020t141652", + "profile": "/home/linuxuser/profile.json", + "profile_sha256": "5e7946a6a506e391510e9222021e60c80d184742b030ee6e2978ab0c0b1e8ac5", + "source_revision": "8957ba27aa24b79ec2ea59e0c37b5ce662d434ab", + "native_binary_sha256": "edc283f130ff5c802fe5a80d56730cc9fe377c01c9c90340e08a68eb0866e2cc", + "smoke": false, + "idle_only": false, + "machine": { + "kernel": "6.8.0-146-generic", + "logical_cpus": 4, + "physical_memory_bytes": 8320786432, + "cpu_model": "AMD EPYC-Rome Processor", + "docker": "29.1.3", + "cgroup_driver": "systemd", + "docker_storage_driver": "overlayfs", + "docker_root_filesystem": "ext4" + } + }, + "windows": { + "setup": { + "seconds": 5.4, + "health": { + "replies": 8, + "failures": 0, + "slowest_ok_seconds": 0.302 + }, + "cpu_core_seconds": { + "mysql": 0.876176, + "native-server": 0.408746, + "probe": 0.0, + "sdk-worker": 0.039165 + }, + "max_memory_bytes": { + "mysql": 527196160, + "native-server": 4235264, + "probe": 16654336, + "sdk-worker": 19292160 + }, + "max_runtime_memory_bytes": 4235264, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 2, + "accepted_starts": 2, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 2, + "pending_at_deadline": 0, + "closed_during_offer_window": 2, + "latency_p50_seconds": 0.2922959327697754, + "latency_p95_seconds": 0.5931649208068848, + "latency_p99_seconds": 0.5931649208068848, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.2044, + "mysql_core_seconds_per_workflow": 0.4381 + }, + "warmup": { + "seconds": 59.3, + "health": { + "replies": 88, + "failures": 0, + "slowest_ok_seconds": 0.476 + }, + "cpu_core_seconds": { + "mysql": 7.408028, + "native-server": 3.565426, + "probe": 0.383084, + "sdk-worker": 0.367419 + }, + "max_memory_bytes": { + "mysql": 540975104, + "native-server": 6533120, + "probe": 16683008, + "sdk-worker": 19447808 + }, + "max_runtime_memory_bytes": 6533120, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 15, + "accepted_starts": 15, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 15, + "pending_at_deadline": 0, + "closed_during_offer_window": 15, + "latency_p50_seconds": 1.2313079833984375, + "latency_p95_seconds": 2.112252950668335, + "latency_p99_seconds": 2.112252950668335, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.2377, + "mysql_core_seconds_per_workflow": 0.4939 + }, + "low-before-1": { + "seconds": 120.5, + "health": { + "replies": 181, + "failures": 0, + "slowest_ok_seconds": 0.431 + }, + "cpu_core_seconds": { + "mysql": 14.753484, + "native-server": 7.199888, + "probe": 0.365237, + "sdk-worker": 0.774945 + }, + "max_memory_bytes": { + "mysql": 556244992, + "native-server": 6656000, + "probe": 16691200, + "sdk-worker": 21241856 + }, + "max_runtime_memory_bytes": 6656000, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 30, + "accepted_starts": 30, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 30, + "pending_at_deadline": 0, + "closed_during_offer_window": 30, + "latency_p50_seconds": 0.8146710395812988, + "latency_p95_seconds": 1.784688949584961, + "latency_p99_seconds": 1.8666729927062988, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.24, + "mysql_core_seconds_per_workflow": 0.4918 + }, + "low-before-2": { + "seconds": 119.5, + "health": { + "replies": 178, + "failures": 0, + "slowest_ok_seconds": 0.435 + }, + "cpu_core_seconds": { + "mysql": 15.181942, + "native-server": 7.39961, + "probe": 0.023146, + "sdk-worker": 0.804998 + }, + "max_memory_bytes": { + "mysql": 569720832, + "native-server": 5312512, + "probe": 16683008, + "sdk-worker": 23449600 + }, + "max_runtime_memory_bytes": 5312512, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 30, + "accepted_starts": 30, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 30, + "pending_at_deadline": 0, + "closed_during_offer_window": 30, + "latency_p50_seconds": 1.170957088470459, + "latency_p95_seconds": 1.6039619445800781, + "latency_p99_seconds": 2.2019379138946533, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.2467, + "mysql_core_seconds_per_workflow": 0.5061 + }, + "low-before-3": { + "seconds": 120.1, + "health": { + "replies": 183, + "failures": 0, + "slowest_ok_seconds": 0.342 + }, + "cpu_core_seconds": { + "mysql": 14.658095, + "native-server": 7.169432, + "probe": -0.017477, + "sdk-worker": 0.760215 + }, + "max_memory_bytes": { + "mysql": 580112384, + "native-server": 5115904, + "probe": 16936960, + "sdk-worker": 24776704 + }, + "max_runtime_memory_bytes": 5115904, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 30, + "accepted_starts": 30, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 30, + "pending_at_deadline": 0, + "closed_during_offer_window": 30, + "latency_p50_seconds": 0.8877580165863037, + "latency_p95_seconds": 1.4847869873046875, + "latency_p99_seconds": 2.122774839401245, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.239, + "mysql_core_seconds_per_workflow": 0.4886 + }, + "rate-0.5-1": { + "seconds": 120.2, + "health": { + "replies": 174, + "failures": 0, + "slowest_ok_seconds": 0.527 + }, + "cpu_core_seconds": { + "mysql": 19.666724, + "native-server": 8.952809, + "probe": 0.261888, + "sdk-worker": 1.170249 + }, + "max_memory_bytes": { + "mysql": 604008448, + "native-server": 7004160, + "probe": 16703488, + "sdk-worker": 28123136 + }, + "max_runtime_memory_bytes": 7004160, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.5, + "planned_starts": 60, + "accepted_starts": 60, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 60, + "pending_at_deadline": 0, + "closed_during_offer_window": 60, + "latency_p50_seconds": 0.8057119846343994, + "latency_p95_seconds": 1.4087231159210205, + "latency_p99_seconds": 1.4897749423980713, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.1492, + "mysql_core_seconds_per_workflow": 0.3278 + }, + "rate-0.5-2": { + "seconds": 120.6, + "health": { + "replies": 169, + "failures": 0, + "slowest_ok_seconds": 0.606 + }, + "cpu_core_seconds": { + "mysql": 20.18183, + "native-server": 9.157594, + "probe": -0.046921, + "sdk-worker": 1.216679 + }, + "max_memory_bytes": { + "mysql": 626311168, + "native-server": 6995968, + "probe": 16932864, + "sdk-worker": 30769152 + }, + "max_runtime_memory_bytes": 6995968, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.5, + "planned_starts": 60, + "accepted_starts": 60, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 60, + "pending_at_deadline": 0, + "closed_during_offer_window": 60, + "latency_p50_seconds": 0.8616900444030762, + "latency_p95_seconds": 1.3606820106506348, + "latency_p99_seconds": 1.7296950817108154, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.1526, + "mysql_core_seconds_per_workflow": 0.3364 + }, + "rate-0.5-3": { + "seconds": 119.8, + "health": { + "replies": 168, + "failures": 0, + "slowest_ok_seconds": 0.595 + }, + "cpu_core_seconds": { + "mysql": 19.68832, + "native-server": 8.950379, + "probe": -0.012808, + "sdk-worker": 1.201273 + }, + "max_memory_bytes": { + "mysql": 647774208, + "native-server": 7008256, + "probe": 16949248, + "sdk-worker": 33390592 + }, + "max_runtime_memory_bytes": 7008256, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.5, + "planned_starts": 60, + "accepted_starts": 60, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 60, + "pending_at_deadline": 0, + "closed_during_offer_window": 60, + "latency_p50_seconds": 0.8583979606628418, + "latency_p95_seconds": 1.4343540668487549, + "latency_p99_seconds": 1.5413579940795898, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.1492, + "mysql_core_seconds_per_workflow": 0.3281 + }, + "rate-1-1": { + "seconds": 121.3, + "health": { + "replies": 158, + "failures": 0, + "slowest_ok_seconds": 0.763 + }, + "cpu_core_seconds": { + "mysql": 28.940648, + "native-server": 12.366946, + "probe": 0.301992, + "sdk-worker": 2.071431 + }, + "max_memory_bytes": { + "mysql": 691490816, + "native-server": 7041024, + "probe": 16936960, + "sdk-worker": 36601856 + }, + "max_runtime_memory_bytes": 7041024, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 1, + "planned_starts": 120, + "accepted_starts": 120, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 120, + "pending_at_deadline": 0, + "closed_during_offer_window": 120, + "latency_p50_seconds": 0.642268180847168, + "latency_p95_seconds": 1.1391160488128662, + "latency_p99_seconds": 1.6041810512542725, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.1031, + "mysql_core_seconds_per_workflow": 0.2412 + }, + "rate-1-2": { + "seconds": 121.3, + "health": { + "replies": 156, + "failures": 0, + "slowest_ok_seconds": 0.764 + }, + "cpu_core_seconds": { + "mysql": 29.034662, + "native-server": 12.278764, + "probe": -0.090032, + "sdk-worker": 2.080174 + }, + "max_memory_bytes": { + "mysql": 735203328, + "native-server": 7110656, + "probe": 18374656, + "sdk-worker": 36524032 + }, + "max_runtime_memory_bytes": 7110656, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 1, + "planned_starts": 120, + "accepted_starts": 120, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 120, + "pending_at_deadline": 0, + "closed_during_offer_window": 120, + "latency_p50_seconds": 0.6831719875335693, + "latency_p95_seconds": 2.1111860275268555, + "latency_p99_seconds": 2.5451819896698, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.1023, + "mysql_core_seconds_per_workflow": 0.242 + }, + "rate-1-3": { + "seconds": 121.6, + "health": { + "replies": 156, + "failures": 0, + "slowest_ok_seconds": 0.717 + }, + "cpu_core_seconds": { + "mysql": 29.387882, + "native-server": 12.440495, + "probe": 0.039508, + "sdk-worker": 2.065757 + }, + "max_memory_bytes": { + "mysql": 777027584, + "native-server": 7311360, + "probe": 16703488, + "sdk-worker": 37605376 + }, + "max_runtime_memory_bytes": 7311360, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 1, + "planned_starts": 120, + "accepted_starts": 120, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 120, + "pending_at_deadline": 0, + "closed_during_offer_window": 120, + "latency_p50_seconds": 0.6876001358032227, + "latency_p95_seconds": 1.6906061172485352, + "latency_p99_seconds": 2.5086309909820557, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.1037, + "mysql_core_seconds_per_workflow": 0.2449 + }, + "low-after-1": { + "seconds": 120.1, + "health": { + "replies": 180, + "failures": 0, + "slowest_ok_seconds": 0.387 + }, + "cpu_core_seconds": { + "mysql": 15.418894, + "native-server": 7.249945, + "probe": -0.469773, + "sdk-worker": 0.772241 + }, + "max_memory_bytes": { + "mysql": 788262912, + "native-server": 6823936, + "probe": 16687104, + "sdk-worker": 37871616 + }, + "max_runtime_memory_bytes": 6823936, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 30, + "accepted_starts": 30, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 30, + "pending_at_deadline": 0, + "closed_during_offer_window": 30, + "latency_p50_seconds": 1.120251178741455, + "latency_p95_seconds": 1.8871901035308838, + "latency_p99_seconds": 1.9591500759124756, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.2417, + "mysql_core_seconds_per_workflow": 0.514 + }, + "low-after-2": { + "seconds": 120.2, + "health": { + "replies": 180, + "failures": 0, + "slowest_ok_seconds": 0.429 + }, + "cpu_core_seconds": { + "mysql": 15.669081, + "native-server": 7.285336, + "probe": 0.020311, + "sdk-worker": 0.811745 + }, + "max_memory_bytes": { + "mysql": 794497024, + "native-server": 7094272, + "probe": 16699392, + "sdk-worker": 38146048 + }, + "max_runtime_memory_bytes": 7094272, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 30, + "accepted_starts": 30, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 30, + "pending_at_deadline": 0, + "closed_during_offer_window": 30, + "latency_p50_seconds": 1.0607280731201172, + "latency_p95_seconds": 1.8657190799713135, + "latency_p99_seconds": 1.9156861305236816, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.2428, + "mysql_core_seconds_per_workflow": 0.5223 + }, + "low-after-3": { + "seconds": 120.2, + "health": { + "replies": 180, + "failures": 0, + "slowest_ok_seconds": 0.388 + }, + "cpu_core_seconds": { + "mysql": 15.794167, + "native-server": 7.405617, + "probe": 0.017688, + "sdk-worker": 0.800382 + }, + "max_memory_bytes": { + "mysql": 800604160, + "native-server": 6619136, + "probe": 16941056, + "sdk-worker": 38514688 + }, + "max_runtime_memory_bytes": 6619136, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 30, + "accepted_starts": 30, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 30, + "pending_at_deadline": 0, + "closed_during_offer_window": 30, + "latency_p50_seconds": 1.1393530368804932, + "latency_p95_seconds": 1.6913490295410156, + "latency_p99_seconds": 2.042344093322754, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.2469, + "mysql_core_seconds_per_workflow": 0.5265 + } + }, + "idle": { + "idle-6-1": { + "seconds": 12.3, + "health": { + "replies": 20, + "failures": 0, + "slowest_ok_seconds": 0.104 + }, + "cpu_core_seconds": { + "mysql": 1.271815, + "native-server": 0.678925, + "probe": -0.55062, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 801001472, + "native-server": 5025792, + "probe": 31444992, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 5025792, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "requested_polls": 6, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "empty": 6 + } + }, + "idle-6-2": { + "seconds": 11.2, + "health": { + "replies": 18, + "failures": 0, + "slowest_ok_seconds": 0.104 + }, + "cpu_core_seconds": { + "mysql": 1.277346, + "native-server": 0.682507, + "probe": -0.004253, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 804032512, + "native-server": 6746112, + "probe": 31248384, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 6746112, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "requested_polls": 6, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "empty": 6 + } + }, + "idle-6-3": { + "seconds": 11.3, + "health": { + "replies": 17, + "failures": 0, + "slowest_ok_seconds": 0.105 + }, + "cpu_core_seconds": { + "mysql": 1.187882, + "native-server": 0.644606, + "probe": 0.024741, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 802537472, + "native-server": 6971392, + "probe": 31236096, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 6971392, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "requested_polls": 6, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "empty": 6 + } + }, + "idle-12-1": { + "seconds": 11.5, + "health": { + "replies": 17, + "failures": 0, + "slowest_ok_seconds": 0.105 + }, + "cpu_core_seconds": { + "mysql": 1.518828, + "native-server": 0.80815, + "probe": 0.14792, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 801198080, + "native-server": 5578752, + "probe": 45658112, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 5578752, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "requested_polls": 12, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "empty": 12 + } + }, + "idle-12-2": { + "seconds": 11.3, + "health": { + "replies": 18, + "failures": 0, + "slowest_ok_seconds": 0.016 + }, + "cpu_core_seconds": { + "mysql": 1.467164, + "native-server": 0.789659, + "probe": -0.105134, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 801349632, + "native-server": 5386240, + "probe": 44597248, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 5386240, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "requested_polls": 12, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "empty": 12 + } + }, + "idle-12-3": { + "seconds": 11.5, + "health": { + "replies": 16, + "failures": 0, + "slowest_ok_seconds": 0.104 + }, + "cpu_core_seconds": { + "mysql": 1.661435, + "native-server": 0.916948, + "probe": 0.13414, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 801509376, + "native-server": 5398528, + "probe": 43933696, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 5398528, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "requested_polls": 12, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "empty": 12 + } + }, + "idle-24-1": { + "seconds": 11.9, + "health": { + "replies": 18, + "failures": 0, + "slowest_ok_seconds": 0.019 + }, + "cpu_core_seconds": { + "mysql": 2.294075, + "native-server": 1.256186, + "probe": 0.249966, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 801611776, + "native-server": 5935104, + "probe": 68399104, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 5935104, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "requested_polls": 24, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "empty": 24 + } + }, + "idle-24-2": { + "seconds": 12.2, + "health": { + "replies": 18, + "failures": 0, + "slowest_ok_seconds": 0.017 + }, + "cpu_core_seconds": { + "mysql": 2.315425, + "native-server": 1.232444, + "probe": -0.043559, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 801931264, + "native-server": 5935104, + "probe": 67612672, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 5935104, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "requested_polls": 24, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "empty": 24 + } + }, + "idle-24-3": { + "seconds": 12.2, + "health": { + "replies": 19, + "failures": 0, + "slowest_ok_seconds": 0.019 + }, + "cpu_core_seconds": { + "mysql": 2.217726, + "native-server": 1.206354, + "probe": 0.083076, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 801779712, + "native-server": 6131712, + "probe": 73183232, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 6131712, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "requested_polls": 24, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "empty": 24 + } + } + } +} diff --git a/benchmarks/server-port/v3/summary-repeated-native-2-2026-10-20.json b/benchmarks/server-port/v3/summary-repeated-native-2-2026-10-20.json new file mode 100644 index 00000000..36aa1e61 --- /dev/null +++ b/benchmarks/server-port/v3/summary-repeated-native-2-2026-10-20.json @@ -0,0 +1,911 @@ +{ + "identity": { + "implementation": "native", + "compose_project": "dw-port-native-20261020t151845", + "profile": "/home/linuxuser/profile.json", + "profile_sha256": "5e7946a6a506e391510e9222021e60c80d184742b030ee6e2978ab0c0b1e8ac5", + "source_revision": "8957ba27aa24b79ec2ea59e0c37b5ce662d434ab", + "native_binary_sha256": "edc283f130ff5c802fe5a80d56730cc9fe377c01c9c90340e08a68eb0866e2cc", + "smoke": false, + "idle_only": false, + "machine": { + "kernel": "6.8.0-146-generic", + "logical_cpus": 4, + "physical_memory_bytes": 8320786432, + "cpu_model": "AMD EPYC-Rome Processor", + "docker": "29.1.3", + "cgroup_driver": "systemd", + "docker_storage_driver": "overlayfs", + "docker_root_filesystem": "ext4" + } + }, + "windows": { + "setup": { + "seconds": 5.5, + "health": { + "replies": 8, + "failures": 0, + "slowest_ok_seconds": 0.185 + }, + "cpu_core_seconds": { + "mysql": 0.78864, + "native-server": 0.406774, + "probe": 0.006179, + "sdk-worker": 0.046886 + }, + "max_memory_bytes": { + "mysql": 529104896, + "native-server": 4411392, + "probe": 16662528, + "sdk-worker": 18796544 + }, + "max_runtime_memory_bytes": 4411392, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 2, + "accepted_starts": 2, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 2, + "pending_at_deadline": 0, + "closed_during_offer_window": 2, + "latency_p50_seconds": 0.264415979385376, + "latency_p95_seconds": 0.4201939105987549, + "latency_p99_seconds": 0.4201939105987549, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.2034, + "mysql_core_seconds_per_workflow": 0.3943 + }, + "warmup": { + "seconds": 60.4, + "health": { + "replies": 90, + "failures": 0, + "slowest_ok_seconds": 0.434 + }, + "cpu_core_seconds": { + "mysql": 7.903876, + "native-server": 3.848973, + "probe": 0.431635, + "sdk-worker": 0.430859 + }, + "max_memory_bytes": { + "mysql": 544555008, + "native-server": 6356992, + "probe": 16945152, + "sdk-worker": 18952192 + }, + "max_runtime_memory_bytes": 6356992, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 15, + "accepted_starts": 15, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 15, + "pending_at_deadline": 0, + "closed_during_offer_window": 15, + "latency_p50_seconds": 0.7263832092285156, + "latency_p95_seconds": 1.335334062576294, + "latency_p99_seconds": 1.335334062576294, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.2566, + "mysql_core_seconds_per_workflow": 0.5269 + }, + "low-before-1": { + "seconds": 120.3, + "health": { + "replies": 180, + "failures": 0, + "slowest_ok_seconds": 0.344 + }, + "cpu_core_seconds": { + "mysql": 15.542938, + "native-server": 7.685158, + "probe": 0.347865, + "sdk-worker": 0.825953 + }, + "max_memory_bytes": { + "mysql": 557219840, + "native-server": 7204864, + "probe": 18534400, + "sdk-worker": 20742144 + }, + "max_runtime_memory_bytes": 7204864, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 30, + "accepted_starts": 30, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 30, + "pending_at_deadline": 0, + "closed_during_offer_window": 30, + "latency_p50_seconds": 0.9674599170684814, + "latency_p95_seconds": 1.6804499626159668, + "latency_p99_seconds": 1.8725829124450684, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.2562, + "mysql_core_seconds_per_workflow": 0.5181 + }, + "low-before-2": { + "seconds": 119.7, + "health": { + "replies": 180, + "failures": 0, + "slowest_ok_seconds": 0.428 + }, + "cpu_core_seconds": { + "mysql": 15.094367, + "native-server": 7.400587, + "probe": 0.013393, + "sdk-worker": 0.78088 + }, + "max_memory_bytes": { + "mysql": 569982976, + "native-server": 5160960, + "probe": 16703488, + "sdk-worker": 24485888 + }, + "max_runtime_memory_bytes": 5160960, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 30, + "accepted_starts": 30, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 30, + "pending_at_deadline": 0, + "closed_during_offer_window": 30, + "latency_p50_seconds": 0.989130973815918, + "latency_p95_seconds": 1.6551799774169922, + "latency_p99_seconds": 1.8601720333099365, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.2467, + "mysql_core_seconds_per_workflow": 0.5031 + }, + "low-before-3": { + "seconds": 120.0, + "health": { + "replies": 178, + "failures": 0, + "slowest_ok_seconds": 0.519 + }, + "cpu_core_seconds": { + "mysql": 15.215566, + "native-server": 7.359013, + "probe": -0.015717, + "sdk-worker": 0.806679 + }, + "max_memory_bytes": { + "mysql": 581537792, + "native-server": 5169152, + "probe": 16691200, + "sdk-worker": 24260608 + }, + "max_runtime_memory_bytes": 5169152, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 30, + "accepted_starts": 30, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 30, + "pending_at_deadline": 0, + "closed_during_offer_window": 30, + "latency_p50_seconds": 0.9421291351318359, + "latency_p95_seconds": 1.8761250972747803, + "latency_p99_seconds": 1.94413423538208, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.2453, + "mysql_core_seconds_per_workflow": 0.5072 + }, + "rate-0.5-1": { + "seconds": 119.9, + "health": { + "replies": 170, + "failures": 0, + "slowest_ok_seconds": 0.513 + }, + "cpu_core_seconds": { + "mysql": 21.17952, + "native-server": 9.658587, + "probe": 0.251825, + "sdk-worker": 1.308499 + }, + "max_memory_bytes": { + "mysql": 604405760, + "native-server": 6815744, + "probe": 16932864, + "sdk-worker": 27607040 + }, + "max_runtime_memory_bytes": 6815744, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.5, + "planned_starts": 60, + "accepted_starts": 60, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 60, + "pending_at_deadline": 0, + "closed_during_offer_window": 60, + "latency_p50_seconds": 0.874359130859375, + "latency_p95_seconds": 1.5553700923919678, + "latency_p99_seconds": 1.6183700561523438, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.161, + "mysql_core_seconds_per_workflow": 0.353 + }, + "rate-0.5-2": { + "seconds": 120.4, + "health": { + "replies": 166, + "failures": 0, + "slowest_ok_seconds": 0.548 + }, + "cpu_core_seconds": { + "mysql": 21.947495, + "native-server": 9.851621, + "probe": 0.012017, + "sdk-worker": 1.336684 + }, + "max_memory_bytes": { + "mysql": 628731904, + "native-server": 6828032, + "probe": 18669568, + "sdk-worker": 30281728 + }, + "max_runtime_memory_bytes": 6828032, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.5, + "planned_starts": 60, + "accepted_starts": 60, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 60, + "pending_at_deadline": 0, + "closed_during_offer_window": 60, + "latency_p50_seconds": 0.8867721557617188, + "latency_p95_seconds": 1.3797781467437744, + "latency_p99_seconds": 1.711742877960205, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.1642, + "mysql_core_seconds_per_workflow": 0.3658 + }, + "rate-0.5-3": { + "seconds": 120.6, + "health": { + "replies": 165, + "failures": 0, + "slowest_ok_seconds": 0.518 + }, + "cpu_core_seconds": { + "mysql": 24.106008, + "native-server": 10.781074, + "probe": 0.075858, + "sdk-worker": 1.460856 + }, + "max_memory_bytes": { + "mysql": 649342976, + "native-server": 7081984, + "probe": 16957440, + "sdk-worker": 32976896 + }, + "max_runtime_memory_bytes": 7081984, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.5, + "planned_starts": 60, + "accepted_starts": 60, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 60, + "pending_at_deadline": 0, + "closed_during_offer_window": 60, + "latency_p50_seconds": 0.9854328632354736, + "latency_p95_seconds": 1.5133790969848633, + "latency_p99_seconds": 1.9184279441833496, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.1797, + "mysql_core_seconds_per_workflow": 0.4018 + }, + "rate-1-1": { + "seconds": 121.0, + "health": { + "replies": 150, + "failures": 0, + "slowest_ok_seconds": 0.719 + }, + "cpu_core_seconds": { + "mysql": 33.022403, + "native-server": 13.945428, + "probe": 0.224814, + "sdk-worker": 2.466644 + }, + "max_memory_bytes": { + "mysql": 692928512, + "native-server": 7155712, + "probe": 16941056, + "sdk-worker": 36147200 + }, + "max_runtime_memory_bytes": 7155712, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 1, + "planned_starts": 120, + "accepted_starts": 120, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 120, + "pending_at_deadline": 0, + "closed_during_offer_window": 120, + "latency_p50_seconds": 0.7154889106750488, + "latency_p95_seconds": 1.5674889087677002, + "latency_p99_seconds": 2.94549298286438, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.1162, + "mysql_core_seconds_per_workflow": 0.2752 + }, + "rate-1-2": { + "seconds": 122.0, + "health": { + "replies": 153, + "failures": 0, + "slowest_ok_seconds": 0.86 + }, + "cpu_core_seconds": { + "mysql": 32.588333, + "native-server": 13.690904, + "probe": -0.024711, + "sdk-worker": 2.364287 + }, + "max_memory_bytes": { + "mysql": 736538624, + "native-server": 7127040, + "probe": 16945152, + "sdk-worker": 36696064 + }, + "max_runtime_memory_bytes": 7127040, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 1, + "planned_starts": 120, + "accepted_starts": 120, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 120, + "pending_at_deadline": 0, + "closed_during_offer_window": 119, + "latency_p50_seconds": 0.6948199272155762, + "latency_p95_seconds": 1.394827127456665, + "latency_p99_seconds": 1.7588160037994385, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.1141, + "mysql_core_seconds_per_workflow": 0.2716 + }, + "rate-1-3": { + "seconds": 121.1, + "health": { + "replies": 150, + "failures": 0, + "slowest_ok_seconds": 0.728 + }, + "cpu_core_seconds": { + "mysql": 31.711915, + "native-server": 13.246497, + "probe": -0.056814, + "sdk-worker": 2.334332 + }, + "max_memory_bytes": { + "mysql": 778280960, + "native-server": 7151616, + "probe": 16691200, + "sdk-worker": 37113856 + }, + "max_runtime_memory_bytes": 7151616, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 1, + "planned_starts": 120, + "accepted_starts": 120, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 120, + "pending_at_deadline": 0, + "closed_during_offer_window": 120, + "latency_p50_seconds": 0.6748061180114746, + "latency_p95_seconds": 1.1207969188690186, + "latency_p99_seconds": 2.0817408561706543, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.1104, + "mysql_core_seconds_per_workflow": 0.2643 + }, + "low-after-1": { + "seconds": 119.9, + "health": { + "replies": 174, + "failures": 0, + "slowest_ok_seconds": 0.48 + }, + "cpu_core_seconds": { + "mysql": 18.335757, + "native-server": 8.539634, + "probe": -0.419468, + "sdk-worker": 0.943311 + }, + "max_memory_bytes": { + "mysql": 789663744, + "native-server": 7131136, + "probe": 16699392, + "sdk-worker": 37392384 + }, + "max_runtime_memory_bytes": 7131136, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 30, + "accepted_starts": 30, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 30, + "pending_at_deadline": 0, + "closed_during_offer_window": 30, + "latency_p50_seconds": 1.0487730503082275, + "latency_p95_seconds": 1.6618030071258545, + "latency_p99_seconds": 1.9577748775482178, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.2847, + "mysql_core_seconds_per_workflow": 0.6112 + }, + "low-after-2": { + "seconds": 120.2, + "health": { + "replies": 176, + "failures": 0, + "slowest_ok_seconds": 0.483 + }, + "cpu_core_seconds": { + "mysql": 17.32578, + "native-server": 8.043092, + "probe": -0.002517, + "sdk-worker": 0.906545 + }, + "max_memory_bytes": { + "mysql": 795537408, + "native-server": 7127040, + "probe": 16687104, + "sdk-worker": 37650432 + }, + "max_runtime_memory_bytes": 7127040, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 30, + "accepted_starts": 30, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 30, + "pending_at_deadline": 0, + "closed_during_offer_window": 30, + "latency_p50_seconds": 0.989509105682373, + "latency_p95_seconds": 1.8215291500091553, + "latency_p99_seconds": 1.8415191173553467, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.2681, + "mysql_core_seconds_per_workflow": 0.5775 + }, + "low-after-3": { + "seconds": 119.1, + "health": { + "replies": 174, + "failures": 0, + "slowest_ok_seconds": 0.356 + }, + "cpu_core_seconds": { + "mysql": 17.427466, + "native-server": 8.173384, + "probe": -0.029393, + "sdk-worker": 0.915135 + }, + "max_memory_bytes": { + "mysql": 801325056, + "native-server": 7168000, + "probe": 18608128, + "sdk-worker": 39112704 + }, + "max_runtime_memory_bytes": 7168000, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 30, + "accepted_starts": 30, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 30, + "pending_at_deadline": 0, + "closed_during_offer_window": 30, + "latency_p50_seconds": 0.9510529041290283, + "latency_p95_seconds": 1.502997875213623, + "latency_p99_seconds": 1.7160539627075195, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.2724, + "mysql_core_seconds_per_workflow": 0.5809 + } + }, + "idle": { + "idle-6-1": { + "seconds": 12.4, + "health": { + "replies": 18, + "failures": 0, + "slowest_ok_seconds": 0.601 + }, + "cpu_core_seconds": { + "mysql": 1.388882, + "native-server": 0.694483, + "probe": -0.556331, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 801333248, + "native-server": 6881280, + "probe": 31330304, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 6881280, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "requested_polls": 6, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "empty": 6 + } + }, + "idle-6-2": { + "seconds": 11.2, + "health": { + "replies": 17, + "failures": 0, + "slowest_ok_seconds": 0.107 + }, + "cpu_core_seconds": { + "mysql": 1.305827, + "native-server": 0.755374, + "probe": -0.025734, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 801746944, + "native-server": 5115904, + "probe": 30846976, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 5115904, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "requested_polls": 6, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "empty": 6 + } + }, + "idle-6-3": { + "seconds": 11.4, + "health": { + "replies": 18, + "failures": 0, + "slowest_ok_seconds": 0.09 + }, + "cpu_core_seconds": { + "mysql": 1.233457, + "native-server": 0.683825, + "probe": 0.043972, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 801890304, + "native-server": 5021696, + "probe": 31555584, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 5021696, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "requested_polls": 6, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "empty": 6 + } + }, + "idle-12-1": { + "seconds": 11.8, + "health": { + "replies": 19, + "failures": 0, + "slowest_ok_seconds": 0.101 + }, + "cpu_core_seconds": { + "mysql": 1.61893, + "native-server": 0.822211, + "probe": 0.205678, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 802017280, + "native-server": 5345280, + "probe": 45867008, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 5345280, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "requested_polls": 12, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "empty": 12 + } + }, + "idle-12-2": { + "seconds": 11.4, + "health": { + "replies": 18, + "failures": 0, + "slowest_ok_seconds": 0.105 + }, + "cpu_core_seconds": { + "mysql": 1.645461, + "native-server": 0.921932, + "probe": -0.173109, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 802181120, + "native-server": 5292032, + "probe": 42070016, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 5292032, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "requested_polls": 12, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "empty": 12 + } + }, + "idle-12-3": { + "seconds": 11.5, + "health": { + "replies": 18, + "failures": 0, + "slowest_ok_seconds": 0.098 + }, + "cpu_core_seconds": { + "mysql": 1.590614, + "native-server": 0.873969, + "probe": 0.067627, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 802652160, + "native-server": 5328896, + "probe": 42647552, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 5328896, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "requested_polls": 12, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "empty": 12 + } + }, + "idle-24-1": { + "seconds": 12.2, + "health": { + "replies": 20, + "failures": 0, + "slowest_ok_seconds": 0.102 + }, + "cpu_core_seconds": { + "mysql": 2.273613, + "native-server": 1.241196, + "probe": 0.300321, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 802402304, + "native-server": 7852032, + "probe": 70094848, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 7852032, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "requested_polls": 24, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "empty": 24 + } + }, + "idle-24-2": { + "seconds": 11.7, + "health": { + "replies": 17, + "failures": 0, + "slowest_ok_seconds": 0.107 + }, + "cpu_core_seconds": { + "mysql": 2.244408, + "native-server": 1.18888, + "probe": -0.185103, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 802770944, + "native-server": 8011776, + "probe": 58925056, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 8011776, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "requested_polls": 24, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "empty": 24 + } + }, + "idle-24-3": { + "seconds": 12.2, + "health": { + "replies": 18, + "failures": 0, + "slowest_ok_seconds": 0.101 + }, + "cpu_core_seconds": { + "mysql": 2.191166, + "native-server": 1.235365, + "probe": 0.126906, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 802361344, + "native-server": 5984256, + "probe": 69505024, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 5984256, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "requested_polls": 24, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "empty": 24 + } + } + } +} diff --git a/benchmarks/server-port/v3/summary-repeated-php-1-2026-10-20.json b/benchmarks/server-port/v3/summary-repeated-php-1-2026-10-20.json new file mode 100644 index 00000000..62de913d --- /dev/null +++ b/benchmarks/server-port/v3/summary-repeated-php-1-2026-10-20.json @@ -0,0 +1,1124 @@ +{ + "identity": { + "implementation": "php", + "compose_project": "dw-port-php-20261020t134400", + "profile": "/home/linuxuser/profile.json", + "profile_sha256": "5e7946a6a506e391510e9222021e60c80d184742b030ee6e2978ab0c0b1e8ac5", + "source_revision": "8957ba27aa24b79ec2ea59e0c37b5ce662d434ab", + "native_binary_sha256": "edc283f130ff5c802fe5a80d56730cc9fe377c01c9c90340e08a68eb0866e2cc", + "smoke": false, + "idle_only": false, + "machine": { + "kernel": "6.8.0-146-generic", + "logical_cpus": 4, + "physical_memory_bytes": 8320786432, + "cpu_model": "AMD EPYC-Rome Processor", + "docker": "29.1.3", + "cgroup_driver": "systemd", + "docker_storage_driver": "overlayfs", + "docker_root_filesystem": "ext4" + } + }, + "windows": { + "setup": { + "seconds": 6.6, + "health": { + "replies": 8, + "failures": 0, + "slowest_ok_seconds": 0.874 + }, + "cpu_core_seconds": { + "mysql": 1.504809, + "php-scheduler": 1.649437, + "php-server": 5.056903, + "php-worker": 0.023939, + "probe": 0.007121, + "redis": 0.182108, + "sdk-worker": 0.052484 + }, + "max_memory_bytes": { + "mysql": 569229312, + "php-scheduler": 43175936, + "php-server": 91193344, + "php-worker": 49610752, + "probe": 19148800, + "redis": 3719168, + "sdk-worker": 20959232 + }, + "max_runtime_memory_bytes": 187621376, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 2, + "accepted_starts": 2, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 2, + "pending_at_deadline": 0, + "closed_during_offer_window": 1, + "latency_p50_seconds": 1.3568880558013916, + "latency_p95_seconds": 2.585994005203247, + "latency_p99_seconds": 2.585994005203247, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 3.4562, + "mysql_core_seconds_per_workflow": 0.7524 + }, + "warmup": { + "seconds": 59.2, + "health": { + "replies": 78, + "failures": 0, + "slowest_ok_seconds": 0.863 + }, + "cpu_core_seconds": { + "mysql": 9.797015, + "php-scheduler": 6.664555, + "php-server": 30.112366, + "php-worker": 0.394338, + "probe": 0.365943, + "redis": 1.656685, + "sdk-worker": 0.485625 + }, + "max_memory_bytes": { + "mysql": 585056256, + "php-scheduler": 48877568, + "php-server": 97693696, + "php-worker": 49614848, + "probe": 19161088, + "redis": 4448256, + "sdk-worker": 22761472 + }, + "max_runtime_memory_bytes": 195813376, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 15, + "accepted_starts": 15, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 15, + "pending_at_deadline": 0, + "closed_during_offer_window": 15, + "latency_p50_seconds": 1.0513579845428467, + "latency_p95_seconds": 1.5864241123199463, + "latency_p99_seconds": 1.5864241123199463, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 2.5885, + "mysql_core_seconds_per_workflow": 0.6531 + }, + "low-before-1": { + "seconds": 120.2, + "health": { + "replies": 158, + "failures": 0, + "slowest_ok_seconds": 0.584 + }, + "cpu_core_seconds": { + "mysql": 20.040341, + "php-scheduler": 13.497456, + "php-server": 60.623072, + "php-worker": 0.798438, + "probe": 0.37686, + "redis": 3.363789, + "sdk-worker": 0.944666 + }, + "max_memory_bytes": { + "mysql": 605970432, + "php-scheduler": 52334592, + "php-server": 109006848, + "php-worker": 50028544, + "probe": 20021248, + "redis": 6221824, + "sdk-worker": 25751552 + }, + "max_runtime_memory_bytes": 198287360, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 30, + "accepted_starts": 30, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 30, + "pending_at_deadline": 0, + "closed_during_offer_window": 30, + "latency_p50_seconds": 1.128861904144287, + "latency_p95_seconds": 1.4585249423980713, + "latency_p99_seconds": 1.537226915359497, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 2.6094, + "mysql_core_seconds_per_workflow": 0.668 + }, + "low-before-2": { + "seconds": 120.2, + "health": { + "replies": 158, + "failures": 0, + "slowest_ok_seconds": 0.702 + }, + "cpu_core_seconds": { + "mysql": 19.857788, + "php-scheduler": 12.698215, + "php-server": 60.391955, + "php-worker": 0.785859, + "probe": 0.01672, + "redis": 3.357225, + "sdk-worker": 0.950209 + }, + "max_memory_bytes": { + "mysql": 621260800, + "php-scheduler": 51154944, + "php-server": 104452096, + "php-worker": 50839552, + "probe": 19791872, + "redis": 6172672, + "sdk-worker": 28938240 + }, + "max_runtime_memory_bytes": 195870720, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 30, + "accepted_starts": 30, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 30, + "pending_at_deadline": 0, + "closed_during_offer_window": 30, + "latency_p50_seconds": 1.1518518924713135, + "latency_p95_seconds": 1.4961800575256348, + "latency_p99_seconds": 1.5145728588104248, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 2.5744, + "mysql_core_seconds_per_workflow": 0.6619 + }, + "low-before-3": { + "seconds": 120.2, + "health": { + "replies": 156, + "failures": 0, + "slowest_ok_seconds": 0.858 + }, + "cpu_core_seconds": { + "mysql": 21.376865, + "php-scheduler": 14.672424, + "php-server": 63.623704, + "php-worker": 0.823863, + "probe": 0.05507, + "redis": 3.531226, + "sdk-worker": 1.027329 + }, + "max_memory_bytes": { + "mysql": 638160896, + "php-scheduler": 51429376, + "php-server": 112340992, + "php-worker": 51687424, + "probe": 21409792, + "redis": 6946816, + "sdk-worker": 31318016 + }, + "max_runtime_memory_bytes": 201371648, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 30, + "accepted_starts": 30, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 30, + "pending_at_deadline": 0, + "closed_during_offer_window": 30, + "latency_p50_seconds": 1.187798023223877, + "latency_p95_seconds": 1.6202080249786377, + "latency_p99_seconds": 1.651698112487793, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 2.755, + "mysql_core_seconds_per_workflow": 0.7126 + }, + "rate-0.5-1": { + "seconds": 122.4, + "health": { + "replies": 154, + "failures": 0, + "slowest_ok_seconds": 0.787 + }, + "cpu_core_seconds": { + "mysql": 34.159995, + "php-scheduler": 14.081556, + "php-server": 90.804063, + "php-worker": 0.827854, + "probe": 0.356474, + "redis": 4.042836, + "sdk-worker": 1.722026 + }, + "max_memory_bytes": { + "mysql": 672313344, + "php-scheduler": 50196480, + "php-server": 106352640, + "php-worker": 53936128, + "probe": 20967424, + "redis": 7548928, + "sdk-worker": 35209216 + }, + "max_runtime_memory_bytes": 204369920, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.5, + "planned_starts": 60, + "accepted_starts": 60, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 60, + "pending_at_deadline": 0, + "closed_during_offer_window": 60, + "latency_p50_seconds": 1.1291561126708984, + "latency_p95_seconds": 1.526949167251587, + "latency_p99_seconds": 1.633439064025879, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 1.8293, + "mysql_core_seconds_per_workflow": 0.5693 + }, + "rate-0.5-2": { + "seconds": 122.9, + "health": { + "replies": 151, + "failures": 0, + "slowest_ok_seconds": 0.809 + }, + "cpu_core_seconds": { + "mysql": 34.1152, + "php-scheduler": 13.440402, + "php-server": 90.610046, + "php-worker": 0.799173, + "probe": 0.01692, + "redis": 4.091798, + "sdk-worker": 1.702909 + }, + "max_memory_bytes": { + "mysql": 702459904, + "php-scheduler": 49590272, + "php-server": 110915584, + "php-worker": 52420608, + "probe": 20811776, + "redis": 6348800, + "sdk-worker": 38711296 + }, + "max_runtime_memory_bytes": 202567680, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.5, + "planned_starts": 60, + "accepted_starts": 60, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 60, + "pending_at_deadline": 0, + "closed_during_offer_window": 60, + "latency_p50_seconds": 1.1356868743896484, + "latency_p95_seconds": 1.5167322158813477, + "latency_p99_seconds": 1.6396760940551758, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 1.8157, + "mysql_core_seconds_per_workflow": 0.5686 + }, + "rate-0.5-3": { + "seconds": 122.1, + "health": { + "replies": 147, + "failures": 0, + "slowest_ok_seconds": 0.868 + }, + "cpu_core_seconds": { + "mysql": 35.826468, + "php-scheduler": 14.224871, + "php-server": 93.479474, + "php-worker": 0.827446, + "probe": -0.005189, + "redis": 4.158363, + "sdk-worker": 1.809416 + }, + "max_memory_bytes": { + "mysql": 733597696, + "php-scheduler": 51208192, + "php-server": 120877056, + "php-worker": 52404224, + "probe": 19980288, + "redis": 7733248, + "sdk-worker": 38957056 + }, + "max_runtime_memory_bytes": 226398208, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.5, + "planned_starts": 60, + "accepted_starts": 60, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 60, + "pending_at_deadline": 0, + "closed_during_offer_window": 60, + "latency_p50_seconds": 1.2154319286346436, + "latency_p95_seconds": 1.5453448295593262, + "latency_p99_seconds": 1.6753089427947998, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 1.8782, + "mysql_core_seconds_per_workflow": 0.5971 + }, + "rate-1-1": { + "seconds": 176.8, + "health": { + "replies": 204, + "failures": 0, + "slowest_ok_seconds": 0.884 + }, + "cpu_core_seconds": { + "mysql": 66.727563, + "php-scheduler": 21.013864, + "php-server": 164.107583, + "php-worker": 1.188324, + "probe": 0.830886, + "redis": 6.423366, + "sdk-worker": 3.396652 + }, + "max_memory_bytes": { + "mysql": 781090816, + "php-scheduler": 51212288, + "php-server": 118976512, + "php-worker": 52404224, + "probe": 19783680, + "redis": 8462336, + "sdk-worker": 39272448 + }, + "max_runtime_memory_bytes": 207253504, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 1, + "planned_starts": 120, + "accepted_starts": 120, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 120, + "pending_at_deadline": 0, + "closed_during_offer_window": 69, + "latency_p50_seconds": 44.781501054763794, + "latency_p95_seconds": 64.77137804031372, + "latency_p99_seconds": 66.02623295783997, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 1.6061, + "mysql_core_seconds_per_workflow": 0.5561 + }, + "rate-1-2": { + "seconds": 178.3, + "health": { + "replies": 210, + "failures": 0, + "slowest_ok_seconds": 0.787 + }, + "cpu_core_seconds": { + "mysql": 66.481232, + "php-scheduler": 20.010903, + "php-server": 164.224564, + "php-worker": 1.228025, + "probe": 0.038332, + "redis": 6.447414, + "sdk-worker": 3.377742 + }, + "max_memory_bytes": { + "mysql": 818790400, + "php-scheduler": 48791552, + "php-server": 114503680, + "php-worker": 52760576, + "probe": 20025344, + "redis": 8364032, + "sdk-worker": 40349696 + }, + "max_runtime_memory_bytes": 210096128, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 1, + "planned_starts": 120, + "accepted_starts": 120, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 120, + "pending_at_deadline": 0, + "closed_during_offer_window": 68, + "latency_p50_seconds": 46.07006311416626, + "latency_p95_seconds": 65.88363909721375, + "latency_p99_seconds": 67.44143986701965, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 1.5993, + "mysql_core_seconds_per_workflow": 0.554 + }, + "rate-1-3": { + "seconds": 181.3, + "health": { + "replies": 208, + "failures": 0, + "slowest_ok_seconds": 0.875 + }, + "cpu_core_seconds": { + "mysql": 67.915662, + "php-scheduler": 20.903777, + "php-server": 167.68432, + "php-worker": 1.2719, + "probe": 0.061674, + "redis": 6.673205, + "sdk-worker": 3.479508 + }, + "max_memory_bytes": { + "mysql": 856571904, + "php-scheduler": 52236288, + "php-server": 118317056, + "php-worker": 54628352, + "probe": 19775488, + "redis": 8306688, + "sdk-worker": 41897984 + }, + "max_runtime_memory_bytes": 210964480, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 1, + "planned_starts": 120, + "accepted_starts": 120, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 120, + "pending_at_deadline": 0, + "closed_during_offer_window": 66, + "latency_p50_seconds": 48.342694997787476, + "latency_p95_seconds": 67.38858103752136, + "latency_p99_seconds": 69.02710199356079, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 1.6378, + "mysql_core_seconds_per_workflow": 0.566 + }, + "low-after-1": { + "seconds": 119.4, + "health": { + "replies": 152, + "failures": 0, + "slowest_ok_seconds": 0.782 + }, + "cpu_core_seconds": { + "mysql": 22.179925, + "php-scheduler": 14.484918, + "php-server": 64.695478, + "php-worker": 0.829711, + "probe": -1.265806, + "redis": 3.54183, + "sdk-worker": 1.127742 + }, + "max_memory_bytes": { + "mysql": 866603008, + "php-scheduler": 47493120, + "php-server": 114991104, + "php-worker": 54050816, + "probe": 19767296, + "redis": 7155712, + "sdk-worker": 41742336 + }, + "max_runtime_memory_bytes": 201912320, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 30, + "accepted_starts": 30, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 30, + "pending_at_deadline": 0, + "closed_during_offer_window": 30, + "latency_p50_seconds": 1.2256431579589844, + "latency_p95_seconds": 1.61228609085083, + "latency_p99_seconds": 1.7355849742889404, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 2.7851, + "mysql_core_seconds_per_workflow": 0.7393 + }, + "low-after-2": { + "seconds": 120.2, + "health": { + "replies": 154, + "failures": 0, + "slowest_ok_seconds": 0.683 + }, + "cpu_core_seconds": { + "mysql": 21.854782, + "php-scheduler": 13.381766, + "php-server": 63.860201, + "php-worker": 0.806617, + "probe": -0.048811, + "redis": 3.524795, + "sdk-worker": 1.107556 + }, + "max_memory_bytes": { + "mysql": 875786240, + "php-scheduler": 52068352, + "php-server": 111415296, + "php-worker": 52908032, + "probe": 19775488, + "redis": 6717440, + "sdk-worker": 41689088 + }, + "max_runtime_memory_bytes": 197271552, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 30, + "accepted_starts": 30, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 30, + "pending_at_deadline": 0, + "closed_during_offer_window": 30, + "latency_p50_seconds": 1.197011947631836, + "latency_p95_seconds": 1.7318499088287354, + "latency_p99_seconds": 1.8175239562988281, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 2.7191, + "mysql_core_seconds_per_workflow": 0.7285 + }, + "low-after-3": { + "seconds": 120.0, + "health": { + "replies": 156, + "failures": 0, + "slowest_ok_seconds": 0.686 + }, + "cpu_core_seconds": { + "mysql": 21.31017, + "php-scheduler": 13.8905, + "php-server": 62.803105, + "php-worker": 0.825999, + "probe": 0.016519, + "redis": 3.496282, + "sdk-worker": 1.113688 + }, + "max_memory_bytes": { + "mysql": 885342208, + "php-scheduler": 47980544, + "php-server": 122150912, + "php-worker": 52764672, + "probe": 20742144, + "redis": 7487488, + "sdk-worker": 43237376 + }, + "max_runtime_memory_bytes": 195055616, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 30, + "accepted_starts": 30, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 30, + "pending_at_deadline": 0, + "closed_during_offer_window": 30, + "latency_p50_seconds": 1.1954889297485352, + "latency_p95_seconds": 1.56825590133667, + "latency_p99_seconds": 1.68644118309021, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 2.7005, + "mysql_core_seconds_per_workflow": 0.7103 + } + }, + "idle": { + "idle-6-1": { + "seconds": 12.9, + "health": { + "replies": 16, + "failures": 0, + "slowest_ok_seconds": 1.431 + }, + "cpu_core_seconds": { + "mysql": 1.119762, + "php-scheduler": 1.189401, + "php-server": 4.417996, + "php-worker": 0.084415, + "probe": -0.557917, + "redis": 0.425557, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 886943744, + "php-scheduler": 43503616, + "php-server": 117972992, + "php-worker": 54337536, + "probe": 31399936, + "redis": 4829184, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 205033472, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "requested_polls": 6, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "empty": 6 + } + }, + "idle-6-2": { + "seconds": 13.0, + "health": { + "replies": 16, + "failures": 0, + "slowest_ok_seconds": 1.217 + }, + "cpu_core_seconds": { + "mysql": 1.085944, + "php-scheduler": 2.023353, + "php-server": 4.351547, + "php-worker": 0.100568, + "probe": -0.046043, + "redis": 0.401379, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 887758848, + "php-scheduler": 43229184, + "php-server": 104980480, + "php-worker": 52760576, + "probe": 30400512, + "redis": 4653056, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 190066688, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "requested_polls": 6, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "empty": 6 + } + }, + "idle-6-3": { + "seconds": 13.1, + "health": { + "replies": 16, + "failures": 0, + "slowest_ok_seconds": 0.549 + }, + "cpu_core_seconds": { + "mysql": 1.005154, + "php-scheduler": 1.77791, + "php-server": 4.344363, + "php-worker": 0.079783, + "probe": -0.002653, + "redis": 0.39114, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 887844864, + "php-scheduler": 42561536, + "php-server": 101949440, + "php-worker": 52764672, + "probe": 31424512, + "redis": 4689920, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 183525376, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "requested_polls": 6, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "empty": 6 + } + }, + "idle-12-1": { + "seconds": 15.3, + "health": { + "replies": 16, + "failures": 1, + "slowest_ok_seconds": 1.301 + }, + "cpu_core_seconds": { + "mysql": 1.450981, + "php-scheduler": 1.344831, + "php-server": 5.344161, + "php-worker": 0.089381, + "probe": 0.11171, + "redis": 0.501114, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 925487104, + "php-scheduler": 46772224, + "php-server": 139833344, + "php-worker": 52760576, + "probe": 46059520, + "redis": 4550656, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 198746112, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "requested_polls": 12, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "backpressure": 4, + "empty": 8 + } + }, + "idle-12-2": { + "seconds": 16.1, + "health": { + "replies": 16, + "failures": 1, + "slowest_ok_seconds": 1.708 + }, + "cpu_core_seconds": { + "mysql": 1.617205, + "php-scheduler": 2.118904, + "php-server": 5.849974, + "php-worker": 0.090799, + "probe": 0.101958, + "redis": 0.582651, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 925876224, + "php-scheduler": 36306944, + "php-server": 131977216, + "php-worker": 52764672, + "probe": 46710784, + "redis": 4497408, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 218853376, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "requested_polls": 12, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "empty": 8, + "backpressure": 4 + } + }, + "idle-12-3": { + "seconds": 16.1, + "health": { + "replies": 17, + "failures": 1, + "slowest_ok_seconds": 1.476 + }, + "cpu_core_seconds": { + "mysql": 1.445036, + "php-scheduler": 1.548843, + "php-server": 5.645729, + "php-worker": 0.130054, + "probe": 0.086832, + "redis": 0.520426, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 924483584, + "php-scheduler": 44150784, + "php-server": 134873088, + "php-worker": 52760576, + "probe": 46292992, + "redis": 4390912, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 232247296, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "requested_polls": 12, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "empty": 8, + "backpressure": 4 + } + }, + "idle-24-1": { + "seconds": 17.2, + "health": { + "replies": 18, + "failures": 2, + "slowest_ok_seconds": 0.896 + }, + "cpu_core_seconds": { + "mysql": 1.803645, + "php-scheduler": 2.060201, + "php-server": 6.451095, + "php-worker": 0.092034, + "probe": 0.219455, + "redis": 0.55685, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 931160064, + "php-scheduler": 51216384, + "php-server": 132136960, + "php-worker": 52756480, + "probe": 67342336, + "redis": 4497408, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 224530432, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "requested_polls": 24, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "backpressure": 16, + "empty": 8 + } + }, + "idle-24-2": { + "seconds": 17.1, + "health": { + "replies": 17, + "failures": 2, + "slowest_ok_seconds": 1.676 + }, + "cpu_core_seconds": { + "mysql": 1.906241, + "php-scheduler": 1.901284, + "php-server": 6.692275, + "php-worker": 0.122455, + "probe": -0.035132, + "redis": 0.579389, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 936239104, + "php-scheduler": 35033088, + "php-server": 152723456, + "php-worker": 52756480, + "probe": 69390336, + "redis": 5758976, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 217575424, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "requested_polls": 24, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "backpressure": 16, + "empty": 8 + } + }, + "idle-24-3": { + "seconds": 17.2, + "health": { + "replies": 16, + "failures": 2, + "slowest_ok_seconds": 0.916 + }, + "cpu_core_seconds": { + "mysql": 1.92142, + "php-scheduler": 2.029289, + "php-server": 6.846726, + "php-worker": 0.124736, + "probe": 0.083039, + "redis": 0.563508, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 937267200, + "php-scheduler": 43589632, + "php-server": 168722432, + "php-worker": 52764672, + "probe": 72994816, + "redis": 4243456, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 246112256, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "requested_polls": 24, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "backpressure": 16, + "empty": 8 + } + } + } +} diff --git a/benchmarks/server-port/v3/summary-repeated-php-2-2026-10-20.json b/benchmarks/server-port/v3/summary-repeated-php-2-2026-10-20.json new file mode 100644 index 00000000..5d054322 --- /dev/null +++ b/benchmarks/server-port/v3/summary-repeated-php-2-2026-10-20.json @@ -0,0 +1,1124 @@ +{ + "identity": { + "implementation": "php", + "compose_project": "dw-port-php-20261020t144545", + "profile": "/home/linuxuser/profile.json", + "profile_sha256": "5e7946a6a506e391510e9222021e60c80d184742b030ee6e2978ab0c0b1e8ac5", + "source_revision": "8957ba27aa24b79ec2ea59e0c37b5ce662d434ab", + "native_binary_sha256": "edc283f130ff5c802fe5a80d56730cc9fe377c01c9c90340e08a68eb0866e2cc", + "smoke": false, + "idle_only": false, + "machine": { + "kernel": "6.8.0-146-generic", + "logical_cpus": 4, + "physical_memory_bytes": 8320786432, + "cpu_model": "AMD EPYC-Rome Processor", + "docker": "29.1.3", + "cgroup_driver": "systemd", + "docker_storage_driver": "overlayfs", + "docker_root_filesystem": "ext4" + } + }, + "windows": { + "setup": { + "seconds": 6.1, + "health": { + "replies": 8, + "failures": 0, + "slowest_ok_seconds": 0.403 + }, + "cpu_core_seconds": { + "mysql": 1.170159, + "php-scheduler": 1.375, + "php-server": 3.530644, + "php-worker": 0.017897, + "probe": 0.007011, + "redis": 0.170855, + "sdk-worker": 0.042902 + }, + "max_memory_bytes": { + "mysql": 572542976, + "php-scheduler": 43524096, + "php-server": 88588288, + "php-worker": 49451008, + "probe": 19144704, + "redis": 3624960, + "sdk-worker": 20951040 + }, + "max_runtime_memory_bytes": 182153216, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 2, + "accepted_starts": 2, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 2, + "pending_at_deadline": 0, + "closed_during_offer_window": 2, + "latency_p50_seconds": 0.8652470111846924, + "latency_p95_seconds": 1.8545799255371094, + "latency_p99_seconds": 1.8545799255371094, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 2.5472, + "mysql_core_seconds_per_workflow": 0.5851 + }, + "warmup": { + "seconds": 60.8, + "health": { + "replies": 78, + "failures": 0, + "slowest_ok_seconds": 0.723 + }, + "cpu_core_seconds": { + "mysql": 10.668871, + "php-scheduler": 6.668423, + "php-server": 32.386096, + "php-worker": 0.41917, + "probe": 0.372215, + "redis": 1.763566, + "sdk-worker": 0.500587 + }, + "max_memory_bytes": { + "mysql": 593006592, + "php-scheduler": 44797952, + "php-server": 96911360, + "php-worker": 50581504, + "probe": 21094400, + "redis": 5603328, + "sdk-worker": 22888448 + }, + "max_runtime_memory_bytes": 189968384, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 15, + "accepted_starts": 15, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 15, + "pending_at_deadline": 0, + "closed_during_offer_window": 15, + "latency_p50_seconds": 1.2237191200256348, + "latency_p95_seconds": 1.5795209407806396, + "latency_p99_seconds": 1.5795209407806396, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 2.7492, + "mysql_core_seconds_per_workflow": 0.7113 + }, + "low-before-1": { + "seconds": 120.3, + "health": { + "replies": 154, + "failures": 0, + "slowest_ok_seconds": 0.713 + }, + "cpu_core_seconds": { + "mysql": 21.296271, + "php-scheduler": 13.985756, + "php-server": 63.837221, + "php-worker": 0.833778, + "probe": 0.438814, + "redis": 3.467972, + "sdk-worker": 1.000538 + }, + "max_memory_bytes": { + "mysql": 614154240, + "php-scheduler": 47267840, + "php-server": 114008064, + "php-worker": 50053120, + "probe": 20639744, + "redis": 6180864, + "sdk-worker": 25747456 + }, + "max_runtime_memory_bytes": 199602176, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 30, + "accepted_starts": 30, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 30, + "pending_at_deadline": 0, + "closed_during_offer_window": 30, + "latency_p50_seconds": 1.2357759475708008, + "latency_p95_seconds": 1.679650068283081, + "latency_p99_seconds": 1.8180029392242432, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 2.7375, + "mysql_core_seconds_per_workflow": 0.7099 + }, + "low-before-2": { + "seconds": 120.4, + "health": { + "replies": 152, + "failures": 0, + "slowest_ok_seconds": 0.695 + }, + "cpu_core_seconds": { + "mysql": 21.78718, + "php-scheduler": 13.668353, + "php-server": 64.700526, + "php-worker": 0.852338, + "probe": 0.01564, + "redis": 3.596517, + "sdk-worker": 1.001945 + }, + "max_memory_bytes": { + "mysql": 631521280, + "php-scheduler": 52035584, + "php-server": 120164352, + "php-worker": 50896896, + "probe": 19759104, + "redis": 6782976, + "sdk-worker": 28925952 + }, + "max_runtime_memory_bytes": 212869120, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 30, + "accepted_starts": 30, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 30, + "pending_at_deadline": 0, + "closed_during_offer_window": 30, + "latency_p50_seconds": 1.2468769550323486, + "latency_p95_seconds": 1.6789839267730713, + "latency_p99_seconds": 1.6973340511322021, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 2.7606, + "mysql_core_seconds_per_workflow": 0.7262 + }, + "low-before-3": { + "seconds": 120.7, + "health": { + "replies": 152, + "failures": 0, + "slowest_ok_seconds": 0.806 + }, + "cpu_core_seconds": { + "mysql": 22.43567, + "php-scheduler": 14.76506, + "php-server": 65.670336, + "php-worker": 0.866131, + "probe": -0.000866, + "redis": 3.581031, + "sdk-worker": 1.043077 + }, + "max_memory_bytes": { + "mysql": 647872512, + "php-scheduler": 49594368, + "php-server": 113401856, + "php-worker": 52457472, + "probe": 19767296, + "redis": 6266880, + "sdk-worker": 30670848 + }, + "max_runtime_memory_bytes": 201875456, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 30, + "accepted_starts": 30, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 30, + "pending_at_deadline": 0, + "closed_during_offer_window": 30, + "latency_p50_seconds": 1.3055198192596436, + "latency_p95_seconds": 1.649399995803833, + "latency_p99_seconds": 1.7223989963531494, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 2.8294, + "mysql_core_seconds_per_workflow": 0.7479 + }, + "rate-0.5-1": { + "seconds": 122.9, + "health": { + "replies": 150, + "failures": 0, + "slowest_ok_seconds": 0.825 + }, + "cpu_core_seconds": { + "mysql": 35.34101, + "php-scheduler": 13.665336, + "php-server": 93.721084, + "php-worker": 0.846103, + "probe": 0.383153, + "redis": 4.218765, + "sdk-worker": 1.686126 + }, + "max_memory_bytes": { + "mysql": 681295872, + "php-scheduler": 49901568, + "php-server": 117063680, + "php-worker": 54022144, + "probe": 20889600, + "redis": 6488064, + "sdk-worker": 35143680 + }, + "max_runtime_memory_bytes": 207396864, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.5, + "planned_starts": 60, + "accepted_starts": 60, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 60, + "pending_at_deadline": 0, + "closed_during_offer_window": 60, + "latency_p50_seconds": 1.1792030334472656, + "latency_p95_seconds": 1.609131097793579, + "latency_p99_seconds": 1.6938180923461914, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 1.8742, + "mysql_core_seconds_per_workflow": 0.589 + }, + "rate-0.5-2": { + "seconds": 122.1, + "health": { + "replies": 146, + "failures": 0, + "slowest_ok_seconds": 0.793 + }, + "cpu_core_seconds": { + "mysql": 36.534184, + "php-scheduler": 14.546593, + "php-server": 96.016586, + "php-worker": 0.927426, + "probe": 0.072128, + "redis": 4.226208, + "sdk-worker": 1.752287 + }, + "max_memory_bytes": { + "mysql": 713097216, + "php-scheduler": 48873472, + "php-server": 99962880, + "php-worker": 52498432, + "probe": 20951040, + "redis": 8265728, + "sdk-worker": 39034880 + }, + "max_runtime_memory_bytes": 201519104, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.5, + "planned_starts": 60, + "accepted_starts": 60, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 60, + "pending_at_deadline": 0, + "closed_during_offer_window": 60, + "latency_p50_seconds": 1.2057840824127197, + "latency_p95_seconds": 1.6348719596862793, + "latency_p99_seconds": 1.7296020984649658, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 1.9286, + "mysql_core_seconds_per_workflow": 0.6089 + }, + "rate-0.5-3": { + "seconds": 121.9, + "health": { + "replies": 144, + "failures": 0, + "slowest_ok_seconds": 0.812 + }, + "cpu_core_seconds": { + "mysql": 36.688681, + "php-scheduler": 14.632643, + "php-server": 95.642259, + "php-worker": 0.866217, + "probe": -0.083048, + "redis": 4.215405, + "sdk-worker": 1.7989 + }, + "max_memory_bytes": { + "mysql": 742236160, + "php-scheduler": 50565120, + "php-server": 118951936, + "php-worker": 54308864, + "probe": 21495808, + "redis": 7311360, + "sdk-worker": 39960576 + }, + "max_runtime_memory_bytes": 204169216, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.5, + "planned_starts": 60, + "accepted_starts": 60, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 60, + "pending_at_deadline": 0, + "closed_during_offer_window": 60, + "latency_p50_seconds": 1.2680859565734863, + "latency_p95_seconds": 1.557013988494873, + "latency_p99_seconds": 1.8666460514068604, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 1.9226, + "mysql_core_seconds_per_workflow": 0.6115 + }, + "rate-1-1": { + "seconds": 185.9, + "health": { + "replies": 212, + "failures": 0, + "slowest_ok_seconds": 0.911 + }, + "cpu_core_seconds": { + "mysql": 68.790216, + "php-scheduler": 22.00776, + "php-server": 171.008503, + "php-worker": 1.330057, + "probe": 0.87524, + "redis": 6.809588, + "sdk-worker": 3.413805 + }, + "max_memory_bytes": { + "mysql": 788774912, + "php-scheduler": 52244480, + "php-server": 118599680, + "php-worker": 52498432, + "probe": 19771392, + "redis": 7471104, + "sdk-worker": 40652800 + }, + "max_runtime_memory_bytes": 214515712, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 1, + "planned_starts": 120, + "accepted_starts": 120, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 120, + "pending_at_deadline": 0, + "closed_during_offer_window": 62, + "latency_p50_seconds": 55.74392104148865, + "latency_p95_seconds": 72.86986804008484, + "latency_p99_seconds": 74.18884301185608, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 1.6763, + "mysql_core_seconds_per_workflow": 0.5733 + }, + "rate-1-2": { + "seconds": 178.4, + "health": { + "replies": 204, + "failures": 0, + "slowest_ok_seconds": 0.834 + }, + "cpu_core_seconds": { + "mysql": 65.858819, + "php-scheduler": 20.794849, + "php-server": 163.897122, + "php-worker": 1.237972, + "probe": -0.027616, + "redis": 6.473824, + "sdk-worker": 3.309925 + }, + "max_memory_bytes": { + "mysql": 827719680, + "php-scheduler": 51093504, + "php-server": 119918592, + "php-worker": 52858880, + "probe": 21254144, + "redis": 9441280, + "sdk-worker": 41283584 + }, + "max_runtime_memory_bytes": 209801216, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 1, + "planned_starts": 120, + "accepted_starts": 120, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 120, + "pending_at_deadline": 0, + "closed_during_offer_window": 67, + "latency_p50_seconds": 48.43381905555725, + "latency_p95_seconds": 65.49115705490112, + "latency_p99_seconds": 66.32482099533081, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 1.6034, + "mysql_core_seconds_per_workflow": 0.5488 + }, + "rate-1-3": { + "seconds": 181.6, + "health": { + "replies": 208, + "failures": 0, + "slowest_ok_seconds": 0.805 + }, + "cpu_core_seconds": { + "mysql": 67.678031, + "php-scheduler": 21.939475, + "php-server": 167.652277, + "php-worker": 1.27479, + "probe": 0.059808, + "redis": 6.565551, + "sdk-worker": 3.383119 + }, + "max_memory_bytes": { + "mysql": 864178176, + "php-scheduler": 50556928, + "php-server": 119033856, + "php-worker": 52858880, + "probe": 20824064, + "redis": 8126464, + "sdk-worker": 41496576 + }, + "max_runtime_memory_bytes": 223461376, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 1, + "planned_starts": 120, + "accepted_starts": 120, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 120, + "pending_at_deadline": 0, + "closed_during_offer_window": 65, + "latency_p50_seconds": 49.91317009925842, + "latency_p95_seconds": 69.5694899559021, + "latency_p99_seconds": 71.26597094535828, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 1.6453, + "mysql_core_seconds_per_workflow": 0.564 + }, + "low-after-1": { + "seconds": 119.3, + "health": { + "replies": 152, + "failures": 0, + "slowest_ok_seconds": 0.826 + }, + "cpu_core_seconds": { + "mysql": 21.861028, + "php-scheduler": 13.144639, + "php-server": 64.432165, + "php-worker": 0.842851, + "probe": -1.237331, + "redis": 3.57914, + "sdk-worker": 1.100568 + }, + "max_memory_bytes": { + "mysql": 873193472, + "php-scheduler": 51814400, + "php-server": 104218624, + "php-worker": 52879360, + "probe": 19767296, + "redis": 8040448, + "sdk-worker": 40128512 + }, + "max_runtime_memory_bytes": 200564736, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 30, + "accepted_starts": 30, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 30, + "pending_at_deadline": 0, + "closed_during_offer_window": 30, + "latency_p50_seconds": 1.2581219673156738, + "latency_p95_seconds": 1.5643978118896484, + "latency_p99_seconds": 1.730724811553955, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 2.7333, + "mysql_core_seconds_per_workflow": 0.7287 + }, + "low-after-2": { + "seconds": 120.4, + "health": { + "replies": 156, + "failures": 0, + "slowest_ok_seconds": 0.816 + }, + "cpu_core_seconds": { + "mysql": 21.271663, + "php-scheduler": 14.092062, + "php-server": 63.232411, + "php-worker": 0.818295, + "probe": -0.057113, + "redis": 3.492146, + "sdk-worker": 1.08394 + }, + "max_memory_bytes": { + "mysql": 881983488, + "php-scheduler": 50245632, + "php-server": 112705536, + "php-worker": 52858880, + "probe": 19750912, + "redis": 7168000, + "sdk-worker": 40996864 + }, + "max_runtime_memory_bytes": 209444864, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 30, + "accepted_starts": 30, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 30, + "pending_at_deadline": 0, + "closed_during_offer_window": 30, + "latency_p50_seconds": 1.1640570163726807, + "latency_p95_seconds": 1.6432580947875977, + "latency_p99_seconds": 1.6643199920654297, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 2.7212, + "mysql_core_seconds_per_workflow": 0.7091 + }, + "low-after-3": { + "seconds": 120.0, + "health": { + "replies": 152, + "failures": 0, + "slowest_ok_seconds": 0.699 + }, + "cpu_core_seconds": { + "mysql": 21.127632, + "php-scheduler": 14.040819, + "php-server": 62.560674, + "php-worker": 0.81184, + "probe": -0.023795, + "redis": 3.422797, + "sdk-worker": 1.044698 + }, + "max_memory_bytes": { + "mysql": 891281408, + "php-scheduler": 49897472, + "php-server": 114806784, + "php-worker": 54472704, + "probe": 20803584, + "redis": 6688768, + "sdk-worker": 41263104 + }, + "max_runtime_memory_bytes": 207220736, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 30, + "accepted_starts": 30, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 30, + "pending_at_deadline": 0, + "closed_during_offer_window": 30, + "latency_p50_seconds": 1.232442855834961, + "latency_p95_seconds": 1.440964937210083, + "latency_p99_seconds": 1.5305070877075195, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 2.6945, + "mysql_core_seconds_per_workflow": 0.7043 + } + }, + "idle": { + "idle-6-1": { + "seconds": 13.7, + "health": { + "replies": 16, + "failures": 0, + "slowest_ok_seconds": 1.375 + }, + "cpu_core_seconds": { + "mysql": 1.104864, + "php-scheduler": 1.495123, + "php-server": 4.535996, + "php-worker": 0.126425, + "probe": -0.518748, + "redis": 0.493259, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 893804544, + "php-scheduler": 48877568, + "php-server": 104476672, + "php-worker": 52858880, + "probe": 31391744, + "redis": 6451200, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 208138240, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "requested_polls": 6, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "empty": 6 + } + }, + "idle-6-2": { + "seconds": 13.0, + "health": { + "replies": 17, + "failures": 0, + "slowest_ok_seconds": 0.516 + }, + "cpu_core_seconds": { + "mysql": 1.089478, + "php-scheduler": 0.78164, + "php-server": 4.397601, + "php-worker": 0.078596, + "probe": -0.063075, + "redis": 0.441157, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 894009344, + "php-scheduler": 34197504, + "php-server": 124526592, + "php-worker": 52858880, + "probe": 31281152, + "redis": 6389760, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 183857152, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "requested_polls": 6, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "empty": 6 + } + }, + "idle-6-3": { + "seconds": 13.0, + "health": { + "replies": 16, + "failures": 0, + "slowest_ok_seconds": 1.093 + }, + "cpu_core_seconds": { + "mysql": 1.008821, + "php-scheduler": 1.749659, + "php-server": 3.779992, + "php-worker": 0.081498, + "probe": 0.004168, + "redis": 0.373502, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 894189568, + "php-scheduler": 43732992, + "php-server": 121151488, + "php-worker": 52854784, + "probe": 30355456, + "redis": 4386816, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 201003008, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "requested_polls": 6, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "empty": 6 + } + }, + "idle-12-1": { + "seconds": 15.6, + "health": { + "replies": 16, + "failures": 1, + "slowest_ok_seconds": 1.104 + }, + "cpu_core_seconds": { + "mysql": 1.584993, + "php-scheduler": 2.108578, + "php-server": 5.409933, + "php-worker": 0.128848, + "probe": 0.099836, + "redis": 0.543311, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 932577280, + "php-scheduler": 50565120, + "php-server": 124182528, + "php-worker": 52858880, + "probe": 45785088, + "redis": 4153344, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 213549056, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "requested_polls": 12, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "backpressure": 4, + "empty": 8 + } + }, + "idle-12-2": { + "seconds": 16.1, + "health": { + "replies": 16, + "failures": 1, + "slowest_ok_seconds": 1.15 + }, + "cpu_core_seconds": { + "mysql": 1.471128, + "php-scheduler": 1.61049, + "php-server": 5.549401, + "php-worker": 0.099235, + "probe": 0.076546, + "redis": 0.543377, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 934100992, + "php-scheduler": 36921344, + "php-server": 132620288, + "php-worker": 52854784, + "probe": 45920256, + "redis": 5201920, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 196657152, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "requested_polls": 12, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "backpressure": 4, + "empty": 8 + } + }, + "idle-12-3": { + "seconds": 15.9, + "health": { + "replies": 16, + "failures": 1, + "slowest_ok_seconds": 1.554 + }, + "cpu_core_seconds": { + "mysql": 1.400935, + "php-scheduler": 2.024423, + "php-server": 5.508323, + "php-worker": 0.121851, + "probe": 0.015692, + "redis": 0.543894, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 935186432, + "php-scheduler": 44457984, + "php-server": 126164992, + "php-worker": 52858880, + "probe": 46223360, + "redis": 5644288, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 204046336, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "requested_polls": 12, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "backpressure": 4, + "empty": 8 + } + }, + "idle-24-1": { + "seconds": 17.3, + "health": { + "replies": 18, + "failures": 2, + "slowest_ok_seconds": 0.844 + }, + "cpu_core_seconds": { + "mysql": 1.972297, + "php-scheduler": 1.896292, + "php-server": 6.795804, + "php-worker": 0.128665, + "probe": 0.437206, + "redis": 0.587129, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 941764608, + "php-scheduler": 52084736, + "php-server": 137228288, + "php-worker": 52858880, + "probe": 77619200, + "redis": 4280320, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 198823936, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "requested_polls": 24, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "backpressure": 16, + "empty": 8 + } + }, + "idle-24-2": { + "seconds": 17.2, + "health": { + "replies": 16, + "failures": 2, + "slowest_ok_seconds": 0.937 + }, + "cpu_core_seconds": { + "mysql": 2.037846, + "php-scheduler": 2.212841, + "php-server": 6.790381, + "php-worker": 0.097124, + "probe": -0.146052, + "redis": 0.594822, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 947671040, + "php-scheduler": 39317504, + "php-server": 157556736, + "php-worker": 52858880, + "probe": 66371584, + "redis": 4366336, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 235614208, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "requested_polls": 24, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "empty": 8, + "backpressure": 16 + } + }, + "idle-24-3": { + "seconds": 17.3, + "health": { + "replies": 15, + "failures": 3, + "slowest_ok_seconds": 0.441 + }, + "cpu_core_seconds": { + "mysql": 2.059356, + "php-scheduler": 1.905943, + "php-server": 6.98372, + "php-worker": 0.123942, + "probe": -0.076262, + "redis": 0.59964, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 947970048, + "php-scheduler": 50569216, + "php-server": 147697664, + "php-worker": 52854784, + "probe": 61833216, + "redis": 4112384, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 206655488, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "requested_polls": 24, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "backpressure": 16, + "empty": 8 + } + } + } +} From 70f610bef99f1bf1171cba9838f3d0d5a7089198 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 17:27:18 +0000 Subject: [PATCH 42/42] Compare restart history without the public principal; scope the SQLite 2.9.7 cohort Public history now lifts each command's principal onto its event, and worker task history has none, so the recovered terminal resumption compares the fields both views share. The SQLite PHP 2.9.7 upgrade cohort steps set the job's resource scope like their neighbours; the cleanup step failed without it. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/server-parity.yml | 2 ++ scripts/conformance/server-parity/restart.php | 9 ++++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/server-parity.yml b/.github/workflows/server-parity.yml index 681aa9e1..699b5964 100644 --- a/.github/workflows/server-parity.yml +++ b/.github/workflows/server-parity.yml @@ -1671,6 +1671,7 @@ jobs: - name: Back up, take over and resume a published PHP 2.9.7 upgrade source timeout-minutes: 8 env: + RESOURCE_SCOPE: dw-parity-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }} DW_PARITY_UPGRADE_LABEL: php297 DW_PARITY_UPGRADE_EXPECTED_LAYOUT: php-2.9.7 DW_PARITY_UPGRADE_PHP_IMAGE: ghcr.io/durable-workflow/server@sha256:30c34c5f3256a812fbccbe505a068309b58b0c14d01ffff0465d88604cb36d50 @@ -1679,6 +1680,7 @@ jobs: if: always() timeout-minutes: 3 env: + RESOURCE_SCOPE: dw-parity-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }} DW_PARITY_UPGRADE_LABEL: php297 DW_PARITY_UPGRADE_EXPECTED_LAYOUT: php-2.9.7 DW_PARITY_UPGRADE_PHP_IMAGE: ghcr.io/durable-workflow/server@sha256:30c34c5f3256a812fbccbe505a068309b58b0c14d01ffff0465d88604cb36d50 diff --git a/scripts/conformance/server-parity/restart.php b/scripts/conformance/server-parity/restart.php index a72f9497..5ded7dc5 100644 --- a/scripts/conformance/server-parity/restart.php +++ b/scripts/conformance/server-parity/restart.php @@ -53,6 +53,13 @@ function restartControlHistory(array $events): array 'event_type' => $event['event_type'], 'timestamp' => $event['recorded_at'], 'payload' => $event['payload']], $events); } +// Public history lifts each command's principal onto its event; worker task +// history has no principal. Compare the fields both views share. +function restartWithoutPrincipal(array $events): array +{ + return array_map(static fn (array $event): array => array_diff_key($event, ['principal' => true]), $events); +} + function restartSignalWorkflow(WorkflowContext $context, array $input): array { $context->waitCondition(fn (): bool => count($context->signals('payload')) > 0, 'restart.payload'); @@ -336,7 +343,7 @@ function restartTerminalWorkflow(WorkflowContext $context, array $input): array $terminalClaim = $client->pollWorkflowTask('restart-terminal-new', 'restart-terminal-v1', 0); assertRestart(is_array($terminalClaim) && $terminalClaim['task_id'] === $terminal['failure_receipt']['next_task_id'] && $terminalClaim['run_id'] === $terminal['run_id'] - && sameRestartValue(restartControlHistory($terminalClaim['history_events']), $terminal['history']) + && sameRestartValue(restartControlHistory($terminalClaim['history_events']), restartWithoutPrincipal($terminal['history'])) && $terminalClaim['history_events'][4]['workflow_task_id'] === $terminalActivity['task_id'], 'Recovered terminal resumption lost original task/run/history.'); $terminalReplay = (new Replayer($client->payloadCodec()))->replay(restartTerminalWorkflow(...), $terminalClaim['history_events'], $client->payloadCodec()->decodeEnvelope($terminalClaim['arguments']), 'restart-terminal-v1', $terminalClaim);