From b07164e843c86586229f77a9393dec62e84faa87 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 03:22:29 +0000 Subject: [PATCH 01/37] Give codec work under the transition lock its own permit Five transitions run Avro work while holding the global transition lock: the rejected-signal command, signal arguments applied by a resumed task, cooperative cancellation requests, task repair and fenced deregistration. They shared the single codec permit with describe, query and listing reads, so under read load a transition waited on the lock for unrelated decodes. A per-call-site probe on the CPU deadline-health cell measured 0.58 to 0.80 s of such waiting in a cancellation request. These transitions now take a separate transition_codec permit. Reads keep signal_codec. Encodings, values and refusals are unchanged. New tests hold the read permit and require a cancellation request, a signal-resumed completion, a fenced deregistration and a range repair to finish within one second. They time out with the old wiring. Part of #325. Co-Authored-By: Claude Opus 5.5 --- docs/rust-server-transition-locking.md | 25 ++- rust/src/runtime/execution.rs | 41 +++-- rust/src/runtime/store.rs | 12 +- rust/src/runtime/transition_codec_tests.rs | 202 +++++++++++++++++++++ 4 files changed, 254 insertions(+), 26 deletions(-) create mode 100644 rust/src/runtime/transition_codec_tests.rs diff --git a/docs/rust-server-transition-locking.md b/docs/rust-server-transition-locking.md index e593eab1a..89eeabe65 100644 --- a/docs/rust-server-transition-locking.md +++ b/docs/rust-server-transition-locking.md @@ -36,6 +36,21 @@ scheduler from sleeping after each pass to a fixed tick made no measurable difference. Deadline health and throughput under load are bounded by how long other transitions hold the lock. +A scratch build that logged lock wait and hold per call site then ran the same +cell on a host shared with other work, so absolute times are approximate: + +- A cancellation request held the lock 0.65 to 0.86 s, almost all of it + waiting for the single codec permit behind describe decodes of the dense + input (0.58 to 0.80 s). With a permit of its own the hold fell to 0.05 to + 0.09 s. Codec work under the lock now has its own permit. +- Activity status checks spent about 1.6 s per run waiting for the lock. They + no longer take it. +- Cancellation delivery and expiry, workflow completions, heartbeats and + activity expiry held the lock 0.2 to 0.9 s during the read wave without codec + work, several times longer than the same transitions before it. Their summary + projections took under 30 ms each. Lowering the decode thread's CPU priority + changed nothing measurable. This remaining cost is not yet attributed. + ## What lengthens lock hold time - **Summary projection at commit.** `Transaction::commit` reorders open waits @@ -43,10 +58,12 @@ other transitions hold the lock. `summary_history` pages through the run's entire history to do so (`terminal_summary.rs`). Every heartbeat and claim therefore costs time proportional to history length while holding the lock. -- **Codec work inside transactions.** Several paths take the single codec - permit and run Avro work on the blocking pool while the transaction is open, - for example the rejected-signal path, continue-as-new, cooperative requests - and repair. +- **Codec work inside transactions.** Five paths run Avro work on the blocking + pool while the transaction is open: the rejected-signal command, signal + arguments applied by a resumed workflow task, cooperative cancellation + requests, task repair and fenced worker deregistration. They take + `transition_codec`, never the read permit `signal_codec`, so they do not + queue behind reads. The CPU work itself still runs under the lock. - **Claim reads.** Workflow claims load the first history page (the requested page size, 100 by default) and compute the history budget under the lock. Query claims parse cached query JSON values. diff --git a/rust/src/runtime/execution.rs b/rust/src/runtime/execution.rs index edb69ac8e..5715137b8 100644 --- a/rust/src/runtime/execution.rs +++ b/rust/src/runtime/execution.rs @@ -25,6 +25,10 @@ enum Storage { #[path = "query_codec_tests.rs"] mod query_codec_tests; +#[cfg(test)] +#[path = "transition_codec_tests.rs"] +mod transition_codec_tests; + #[derive(Clone)] pub struct Runtime { storage: Arc, @@ -32,6 +36,10 @@ pub struct Runtime { pub(crate) role_tokens: [Option>; 3], scheduler: Arc, signal_codec: Arc, + // Codec work done while holding the transition lock takes this permit, + // never `signal_codec`, so a transition never queues behind reads for + // codec capacity. + transition_codec: Arc, pub(super) external: Arc, queries: Arc, } @@ -194,6 +202,7 @@ impl Runtime { role_tokens: [None, None, None], scheduler: Arc::new(TimerScheduler::new()), signal_codec: Arc::new(Semaphore::new(1)), + transition_codec: Arc::new(Semaphore::new(1)), external: Arc::new(external), queries: Arc::new(queries), }; @@ -262,6 +271,7 @@ impl Runtime { role_tokens: self.role_tokens.clone(), scheduler: self.scheduler.clone(), signal_codec: self.signal_codec.clone(), + transition_codec: self.transition_codec.clone(), external: self.external.clone(), queries: self.queries.clone(), } @@ -376,7 +386,7 @@ impl Runtime { name, body, prepared, - self.signal_codec.clone(), + self.transition_codec.clone(), ) .await } @@ -388,7 +398,7 @@ impl Runtime { name, body, prepared, - self.signal_codec.clone(), + self.transition_codec.clone(), ) .await } @@ -400,7 +410,7 @@ impl Runtime { name, body, prepared, - self.signal_codec.clone(), + self.transition_codec.clone(), ) .await } @@ -421,7 +431,7 @@ impl Runtime { run_id, body, context, - self.signal_codec.clone(), + self.transition_codec.clone(), ) .await } @@ -432,7 +442,7 @@ impl Runtime { run_id, body, context, - self.signal_codec.clone(), + self.transition_codec.clone(), ) .await } @@ -443,7 +453,7 @@ impl Runtime { run_id, body, context, - self.signal_codec.clone(), + self.transition_codec.clone(), ) .await } @@ -658,9 +668,8 @@ impl Runtime { ) -> Result<(Vec, Vec)> ); pub(crate) async fn activities(&self, workflow_id: &str, run_id: &str) -> Result { - // Writers take this permit while holding the writer transaction. Finish - // every read before taking it so a listing can never wait on a writer - // that is itself waiting on the permit. + // Finish every read before taking the codec permit, so a listing never + // holds codec capacity while it waits on the database. let (events, executions) = self.activity_metadata_input(workflow_id, run_id).await?; let permit = self .signal_codec @@ -700,7 +709,7 @@ impl Runtime { delegate!(worker_heartbeat(body: Value) -> Result); delegate!(deregister(worker_id: &str) -> Result); pub(crate) async fn deregister_fenced(&self, worker_id: &str, body: Value) -> Result { - let codec = self.signal_codec.clone(); + let codec = self.transition_codec.clone(); match &*self.storage { Storage::Sqlite(store) => store.deregister_fenced(worker_id, body, codec).await, Storage::Postgres(store) => store.deregister_fenced(worker_id, body, codec).await, @@ -761,17 +770,17 @@ impl Runtime { match &*self.storage { Storage::Sqlite(store) => { store - .complete_workflow(task_id, body, self.signal_codec.clone(), external) + .complete_workflow(task_id, body, self.transition_codec.clone(), external) .await } Storage::Postgres(store) => { store - .complete_workflow(task_id, body, self.signal_codec.clone(), external) + .complete_workflow(task_id, body, self.transition_codec.clone(), external) .await } Storage::MySql(store) => { store - .complete_workflow(task_id, body, self.signal_codec.clone(), external) + .complete_workflow(task_id, body, self.transition_codec.clone(), external) .await } } @@ -817,7 +826,7 @@ impl Runtime { run_id, reason, budget, - self.signal_codec.clone(), + self.transition_codec.clone(), ) .await } @@ -828,7 +837,7 @@ impl Runtime { run_id, reason, budget, - self.signal_codec.clone(), + self.transition_codec.clone(), ) .await } @@ -839,7 +848,7 @@ impl Runtime { run_id, reason, budget, - self.signal_codec.clone(), + self.transition_codec.clone(), ) .await } diff --git a/rust/src/runtime/store.rs b/rust/src/runtime/store.rs index 6490d0f95..24ae515ad 100644 --- a/rust/src/runtime/store.rs +++ b/rust/src/runtime/store.rs @@ -402,7 +402,7 @@ where name: &str, body: Value, mut prepared: PreparedControlArguments, - signal_codec: Arc, + transition_codec: Arc, ) -> Result { reject_fields(&body, &["input", "request_id"])?; if name.trim().is_empty() || name.len() > 255 { @@ -505,7 +505,7 @@ where "message":format!("Workflow signal [{name}] is unknown for workflow type [{workflow_type}]: the durable command contract does not declare that handler. Command contract source [durable_history], declared signals [{labels}].")}); } prepared.command = prepare_rejected_signal_command( - signal_codec, + transition_codec, name.to_owned(), prepared.values.clone(), validation_errors.clone(), @@ -696,7 +696,7 @@ where run_id: &str, task_id: &str, task: &DB::Row, - signal_codec: Arc, + transition_codec: Arc, ) -> Result<()> { let payload = DB::document_row(task, "payload")?; if payload["resume_source_kind"] != "workflow_signal" { @@ -723,7 +723,7 @@ where return Err(refuse(StatusCode::CONFLICT, "signal_wait_mismatch")); } let prepared = prepare_control_arguments( - signal_codec, + transition_codec, name.clone(), DB::string(&signal, "arguments")?, "invalid_signal_arguments", @@ -1499,7 +1499,7 @@ where &self, task_id: &str, body: Value, - signal_codec: Arc, + transition_codec: Arc, external: super::external_resolution::PreparedCommands, ) -> Result { let commands = body["commands"] @@ -1699,7 +1699,7 @@ where return Err(refuse(StatusCode::CONFLICT, "update_task_mismatch")); } let mut sequence = Self::authored_sequence(&mut tx, &run_id).await?; - Self::apply_task_signal(&mut tx, &run_id, task_id, &task, signal_codec).await?; + Self::apply_task_signal(&mut tx, &run_id, task_id, &task, transition_codec).await?; for (index, command) in commands.iter().enumerate() { if command["type"] == "fail_workflow" { // Published parseCommands applies every nonterminal command diff --git a/rust/src/runtime/transition_codec_tests.rs b/rust/src/runtime/transition_codec_tests.rs new file mode 100644 index 000000000..becf90d87 --- /dev/null +++ b/rust/src/runtime/transition_codec_tests.rs @@ -0,0 +1,202 @@ +//! Transitions that encode or decode while holding the transition lock use +//! their own codec permit. Each test holds the read codec permit, as a long +//! describe decode would, and requires the transition to finish anyway. +use super::*; +use crate::codec::{Value as Payload, ValueCodec}; +use serde_json::json; + +fn envelope(value: Payload) -> Value { + json!({"codec":"avro","blob":ValueCodec::new().unwrap().encode(&value).unwrap()}) +} + +const DECISIVE: Duration = Duration::from_secs(1); + +async fn runtime(directory: &tempfile::TempDir) -> Runtime { + let runtime = Runtime::open( + directory.path().join("runtime.sqlite").to_str().unwrap(), + "test-token".into(), + ) + .await + .unwrap(); + runtime + .register(json!({"worker_id":"codec-worker","task_queue":"test","runtime":"php", + "supported_workflow_types":["echo"],"supported_activity_types":[], + "workflow_command_contracts":{"echo":{"signals":["payload"],"signal_contracts":[{"name":"payload", + "parameters":[{"name":"value","position":0,"type":"array","required":true,"variadic":false, + "default_available":false,"allows_null":false}]}]}}})) + .await + .unwrap(); + runtime +} + +async fn start(runtime: &Runtime, workflow_id: &str) { + runtime + .start(json!({"workflow_id":workflow_id,"workflow_type":"echo","task_queue":"test", + "input":envelope(Payload::Array(vec![]))})) + .await + .unwrap(); +} + +async fn poll(runtime: &Runtime) -> Value { + let task = runtime + .poll( + json!({"worker_id":"codec-worker","task_queue":"test","timeout_seconds":0}), + "workflow", + "1.20", + ) + .await + .unwrap()["task"] + .clone(); + assert!(!task.is_null(), "workflow task is ready"); + task +} + +async fn complete(runtime: &Runtime, task: &Value, commands: Value) -> Result { + runtime + .complete_workflow( + task["task_id"].as_str().unwrap(), + json!({"lease_owner":task["lease_owner"],"workflow_task_attempt":task["workflow_task_attempt"], + "commands":commands}), + ) + .await +} + +#[tokio::test] +async fn cancellation_request_does_not_wait_for_read_codec_capacity() { + let directory = tempfile::tempdir().unwrap(); + let runtime = runtime(&directory).await; + start(&runtime, "codec-cancel").await; + let _held = runtime.signal_codec.clone().acquire_owned().await.unwrap(); + let (status, accepted) = tokio::time::timeout( + DECISIVE, + runtime.request_cancellation( + "codec-cancel", + None, + json!({"reason":"held read permit","cleanup_timeout_seconds":30}), + ), + ) + .await + .expect("the cancellation encode must not wait for read codec capacity") + .unwrap(); + assert_eq!(status, StatusCode::ACCEPTED, "{accepted}"); + runtime.close().await; +} + +#[tokio::test] +async fn signal_resumed_completion_does_not_wait_for_read_codec_capacity() { + let directory = tempfile::tempdir().unwrap(); + let runtime = runtime(&directory).await; + start(&runtime, "codec-signal").await; + let waiting = poll(&runtime).await; + complete( + &runtime, + &waiting, + json!([{"type":"open_signal_wait","signal_name":"payload"}]), + ) + .await + .unwrap(); + runtime + .signal( + "codec-signal", + None, + "payload", + json!({"input":envelope(Payload::Array(vec![Payload::Map( + std::collections::BTreeMap::from([("count".into(), Payload::Long(7))]), + )]))}), + ) + .await + .unwrap(); + let resumed = poll(&runtime).await; + let _held = runtime.signal_codec.clone().acquire_owned().await.unwrap(); + let completed = tokio::time::timeout( + DECISIVE, + complete( + &runtime, + &resumed, + json!([{"type":"complete_workflow","result":envelope(Payload::Long(7))}]), + ), + ) + .await + .expect("applying the signal must not wait for read codec capacity") + .unwrap(); + assert_eq!(completed["recorded"], true, "{completed}"); + runtime.close().await; +} + +#[tokio::test] +async fn fenced_deregistration_repair_does_not_wait_for_read_codec_capacity() { + let directory = tempfile::tempdir().unwrap(); + let runtime = Runtime::open( + directory.path().join("runtime.sqlite").to_str().unwrap(), + "test-token".into(), + ) + .await + .unwrap(); + let registration = runtime + .register(json!({"worker_id":"codec-worker","task_queue":"test","runtime":"php", + "supported_workflow_types":["echo"],"supported_activity_types":[]})) + .await + .unwrap(); + start(&runtime, "codec-repair").await; + poll(&runtime).await; + let _held = runtime.signal_codec.clone().acquire_owned().await.unwrap(); + let deregistered = tokio::time::timeout( + DECISIVE, + runtime.deregister_fenced( + "codec-worker", + json!({"registration_token":registration["registration_token"]}), + ), + ) + .await + .expect("repairing the leased task must not wait for read codec capacity") + .unwrap(); + assert_eq!(deregistered["recovered_workflow_task_count"], 1, "{deregistered}"); + runtime.close().await; +} + +#[tokio::test] +async fn range_repair_does_not_wait_for_read_codec_capacity() { + use axum::{ + body::{Body, to_bytes}, + http::Request, + }; + use tower::ServiceExt; + let directory = tempfile::tempdir().unwrap(); + let runtime = runtime(&directory).await; + start(&runtime, "codec-range").await; + let task = poll(&runtime).await; + let reported = runtime + .fail_workflow_task( + task["task_id"].as_str().unwrap(), + json!({"lease_owner":task["lease_owner"],"workflow_task_attempt":task["workflow_task_attempt"], + "failure":{"message":"Recorded version 1 is outside supported range 2..2.", + "type":"DurableWorkflow\\Exception\\NonDeterministicWorkflow", + "reason":"version_marker_incompatible_range","sequence":1}}), + ) + .await + .unwrap(); + assert_eq!(reported["recorded"], true, "{reported}"); + // The repair context comes from the authenticated request, so use the route. + let request = Request::builder() + .method("POST") + .uri("/api/workflows/codec-range/repair") + .header("authorization", "Bearer test-token") + .header("content-type", "application/json") + .header("x-namespace", "default") + .header("x-durable-workflow-control-plane-version", "2") + .body(Body::empty()) + .unwrap(); + let _held = runtime.signal_codec.clone().acquire_owned().await.unwrap(); + let response = tokio::time::timeout( + DECISIVE, + super::super::router(runtime.clone()).oneshot(request), + ) + .await + .expect("the repair payload encode must not wait for read codec capacity") + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let repaired: Value = + serde_json::from_slice(&to_bytes(response.into_body(), 1 << 20).await.unwrap()).unwrap(); + assert_eq!(repaired["outcome"], "repair_dispatched", "{repaired}"); + runtime.close().await; +} From 095bf3e0055af6425d770e9a1e7f5bec962f7e44 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Thu, 15 Oct 2026 06:06:28 +0000 Subject: [PATCH 02/37] Reproduce native empty and malformed command declaration boundaries --- rust/tests/execution.rs | 3 + rust/tests/execution/worker_contracts.rs | 106 +++++++++++++++++++++++ 2 files changed, 109 insertions(+) create mode 100644 rust/tests/execution/worker_contracts.rs diff --git a/rust/tests/execution.rs b/rust/tests/execution.rs index 8c2b7ba54..0c1f6a19e 100644 --- a/rust/tests/execution.rs +++ b/rust/tests/execution.rs @@ -49,6 +49,9 @@ mod selected_run_signals; #[path = "execution/worker_incarnations.rs"] mod worker_incarnations; +#[path = "execution/worker_contracts.rs"] +mod worker_contracts; + #[path = "execution/worker_deregistration.rs"] mod worker_deregistration; diff --git a/rust/tests/execution/worker_contracts.rs b/rust/tests/execution/worker_contracts.rs new file mode 100644 index 000000000..72f730ee0 --- /dev/null +++ b/rust/tests/execution/worker_contracts.rs @@ -0,0 +1,106 @@ +use super::*; + +fn definition() -> Value { + json!({"worker_id":"contract-worker","task_queue":"test","runtime":"python", + "supported_workflow_types":["echo"],"supported_activity_types":[]}) +} + +async fn register_contract(app: &Router, declaration: Option) -> (StatusCode, Value) { + let mut body = definition(); + if let Some(declaration) = declaration { + body["workflow_command_contracts"] = declaration; + } + request(app, "POST", "/api/worker/register", body).await +} + +async fn snapshot(app: &Router, started: &Value) -> Value { + let history = run_history(app, &started["workflow_id"], &started["run_id"]).await; + history.as_array().unwrap().iter() + .find(|event| event["event_type"] == "WorkflowStarted").unwrap()["payload"].clone() +} + +fn assert_empty(payload: &Value) { + for field in ["declared_queries", "declared_signals", "declared_updates", + "declared_query_contracts", "declared_signal_contracts", "declared_update_contracts"] { + assert_eq!(payload.get(field), Some(&json!([])), "{field}: {payload}"); + } +} + +#[tokio::test] +async fn explicit_empty_declarations_survive_cold_reads_and_later_registration() { + let database = TestDatabase::new().await; + let runtime = database.open().await.unwrap(); + let app = router(runtime.clone()); + let mut originals = Vec::new(); + for (index, declaration) in [json!({}), json!([]), json!({"queries":[],"signals":[], + "updates":[],"query_contracts":[],"signal_contracts":[],"update_contracts":[]})].into_iter().enumerate() { + let registered = register_contract(&app, Some(json!({"echo":declaration}))).await; + assert_eq!(registered.0, StatusCode::CREATED, "{}", registered.1); + let started = start(&app, &format!("empty-contract-{index}")).await; + let before = snapshot(&app, &started).await; + assert_empty(&before); + originals.push((started, before)); + } + assert_eq!(register_contract(&app, Some(json!({"echo":{"query_contracts":[ + {"name":"state","parameters":[]}],"signals":["z","a","z"]}}))).await.0, StatusCode::CREATED); + let later = start(&app, "named-contract").await; + let named = snapshot(&app, &later).await; + assert_eq!(named["declared_queries"], json!(["state"])); + assert_eq!(named["declared_signals"], json!(["a","z"])); + assert_eq!(register_contract(&app, None).await.0, StatusCode::CREATED); + let omitted = start(&app, "reregister-omitted").await; + assert_eq!(snapshot(&app, &omitted).await["declared_queries"], json!(["state"])); + drop(app); + drop(runtime); + let cold = router(database.open().await.unwrap()); + for (started, before) in originals { + assert_eq!(snapshot(&cold, &started).await, before); + } + drop(cold); + database.remove().await; +} + +#[tokio::test] +async fn legacy_missing_entries_remain_unknown_instead_of_becoming_empty() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + for (index, declaration) in [None, Some(Value::Null), Some(json!({})), + Some(json!([])), Some(json!({"other":{}}))].into_iter().enumerate() { + let registered = register_contract(&app, declaration).await; + assert_eq!(registered.0, StatusCode::CREATED, "{}", registered.1); + let original = start(&app, &format!("legacy-contract-{index}")).await; + let before = snapshot(&app, &original).await; + assert!(before.get("declared_queries").is_none(), "{before}"); + assert_eq!(register_contract(&app, Some(json!({"echo":{}}))).await.0, StatusCode::CREATED); + assert_eq!(snapshot(&app, &original).await, before); + assert_eq!(register_contract(&app, Some(json!({}))).await.0, StatusCode::CREATED); + } + drop(app); + database.remove().await; +} + +#[tokio::test] +async fn invalid_declarations_refuse_before_registration_rotation_or_lease_release() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + let accepted = register_contract(&app, Some(json!({"echo":{}}))).await; + assert_eq!(accepted.0, StatusCode::CREATED); + let started = start(&app, "contract-lease").await; + let task = poll(&app, "contract-worker", "workflow").await; + let before = snapshot(&app, &started).await; + for declaration in [Value::Null, json!("bad"), json!({"queries":null}), json!({"queries":"state"}), + json!({"queries":[17]}), json!({"signals":[" "]}), json!({"updates":{"key":"set"}}), + json!({"update_validators":null}), json!({"query_contracts":"bad"}), + json!({"signal_contracts":[null]}), json!({"query_contracts":[{}]}), + json!({"update_contracts":[{"name":17}]})] { + let refused = register_contract(&app, Some(json!({"echo":declaration}))).await; + assert_eq!(refused.0, StatusCode::UNPROCESSABLE_ENTITY, "{}", refused.1); + assert_eq!(snapshot(&app, &started).await, before); + } + let completed = finish_task(&app, &task, json!([{"type":"complete_workflow", + "result":envelope(Payload::Long(9007199254740993))}])).await; + assert_eq!(completed.0, StatusCode::OK, "{}", completed.1); + assert_eq!(completed.1["recorded"], true); + drop(app); + database.remove().await; +} From 1a739d4c46bd1e74846c6110e0b6e870d24927ee Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Thu, 15 Oct 2026 06:15:16 +0000 Subject: [PATCH 03/37] Preserve explicit worker command declaration snapshots --- rust/src/runtime/children.rs | 9 +- rust/src/runtime/mod.rs | 1 + rust/src/runtime/store.rs | 36 ++++---- rust/src/runtime/worker_contracts.rs | 93 +++++++++++++++++++ rust/tests/execution/worker_contracts.rs | 109 ++++++++++++++++++----- 5 files changed, 209 insertions(+), 39 deletions(-) create mode 100644 rust/src/runtime/worker_contracts.rs diff --git a/rust/src/runtime/children.rs b/rust/src/runtime/children.rs index 73fc654b7..c6d194fe2 100644 --- a/rust/src/runtime/children.rs +++ b/rust/src/runtime/children.rs @@ -155,6 +155,11 @@ where { let contracts = DB::document_row(&worker, "workflow_command_contracts")?; if let Some(contract) = contracts.get(workflow_type) { + // Older development registrations may predate validation. + // Never turn a malformed entry into authoritative empties. + let Ok(contract) = super::worker_contracts::declaration(contract) else { + continue; + }; for (source, target) in [ ("signals", "declared_signals"), ("signal_contracts", "declared_signal_contracts"), @@ -164,7 +169,9 @@ where ("update_contracts", "declared_update_contracts"), ("update_validators", "declared_update_validators"), ] { - started[target] = contract[source].clone(); + if let Some(value) = contract.get(source) { + started[target] = value.clone(); + } } break; } diff --git a/rust/src/runtime/mod.rs b/rust/src/runtime/mod.rs index 0d051c048..2fa6aa8fe 100644 --- a/rust/src/runtime/mod.rs +++ b/rust/src/runtime/mod.rs @@ -63,6 +63,7 @@ pub mod upgrade; mod visibility; mod wait_projection; mod wait_timeouts; +mod worker_contracts; mod worker_incarnations; mod workflow_failures; diff --git a/rust/src/runtime/store.rs b/rust/src/runtime/store.rs index 6490d0f95..04349dd65 100644 --- a/rust/src/runtime/store.rs +++ b/rust/src/runtime/store.rs @@ -813,27 +813,29 @@ where } } let runtime = text(&body, "runtime")?; - let contracts = body + let supplied_contracts = body .get("workflow_command_contracts") - .cloned() - .unwrap_or_else(|| json!({})); - // PHP encodes an empty associative map as []; activity-only published - // workers send this shape. Preserve a map internally for lookups. - let contracts = if contracts.as_array().is_some_and(|values| values.is_empty()) { - json!({}) - } else { - contracts - }; - if !contracts.is_object() { - return Err(refuse( - StatusCode::UNPROCESSABLE_ENTITY, - "invalid_worker_definition", - )); - } + .map(|contracts| { + super::worker_contracts::normalize(contracts, &body["supported_workflow_types"]) + }) + .transpose()?; let token = super::worker_incarnations::token().await?; let mut tx = self.begin().await?; - let previous = Self::query("SELECT worker_id FROM workflow_worker_registrations WHERE namespace=$1 AND worker_id=$2") + let previous = Self::query("SELECT worker_id,workflow_command_contracts FROM workflow_worker_registrations WHERE namespace=$1 AND worker_id=$2") .bind(self.namespace.as_ref()).bind(worker_id).fetch_optional(&mut *tx).await?; + let contracts = match supplied_contracts { + Some(contracts) => contracts, + None => { + let existing = match &previous { + Some(worker) => { + DB::optional_document_row(worker, "workflow_command_contracts")? + .unwrap_or(Value::Null) + } + None => Value::Null, + }; + super::worker_contracts::normalize(&existing, &body["supported_workflow_types"])? + } + }; if previous.is_some() { // Match PHP's registration boundary: return original workflow leases // to ready without a repair command/history event. Token rotation diff --git a/rust/src/runtime/worker_contracts.rs b/rust/src/runtime/worker_contracts.rs new file mode 100644 index 000000000..871249dc3 --- /dev/null +++ b/rust/src/runtime/worker_contracts.rs @@ -0,0 +1,93 @@ +//! Registration declarations become immutable admission snapshots. A missing +//! workflow entry stays unknown; a valid empty entry declares empty lists. +use super::{Result, refuse}; +use axum::http::StatusCode; +use serde_json::{Value, json}; +use std::collections::BTreeSet; + +fn invalid() -> super::RuntimeError { + refuse( + StatusCode::UNPROCESSABLE_ENTITY, + "invalid_worker_definition", + ) +} + +fn name(value: &Value) -> Result<&str> { + let text = value.as_str().ok_or_else(invalid)?; + let text = text.trim_matches([' ', '\t', '\n', '\r', '\0', '\u{b}']); + if text.is_empty() || text.chars().count() > 255 { + return Err(invalid()); + } + Ok(text) +} + +fn list<'a>(entry: &'a Value, field: &str) -> Result<&'a [Value]> { + match entry.get(field) { + None => Ok(&[]), + Some(Value::Array(values)) => Ok(values), + Some(_) => Err(invalid()), + } +} + +pub(super) fn declaration(entry: &Value) -> Result { + if !entry.is_object() && !entry.as_array().is_some_and(Vec::is_empty) { + return Err(invalid()); + } + let mut result = json!({}); + for (names, handlers) in [ + ("queries", "query_contracts"), + ("signals", "signal_contracts"), + ("updates", "update_contracts"), + ] { + let mut declared = BTreeSet::new(); + for value in list(entry, names)? { + declared.insert(name(value)?.to_owned()); + } + let mut seen = BTreeSet::new(); + let mut contracts = Vec::new(); + for handler in list(entry, handlers)? { + let handler_name = name(&handler["name"])?; + declared.insert(handler_name.to_owned()); + if seen.insert(handler_name.to_owned()) { + contracts.push( + json!({"name":handler_name,"parameters":handler.get("parameters") + .filter(|value| value.is_array()).cloned().unwrap_or_else(|| json!([]))}), + ); + } + } + result[names] = json!(declared); + result[handlers] = json!(contracts); + } + if entry.get("update_validators").is_some() { + let mut validators = BTreeSet::new(); + for value in list(entry, "update_validators")? { + validators.insert(name(value)?.to_owned()); + } + result["update_validators"] = json!(validators); + } + Ok(result) +} + +pub(super) fn normalize(value: &Value, supported: &Value) -> Result { + if value.is_null() || value.as_array().is_some_and(Vec::is_empty) { + return Ok(json!({})); + } + let entries = value.as_object().ok_or_else(invalid)?; + let mut result = json!({}); + for (workflow_type, entry) in entries { + // Validate even an unadvertised declaration before filtering it out. + // Otherwise malformed metadata could rotate an existing registration. + let entry = declaration(entry)?; + let workflow_type = workflow_type.trim_matches([' ', '\t', '\n', '\r', '\0', '\u{b}']); + if !workflow_type.is_empty() + && supported.as_array().is_some_and(|types| { + types + .iter() + .any(|value| value.as_str() == Some(workflow_type)) + }) + { + result[workflow_type] = entry; + } + } + Ok(result) +} diff --git a/rust/tests/execution/worker_contracts.rs b/rust/tests/execution/worker_contracts.rs index 72f730ee0..9dac8ad68 100644 --- a/rust/tests/execution/worker_contracts.rs +++ b/rust/tests/execution/worker_contracts.rs @@ -15,13 +15,24 @@ async fn register_contract(app: &Router, declaration: Option) -> (StatusC async fn snapshot(app: &Router, started: &Value) -> Value { let history = run_history(app, &started["workflow_id"], &started["run_id"]).await; - history.as_array().unwrap().iter() - .find(|event| event["event_type"] == "WorkflowStarted").unwrap()["payload"].clone() + history + .as_array() + .unwrap() + .iter() + .find(|event| event["event_type"] == "WorkflowStarted") + .unwrap()["payload"] + .clone() } fn assert_empty(payload: &Value) { - for field in ["declared_queries", "declared_signals", "declared_updates", - "declared_query_contracts", "declared_signal_contracts", "declared_update_contracts"] { + for field in [ + "declared_queries", + "declared_signals", + "declared_updates", + "declared_query_contracts", + "declared_signal_contracts", + "declared_update_contracts", + ] { assert_eq!(payload.get(field), Some(&json!([])), "{field}: {payload}"); } } @@ -32,8 +43,15 @@ async fn explicit_empty_declarations_survive_cold_reads_and_later_registration() let runtime = database.open().await.unwrap(); let app = router(runtime.clone()); let mut originals = Vec::new(); - for (index, declaration) in [json!({}), json!([]), json!({"queries":[],"signals":[], - "updates":[],"query_contracts":[],"signal_contracts":[],"update_contracts":[]})].into_iter().enumerate() { + for (index, declaration) in [ + json!({}), + json!([]), + json!({"queries":[],"signals":[], + "updates":[],"query_contracts":[],"signal_contracts":[],"update_contracts":[]}), + ] + .into_iter() + .enumerate() + { let registered = register_contract(&app, Some(json!({"echo":declaration}))).await; assert_eq!(registered.0, StatusCode::CREATED, "{}", registered.1); let started = start(&app, &format!("empty-contract-{index}")).await; @@ -41,15 +59,26 @@ async fn explicit_empty_declarations_survive_cold_reads_and_later_registration() assert_empty(&before); originals.push((started, before)); } - assert_eq!(register_contract(&app, Some(json!({"echo":{"query_contracts":[ - {"name":"state","parameters":[]}],"signals":["z","a","z"]}}))).await.0, StatusCode::CREATED); + assert_eq!( + register_contract( + &app, + Some(json!({"echo":{"query_contracts":[ + {"name":"state","parameters":[]}],"signals":["z","a","z"]}})) + ) + .await + .0, + StatusCode::CREATED + ); let later = start(&app, "named-contract").await; let named = snapshot(&app, &later).await; assert_eq!(named["declared_queries"], json!(["state"])); - assert_eq!(named["declared_signals"], json!(["a","z"])); + assert_eq!(named["declared_signals"], json!(["a", "z"])); assert_eq!(register_contract(&app, None).await.0, StatusCode::CREATED); let omitted = start(&app, "reregister-omitted").await; - assert_eq!(snapshot(&app, &omitted).await["declared_queries"], json!(["state"])); + assert_eq!( + snapshot(&app, &omitted).await["declared_queries"], + json!(["state"]) + ); drop(app); drop(runtime); let cold = router(database.open().await.unwrap()); @@ -64,16 +93,30 @@ async fn explicit_empty_declarations_survive_cold_reads_and_later_registration() async fn legacy_missing_entries_remain_unknown_instead_of_becoming_empty() { let database = TestDatabase::new().await; let app = router(database.open().await.unwrap()); - for (index, declaration) in [None, Some(Value::Null), Some(json!({})), - Some(json!([])), Some(json!({"other":{}}))].into_iter().enumerate() { + for (index, declaration) in [ + None, + Some(Value::Null), + Some(json!({})), + Some(json!([])), + Some(json!({"other":{}})), + ] + .into_iter() + .enumerate() + { let registered = register_contract(&app, declaration).await; assert_eq!(registered.0, StatusCode::CREATED, "{}", registered.1); let original = start(&app, &format!("legacy-contract-{index}")).await; let before = snapshot(&app, &original).await; assert!(before.get("declared_queries").is_none(), "{before}"); - assert_eq!(register_contract(&app, Some(json!({"echo":{}}))).await.0, StatusCode::CREATED); + assert_eq!( + register_contract(&app, Some(json!({"echo":{}}))).await.0, + StatusCode::CREATED + ); assert_eq!(snapshot(&app, &original).await, before); - assert_eq!(register_contract(&app, Some(json!({}))).await.0, StatusCode::CREATED); + assert_eq!( + register_contract(&app, Some(json!({}))).await.0, + StatusCode::CREATED + ); } drop(app); database.remove().await; @@ -88,19 +131,43 @@ async fn invalid_declarations_refuse_before_registration_rotation_or_lease_relea let started = start(&app, "contract-lease").await; let task = poll(&app, "contract-worker", "workflow").await; let before = snapshot(&app, &started).await; - for declaration in [Value::Null, json!("bad"), json!({"queries":null}), json!({"queries":"state"}), - json!({"queries":[17]}), json!({"signals":[" "]}), json!({"updates":{"key":"set"}}), - json!({"update_validators":null}), json!({"query_contracts":"bad"}), - json!({"signal_contracts":[null]}), json!({"query_contracts":[{}]}), - json!({"update_contracts":[{"name":17}]})] { + for declaration in [ + Value::Null, + json!("bad"), + json!({"queries":null}), + json!({"queries":"state"}), + json!({"queries":[17]}), + json!({"signals":[" "]}), + json!({"updates":{"key":"set"}}), + json!({"update_validators":null}), + json!({"query_contracts":"bad"}), + json!({"signal_contracts":[null]}), + json!({"query_contracts":[{}]}), + json!({"update_contracts":[{"name":17}]}), + ] { let refused = register_contract(&app, Some(json!({"echo":declaration}))).await; assert_eq!(refused.0, StatusCode::UNPROCESSABLE_ENTITY, "{}", refused.1); assert_eq!(snapshot(&app, &started).await, before); } - let completed = finish_task(&app, &task, json!([{"type":"complete_workflow", - "result":envelope(Payload::Long(9007199254740993))}])).await; + let completed = finish_task( + &app, + &task, + json!([{"type":"complete_workflow", + "result":envelope(Payload::Long(9007199254740993))}]), + ) + .await; assert_eq!(completed.0, StatusCode::OK, "{}", completed.1); assert_eq!(completed.1["recorded"], true); + let retired = request_protocol( + &app, + "POST", + "/api/worker/registrations/contract-worker/deregister", + json!({"registration_token":accepted.1["registration_token"]}), + "1.20", + ) + .await; + assert_eq!(retired.0, StatusCode::OK, "{}", retired.1); + assert_eq!(retired.1["recovered_workflow_task_count"], 0); drop(app); database.remove().await; } From c4e37fd283adb43d94dea30561d661f8b2cd7054 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 03:58:38 +0000 Subject: [PATCH 04/37] Report leased and empty poll statuses like the worker protocol The worker protocol requires poll_status "leased" on a workflow task claim, and published PHP (2.5.14 and 2.8) reports "leased" and "empty" for workflow, activity and query polls alike. Native reported "task" and "timeout". It now reports the same two values. The wake hint task_queue_changed and every refusal status are unchanged. The cross-kind poll contract, which accepted either claim status, now requires "leased". Part of #325. Co-Authored-By: Claude Opus 5.5 --- rust/src/runtime/queries.rs | 4 ++-- rust/src/runtime/store.rs | 4 ++-- rust/tests/execution/cross_kind_poll.rs | 6 +++--- .../conformance/server-parity/cross-kind-poll-contract.mjs | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/rust/src/runtime/queries.rs b/rust/src/runtime/queries.rs index f91c4e04e..861532c37 100644 --- a/rust/src/runtime/queries.rs +++ b/rust/src/runtime/queries.rs @@ -357,14 +357,14 @@ where loop { let wake = self.wake.notified(); match self.claim_query(&body, config, &proof).await? { - QueryClaim::Task(task) => return Ok(poll_result(Some(task), "task")), + QueryClaim::Task(task) => return Ok(poll_result(Some(task), "leased")), QueryClaim::StaleWorker => { return Ok(poll_result(None, "stale_worker_registration")); } QueryClaim::Empty => {} } if tokio::time::Instant::now() >= deadline { - return Ok(poll_result(None, "timeout")); + return Ok(poll_result(None, "empty")); } if self .poll_queue_changed(&body, "query", &mut cross_kind) diff --git a/rust/src/runtime/store.rs b/rust/src/runtime/store.rs index 6490d0f95..2f25c77c3 100644 --- a/rust/src/runtime/store.rs +++ b/rust/src/runtime/store.rs @@ -939,12 +939,12 @@ where let wake = self.wake.notified(); if let Some(task) = self.claim(&body, kind, protocol_version).await? { return Ok( - json!({"task": task, "poll_status": "task", "server_capabilities": capabilities()}), + json!({"task": task, "poll_status": "leased", "server_capabilities": capabilities()}), ); } if tokio::time::Instant::now() >= deadline { return Ok( - json!({"task": null, "poll_status": "timeout", "server_capabilities": capabilities()}), + json!({"task": null, "poll_status": "empty", "server_capabilities": capabilities()}), ); } if self diff --git a/rust/tests/execution/cross_kind_poll.rs b/rust/tests/execution/cross_kind_poll.rs index 754af04ff..5986b0ce4 100644 --- a/rust/tests/execution/cross_kind_poll.rs +++ b/rust/tests/execution/cross_kind_poll.rs @@ -177,7 +177,7 @@ async fn scoped_hints_require_original_opt_in_and_a_new_ready_identity() { let result = held.await.unwrap(); assert_eq!(result.0, StatusCode::OK, "{scenario}: {}", result.1); assert_eq!( - result.1["poll_status"], "timeout", + result.1["poll_status"], "empty", "{scenario}: {}", result.1 ); @@ -216,7 +216,7 @@ async fn activity_admission_wakes_other_polls_but_same_kind_claim_wins() { json!({"lease_owner":"mixed","activity_attempt_id":activity["activity_attempt_id"],"result":envelope(Payload::Null)})).await.0,StatusCode::OK); let result = held.await.unwrap(); assert_eq!(result.0, StatusCode::OK, "{}", result.1); - assert_eq!(result.1["poll_status"], "task"); + assert_eq!(result.1["poll_status"], "leased"); assert!(result.1["task"].is_object()); runtime.close().await; database.remove().await; @@ -311,7 +311,7 @@ async fn zero_timeout_polls_keep_the_existing_empty_reply() { ) .await; assert_eq!(result.0, StatusCode::OK, "{}", result.1); - assert_eq!(result.1["poll_status"], "timeout"); + assert_eq!(result.1["poll_status"], "empty"); assert_eq!(result.1.get("task"), Some(&Value::Null)); } runtime.close().await; diff --git a/scripts/conformance/server-parity/cross-kind-poll-contract.mjs b/scripts/conformance/server-parity/cross-kind-poll-contract.mjs index 5dceb69a9..adf89f2d3 100644 --- a/scripts/conformance/server-parity/cross-kind-poll-contract.mjs +++ b/scripts/conformance/server-parity/cross-kind-poll-contract.mjs @@ -44,7 +44,7 @@ export function checkCrossKindPollWake(record, profile) { } const sweep = following[sweepIndex]; assert.equal(sweep.http, 200); - assert(['task', 'leased'].includes(sweep.poll_status)); + assert.equal(sweep.poll_status, 'leased', 'a claim reports leased, as the worker protocol requires'); assert.equal(sweep.task_is_null, false); assert.equal(typeof sweep.task_id, 'string'); assert(sweep.task_id.length > 0, 'the ordinary poll separately grants a task'); From d5d880fd754b044f1c13ba48cee4fcfd87f054df Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 07:41:11 +0000 Subject: [PATCH 05/37] Let held long polls wait without the transition lock Each held native poll used to repeat its full claim every 50 ms: a transaction under the global transition lock that read the registration, deleted expired poll receipts and selected candidates. With no work, idle cost grew by about 0.035 cores per waiting poll, two-thirds of it in MySQL, and every probe queued behind, and delayed, real transitions. A held poll now claims under the lock on its first probe. Afterwards it claims only when a lock-free read finds something it could claim or must refuse: an eligible task or query, a receipt for its own poll request, or a removed, moved or replaced registration. The read runs at once when a commit on the same node wakes the poll, and otherwise every 50 ms, which is how a poll still observes other server processes. A false positive costs one empty claim, and a commit racing a read is seen on the next read. New tests hold the transition lock from outside the runtime and check that idle workflow, activity and query polls still end at their deadline (each fails with the previous probing), and that a held poll still claims a task or query, and refuses a removed registration, committed by another node. Co-Authored-By: Claude Opus 5.5 --- docs/rust-server-transition-locking.md | 9 ++ rust/src/runtime/backend.rs | 4 +- rust/src/runtime/mod.rs | 1 + rust/src/runtime/poll_readiness.rs | 97 +++++++++++++++++ rust/src/runtime/queries.rs | 22 ++-- rust/src/runtime/store.rs | 17 ++- rust/tests/execution.rs | 2 + rust/tests/execution/lock_free_waiting.rs | 123 ++++++++++++++++++++++ rust/tests/execution/read_snapshots.rs | 6 +- 9 files changed, 265 insertions(+), 16 deletions(-) create mode 100644 rust/src/runtime/poll_readiness.rs create mode 100644 rust/tests/execution/lock_free_waiting.rs diff --git a/docs/rust-server-transition-locking.md b/docs/rust-server-transition-locking.md index e593eab1a..4e6b6bad5 100644 --- a/docs/rust-server-transition-locking.md +++ b/docs/rust-server-transition-locking.md @@ -26,6 +26,15 @@ read-only snapshot (`Store::begin_read()`, `READ_SNAPSHOT_BEGIN` in `backend.rs`). Control-plane reads such as describe and run history query the pool without a transaction. +A held long poll takes the lock for its first probe, and afterwards only when +a lock-free read finds something to claim or refuse: an eligible task (or +query), a receipt for the poll's own request ID, or a removed, moved or +replaced registration (`poll_readiness.rs`). The read runs at once when a +commit on the same node wakes the poll, and otherwise every 50 ms, which is +how a poll observes another server process. An idle poll therefore neither +waits for the lock nor delays transitions, and an unrelated commit does not +send every waiting poll to the lock. + ## Measured cost Under the CPU deadline-health profile on a local MySQL 8.0 cell, per-step diff --git a/rust/src/runtime/backend.rs b/rust/src/runtime/backend.rs index 7aaa69405..7dc404387 100644 --- a/rust/src/runtime/backend.rs +++ b/rust/src/runtime/backend.rs @@ -337,8 +337,8 @@ impl Backend for Postgres { async fn begin(pool: &Pool) -> Result> { let mut tx = pool.begin().await?; // The first single-namespace slice serializes its durable transitions - // across independent pools/processes. Empty long polls release this - // transaction before waiting. Throughput/idle cost remain unqualified. + // across independent pools/processes. Held long polls wait without it + // (poll_readiness.rs). Throughput/idle cost remain unqualified. sqlx::query("SELECT pg_advisory_xact_lock(4924501473315676163)") .execute(&mut *tx) .await?; diff --git a/rust/src/runtime/mod.rs b/rust/src/runtime/mod.rs index 0d051c048..e83d15aa2 100644 --- a/rust/src/runtime/mod.rs +++ b/rust/src/runtime/mod.rs @@ -35,6 +35,7 @@ mod memos; pub mod mysql; mod namespaces; pub mod postgres; +mod poll_readiness; mod previews; mod queries; mod query_authority; diff --git a/rust/src/runtime/poll_readiness.rs b/rust/src/runtime/poll_readiness.rs new file mode 100644 index 000000000..f0727bf34 --- /dev/null +++ b/rust/src/runtime/poll_readiness.rs @@ -0,0 +1,97 @@ +//! Lock-free waiting for held long polls. +//! +//! A held poll takes the transition lock only to claim or refuse. After its +//! first claim it reads, without the lock, whether anything now exists that it +//! could claim or must refuse: an eligible task, a receipt for its own request, +//! or a changed registration. A same-node commit wakes it to read at once; +//! timed reads observe other server processes. A false positive costs one locked +//! claim that finds nothing, and a commit racing a read is seen on the next one. +use super::{ + Result, backend::Backend, query_authority::{self, QueryTaskConfig}, store::Store, text, +}; +use serde_json::Value; +use sqlx::{ColumnIndex, Decode, Encode, Executor, IntoArguments, Type}; +use std::time::Duration; + +/// How often a held poll reads for work committed by another server process. +pub(super) const READ_INTERVAL: Duration = Duration::from_millis(50); + +impl Store +where + for<'c> &'c mut DB::Connection: Executor<'c, Database = DB>, + DB::Arguments: IntoArguments, + for<'q> &'q str: Encode<'q, DB> + Type, + for<'q> Option<&'q str>: Encode<'q, DB>, + for<'q> Option: Encode<'q, DB>, + for<'q> String: Encode<'q, DB> + Type, + for<'q> i64: Encode<'q, DB> + Type, + for<'r> i64: Decode<'r, DB>, + usize: ColumnIndex, +{ + /// Whether a held workflow or activity poll should claim again. + pub(super) async fn task_poll_ready(&self, body: &Value, kind: &str) -> Result { + let worker_id = text(body, "worker_id")?; + let queue = text(body, "task_queue")?; + // The claim refuses a removed or moved registration. + let registration = Self::query( + "SELECT task_queue FROM workflow_worker_registrations WHERE worker_id=$1 AND namespace=$2", + ) + .bind(worker_id) + .bind(self.namespace.as_ref()) + .fetch_optional(&self.pool) + .await?; + match registration { + Some(row) if DB::string(&row, "task_queue")? == queue => {} + _ => return Ok(true), + } + // A concurrent poll with the same request ID may have claimed already; + // the claim replays its receipt. + if let Some(request_id) = body.get("poll_request_id").and_then(Value::as_str) + && Self::query("SELECT task_id FROM dw_poll_receipts WHERE worker_id=$1 AND kind=$2 AND request_id=$3 AND namespace=$4") + .bind(worker_id).bind(kind).bind(request_id).bind(self.namespace.as_ref()) + .fetch_optional(&self.pool).await?.is_some() + { + return Ok(true); + } + let at = super::store::now(); + Ok(Self::query(DB::TASK_CANDIDATES_SQL) + .bind(self.namespace.as_ref()) + .bind(queue) + .bind(kind) + .bind(DB::bind_time(at)) + .bind(DB::bind_time(at)) + .fetch_optional(&self.pool) + .await? + .is_some()) + } + + /// Whether a held query poll should claim again. `proof` is the + /// registration this poll belongs to. + pub(super) async fn query_poll_ready( + &self, + body: &Value, + config: &QueryTaskConfig, + proof: &str, + ) -> Result { + let owner = text(body, "worker_id")?; + // The claim refuses a replaced, removed or stale registration. + if Self::query_worker_proof(&self.pool, self.namespace.as_ref(), owner, config) + .await? + .as_deref() + != Some(proof) + { + return Ok(true); + } + let at = super::store::now(); + let query_now = query_authority::wire(at); + Ok(Self::query(DB::POLL_WAKE_QUERIES_SQL) + .bind(at.timestamp()) + .bind(self.namespace.as_ref()) + .bind(text(body, "task_queue")?) + .bind(query_now.clone()) + .bind(query_now) + .fetch_optional(&self.pool) + .await? + .is_some()) + } +} diff --git a/rust/src/runtime/queries.rs b/rust/src/runtime/queries.rs index f91c4e04e..ed9a5588e 100644 --- a/rust/src/runtime/queries.rs +++ b/rust/src/runtime/queries.rs @@ -354,14 +354,19 @@ where } else { None }; + // Only the first probe claims under the lock unconditionally; later + // probes claim after a lock-free read finds something to claim or refuse. + let mut first = true; loop { let wake = self.wake.notified(); - match self.claim_query(&body, config, &proof).await? { - QueryClaim::Task(task) => return Ok(poll_result(Some(task), "task")), - QueryClaim::StaleWorker => { - return Ok(poll_result(None, "stale_worker_registration")); + if std::mem::take(&mut first) || self.query_poll_ready(&body, config, &proof).await? { + match self.claim_query(&body, config, &proof).await? { + QueryClaim::Task(task) => return Ok(poll_result(Some(task), "task")), + QueryClaim::StaleWorker => { + return Ok(poll_result(None, "stale_worker_registration")); + } + QueryClaim::Empty => {} } - QueryClaim::Empty => {} } if tokio::time::Instant::now() >= deadline { return Ok(poll_result(None, "timeout")); @@ -375,11 +380,14 @@ where json!({"task":null,"poll_status":"task_queue_changed","server_capabilities":super::store::capabilities()}), ); } - tokio::select! { _ = wake => {}, _ = tokio::time::sleep(Duration::from_millis(50)) => {} } + tokio::select! { + _ = wake => {}, + _ = tokio::time::sleep(super::poll_readiness::READ_INTERVAL) => {}, + } } } - async fn query_worker_proof<'e, E>( + pub(super) async fn query_worker_proof<'e, E>( executor: E, namespace: &str, owner: &str, diff --git a/rust/src/runtime/store.rs b/rust/src/runtime/store.rs index 6490d0f95..1dddc5dab 100644 --- a/rust/src/runtime/store.rs +++ b/rust/src/runtime/store.rs @@ -933,11 +933,17 @@ where } else { None }; + // Only the first probe claims under the lock unconditionally. Later + // probes, on a same-node wake or a timed read that also observes + // other server processes, claim only after a lock-free read finds + // something to claim or refuse. + let mut first = true; loop { - // Register the notification before probing; periodic probing also - // observes work committed by an independent process. + // Register the notification before probing. let wake = self.wake.notified(); - if let Some(task) = self.claim(&body, kind, protocol_version).await? { + if (std::mem::take(&mut first) || self.task_poll_ready(&body, kind).await?) + && let Some(task) = self.claim(&body, kind, protocol_version).await? + { return Ok( json!({"task": task, "poll_status": "task", "server_capabilities": capabilities()}), ); @@ -956,7 +962,10 @@ where json!({"task":null,"poll_status":"task_queue_changed","server_capabilities":capabilities()}), ); } - tokio::select! { _ = wake => {}, _ = tokio::time::sleep(Duration::from_millis(50)) => {} } + tokio::select! { + _ = wake => {}, + _ = tokio::time::sleep(super::poll_readiness::READ_INTERVAL) => {}, + } } } diff --git a/rust/tests/execution.rs b/rust/tests/execution.rs index 8c2b7ba54..7eeceb171 100644 --- a/rust/tests/execution.rs +++ b/rust/tests/execution.rs @@ -61,6 +61,8 @@ mod version_markers; #[path = "execution/activity_deadlines.rs"] mod activity_deadlines; +#[path = "execution/lock_free_waiting.rs"] +mod lock_free_waiting; #[path = "execution/read_snapshots.rs"] mod read_snapshots; diff --git a/rust/tests/execution/lock_free_waiting.rs b/rust/tests/execution/lock_free_waiting.rs new file mode 100644 index 000000000..bd22759dd --- /dev/null +++ b/rust/tests/execution/lock_free_waiting.rs @@ -0,0 +1,123 @@ +//! A held long poll waits without the transition lock, so idle polls neither +//! wait behind transitions nor delay them. It still claims work, and refuses +//! a removed registration, committed by another server process. +use super::read_snapshots::hold_transition_lock; +use super::*; +use std::time::Duration; + +fn held_poll( + app: &Router, + worker: &str, + kind: &str, + seconds: u64, +) -> tokio::task::JoinHandle<(StatusCode, Value)> { + let app = app.clone(); + let path = format!("/api/worker/{kind}-tasks/poll"); + let body = json!({"worker_id":worker,"task_queue":"test","timeout_seconds":seconds}); + tokio::spawn(async move { request(&app, "POST", &path, body).await }) +} + +#[tokio::test] +async fn held_polls_reach_their_deadline_while_a_transition_holds_the_lock() { + let database = TestDatabase::new().await; + let runtime = database.open().await.unwrap(); + let app = router(runtime.clone()); + register(&app, "waiting-worker", json!(["echo"]), json!(["echo-activity"])).await; + query_worker(&app, "waiting-query-worker").await; + let polls = [ + held_poll(&app, "waiting-worker", "workflow", 2), + held_poll(&app, "waiting-worker", "activity", 2), + held_poll(&app, "waiting-query-worker", "query", 2), + ]; + // Each poll's first claim takes the lock; let them finish and wait. + tokio::time::sleep(Duration::from_millis(500)).await; + let writer = hold_transition_lock(&database).await; + for poll in polls { + // The polls end about 1.5 s from now. A probe that took the lock + // would wait for it, and the lock is held past this bound. + let (status, body) = tokio::time::timeout(Duration::from_secs(3), poll) + .await + .expect("an idle poll must reach its deadline without the transition lock") + .unwrap(); + assert_eq!(status, StatusCode::OK, "{body}"); + assert!(body["task"].is_null(), "{body}"); + } + writer.release().await; + runtime.close().await; + database.remove().await; +} + +#[tokio::test] +async fn held_polls_observe_work_and_removed_registrations_from_another_node() { + let database = TestDatabase::new().await; + let node_a = database.open().await.unwrap(); + let node_b = database.open().await.unwrap(); + let app_a = router(node_a.clone()); + let app_b = router(node_b.clone()); + register(&app_a, "observing-worker", json!(["echo"]), json!([])).await; + query_worker(&app_a, "observing-query-worker").await; + + // Node B's commits cannot wake node A's poll in process. + let waiting = held_poll(&app_a, "observing-worker", "workflow", 20); + tokio::time::sleep(Duration::from_millis(300)).await; + let started = start(&app_b, "observed-elsewhere").await; + let (status, claimed) = tokio::time::timeout(Duration::from_secs(5), waiting) + .await + .expect("a held poll claims work another node committed") + .unwrap(); + assert_eq!(status, StatusCode::OK, "{claimed}"); + assert_eq!(claimed["task"]["run_id"], started["run_id"], "{claimed}"); + let completed = request( + &app_a, + "POST", + &format!( + "/api/worker/workflow-tasks/{}/complete", + claimed["task"]["task_id"].as_str().unwrap() + ), + completion( + &claimed["task"], + json!([{"type":"complete_workflow","result":envelope(Payload::Null)}]), + ), + ) + .await; + assert_eq!(completed.0, StatusCode::OK, "{}", completed.1); + + let waiting = held_poll(&app_a, "observing-query-worker", "query", 5); + tokio::time::sleep(Duration::from_millis(300)).await; + let query = { + let app = app_b.clone(); + let path = format!( + "/api/workflows/observed-elsewhere/runs/{}/query/state", + started["run_id"].as_str().unwrap() + ); + let input = json!({"input":envelope(Payload::Array(vec![Payload::Long(1)]))}); + tokio::spawn(async move { request(&app, "POST", &path, input).await }) + }; + let (status, claimed) = tokio::time::timeout(Duration::from_secs(3), waiting) + .await + .expect("a held query poll claims a query another node admitted") + .unwrap(); + assert_eq!(status, StatusCode::OK, "{claimed}"); + assert_eq!(claimed["task"]["query_name"], "state", "{claimed}"); + query.abort(); + + let waiting = held_poll(&app_a, "observing-worker", "workflow", 20); + tokio::time::sleep(Duration::from_millis(300)).await; + let removed = request( + &app_b, + "DELETE", + "/api/worker/registrations/observing-worker", + Value::Null, + ) + .await; + assert_eq!(removed.0, StatusCode::OK, "{}", removed.1); + let (status, refused) = tokio::time::timeout(Duration::from_secs(5), waiting) + .await + .expect("a held poll refuses a registration another node removed") + .unwrap(); + assert_eq!(status, StatusCode::CONFLICT, "{refused}"); + assert_eq!(refused["reason"], "worker_not_registered", "{refused}"); + node_a.close().await; + node_b.close().await; + database.remove().await; +} diff --git a/rust/tests/execution/read_snapshots.rs b/rust/tests/execution/read_snapshots.rs index 8dcb0d814..2c0fdcfb9 100644 --- a/rust/tests/execution/read_snapshots.rs +++ b/rust/tests/execution/read_snapshots.rs @@ -3,7 +3,7 @@ use super::*; use sqlx::Connection; -enum Writer { +pub(super) enum Writer { Sqlite(sqlx::SqliteConnection), Postgres(sqlx::PgConnection), MySql(sqlx::MySqlConnection), @@ -11,7 +11,7 @@ enum Writer { // Takes the lock a native transition takes, on a connection outside the // runtime's pool, as a transition from another server process would. -async fn hold_transition_lock(database: &TestDatabase) -> Writer { +pub(super) async fn hold_transition_lock(database: &TestDatabase) -> Writer { match database { TestDatabase::Sqlite(dir) => { let mut connection = sqlx::SqliteConnection::connect(&format!( @@ -56,7 +56,7 @@ async fn hold_transition_lock(database: &TestDatabase) -> Writer { } impl Writer { - async fn release(self) { + pub(super) async fn release(self) { match self { Self::Sqlite(mut connection) => { sqlx::raw_sql("ROLLBACK") From 7e4919313a0d5ac0512e22c5e181988a4ddb9902 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 07:50:39 +0000 Subject: [PATCH 06/37] Import the codec test names explicitly Co-Authored-By: Claude Opus 5.5 --- rust/src/runtime/transition_codec_tests.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/rust/src/runtime/transition_codec_tests.rs b/rust/src/runtime/transition_codec_tests.rs index becf90d87..9f02385d6 100644 --- a/rust/src/runtime/transition_codec_tests.rs +++ b/rust/src/runtime/transition_codec_tests.rs @@ -1,9 +1,11 @@ //! Transitions that encode or decode while holding the transition lock use //! their own codec permit. Each test holds the read codec permit, as a long //! describe decode would, and requires the transition to finish anyway. -use super::*; +use super::{Result, Runtime}; use crate::codec::{Value as Payload, ValueCodec}; -use serde_json::json; +use axum::http::StatusCode; +use serde_json::{Value, json}; +use std::time::Duration; fn envelope(value: Payload) -> Value { json!({"codec":"avro","blob":ValueCodec::new().unwrap().encode(&value).unwrap()}) From 0ac1b8feacb57b38b7cbfdfa08cf0c8e8e92b3bd Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 07:56:11 +0000 Subject: [PATCH 07/37] Retry the PHP reference's lock-pressure refusals below the SDK transport The published PHP SQLite reference shares its database file with its maintenance loop and sometimes refuses a request with its documented HTTP 503 backend_lock_pressure reply: retryable, nothing committed, worker mutations fenced by task and attempt. Probe runs failed on these refusals for polls, worker operations and a Rust SDK actor's deregistration. - Parity probes now build their SDK clients through parityClient(), whose default transport is the SDK's own Psr18Transport over Guzzle with an innermost retry middleware. The transport keeps every capability the SDK checks (bounded requests and payload transfer), unlike the earlier wrapper. Observed-transport stacks add the same middleware innermost, so recording middleware sees only the final reply. - lockPressureOperationRetryDelay retries any non-poll request only on status 503, reason backend_lock_pressure and retryable true, within 15 retries, 30 s of waiting and the request's own timeout. Polls keep the stricter rule that no task was handed out. Bodies that cannot be replayed are never retried. Retries are reported once on stderr. - The Rust SDK actor retries its deregistration on the same refusal, under the same bounds. Co-Authored-By: Claude Opus 5.5 --- .../server-parity/activity-deadline-probe.php | 3 +- .../server-parity/admission-probe.php | 3 +- .../server-parity/cancellation-probe.php | 5 +- .../child-cancellation-probe.php | 3 +- .../server-parity/cooperative-probe.php | 6 +- .../server-parity/cooperative-restart.php | 7 +- .../rust/src/lock_pressure.rs | 94 ++++++++++++++++++ .../corrected-markers/rust/src/main.rs | 6 +- .../corrected-markers/rust/src/memo_author.rs | 5 +- .../lock-pressure-middleware.php | 95 +++++++++++++++++++ .../server-parity/lock-pressure-retry.php | 42 ++++++++ .../server-parity/namespace-probe.php | 4 +- .../server-parity/patch-deployment-probe.php | 6 +- scripts/conformance/server-parity/probe.php | 8 +- .../server-parity/sdk-direction-probe.php | 5 +- .../sdk-namespace-signal-probe.php | 5 +- .../sdk-query-direction-probe.php | 5 +- .../sdk-selected-run-signal-probe.php | 5 +- .../sdk-update-direction-probe.php | 5 +- .../server-parity/task-retry-probe.php | 3 +- .../server-parity/waiting-history-probe.php | 3 +- .../ServerParityLockPressureRetryTest.php | 24 +++++ 22 files changed, 312 insertions(+), 30 deletions(-) create mode 100644 scripts/conformance/server-parity/corrected-markers/rust/src/lock_pressure.rs create mode 100644 scripts/conformance/server-parity/lock-pressure-middleware.php diff --git a/scripts/conformance/server-parity/activity-deadline-probe.php b/scripts/conformance/server-parity/activity-deadline-probe.php index b4bd61dfe..4728f6264 100644 --- a/scripts/conformance/server-parity/activity-deadline-probe.php +++ b/scripts/conformance/server-parity/activity-deadline-probe.php @@ -3,6 +3,7 @@ declare(strict_types=1); use DurableWorkflow\Client; +require_once __DIR__.'/lock-pressure-middleware.php'; require_once __DIR__.'/namespace-probe.php'; require_once __DIR__.'/activity-deadline-poll.php'; @@ -34,7 +35,7 @@ function activityDeadlineHttpObservation(array $fixture, array $options): array if (!preg_match('#^http://127\.0\.0\.1:[0-9]+$#', $url)) { throw new RuntimeException('Activity deadline recorder did not bind its local port'); } - $client = new Client($url, namespace:'default', token:getenv('DW_PARITY_TOKEN') ?: null); + $client = parityClient($url, namespace:'default', token:getenv('DW_PARITY_TOKEN') ?: null); $checkBudget = static function () use ($deadline): void { if (microtime(true) >= $deadline) { throw new RuntimeException('Original activity deadline observation expired'); diff --git a/scripts/conformance/server-parity/admission-probe.php b/scripts/conformance/server-parity/admission-probe.php index 0e681a1db..337bd9564 100644 --- a/scripts/conformance/server-parity/admission-probe.php +++ b/scripts/conformance/server-parity/admission-probe.php @@ -8,6 +8,7 @@ use Workflow\V2\StartOptions; use Workflow\V2\WorkflowStub; use Workflow\WorkflowOptions; +require_once __DIR__.'/lock-pressure-middleware.php'; function parityAdmissionToken(string $auth): string { @@ -86,7 +87,7 @@ function finishHttpAdmission(Client $client, array $fixture, string $workflowId, throw $error; } finally { if (isset($definition['cleanup_auth'])) { - $cleanupClient = new Client($url, namespace: $namespace, token: parityAdmissionToken($definition['cleanup_auth'])); + $cleanupClient = parityClient($url, namespace: $namespace, token: parityAdmissionToken($definition['cleanup_auth'])); $state['cleanup_auth'] = $definition['cleanup_auth']; $state['cleanup'] = $cleanupClient->cancelWorkflow($peerId, $definition['cleanup_reason'], $peer->selectedRunId); } else { diff --git a/scripts/conformance/server-parity/cancellation-probe.php b/scripts/conformance/server-parity/cancellation-probe.php index 0292c6abc..98f31d054 100644 --- a/scripts/conformance/server-parity/cancellation-probe.php +++ b/scripts/conformance/server-parity/cancellation-probe.php @@ -17,6 +17,7 @@ use Workflow\V2\WorkflowStub; use Workflow\V2\StartOptions; use Workflow\WorkflowOptions; +require_once __DIR__.'/lock-pressure-middleware.php'; // Application callbacks use only the published APIs. These observations never // mutate engine rows or synthesize a claim, cancellation or terminal outcome. @@ -40,7 +41,7 @@ function cancellationReceipt(callable $submit): array function httpCancellationObservation(array $fixture, string $workflowId, string $namespace, string $queue, string $url): array { $polls = $completions = $activityPolls = $activityOutcomes = []; - $client = new Client($url, namespace: $namespace, token: getenv('DW_PARITY_TOKEN') ?: null, + $client = parityClient($url, namespace: $namespace, token: getenv('DW_PARITY_TOKEN') ?: null, transport: observedChildTransport($polls, $completions, $activityPolls, $activityOutcomes)); $phase = $fixture['immediate_cancellation']['phase']; $reason = $fixture['immediate_cancellation']['reason']; @@ -113,7 +114,7 @@ function httpCancellationObservation(array $fixture, string $workflowId, string $task['activity_attempt_id'], $task['lease_owner'], $fixture['input'])); } // A separate published Client reloads the same durable run and history. - $fresh = new Client($url, namespace: $namespace, token: getenv('DW_PARITY_TOKEN') ?: null); + $fresh = parityClient($url, namespace: $namespace, token: getenv('DW_PARITY_TOKEN') ?: null); $inspectionWorker = $workflowId.':inspection'; $fresh->registerWorker($inspectionWorker, $queue, [$fixture['workflow_type']], ['parity.v1.cancel_activity']); $remaining = ['workflow' => $fresh->pollWorkflowTask($inspectionWorker, $queue, 0), diff --git a/scripts/conformance/server-parity/child-cancellation-probe.php b/scripts/conformance/server-parity/child-cancellation-probe.php index b548be1bb..7b8ee4449 100644 --- a/scripts/conformance/server-parity/child-cancellation-probe.php +++ b/scripts/conformance/server-parity/child-cancellation-probe.php @@ -5,6 +5,7 @@ use DurableWorkflow\Client; use Workflow\V2\Models\WorkflowRun; use Workflow\V2\WorkflowStub; +require_once __DIR__.'/lock-pressure-middleware.php'; final class ChildCancellationProbeState { @@ -54,7 +55,7 @@ function finishHttpChildCancellation(Client $client, string $parentId, string $p $state['terminal_duplicate'] = cancellationReceipt(static fn (): array => $client->cancelWorkflow($id, 'post-parent-completion replacement', $runId)); $state['child_history_after_terminal_duplicate'] = httpHistory($client, $id, $runId); $state['parent_history_after_terminal_duplicate'] = httpHistory($client, $parentId, $parentRunId); - $fresh = new Client($url, namespace: $namespace, token: getenv('DW_PARITY_TOKEN') ?: null); + $fresh = parityClient($url, namespace: $namespace, token: getenv('DW_PARITY_TOKEN') ?: null); $state['fresh_child'] = $fresh->describeWorkflow($id, $runId)->raw; $state['fresh_child_history'] = httpHistory($fresh, $id, $runId); $state['fresh_parent'] = $fresh->describeWorkflow($parentId, $parentRunId)->raw; diff --git a/scripts/conformance/server-parity/cooperative-probe.php b/scripts/conformance/server-parity/cooperative-probe.php index cb2e19d83..b856b308c 100644 --- a/scripts/conformance/server-parity/cooperative-probe.php +++ b/scripts/conformance/server-parity/cooperative-probe.php @@ -21,10 +21,12 @@ use Workflow\V2\WorkflowStub; use Workflow\V2\StartOptions; use Workflow\WorkflowOptions; +require_once __DIR__.'/lock-pressure-middleware.php'; function cooperativeObservedTransport(array &$traffic): Psr18Transport { $stack = HandlerStack::create(); + $stack->unshift(LockPressurePollRetries::middleware(), 'lock_pressure_poll_retry'); $stack->push(static function (callable $handler) use (&$traffic): callable { return static function (RequestInterface $request, array $options) use ($handler, &$traffic) { return $handler($request, $options)->then(static function (ResponseInterface $response) use ($request, &$traffic): ResponseInterface { @@ -75,7 +77,7 @@ function httpCooperativeObservation(array $fixture, string $workflowId, string $ { $traffic = []; $diagnostics = []; - $client = new Client($url, namespace: $namespace, token: getenv('DW_PARITY_TOKEN') ?: null, + $client = parityClient($url, namespace: $namespace, token: getenv('DW_PARITY_TOKEN') ?: null, transport: cooperativeObservedTransport($traffic), workerProtocolVersion: '1.20'); $phase = $fixture['cooperative_cancellation']['phase']; $reason = $fixture['cooperative_cancellation']['reason']; @@ -150,7 +152,7 @@ static function (string $event, array $details) use (&$handle, &$diagnostics, $c } catch (WorkflowCancelled $error) { $outcome = ['type' => $error::class, 'message' => $error->getMessage()]; } - $fresh = new Client($url, namespace: $namespace, token: getenv('DW_PARITY_TOKEN') ?: null, workerProtocolVersion: '1.20'); + $fresh = parityClient($url, namespace: $namespace, token: getenv('DW_PARITY_TOKEN') ?: null, workerProtocolVersion: '1.20'); return ['execution' => $execution->raw, 'workflow_id' => $execution->workflowId, 'run_id' => $execution->runId, 'workflow_type' => $execution->workflowType, 'namespace' => $execution->namespace, 'task_queue' => $execution->taskQueue, diff --git a/scripts/conformance/server-parity/cooperative-restart.php b/scripts/conformance/server-parity/cooperative-restart.php index 6fd9950b3..2e4b81d3b 100644 --- a/scripts/conformance/server-parity/cooperative-restart.php +++ b/scripts/conformance/server-parity/cooperative-restart.php @@ -8,6 +8,7 @@ use DurableWorkflow\Worker\CancellationDelivery; require __DIR__.'/vendor/autoload.php'; +require_once __DIR__.'/lock-pressure-middleware.php'; require __DIR__.'/observation-values.php'; require __DIR__.'/cooperative-probe.php'; @@ -56,7 +57,7 @@ function cooperativeEvent(array $events, string $type): array $workflowId = 'root-restart-'.$outcome; $queue = 'root-restart-'.$outcome; $traffic = []; - $client = new Client($url, namespace: 'default', token: getenv('DW_PARITY_TOKEN') ?: null, + $client = parityClient($url, namespace: 'default', token: getenv('DW_PARITY_TOKEN') ?: null, transport: cooperativeObservedTransport($traffic), workerProtocolVersion: '1.20'); $worker = $handle = $accepted = null; $guard = microtime(true) + 20; @@ -118,7 +119,7 @@ static function (string $event) use (&$handle, $client, $fixture, $workflowId, $ foreach (['deadline_expired', 'completed'] as $outcome) { $receipt = $receipts[$outcome]; $traffic = []; - $client = new Client($url, namespace: 'default', token: getenv('DW_PARITY_TOKEN') ?: null, + $client = parityClient($url, namespace: 'default', token: getenv('DW_PARITY_TOKEN') ?: null, transport: cooperativeObservedTransport($traffic), workerProtocolVersion: '1.20'); $handle = $client->workflowHandle($receipt['workflow_id'], $receipt['run_id']); $before = httpHistory($client, $receipt['workflow_id'], $receipt['run_id']); @@ -188,7 +189,7 @@ static function () use (&$worker, $handle, $guard): float { } $duplicate = $client->requestWorkflowCancellation($receipt['workflow_id'], 'post-terminal replacement', 3600, $receipt['run_id']); assertCooperativeRecovery($duplicate['duplicate'] === true && $duplicate['cancellation_request']['cleanup_deadline_at'] === $receipt['context']['cleanup_deadline_at'], 'Terminal duplicate changed the original budget.'); - $fresh = new Client($url, namespace: 'default', token: getenv('DW_PARITY_TOKEN') ?: null, workerProtocolVersion: '1.20'); + $fresh = parityClient($url, namespace: 'default', token: getenv('DW_PARITY_TOKEN') ?: null, workerProtocolVersion: '1.20'); assertCooperativeRecovery(sameCooperativeValue($events, httpHistory($fresh, $receipt['workflow_id'], $receipt['run_id'])), 'Late calls or fresh reads changed terminal history.'); try { $fresh->workflowHandle($receipt['workflow_id'], $receipt['run_id'])->resultOfSelectedRun(1); diff --git a/scripts/conformance/server-parity/corrected-markers/rust/src/lock_pressure.rs b/scripts/conformance/server-parity/corrected-markers/rust/src/lock_pressure.rs new file mode 100644 index 000000000..5f9f9f9b9 --- /dev/null +++ b/scripts/conformance/server-parity/corrected-markers/rust/src/lock_pressure.rs @@ -0,0 +1,94 @@ +//! The published PHP SQLite reference can refuse a worker operation with its +//! documented HTTP 503 `backend_lock_pressure` reply and `retryable: true` +//! after its transaction failed, so nothing was committed. Only that refusal is +//! retried, within 15 retries and 30 seconds of waiting, as the PHP probes' +//! transport does (`../lock-pressure-retry.php`). Every other outcome is +//! returned unchanged. +use durable_workflow::{Error, Result}; +use serde_json::Value; +use std::{future::Future, time::Duration}; + +const MAX_RETRIES: u32 = 15; +const MAX_WAIT: Duration = Duration::from_secs(30); + +fn delay_for(status: u16, body: &str, retries: u32, waited: Duration) -> Option { + if status != 503 || retries >= MAX_RETRIES { + return None; + } + let body: Value = serde_json::from_str(body).ok()?; + if body.get("reason")?.as_str()? != "backend_lock_pressure" + || body.get("retryable") != Some(&Value::Bool(true)) + { + return None; + } + let seconds = match body.get("retry_after_seconds") { + None => 1.0, + Some(value) => value.as_f64()?, + }; + let delay = Duration::from_secs_f64(seconds.clamp(0.1, 5.0)); + (waited + delay <= MAX_WAIT).then_some(delay) +} + +/// Runs `call`, repeating it only after the documented lock-pressure refusal. +pub async fn retrying(what: &str, mut call: F) -> Result +where + F: FnMut() -> Fut, + Fut: Future>, +{ + let (mut retries, mut waited) = (0, Duration::ZERO); + loop { + let error = match call().await { + Err(error) => error, + done => return done, + }; + let delay = match &error { + Error::Http { status, body } => delay_for(status.as_u16(), body, retries, waited), + _ => None, + }; + let Some(delay) = delay else { + return Err(error); + }; + eprintln!("{what}: retrying backend_lock_pressure refusal after {delay:?}"); + tokio::time::sleep(delay).await; + waited += delay; + retries += 1; + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const REFUSAL: &str = r#"{"reason":"backend_lock_pressure","operation":"destroy","outcome":"deferred","retryable":true,"retry_after_seconds":1}"#; + + #[test] + fn documented_refusal_waits_for_the_advertised_interval() { + assert_eq!(delay_for(503, REFUSAL, 0, Duration::ZERO), Some(Duration::from_secs(1))); + let long = REFUSAL.replace(r#""retry_after_seconds":1"#, r#""retry_after_seconds":60"#); + assert_eq!(delay_for(503, &long, 0, Duration::ZERO), Some(Duration::from_secs(5))); + let absent = REFUSAL.replace(r#","retry_after_seconds":1"#, ""); + assert_eq!(delay_for(503, &absent, 0, Duration::ZERO), Some(Duration::from_secs(1))); + } + + #[test] + fn other_statuses_reasons_and_bodies_are_returned() { + for (status, body) in [ + (409, REFUSAL.to_owned()), + (500, REFUSAL.to_owned()), + (503, REFUSAL.replace("backend_lock_pressure", "backend_unavailable")), + (503, REFUSAL.replace(r#""retryable":true"#, r#""retryable":false"#)), + (503, REFUSAL.replace(r#","retryable":true"#, "")), + (503, REFUSAL.replace(r#""retry_after_seconds":1"#, r#""retry_after_seconds":"soon""#)), + (503, "not json".to_owned()), + ] { + assert_eq!(delay_for(status, &body, 0, Duration::ZERO), None, "{status} {body}"); + } + } + + #[test] + fn retries_stop_at_the_attempt_and_wait_budgets() { + assert_eq!(delay_for(503, REFUSAL, 15, Duration::ZERO), None); + assert_eq!(delay_for(503, REFUSAL, 2, Duration::from_millis(29_500)), None); + assert_eq!(delay_for(503, REFUSAL, 14, Duration::from_secs(29)), Some(Duration::from_secs(1))); + } +} diff --git a/scripts/conformance/server-parity/corrected-markers/rust/src/main.rs b/scripts/conformance/server-parity/corrected-markers/rust/src/main.rs index c5f7bdb7d..3a1d863c7 100644 --- a/scripts/conformance/server-parity/corrected-markers/rust/src/main.rs +++ b/scripts/conformance/server-parity/corrected-markers/rust/src/main.rs @@ -11,6 +11,7 @@ use durable_workflow::{ use serde_json::{json, Value}; mod legacy_range; +mod lock_pressure; mod memo_author; const WORKFLOW: &str = "parity.v1.corrected_markers"; @@ -221,7 +222,10 @@ async fn main() -> Result<()> { } }) .await; - let deregistered = client.deregister_worker_registration(&worker_id).await; + let deregistered = lock_pressure::retrying("deregister", || { + client.deregister_worker_registration(&worker_id) + }) + .await; let result = match outcome { Ok(result) => result, Err(_) => Err(Error::WorkerLoop( diff --git a/scripts/conformance/server-parity/corrected-markers/rust/src/memo_author.rs b/scripts/conformance/server-parity/corrected-markers/rust/src/memo_author.rs index 8520ac901..d2c4c04c2 100644 --- a/scripts/conformance/server-parity/corrected-markers/rust/src/memo_author.rs +++ b/scripts/conformance/server-parity/corrected-markers/rust/src/memo_author.rs @@ -90,7 +90,10 @@ pub async fn run(args: &[String]) -> Result<()> { }) .await .map_err(|_| Error::WorkerLoop("Original memo phase observation expired".into()))?; - let retired = client.deregister_worker_registration(&worker_id).await; + let retired = super::lock_pressure::retrying("deregister", || { + client.deregister_worker_registration(&worker_id) + }) + .await; outcome?; retired?; println!( diff --git a/scripts/conformance/server-parity/lock-pressure-middleware.php b/scripts/conformance/server-parity/lock-pressure-middleware.php new file mode 100644 index 000000000..11cf1d5ef --- /dev/null +++ b/scripts/conformance/server-parity/lock-pressure-middleware.php @@ -0,0 +1,95 @@ + */ + public static array $retries = []; + private static bool $reporting = false; + + public static function middleware(): callable + { + if (!self::$reporting) { + self::$reporting = true; + register_shutdown_function(static function (): void { + if (self::$retries !== []) { + fwrite(STDERR, json_encode(['lock_pressure_poll_retries' => self::$retries], JSON_THROW_ON_ERROR).PHP_EOL); + } + }); + } + + return static fn (callable $handler): callable => static function (RequestInterface $request, array $options) use ($handler) { + $started = microtime(true); + $budget = (float) ($options['timeout'] ?? 0); + $waited = 0.0; + $count = 0; + while (true) { + try { + $response = $handler($request, $options)->wait(); + } catch (\Throwable $error) { + return Create::rejectionFor($error); + } + // A request body that cannot be replayed exactly is never retried. + if (!$response instanceof ResponseInterface || $response->getStatusCode() !== 503 + || (!$request->getBody()->isSeekable() && $request->getBody()->getSize() !== 0)) { + return Create::promiseFor($response); + } + $text = (string) $response->getBody(); + $response = $response->withBody(Utils::streamFor($text)); + $decoded = json_decode($text, true); + $delay = lockPressureOperationRetryDelay($request->getMethod(), (string) $request->getUri(), 503, + is_array($decoded) ? $decoded : null, $count, $waited); + if ($delay === null || ($budget > 0 && microtime(true) - $started + $delay >= $budget)) { + return Create::promiseFor($response); + } + self::$retries[] = ['path' => $request->getUri()->getPath(), 'retry_after_seconds' => $delay]; + usleep((int) round($delay * 1_000_000)); + $waited += $delay; + $count++; + if ($request->getBody()->isSeekable()) { + $request->getBody()->rewind(); + } + } + }; + } + + /** The SDK's default transport, with this retry innermost. */ + public static function transport(): Psr18Transport + { + $stack = HandlerStack::create(); + $stack->unshift(self::middleware(), 'lock_pressure_poll_retry'); + + return new Psr18Transport(client: new GuzzleClient(['http_errors' => false, 'handler' => $stack])); + } +} + +/** An SDK client for a parity probe. A caller-supplied transport is used unchanged. */ +function parityClient(mixed ...$arguments): Client +{ + $arguments['transport'] ??= LockPressurePollRetries::transport(); + + return new Client(...$arguments); +} diff --git a/scripts/conformance/server-parity/lock-pressure-retry.php b/scripts/conformance/server-parity/lock-pressure-retry.php index fa7c28af4..112299d78 100644 --- a/scripts/conformance/server-parity/lock-pressure-retry.php +++ b/scripts/conformance/server-parity/lock-pressure-retry.php @@ -38,3 +38,45 @@ function lockPressurePollRetryDelay( return $waitedSeconds + $delay <= $maxWaitSeconds ? $delay : null; } + +/** + * Decide whether any other request refused for backend lock pressure may be + * retried. + * + * Server returns this refusal as HTTP 503 with `reason: backend_lock_pressure` + * and `retryable: true` only after its transaction failed with a concurrency + * error, so nothing was committed. Worker mutations remain fenced by their + * task and attempt identifiers (`outcome: deferred`), so repeating the request + * cannot record a second outcome. Polls keep lockPressurePollRetryDelay's + * stricter check that no task was handed out. + * + * Returns the delay in seconds before the next attempt, or null to return the + * refusal to the caller unchanged. + */ +function lockPressureOperationRetryDelay( + string $method, + string $uri, + ?int $status, + mixed $response, + int $retriesSoFar, + float $waitedSeconds, + int $maxRetries = 15, + float $maxWaitSeconds = 30.0, +): ?float { + $path = parse_url($uri, PHP_URL_PATH); + if (! is_string($path) || str_ends_with($path, '/poll')) { + return lockPressurePollRetryDelay($method, $uri, $status, $response, $retriesSoFar, $waitedSeconds, + $maxRetries, $maxWaitSeconds); + } + if ($status !== 503 || ! is_array($response) || ($response['reason'] ?? null) !== 'backend_lock_pressure' + || ($response['retryable'] ?? null) !== true || $retriesSoFar >= $maxRetries) { + return null; + } + $retryAfter = $response['retry_after_seconds'] ?? 1; + if (! is_int($retryAfter) && ! is_float($retryAfter)) { + return null; + } + $delay = min(max((float) $retryAfter, 0.1), 5.0); + + return $waitedSeconds + $delay <= $maxWaitSeconds ? $delay : null; +} diff --git a/scripts/conformance/server-parity/namespace-probe.php b/scripts/conformance/server-parity/namespace-probe.php index 7422ec260..4f756671d 100644 --- a/scripts/conformance/server-parity/namespace-probe.php +++ b/scripts/conformance/server-parity/namespace-probe.php @@ -20,6 +20,7 @@ use Workflow\V2\WorkflowStub; use Workflow\WorkflowOptions; use Workflow\Serializers\Serializer; +require_once __DIR__.'/lock-pressure-middleware.php'; function namespaceReceipt(string $id, callable $request): array { @@ -45,6 +46,7 @@ function namespaceHttpRun(Client $client, string $workflowId, string $runId): ar function namespaceTransport(array &$receipts, ?Closure $beforeCompletion = null): Psr18Transport { $stack = HandlerStack::create(); + $stack->unshift(LockPressurePollRetries::middleware(), 'lock_pressure_poll_retry'); $stack->push(static function (callable $handler) use (&$receipts, $beforeCompletion): callable { return static function (RequestInterface $request, array $options) use ($handler, &$receipts, $beforeCompletion) { $path = $request->getUri()->getPath(); @@ -98,7 +100,7 @@ function finishHttpNamespaces(Client $admin, array $fixture, string $workflowId, 'before' => $before, 'after' => namespaceHttpRun($clients[0], $ids[0], $handles[0]->selectedRunId)]; }; foreach ($names as $index => $name) { - $clients[] = new Client($url, namespace: $name, token: getenv('DW_PARITY_TOKEN') ?: null, + $clients[] = parityClient($url, namespace: $name, token: getenv('DW_PARITY_TOKEN') ?: null, transport: namespaceTransport($state['worker_receipts'][$index], $index === 0 ? $beforeCompletion : null)); $state['registrations'][] = $clients[$index]->registerWorker($state['worker_id'], $queue, [$definition['peer_workflow_type']], ['parity.v1.echo_activity']); diff --git a/scripts/conformance/server-parity/patch-deployment-probe.php b/scripts/conformance/server-parity/patch-deployment-probe.php index 032944130..204b073c4 100644 --- a/scripts/conformance/server-parity/patch-deployment-probe.php +++ b/scripts/conformance/server-parity/patch-deployment-probe.php @@ -20,6 +20,7 @@ use Workflow\V2\StartOptions; use Workflow\V2\WorkflowStub; use Workflow\WorkflowOptions; +require_once __DIR__.'/lock-pressure-middleware.php'; final class PatchDeploymentState { @@ -100,6 +101,7 @@ function patchHttpPhase(array $fixture, array $options): array $worker = $handle = null; $requests = $diagnostics = []; $stack = HandlerStack::create(); + $stack->unshift(LockPressurePollRetries::middleware(), 'lock_pressure_poll_retry'); $stack->push(static function (callable $handler) use (&$worker, &$requests, $phase, $fixture): callable { return static function (RequestInterface $request, array $transportOptions) use ($handler, &$worker, &$requests, $phase, $fixture) { return $handler($request, $transportOptions)->then(static function (ResponseInterface $response) use ($request, &$worker, &$requests, $phase, $fixture): ResponseInterface { @@ -126,7 +128,7 @@ function patchHttpPhase(array $fixture, array $options): array }); }; }); - $client = new Client($options['url'], namespace: 'default', token: getenv('DW_PARITY_TOKEN') ?: null, + $client = parityClient($options['url'], namespace: 'default', token: getenv('DW_PARITY_TOKEN') ?: null, transport: new Psr18Transport(client: new GuzzleClient(['http_errors' => false, 'handler' => $stack]))); if ($phase === 'replacement') { $handle = $client->workflowHandle($workflowId, $options['run-id']); @@ -279,7 +281,7 @@ function patchPublishedSdkPhase(array $fixture, array $options): array if ($result['language'] !== $consumer['language'] || $result['sdk_version'] !== $consumer['version']) { throw new RuntimeException('Actual published worker differs from the selected consumer.'); } - $client = new Client($options['url'], namespace: 'default', token: getenv('DW_PARITY_TOKEN') ?: null); + $client = parityClient($options['url'], namespace: 'default', token: getenv('DW_PARITY_TOKEN') ?: null); $runId = $original ? ($result['run_id'] ?? null) : $options['run-id']; if (! is_string($runId) || $runId === '') { throw new RuntimeException('Actual published author returned no original run identity.'); diff --git a/scripts/conformance/server-parity/probe.php b/scripts/conformance/server-parity/probe.php index d63bc0f8d..9c078e331 100644 --- a/scripts/conformance/server-parity/probe.php +++ b/scripts/conformance/server-parity/probe.php @@ -64,12 +64,14 @@ require __DIR__.'/observation-values.php'; require __DIR__.'/probe-failure.php'; +require_once __DIR__.'/lock-pressure-middleware.php'; function observedChildTransport(array &$polls, array &$completions, array &$activityPolls, array &$activityOutcomes, ?Closure $afterCompletion = null, ?array &$scheduleReceipts = null): Psr18Transport { // Observe real published transport I/O. No response or request is changed, // and authentication headers are never retained. $stack = HandlerStack::create(); + $stack->unshift(LockPressurePollRetries::middleware(), 'lock_pressure_poll_retry'); $stack->push(static function (callable $handler) use (&$polls, &$completions, &$activityPolls, &$activityOutcomes, $afterCompletion, &$scheduleReceipts): callable { return static function (RequestInterface $request, array $options) use ($handler, &$polls, &$completions, &$activityPolls, &$activityOutcomes, $afterCompletion, &$scheduleReceipts) { return $handler($request, $options)->then(static function (ResponseInterface $response) use ($request, &$polls, &$completions, &$activityPolls, &$activityOutcomes, $afterCompletion, &$scheduleReceipts): ResponseInterface { @@ -114,7 +116,7 @@ function observedChildTransport(array &$polls, array &$completions, array &$acti // client in an independent PHP process so this probe's worker keeps polling. if ($mode === 'query') { try { - $client = new Client($options['url'], namespace: $namespace, token: getenv('DW_PARITY_TOKEN') ?: null); + $client = parityClient($options['url'], namespace: $namespace, token: getenv('DW_PARITY_TOKEN') ?: null); $result = $client->queryWorkflow($workflowId, $fixture['query_name'], $fixture['query_arguments'], $options['run-id']); echo json_encode(['result' => $result, 'typed_result' => typedValue($result)], JSON_THROW_ON_ERROR | JSON_UNESCAPED_UNICODE)."\n"; exit(0); @@ -142,9 +144,9 @@ function httpObservation(array $fixture, string $workflowId, string $namespace, } : null; $transport = isset($fixture['child_count']) || isset($fixture['child_cancellation']) || isset($fixture['retry_policy']) || isset($fixture['schedule']) ? observedChildTransport($workflowPolls, $workflowCompletions, $activityPolls, $activityOutcomes, $afterCompletion, $scheduleReceipts) : null; - $client = new Client($url, namespace: $namespace, transport: $transport, token: getenv('DW_PARITY_TOKEN') ?: null); + $client = parityClient($url, namespace: $namespace, transport: $transport, token: getenv('DW_PARITY_TOKEN') ?: null); $workerClient = ($fixture['auth_profile'] ?? null) === 'role_tokens' - ? new Client($url, namespace: $namespace, token: parityAdmissionToken('worker')) : $client; + ? parityClient($url, namespace: $namespace, token: parityAdmissionToken('worker')) : $client; $handle = null; $deliveries = []; $queries = []; diff --git a/scripts/conformance/server-parity/sdk-direction-probe.php b/scripts/conformance/server-parity/sdk-direction-probe.php index b7fa2df5d..1a582e4a8 100644 --- a/scripts/conformance/server-parity/sdk-direction-probe.php +++ b/scripts/conformance/server-parity/sdk-direction-probe.php @@ -5,6 +5,7 @@ use DurableWorkflow\Client; use DurableWorkflow\Worker; use DurableWorkflow\Worker\WorkflowContext; +require_once __DIR__.'/lock-pressure-middleware.php'; function sdkDirectionResponseSummary(mixed $response, bool $workflowHistory = false): mixed { @@ -73,7 +74,7 @@ function sdkDirectionHttpObservation(array $fixture, array $options, ?Closure $a $spec = $fixture[$directionKey]; $childWorkflowId = $childRunId = ''; $namespace = $fixture['namespace'] ?? 'default'; - $client = new Client($options['url'], namespace: $namespace, token: getenv('DW_PARITY_TOKEN') ?: null); + $client = parityClient($options['url'], namespace: $namespace, token: getenv('DW_PARITY_TOKEN') ?: null); $deadline = $scenarioDeadline ?? microtime(true) + 35; $observerStart = null; try { @@ -81,7 +82,7 @@ function sdkDirectionHttpObservation(array $fixture, array $options, ?Closure $a if (! preg_match('#^http://127\\.0\\.0\\.1:[0-9]+$#', $url)) { throw new RuntimeException('SDK observer did not bind its local port.'); } - $startClient = $signalDirection ? new Client($url, namespace: $namespace, token: getenv('DW_PARITY_TOKEN') ?: null) : null; + $startClient = $signalDirection ? parityClient($url, namespace: $namespace, token: getenv('DW_PARITY_TOKEN') ?: null) : null; foreach ($spec['phase_order'] as $phase) { $language = $signalDirection ? $spec[in_array($phase, ['signal-1', 'signal-2'], true) ? 'client' : 'workflow'] : ($timerAuthor ? $spec['language'] : ($childDirection ? $spec[$phase === 'child' ? 'child' : 'parent'] diff --git a/scripts/conformance/server-parity/sdk-namespace-signal-probe.php b/scripts/conformance/server-parity/sdk-namespace-signal-probe.php index 082f7364a..1f7e411d5 100644 --- a/scripts/conformance/server-parity/sdk-namespace-signal-probe.php +++ b/scripts/conformance/server-parity/sdk-namespace-signal-probe.php @@ -11,6 +11,7 @@ use Workflow\V2\StartOptions; use Workflow\V2\WorkflowStub; use Workflow\WorkflowOptions; +require_once __DIR__.'/lock-pressure-middleware.php'; function namespaceSignalRunFixture(array $fixture, bool $peer): array { @@ -158,7 +159,7 @@ function namespaceSignalHttpObservation(array $fixture, array $options): array throw new RuntimeException('Namespace control observer did not bind its local port.'); } $checkpoint('client_setup'); - $admin = new Client($url, namespace: 'default', token: getenv('DW_PARITY_TOKEN') ?: null); + $admin = parityClient($url, namespace: 'default', token: getenv('DW_PARITY_TOKEN') ?: null); foreach ($fixtures as $index => $runFixture) { try { $checkpoint('namespace_describe', $runFixture['namespace']); @@ -175,7 +176,7 @@ function namespaceSignalHttpObservation(array $fixture, array $options): array $state['administration'][] = ['created' => $created, 'namespace' => $namespace->raw]; $paths[$index] = tempnam(sys_get_temp_dir(), 'parity-namespace-fixture-'); file_put_contents($paths[$index], json_encode($runFixture, JSON_THROW_ON_ERROR | JSON_UNESCAPED_UNICODE)); - $clients[$index] = new Client($options['url'], namespace: $runFixture['namespace'], token: getenv('DW_PARITY_TOKEN') ?: null); + $clients[$index] = parityClient($options['url'], namespace: $runFixture['namespace'], token: getenv('DW_PARITY_TOKEN') ?: null); $start = microtime(true); $deadline = $start + 35; $state['run_budgets'][$index] = ['started_at_unix' => $start, 'deadline_at_unix' => $deadline]; diff --git a/scripts/conformance/server-parity/sdk-query-direction-probe.php b/scripts/conformance/server-parity/sdk-query-direction-probe.php index 18dce9429..1beb8c232 100644 --- a/scripts/conformance/server-parity/sdk-query-direction-probe.php +++ b/scripts/conformance/server-parity/sdk-query-direction-probe.php @@ -4,6 +4,7 @@ use Composer\InstalledVersions; use DurableWorkflow\Client; +require_once __DIR__.'/lock-pressure-middleware.php'; function sdkQueryActorOpen(string $language, string $phase, array $fixture, array $options, string $runId, float $deadline): array { @@ -242,7 +243,7 @@ function sdkQueryDirectionHttpObservation(array $fixture, array $options): array throw new RuntimeException('SDK query direction requires its evidence directory.'); } $spec = $fixture['sdk_query_direction']; - $client = new Client($options['url'], namespace: 'default', token: getenv('DW_PARITY_TOKEN') ?: null); + $client = parityClient($options['url'], namespace: 'default', token: getenv('DW_PARITY_TOKEN') ?: null); $deadline = microtime(true) + 35; $runId = ''; $phases = $actors = []; @@ -262,7 +263,7 @@ function sdkQueryDirectionHttpObservation(array $fixture, array $options): array if ($request['method'] === 'POST' && $request['path'] === '/api/worker/register' && $request['status'] === 201 && ($request['response']['registered'] ?? null) === true && ($request['request']['worker_id'] ?? null) === $options['workflow-id'].':original') { - $startClient = new Client($author['url'], namespace: 'default', token: getenv('DW_PARITY_TOKEN') ?: null); + $startClient = parityClient($author['url'], namespace: 'default', token: getenv('DW_PARITY_TOKEN') ?: null); $runId = $startClient->startWorkflow($fixture['workflow_type'], $options['workflow-id'], $spec['task_queue'], [$fixture['input'], $fixture['signal_count']])->selectedRunId; $observerStart = ['role' => 'observer', 'language' => 'php', 'pid' => getmypid(), diff --git a/scripts/conformance/server-parity/sdk-selected-run-signal-probe.php b/scripts/conformance/server-parity/sdk-selected-run-signal-probe.php index a97a41a0f..cd11ce1c7 100644 --- a/scripts/conformance/server-parity/sdk-selected-run-signal-probe.php +++ b/scripts/conformance/server-parity/sdk-selected-run-signal-probe.php @@ -10,6 +10,7 @@ use Workflow\V2\StartOptions; use Workflow\V2\WorkflowStub; use Workflow\WorkflowOptions; +require_once __DIR__.'/lock-pressure-middleware.php'; require __DIR__.'/selected-run-signal-state.php'; @@ -70,7 +71,7 @@ function selectedRunSignalHttpObservation(array $fixture, array $options): array if (! preg_match('#^http://127\.0\.0\.1:[0-9]+$#', $url)) { throw new RuntimeException('Selected-run control observer failed to bind.'); } - $admin = new Client($url, namespace: 'default', token: getenv('DW_PARITY_TOKEN') ?: null); + $admin = parityClient($url, namespace: 'default', token: getenv('DW_PARITY_TOKEN') ?: null); foreach ($fixtures as $index => $runFixture) { $checkpoint('namespace_setup_'.$index); try { @@ -84,7 +85,7 @@ function selectedRunSignalHttpObservation(array $fixture, array $options): array $state['administration'][] = ['created' => $created, 'namespace' => $namespace->raw]; $paths[$index] = tempnam(sys_get_temp_dir(), 'parity-selected-run-'); file_put_contents($paths[$index], json_encode($runFixture, JSON_THROW_ON_ERROR | JSON_UNESCAPED_UNICODE)); - $clients[$index] = new Client($options['url'], namespace: $runFixture['namespace'], token: getenv('DW_PARITY_TOKEN') ?: null); + $clients[$index] = parityClient($options['url'], namespace: $runFixture['namespace'], token: getenv('DW_PARITY_TOKEN') ?: null); $start = microtime(true); $deadline = $start + 35; $state['run_budgets'][$index] = ['started_at_unix' => $start, 'deadline_at_unix' => $deadline]; diff --git a/scripts/conformance/server-parity/sdk-update-direction-probe.php b/scripts/conformance/server-parity/sdk-update-direction-probe.php index 9fe3f7bb3..e8ce522c0 100644 --- a/scripts/conformance/server-parity/sdk-update-direction-probe.php +++ b/scripts/conformance/server-parity/sdk-update-direction-probe.php @@ -4,6 +4,7 @@ use Composer\InstalledVersions; use DurableWorkflow\Client; +require_once __DIR__.'/lock-pressure-middleware.php'; function sdkUpdateActorOpen(string $language, string $phase, array $fixture, array $options, string $runId, float $deadline): array { @@ -78,7 +79,7 @@ function sdkUpdateDirectionHttpObservation(array $fixture, array $options): arra throw new RuntimeException('SDK update direction requires its evidence directory.'); } $spec = $fixture['sdk_update_direction']; - $client = new Client($options['url'], namespace: 'default', token: getenv('DW_PARITY_TOKEN') ?: null); + $client = parityClient($options['url'], namespace: 'default', token: getenv('DW_PARITY_TOKEN') ?: null); $deadline = microtime(true) + 35; $workflowId = $options['workflow-id']; $runId = ''; @@ -113,7 +114,7 @@ function sdkUpdateDirectionHttpObservation(array $fixture, array $options): arra if ($request['method'] === 'POST' && $request['path'] === '/api/worker/register' && $request['status'] === 201 && ($request['response']['registered'] ?? null) === true && ($request['request']['worker_id'] ?? null) === $workflowId.':original') { - $start = new Client($actor['url'], namespace: 'default', token: getenv('DW_PARITY_TOKEN') ?: null); + $start = parityClient($actor['url'], namespace: 'default', token: getenv('DW_PARITY_TOKEN') ?: null); $runId = $start->startWorkflow($fixture['workflow_type'], $workflowId, $spec['task_queue'], [$fixture['input'], $fixture['signal_count']])->selectedRunId; $observerStart = ['role' => 'observer', 'language' => 'php', 'pid' => getmypid(), diff --git a/scripts/conformance/server-parity/task-retry-probe.php b/scripts/conformance/server-parity/task-retry-probe.php index 42c81935b..fa05a5fd8 100644 --- a/scripts/conformance/server-parity/task-retry-probe.php +++ b/scripts/conformance/server-parity/task-retry-probe.php @@ -3,12 +3,13 @@ declare(strict_types=1); use DurableWorkflow\Client; +require_once __DIR__.'/lock-pressure-middleware.php'; function finishHttpTaskRetry(Client $admin, array $fixture, string $workflowId, string $queue, string $url): array { $definition = $fixture['workflow_task_retry']; $requests = []; - $client = new Client($url, namespace: 'default', token: getenv('DW_PARITY_TOKEN') ?: null, + $client = parityClient($url, namespace: 'default', token: getenv('DW_PARITY_TOKEN') ?: null, transport: namespaceTransport($requests)); $workerId = $workflowId.'-retry-worker'; $peerId = $workflowId.$definition['peer_suffix']; diff --git a/scripts/conformance/server-parity/waiting-history-probe.php b/scripts/conformance/server-parity/waiting-history-probe.php index 6d21e0026..26d4efd99 100644 --- a/scripts/conformance/server-parity/waiting-history-probe.php +++ b/scripts/conformance/server-parity/waiting-history-probe.php @@ -3,12 +3,13 @@ declare(strict_types=1); use DurableWorkflow\Client; +require_once __DIR__.'/lock-pressure-middleware.php'; function finishHttpWaitingHistory(Client $admin, array $fixture, string $workflowId, string $queue, string $url): array { $definition = $fixture['waiting_for_history']; $requests = []; - $client = new Client($url, namespace: 'default', token: getenv('DW_PARITY_TOKEN') ?: null, + $client = parityClient($url, namespace: 'default', token: getenv('DW_PARITY_TOKEN') ?: null, transport: namespaceTransport($requests)); $workerId = $workflowId.'-waiting-worker'; $peerId = $workflowId.$definition['peer_suffix']; diff --git a/tests/Unit/ServerParityLockPressureRetryTest.php b/tests/Unit/ServerParityLockPressureRetryTest.php index e29e3beb9..849e1b310 100644 --- a/tests/Unit/ServerParityLockPressureRetryTest.php +++ b/tests/Unit/ServerParityLockPressureRetryTest.php @@ -50,6 +50,30 @@ public function test_other_requests_statuses_and_bodies_are_never_retried(): voi } } + public function test_documented_operation_refusals_are_retried_and_polls_keep_their_rules(): void + { + $deferred = ['reason' => 'backend_lock_pressure', 'outcome' => 'deferred', 'retryable' => true, 'retry_after_seconds' => 1]; + $complete = 'http://parity-server:8080/api/worker/workflow-tasks/task/complete'; + $start = 'http://parity-server:8080/api/workflows'; + self::assertSame(1.0, \lockPressureOperationRetryDelay('POST', $complete, 503, $deferred, 0, 0.0)); + self::assertSame(1.0, \lockPressureOperationRetryDelay('POST', $start, 503, + ['reason' => 'backend_lock_pressure', 'retryable' => true, 'retry_after_seconds' => 1], 0, 0.0)); + self::assertSame(1.0, \lockPressureOperationRetryDelay('POST', self::POLL, 503, $this->refusal(), 0, 0.0)); + foreach ([ + [$complete, 503, array_replace($deferred, ['retryable' => false])], + [$complete, 503, array_diff_key($deferred, ['retryable' => true])], + [$complete, 503, array_replace($deferred, ['reason' => 'backend_unavailable'])], + [$complete, 409, $deferred], + [$complete, 503, null], + // A poll refusal still needs task null, even through this entry point. + [self::POLL, 503, $deferred], + ] as $index => [$uri, $status, $response]) { + self::assertNull(\lockPressureOperationRetryDelay('POST', $uri, $status, $response, 0, 0.0), (string) $index); + } + self::assertNull(\lockPressureOperationRetryDelay('POST', $complete, 503, $deferred, 15, 0.0)); + self::assertNull(\lockPressureOperationRetryDelay('POST', $complete, 503, $deferred, 2, 29.5)); + } + public function test_retries_stop_at_the_attempt_and_wait_budgets(): void { self::assertNull(\lockPressurePollRetryDelay('POST', self::POLL, 503, $this->refusal(), 15, 0.0)); From e737397b72124d192fcdc0b007d73cea0c95fb79 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 08:02:08 +0000 Subject: [PATCH 08/37] Design the backup-first native takeover The storage audit lists the takeover gates, and the inspection commands keep their backup, quiescence, compatibility and takeover flags false. This note proposes what turns each one true: - a fingerprint of the database's complete logical contents, compared between the stopped source and a restored backup, then checked again inside the takeover's own exclusive transaction before the marker is written; - per-backend writer exclusion and atomicity, with a marker step journal where DDL commits implicitly; - resumption of an interrupted takeover from the marker, refusal of a different fingerprint, and the per-backend upgrade fixtures and checks. It records what is not yet known, such as MySQL's exclusion method. Co-Authored-By: Claude Opus 5.5 --- docs/rust-server-storage-audit.md | 4 +- docs/rust-server-upgrade-takeover.md | 124 +++++++++++++++++++++++++++ 2 files changed, 127 insertions(+), 1 deletion(-) create mode 100644 docs/rust-server-upgrade-takeover.md diff --git a/docs/rust-server-storage-audit.md b/docs/rust-server-storage-audit.md index b1cfd921e..cd97df381 100644 --- a/docs/rust-server-storage-audit.md +++ b/docs/rust-server-storage-audit.md @@ -96,5 +96,7 @@ and takeover flag stays false. There is no mixed PHP/Rust fleet or PHP rollback after upgrade. Restoring a pre-upgrade backup is a separate recovery action with its own data boundary. -All gates above remain open. HTTP export/import remains a supported deployment +All gates above remain open. The [takeover design](rust-server-upgrade-takeover.md) +proposes how the takeover command proves the backup, excludes other writers and +resumes after interruption. HTTP export/import remains a supported deployment transfer surface, but it does not substitute for in-place upgrade qualification. diff --git a/docs/rust-server-upgrade-takeover.md b/docs/rust-server-upgrade-takeover.md new file mode 100644 index 000000000..7795cb75b --- /dev/null +++ b/docs/rust-server-upgrade-takeover.md @@ -0,0 +1,124 @@ +# Native upgrade takeover + +This note designs the one-way, backup-first takeover of an existing PHP Server +database by the native runtime. It is a design input for +[Server #325](https://github.com/durable-workflow/server/issues/325), not a +qualified procedure. The gates it must satisfy are in the +[storage and upgrade audit](rust-server-storage-audit.md). Today +`upgrade-inspect`, `upgrade-queue-inspect` and `upgrade-payload-inspect` only +read, and their `backup_verified`, `php_quiescence_verified`, +`stored_representation_compatibility_verified` and `takeover_allowed` flags are +always false. This note says what makes each one true. + +## Shape + +Takeover is an explicit command, never a side effect of starting the server. +Native startup keeps refusing a PHP or unknown database. + +1. The operator stops every PHP write-capable role: HTTP, bootstrap and + migration, queue workers and the scheduler. +2. `upgrade-fingerprint` prints a digest of the database's complete logical + contents (below). +3. The operator takes a backup with the + [existing procedure](self-hosted-backup-and-restore.md), restores it into a + separate database, and runs `upgrade-fingerprint` there. Equal digests show + that the backup restores to exactly the stopped database. +4. `upgrade-takeover` runs with `DW_UPGRADE_BACKUP_FINGERPRINT` set to that + digest. It refuses unless the live database still has that digest inside + the same exclusive transaction that writes the ownership marker. A write + after the backup, by PHP or anything else, changes the digest and stops the + takeover before any mutation. +5. Native starts normally on the upgraded database. PHP refuses it from the + marker onward ([PHP ownership checks](php-database-ownership.md)). + +There is no PHP rollback after step 4. Recovery restores the verified backup +into a new database for a PHP installation, with the data boundary of that +backup. + +## Fingerprint + +The digest covers every relation in the database, not only the ones PHP +knows: the sorted relation names, each relation's column names and declared +types, and every row in primary-key order (all columns when there is no +primary key). Values are encoded canonically, so a dump and restore reproduces +the same bytes: integers in decimal, text and JSON as stored text, timestamps +at their stored precision, binary as hex, NULL distinct from empty. It is read +in one read-only snapshot (`READ_SNAPSHOT_BEGIN`). The command reports the +digest, the per-relation row counts and the elapsed time, never row contents. + +Takeover reads every row once more, inside its exclusive transaction. That +cost is accepted for a one-time operation. Its duration is reported so an +operator can plan the stop window from the fingerprint run. + +External payload objects are not in the database. Until a qualified object +check exists, takeover refuses when `upgrade-payload-inspect` reports any +external reference, and the operator's object manifest from the backup +procedure remains the record for those bytes. + +## Exclusive transaction per backend + +The marker, the fingerprint check and the schema changes must not interleave +with any other writer. + +| Backend | Exclusion | Atomicity | +| --- | --- | --- | +| SQLite | `BEGIN EXCLUSIVE` on the database file | Fingerprint, marker, native relations and conversions commit in one transaction. | +| PostgreSQL | `LOCK TABLE … IN EXCLUSIVE MODE` on every relation, then the native advisory key | DDL is transactional, so everything commits in one transaction. | +| MySQL and MariaDB | To be qualified: `LOCK TABLES … WRITE` on every relation, or a check that no other session uses the database, repeated after the marker is written | DDL commits implicitly, so the marker records a step journal (below). | + +On SQLite and PostgreSQL, a PHP connection that slipped past the stop blocks on +these locks, and a write it committed earlier changes the fingerprint. Neither +leaves a mixed state. + +## Marker and interruption + +The marker is written first in each takeover transaction, with state +`takeover`, the fingerprint and the step list. Native startup refuses state +`takeover`. Each step is idempotent and records its completion in the marker. +Rerunning `upgrade-takeover` with the same fingerprint resumes after the last +recorded step; a different fingerprint is refused. The final step sets state +`ready` and the native schema version. + +On SQLite and PostgreSQL an interruption rolls the whole transaction back, so +the database is either unchanged PHP or fully native. On MySQL and MariaDB the +step journal makes each committed DDL step visible, and resumption continues +from it. The PHP fence refuses the database from the first step, because the +marker exists. + +## Steps + +1. Refuse unless every inspection command reports `qualified` for this + database and the source catalog matches a qualified published PHP release. +2. Check the fingerprint (above) and write the marker. +3. Create the native relations and indexes (`upgrade.rs`, `native_object`). +4. Convert or adopt stored representations the inspections qualified: + poll-request bindings in task payloads, worker registrations (an + incarnation per active registration), cached query state, and queued + timer wakeups mapped to their durable task rows. Anything not qualified + was already refused by step 1. +5. Record the native migration ledger and set the marker to `ready`. + +## Tests + +Each backend gets an upgrade fixture written by the last published PHP release +with running and waiting workflows, a pending timer, a leased activity, an +acknowledged signal and an expired lease. The tests check: + +- refusal without a fingerprint, with a stale one, and after one extra row; +- interruption after each step, then resumption to the same final state; +- native completes the original work with its original histories, deadlines + and identities, and refuses the stale attempt; +- PHP refuses the upgraded database at startup, bootstrap, migration, worker + and scheduler entry points; +- a backup restored into a new database fingerprints equal to the stopped + source. + +## Open questions + +- Whether MySQL and MariaDB can create the native relations while + `LOCK TABLES` holds every existing one, or need the session check instead. +- Whether the declared column types print identically after a restore into + another minor server version. Until that is shown, the restore check uses + the same server version as the source. +- How long the fingerprint takes on a large installation, measured on a + disposable host before the stop window is documented. From eb8604ae877dcce61d23f8ede09466cb02de098b Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 08:02:45 +0000 Subject: [PATCH 09/37] Require a fenced PHP release before takeover Co-Authored-By: Claude Opus 5.5 --- docs/rust-server-upgrade-takeover.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/rust-server-upgrade-takeover.md b/docs/rust-server-upgrade-takeover.md index 7795cb75b..1b1b7d161 100644 --- a/docs/rust-server-upgrade-takeover.md +++ b/docs/rust-server-upgrade-takeover.md @@ -15,8 +15,10 @@ always false. This note says what makes each one true. Takeover is an explicit command, never a side effect of starting the server. Native startup keeps refusing a PHP or unknown database. -1. The operator stops every PHP write-capable role: HTTP, bootstrap and - migration, queue workers and the scheduler. +1. The installation runs Server 2.5.14 or later, the first release with the + ownership fence, so any PHP role that restarts later refuses the upgraded + database. The operator then stops every PHP write-capable role: HTTP, + bootstrap and migration, queue workers and the scheduler. 2. `upgrade-fingerprint` prints a digest of the database's complete logical contents (below). 3. The operator takes a backup with the @@ -29,7 +31,8 @@ Native startup keeps refusing a PHP or unknown database. after the backup, by PHP or anything else, changes the digest and stops the takeover before any mutation. 5. Native starts normally on the upgraded database. PHP refuses it from the - marker onward ([PHP ownership checks](php-database-ownership.md)). + marker onward ([PHP ownership checks](php-database-ownership.md)). Earlier + PHP releases lack that check, which is why step 1 requires 2.5.14. There is no PHP rollback after step 4. Recovery restores the verified backup into a new database for a PHP installation, with the data boundary of that From 98ef1af7af18bb13c12f15c55452a68b37071399 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 08:09:56 +0000 Subject: [PATCH 10/37] Back off a held poll's timed reads up to 250 ms On the October 20 profile host, lock-free reads every 50 ms cut MySQL's idle cost at 24 waiting polls only from about 0.53 to 0.46 cores, and the native process stayed near 0.28 cores. The cost is the read rate, 20 per second per poll, not the lock. A held poll now reads 50 ms after a probe and then at doubling intervals up to 250 ms. A same-node wake still reads at once and starts again from 50 ms, so same-node work is unaffected, and work committed by another server process is observed within 250 ms. No wait runs past the poll's deadline. Co-Authored-By: Claude Opus 5.5 --- docs/rust-server-transition-locking.md | 9 +++++---- rust/src/runtime/poll_readiness.rs | 28 ++++++++++++++++++++++++-- rust/src/runtime/queries.rs | 6 ++---- rust/src/runtime/store.rs | 6 ++---- 4 files changed, 35 insertions(+), 14 deletions(-) diff --git a/docs/rust-server-transition-locking.md b/docs/rust-server-transition-locking.md index 4e6b6bad5..9bd74a9c8 100644 --- a/docs/rust-server-transition-locking.md +++ b/docs/rust-server-transition-locking.md @@ -30,10 +30,11 @@ A held long poll takes the lock for its first probe, and afterwards only when a lock-free read finds something to claim or refuse: an eligible task (or query), a receipt for the poll's own request ID, or a removed, moved or replaced registration (`poll_readiness.rs`). The read runs at once when a -commit on the same node wakes the poll, and otherwise every 50 ms, which is -how a poll observes another server process. An idle poll therefore neither -waits for the lock nor delays transitions, and an unrelated commit does not -send every waiting poll to the lock. +commit on the same node wakes the poll. Otherwise it runs 50 ms after a probe +and then at doubling intervals up to 250 ms, so a poll observes work committed +by another server process within 250 ms. An idle poll therefore neither waits +for the lock nor delays transitions, and an unrelated commit does not send +every waiting poll to the lock. ## Measured cost diff --git a/rust/src/runtime/poll_readiness.rs b/rust/src/runtime/poll_readiness.rs index f0727bf34..459db2fe1 100644 --- a/rust/src/runtime/poll_readiness.rs +++ b/rust/src/runtime/poll_readiness.rs @@ -12,9 +12,33 @@ use super::{ use serde_json::Value; use sqlx::{ColumnIndex, Decode, Encode, Executor, IntoArguments, Type}; use std::time::Duration; +use tokio::{sync::futures::Notified, time::Instant}; -/// How often a held poll reads for work committed by another server process. -pub(super) const READ_INTERVAL: Duration = Duration::from_millis(50); +/// Paces a held poll's timed reads, which are how it observes another server +/// process. The first read comes 50 ms after a probe, and each later one waits +/// twice as long, up to 250 ms. A same-node wake reads at once and starts again +/// from 50 ms. No wait runs past the poll's deadline. +pub(super) struct ReadPace { + next: Duration, +} + +impl ReadPace { + const FIRST: Duration = Duration::from_millis(50); + const LONGEST: Duration = Duration::from_millis(250); + + pub(super) fn new() -> Self { + Self { next: Self::FIRST } + } + + /// Returns at a same-node wake or when the next timed read is due. + pub(super) async fn wait(&mut self, wake: Notified<'_>, deadline: Instant) { + let pause = self.next.min(deadline.saturating_duration_since(Instant::now())); + tokio::select! { + _ = wake => self.next = Self::FIRST, + _ = tokio::time::sleep(pause) => self.next = (self.next * 2).min(Self::LONGEST), + } + } +} impl Store where diff --git a/rust/src/runtime/queries.rs b/rust/src/runtime/queries.rs index ed9a5588e..cdca5859c 100644 --- a/rust/src/runtime/queries.rs +++ b/rust/src/runtime/queries.rs @@ -357,6 +357,7 @@ where // Only the first probe claims under the lock unconditionally; later // probes claim after a lock-free read finds something to claim or refuse. let mut first = true; + let mut pace = super::poll_readiness::ReadPace::new(); loop { let wake = self.wake.notified(); if std::mem::take(&mut first) || self.query_poll_ready(&body, config, &proof).await? { @@ -380,10 +381,7 @@ where json!({"task":null,"poll_status":"task_queue_changed","server_capabilities":super::store::capabilities()}), ); } - tokio::select! { - _ = wake => {}, - _ = tokio::time::sleep(super::poll_readiness::READ_INTERVAL) => {}, - } + pace.wait(wake, deadline).await; } } diff --git a/rust/src/runtime/store.rs b/rust/src/runtime/store.rs index 1dddc5dab..9b045bee8 100644 --- a/rust/src/runtime/store.rs +++ b/rust/src/runtime/store.rs @@ -938,6 +938,7 @@ where // other server processes, claim only after a lock-free read finds // something to claim or refuse. let mut first = true; + let mut pace = super::poll_readiness::ReadPace::new(); loop { // Register the notification before probing. let wake = self.wake.notified(); @@ -962,10 +963,7 @@ where json!({"task":null,"poll_status":"task_queue_changed","server_capabilities":capabilities()}), ); } - tokio::select! { - _ = wake => {}, - _ = tokio::time::sleep(super::poll_readiness::READ_INTERVAL) => {}, - } + pace.wait(wake, deadline).await; } } From eb272676d352c75fc79c08b0d929170ff3e19402 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 08:46:36 +0000 Subject: [PATCH 11/37] Keep the readiness reads beside the claims they mirror The task read now sits next to the task claim in store.rs and the query read next to the query claim in queries.rs, inside their existing Store impls. poll_readiness.rs keeps only the read pacing. No behavior changes. Co-Authored-By: Claude Opus 5.5 --- docs/rust-server-transition-locking.md | 2 +- rust/src/runtime/backend.rs | 2 +- rust/src/runtime/poll_readiness.rs | 92 ++------------------------ rust/src/runtime/queries.rs | 32 ++++++++- rust/src/runtime/store.rs | 37 +++++++++++ 5 files changed, 74 insertions(+), 91 deletions(-) diff --git a/docs/rust-server-transition-locking.md b/docs/rust-server-transition-locking.md index 9bd74a9c8..392f7743c 100644 --- a/docs/rust-server-transition-locking.md +++ b/docs/rust-server-transition-locking.md @@ -29,7 +29,7 @@ pool without a transaction. A held long poll takes the lock for its first probe, and afterwards only when a lock-free read finds something to claim or refuse: an eligible task (or query), a receipt for the poll's own request ID, or a removed, moved or -replaced registration (`poll_readiness.rs`). The read runs at once when a +replaced registration (`Store::task_poll_ready` and `Store::query_poll_ready`). The read runs at once when a commit on the same node wakes the poll. Otherwise it runs 50 ms after a probe and then at doubling intervals up to 250 ms, so a poll observes work committed by another server process within 250 ms. An idle poll therefore neither waits diff --git a/rust/src/runtime/backend.rs b/rust/src/runtime/backend.rs index 7dc404387..b31a8349d 100644 --- a/rust/src/runtime/backend.rs +++ b/rust/src/runtime/backend.rs @@ -338,7 +338,7 @@ impl Backend for Postgres { let mut tx = pool.begin().await?; // The first single-namespace slice serializes its durable transitions // across independent pools/processes. Held long polls wait without it - // (poll_readiness.rs). Throughput/idle cost remain unqualified. + // (Store::task_poll_ready). Throughput/idle cost remain unqualified. sqlx::query("SELECT pg_advisory_xact_lock(4924501473315676163)") .execute(&mut *tx) .await?; diff --git a/rust/src/runtime/poll_readiness.rs b/rust/src/runtime/poll_readiness.rs index 459db2fe1..6c0265829 100644 --- a/rust/src/runtime/poll_readiness.rs +++ b/rust/src/runtime/poll_readiness.rs @@ -3,14 +3,10 @@ //! A held poll takes the transition lock only to claim or refuse. After its //! first claim it reads, without the lock, whether anything now exists that it //! could claim or must refuse: an eligible task, a receipt for its own request, -//! or a changed registration. A same-node commit wakes it to read at once; -//! timed reads observe other server processes. A false positive costs one locked -//! claim that finds nothing, and a commit racing a read is seen on the next one. -use super::{ - Result, backend::Backend, query_authority::{self, QueryTaskConfig}, store::Store, text, -}; -use serde_json::Value; -use sqlx::{ColumnIndex, Decode, Encode, Executor, IntoArguments, Type}; +//! or a changed registration (`Store::task_poll_ready`, `Store::query_poll_ready`). +//! A same-node commit wakes it to read at once; timed reads observe other +//! server processes. A false positive costs one locked claim that finds +//! nothing, and a commit racing a read is seen on the next one. use std::time::Duration; use tokio::{sync::futures::Notified, time::Instant}; @@ -39,83 +35,3 @@ impl ReadPace { } } } - -impl Store -where - for<'c> &'c mut DB::Connection: Executor<'c, Database = DB>, - DB::Arguments: IntoArguments, - for<'q> &'q str: Encode<'q, DB> + Type, - for<'q> Option<&'q str>: Encode<'q, DB>, - for<'q> Option: Encode<'q, DB>, - for<'q> String: Encode<'q, DB> + Type, - for<'q> i64: Encode<'q, DB> + Type, - for<'r> i64: Decode<'r, DB>, - usize: ColumnIndex, -{ - /// Whether a held workflow or activity poll should claim again. - pub(super) async fn task_poll_ready(&self, body: &Value, kind: &str) -> Result { - let worker_id = text(body, "worker_id")?; - let queue = text(body, "task_queue")?; - // The claim refuses a removed or moved registration. - let registration = Self::query( - "SELECT task_queue FROM workflow_worker_registrations WHERE worker_id=$1 AND namespace=$2", - ) - .bind(worker_id) - .bind(self.namespace.as_ref()) - .fetch_optional(&self.pool) - .await?; - match registration { - Some(row) if DB::string(&row, "task_queue")? == queue => {} - _ => return Ok(true), - } - // A concurrent poll with the same request ID may have claimed already; - // the claim replays its receipt. - if let Some(request_id) = body.get("poll_request_id").and_then(Value::as_str) - && Self::query("SELECT task_id FROM dw_poll_receipts WHERE worker_id=$1 AND kind=$2 AND request_id=$3 AND namespace=$4") - .bind(worker_id).bind(kind).bind(request_id).bind(self.namespace.as_ref()) - .fetch_optional(&self.pool).await?.is_some() - { - return Ok(true); - } - let at = super::store::now(); - Ok(Self::query(DB::TASK_CANDIDATES_SQL) - .bind(self.namespace.as_ref()) - .bind(queue) - .bind(kind) - .bind(DB::bind_time(at)) - .bind(DB::bind_time(at)) - .fetch_optional(&self.pool) - .await? - .is_some()) - } - - /// Whether a held query poll should claim again. `proof` is the - /// registration this poll belongs to. - pub(super) async fn query_poll_ready( - &self, - body: &Value, - config: &QueryTaskConfig, - proof: &str, - ) -> Result { - let owner = text(body, "worker_id")?; - // The claim refuses a replaced, removed or stale registration. - if Self::query_worker_proof(&self.pool, self.namespace.as_ref(), owner, config) - .await? - .as_deref() - != Some(proof) - { - return Ok(true); - } - let at = super::store::now(); - let query_now = query_authority::wire(at); - Ok(Self::query(DB::POLL_WAKE_QUERIES_SQL) - .bind(at.timestamp()) - .bind(self.namespace.as_ref()) - .bind(text(body, "task_queue")?) - .bind(query_now.clone()) - .bind(query_now) - .fetch_optional(&self.pool) - .await? - .is_some()) - } -} diff --git a/rust/src/runtime/queries.rs b/rust/src/runtime/queries.rs index cdca5859c..27b78179c 100644 --- a/rust/src/runtime/queries.rs +++ b/rust/src/runtime/queries.rs @@ -385,7 +385,7 @@ where } } - pub(super) async fn query_worker_proof<'e, E>( + async fn query_worker_proof<'e, E>( executor: E, namespace: &str, owner: &str, @@ -437,6 +437,36 @@ where delivered } + /// Whether a held query poll should claim again. `proof` is the + /// registration this poll belongs to. + async fn query_poll_ready( + &self, + body: &Value, + config: &QueryTaskConfig, + proof: &str, + ) -> Result { + let owner = text(body, "worker_id")?; + // The claim refuses a replaced, removed or stale registration. + if Self::query_worker_proof(&self.pool, self.namespace.as_ref(), owner, config) + .await? + .as_deref() + != Some(proof) + { + return Ok(true); + } + let at = super::store::now(); + let query_now = authority::wire(at); + Ok(Self::query(DB::POLL_WAKE_QUERIES_SQL) + .bind(at.timestamp()) + .bind(self.namespace.as_ref()) + .bind(text(body, "task_queue")?) + .bind(query_now.clone()) + .bind(query_now) + .fetch_optional(&self.pool) + .await? + .is_some()) + } + async fn claim_query( &self, body: &Value, diff --git a/rust/src/runtime/store.rs b/rust/src/runtime/store.rs index 9b045bee8..c593973b6 100644 --- a/rust/src/runtime/store.rs +++ b/rust/src/runtime/store.rs @@ -967,6 +967,43 @@ where } } + /// Whether a held workflow or activity poll should claim again. + async fn task_poll_ready(&self, body: &Value, kind: &str) -> Result { + let worker_id = text(body, "worker_id")?; + let queue = text(body, "task_queue")?; + // The claim refuses a removed or moved registration. + let registration = Self::query( + "SELECT task_queue FROM workflow_worker_registrations WHERE worker_id=$1 AND namespace=$2", + ) + .bind(worker_id) + .bind(self.namespace.as_ref()) + .fetch_optional(&self.pool) + .await?; + match registration { + Some(row) if DB::string(&row, "task_queue")? == queue => {} + _ => return Ok(true), + } + // A concurrent poll with the same request ID may have claimed already; + // the claim replays its receipt. + if let Some(request_id) = body.get("poll_request_id").and_then(Value::as_str) + && Self::query("SELECT task_id FROM dw_poll_receipts WHERE worker_id=$1 AND kind=$2 AND request_id=$3 AND namespace=$4") + .bind(worker_id).bind(kind).bind(request_id).bind(self.namespace.as_ref()) + .fetch_optional(&self.pool).await?.is_some() + { + return Ok(true); + } + let at = now(); + Ok(Self::query(DB::TASK_CANDIDATES_SQL) + .bind(self.namespace.as_ref()) + .bind(queue) + .bind(kind) + .bind(DB::bind_time(at)) + .bind(DB::bind_time(at)) + .fetch_optional(&self.pool) + .await? + .is_some()) + } + async fn claim( &self, body: &Value, From 00b8b65386a9e235cd84646b841d082ce167aba9 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 08:49:05 +0000 Subject: [PATCH 12/37] Record that native's schema must first converge on the source release Co-Authored-By: Claude Opus 5.5 --- docs/rust-server-upgrade-takeover.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/rust-server-upgrade-takeover.md b/docs/rust-server-upgrade-takeover.md index 1b1b7d161..52a8c03e2 100644 --- a/docs/rust-server-upgrade-takeover.md +++ b/docs/rust-server-upgrade-takeover.md @@ -101,6 +101,16 @@ marker exists. was already refused by step 1. 5. Record the native migration ledger and set the marker to `ready`. +Native's frozen physical layout predates the current PHP release, so step 3 +alone does not yield a database native accepts. PHP 2.8.0 adds its own +`workflow_worker_registration_incarnations` table, a `call_deadline_at` +column on `workflow_service_calls` and a changed `workflow_worker_registrations` +(`rust/upgrade/php-2.8.0/`). Before takeover is implemented, native's schema +moves to the layout of the release takeover starts from, and native reads +PHP's incarnation table instead of keeping a second one. A fresh native +database then has the same PHP layout, and takeover only adds native relations +and converts data. + ## Tests Each backend gets an upgrade fixture written by the last published PHP release From 026e25879f36b49815141a28a0d2dc63544f925b Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 09:04:05 +0000 Subject: [PATCH 13/37] Reproduce default and opt-out workflow definition upgrade stalls --- .../WorkflowDefinitionEvolutionTest.php | 109 ++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100644 tests/Feature/WorkflowDefinitionEvolutionTest.php diff --git a/tests/Feature/WorkflowDefinitionEvolutionTest.php b/tests/Feature/WorkflowDefinitionEvolutionTest.php new file mode 100644 index 000000000..93ff05acc --- /dev/null +++ b/tests/Feature/WorkflowDefinitionEvolutionTest.php @@ -0,0 +1,109 @@ +createNamespace('default'); + config(['server.polling.timeout' => 0, 'workflows.v2.compatibility.pin_to_recorded_fingerprint' => true]); + } + + public function test_default_replacement_can_claim_when_the_recorded_definition_is_gone(): void + { + [$run, $task, $started] = $this->original(); + $original = [$started->getRawOriginal('payload'), $run->run_deadline_at, $run->execution_deadline_at]; + $this->deleteJson('/api/workers/original-worker', [], $this->apiHeaders())->assertOk(); + $this->worker('replacement-worker', 'definition-v2'); + + $this->poll('replacement-worker')->assertOk() + ->assertJsonPath('task.run_id', $run->id) + ->assertJsonPath('task.task_id', $task->id); + + self::assertSame($original, [ + $started->fresh()->getRawOriginal('payload'), $run->fresh()->run_deadline_at, $run->fresh()->execution_deadline_at, + ]); + self::assertSame(1, $task->fresh()->attempt_count); + self::assertSame(0, $task->fresh()->repair_count); + } + + public function test_explicit_opt_out_does_not_wait_for_a_matching_peer(): void + { + [$run, $task] = $this->original(); + $this->worker('replacement-worker', 'definition-v2'); + config(['workflows.v2.compatibility.pin_to_recorded_fingerprint' => false]); + + $this->poll('replacement-worker')->assertOk() + ->assertJsonPath('task.run_id', $run->id) + ->assertJsonPath('task.task_id', $task->id); + } + + public function test_default_keeps_the_matching_registered_peer_preference(): void + { + [, $task] = $this->original(); + $this->worker('replacement-worker', 'definition-v2'); + + $this->poll('replacement-worker')->assertOk()->assertJsonPath('task', null); + self::assertSame(TaskStatus::Ready, $task->fresh()->status); + self::assertNull($task->fresh()->lease_owner); + self::assertSame(0, $task->fresh()->attempt_count); + + $this->poll('original-worker')->assertOk()->assertJsonPath('task.task_id', $task->id); + } + + private function original(): array + { + $this->worker('original-worker', 'definition-v1'); + $start = $this->postJson('/api/workflows', [ + 'workflow_id' => 'definition-original', 'workflow_type' => self::TYPE, 'task_queue' => self::QUEUE, + 'run_timeout_seconds' => 60, 'execution_timeout_seconds' => 120, + ], $this->apiHeaders())->assertCreated(); + $run = WorkflowRun::query()->findOrFail($start->json('run_id')); + $task = WorkflowTask::query()->where('workflow_run_id', $run->id)->firstOrFail(); + $started = WorkflowHistoryEvent::query()->where('workflow_run_id', $run->id) + ->where('event_type', 'WorkflowStarted')->firstOrFail(); + self::assertSame('worker', $started->payload['workflow_definition_fingerprint_source']); + self::assertSame('definition-v1', $started->payload['workflow_definition_fingerprint']); + + return [$run, $task, $started]; + } + + private function worker(string $id, string $fingerprint): void + { + $this->postJson('/api/worker/register', [ + 'worker_id' => $id, 'task_queue' => self::QUEUE, 'runtime' => 'python', 'sdk_version' => '2.6.1', + 'capability_manifest' => $this->portableWorkerAffinityRefusalManifest(), + 'supported_workflow_types' => [self::TYPE], + 'workflow_definition_fingerprints' => [self::TYPE => $fingerprint], + 'workflow_command_contracts' => [self::TYPE => ['signals' => [], 'queries' => [], 'updates' => [], 'update_validators' => []]], + ], $this->workerHeaders())->assertCreated(); + } + + private function poll(string $id): \Illuminate\Testing\TestResponse + { + return $this->postJson('/api/worker/workflow-tasks/poll', [ + 'worker_id' => $id, 'task_queue' => self::QUEUE, 'timeout_seconds' => 0, + ], $this->workerHeaders()); + } +} From e03bd5fc39166723a8a80430d6fb47e5b1629376 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 09:21:23 +0000 Subject: [PATCH 14/37] Allow strict replay by replacement workflow definitions and explain routing --- .../Controllers/Api/TaskQueueController.php | 38 ++-- app/Http/Controllers/Api/WorkerController.php | 10 +- .../Controllers/Api/WorkflowController.php | 1 + app/Support/ServerWorkflowControlPlane.php | 26 ++- app/Support/WorkflowDefinitionRouting.php | 179 ++++++++++++++++++ app/Support/WorkflowReplayEligibility.php | 40 ++++ app/Support/WorkflowRunDiagnostics.php | 13 ++ app/Support/WorkflowTaskPoller.php | 68 ++----- config/dw-contract.php | 2 +- docs/server-reference.md | 20 +- .../WorkflowDefinitionEvolutionTest.php | 178 ++++++++++++++++- 11 files changed, 501 insertions(+), 74 deletions(-) create mode 100644 app/Support/WorkflowDefinitionRouting.php create mode 100644 app/Support/WorkflowReplayEligibility.php diff --git a/app/Http/Controllers/Api/TaskQueueController.php b/app/Http/Controllers/Api/TaskQueueController.php index f27274fda..907d18fe7 100644 --- a/app/Http/Controllers/Api/TaskQueueController.php +++ b/app/Http/Controllers/Api/TaskQueueController.php @@ -10,6 +10,7 @@ use App\Support\TaskQueueAdmission; use App\Support\TaskQueueBuildIdRolloutSnapshot; use App\Support\TaskQueuePriorityFairnessSurface; +use App\Support\WorkflowDefinitionRouting; use App\Support\WorkflowQueryTaskBroker; use Illuminate\Http\JsonResponse; use Illuminate\Http\Request; @@ -28,6 +29,7 @@ public function __construct( private readonly TaskQueueBuildIdRolloutSnapshot $buildIdRollouts, private readonly TaskQueuePriorityFairnessSurface $priorityFairness, private readonly DeploymentLifecycleService $deployments, + private readonly WorkflowDefinitionRouting $definitionRouting, ) {} public function index(Request $request): JsonResponse @@ -67,22 +69,26 @@ public function show(Request $request, string $taskQueue): JsonResponse $namespace = (string) $request->attributes->get('namespace'); - return ControlPlaneProtocol::json( - $this->withAdmission( + $payload = $this->withAdmission( + $namespace, + $this->withRecentTaskFlow( $namespace, - $this->withRecentTaskFlow( + $taskQueue, + StandaloneWorkerVisibility::queueDetail( $namespace, $taskQueue, - StandaloneWorkerVisibility::queueDetail( - $namespace, - $taskQueue, - WorkerRegistration::class, - now(), - $this->workerStaleAfterSeconds(), - )->toArray(), - ), + WorkerRegistration::class, + now(), + $this->workerStaleAfterSeconds(), + )->toArray(), ), ); + $payload['definition_routing'] = $this->definitionRouting->forQueue($namespace, $taskQueue); + if ($payload['definition_routing']['blocked_sample_count'] > 0) { + $payload['repair']['needs_attention'] = true; + } + + return ControlPlaneProtocol::json($payload); } /** @@ -263,7 +269,7 @@ private function setBuildIdDrainIntent(Request $request, string $taskQueue, stri } /** - * @param array $deployment + * @param array $deployment * @return array */ private function buildIdLifecyclePayload( @@ -351,7 +357,7 @@ private function normalizeDeploymentLifecycleTimestamps(array $deployment): arra } /** - * @param list> $blockages + * @param list> $blockages */ private function orderBlockages(array &$blockages): void { @@ -630,6 +636,7 @@ private function activityAttemptStateCounts(string $namespace, string $taskQueue if ($attemptStatus === ActivityAttemptStatus::Running->value) { $counts['running_count'] += $count; $counts['open_count'] += $count; + continue; } @@ -639,6 +646,7 @@ private function activityAttemptStateCounts(string $namespace, string $taskQueue ) { $counts['retrying_count'] += $count; $counts['open_count'] += $count; + continue; } @@ -646,6 +654,7 @@ private function activityAttemptStateCounts(string $namespace, string $taskQueue $counts['timed_out_count'] += $count; $counts['expired_count'] += $count; $counts['terminal_count'] += $count; + continue; } @@ -656,18 +665,21 @@ private function activityAttemptStateCounts(string $namespace, string $taskQueue ) { $counts['timed_out_count'] += $count; $counts['terminal_count'] += $count; + continue; } if ($attemptStatus === ActivityAttemptStatus::Failed->value) { $counts['failed_count'] += $count; $counts['terminal_count'] += $count; + continue; } if ($attemptStatus === ActivityAttemptStatus::Completed->value) { $counts['completed_count'] += $count; $counts['terminal_count'] += $count; + continue; } diff --git a/app/Http/Controllers/Api/WorkerController.php b/app/Http/Controllers/Api/WorkerController.php index 9d9e3c6ee..a90ca6ccf 100644 --- a/app/Http/Controllers/Api/WorkerController.php +++ b/app/Http/Controllers/Api/WorkerController.php @@ -46,6 +46,7 @@ use App\Support\WorkflowHistoryPageToken; use App\Support\WorkflowMetadataCapabilityPolicy; use App\Support\WorkflowQueryTaskBroker; +use App\Support\WorkflowReplayEligibility; use App\Support\WorkflowStreamCommandProcessor; use App\Support\WorkflowStreamService; use App\Support\WorkflowTaskLeaseRecovery; @@ -407,7 +408,10 @@ public function register(Request $request): JsonResponse $this->validateWorkerTaskCapacity($maxWorkflowTasks, $maxActivityTasks); $maxWorkerSessions = $validated['max_concurrent_worker_sessions'] ?? 10; $taskSlots = is_array($validated['task_slots'] ?? null) ? $validated['task_slots'] : []; - $processMetrics = $this->normalizeProcessMetrics($validated['process_metrics'] ?? null); + $processMetrics = [ + ...($this->normalizeProcessMetrics($validated['process_metrics'] ?? null) ?? []), + WorkflowReplayEligibility::PROTOCOL_METRIC => WorkerProtocol::requestVersion($request), + ]; $releaseLeasesForRegistration = $this->shouldReleaseLeasesForWorkerRegistration($existing, $processMetrics); try { @@ -935,6 +939,10 @@ public function heartbeat(Request $request): JsonResponse if (array_key_exists('process_metrics', $validated)) { $update['process_metrics'] = $this->normalizeProcessMetrics($validated['process_metrics']); } + $update['process_metrics'] = [ + ...($update['process_metrics'] ?? $worker->process_metrics ?? []), + WorkflowReplayEligibility::PROTOCOL_METRIC => WorkerProtocol::requestVersion($request), + ]; if (array_key_exists('heartbeat_interval_seconds', $validated) && $validated['heartbeat_interval_seconds'] !== null) { diff --git a/app/Http/Controllers/Api/WorkflowController.php b/app/Http/Controllers/Api/WorkflowController.php index 7af65aa5b..d2c323820 100644 --- a/app/Http/Controllers/Api/WorkflowController.php +++ b/app/Http/Controllers/Api/WorkflowController.php @@ -1250,6 +1250,7 @@ private function formatRun(WorkflowRun $run, string $namespace, array $descripti 'compatibility_supported_in_fleet' => $this->compatibilitySupportedInFleet($namespace, $run), 'compatibility_fleet_reason' => $this->compatibilityFleetReason($namespace, $run), 'payload_codec' => $run->payload_codec, + 'definition_routing' => $description['definition_routing'] ?? null, 'execution_timeout_seconds' => $description['execution_timeout_seconds'] ?? null, 'run_timeout_seconds' => $runDescription['run_timeout_seconds'] ?? null, 'execution_deadline_at' => $runDescription['execution_deadline_at'] ?? null, diff --git a/app/Support/ServerWorkflowControlPlane.php b/app/Support/ServerWorkflowControlPlane.php index 162735706..3094a9476 100644 --- a/app/Support/ServerWorkflowControlPlane.php +++ b/app/Support/ServerWorkflowControlPlane.php @@ -26,6 +26,7 @@ public function __construct( private readonly ControlPlaneFailureDiagnostics $failureDiagnostics, private readonly NamespaceDurableStateQuota $durableStateQuota, private readonly WorkflowStartVersionPin $versionPin, + private readonly WorkflowDefinitionRouting $definitionRouting, ) {} public function start(string $workflowType, ?string $instanceId = null, array $options = []): array @@ -148,10 +149,12 @@ public function terminate(string $instanceId, array $options = []): array public function repair(string $instanceId, array $options = []): array { - return $this->mutateCommand( + $result = $this->mutateCommand( $instanceId, 'repair', HistoryEventType::RepairRequested, $options, fn (): array => $this->inner->repair($instanceId, $options), ); + + return $this->withDefinitionRouting($instanceId, $options, $result); } public function redrive(string $instanceId, string $failedRunId, array $options = []): array @@ -309,7 +312,26 @@ private function recoveredCommandResult(Throwable $exception, WorkflowCommand $c public function describe(string $instanceId, array $options = []): array { - return $this->inner->describe($instanceId, $options); + return $this->withDefinitionRouting($instanceId, $options, $this->inner->describe($instanceId, $options)); + } + + private function withDefinitionRouting(string $instanceId, array $options, array $result): array + { + $namespace = $this->namespace($options); + if ($namespace === null || ($result['found'] ?? true) === false + || in_array($result['reason'] ?? null, ['instance_not_found', 'run_not_found'], true)) { + return $result; + } + + $runId = data_get($result, 'run.run_id') ?? $result['workflow_run_id'] ?? $options['run_id'] ?? null; + $run = WorkflowRun::query()->where('namespace', $namespace)->where('workflow_instance_id', $instanceId) + ->when(is_string($runId), static fn ($query) => $query->whereKey($runId)) + ->orderByDesc('run_number')->first(); + if ($run instanceof WorkflowRun) { + $result['definition_routing'] = $this->definitionRouting->inspect($run); + } + + return $result; } /** diff --git a/app/Support/WorkflowDefinitionRouting.php b/app/Support/WorkflowDefinitionRouting.php new file mode 100644 index 000000000..cb1f264fa --- /dev/null +++ b/app/Support/WorkflowDefinitionRouting.php @@ -0,0 +1,179 @@ + */ + public function forWorker( + WorkflowRun $run, + WorkerRegistration $worker, + ?string $taskQueue = null, + ?string $requiredCompatibility = null, + ): array { + $decision = $this->context($run, $taskQueue, $requiredCompatibility); + $recorded = $decision['recorded_fingerprint']; + $advertised = self::stringValue($worker->workflow_definition_fingerprints[$run->workflow_type] ?? null); + $decision['advertised_fingerprint'] = $advertised; + + if (! $decision['worker_pinned']) { + return [...$decision, 'allowed' => true, 'status' => 'not_worker_pinned']; + } + if (! $decision['pinning_enabled']) { + return [...$decision, 'allowed' => true, 'status' => 'pinning_disabled']; + } + if ($recorded === null || $advertised === null) { + return [...$decision, 'allowed' => false, 'status' => $recorded === null + ? 'recorded_definition_missing' : 'worker_definition_missing']; + } + if (hash_equals($recorded, $advertised)) { + return [...$decision, 'allowed' => true, 'status' => 'matching_definition']; + } + + foreach ($this->eligibleWorkers($run, $decision['task_queue'], $decision['required_build_id']) as $peer) { + if (self::stringValue($peer->workflow_definition_fingerprints[$run->workflow_type] ?? null) === $recorded) { + return [...$decision, 'allowed' => false, 'status' => 'matching_definition_preferred', + 'preferred_worker_id' => $peer->worker_id]; + } + } + + // Match the embedded resolver: prefer the recorded source if it is + // available, otherwise let the current definition validate replay. + // This never changes the run's original fingerprint or build pin. + return [...$decision, 'allowed' => true, 'status' => 'replay_required']; + } + + /** @return array */ + public function inspect(WorkflowRun $run, ?string $taskQueue = null, ?string $requiredCompatibility = null): array + { + $task = WorkflowTask::query()->where('workflow_run_id', $run->id) + ->where('task_type', TaskType::Workflow->value)->where('status', TaskStatus::Ready->value) + ->where(static fn ($query) => $query->whereNull('available_at')->orWhere('available_at', '<=', now()->format('Y-m-d H:i:s.u'))) + ->when($taskQueue !== null, static fn ($query) => $query->where('queue', $taskQueue)) + ->orderBy('id')->first(); + if ($task instanceof WorkflowTask) { + $taskQueue ??= $task->queue; + $requiredCompatibility ??= TaskCompatibility::resolve($task, $run); + } + $context = $this->context($run, $taskQueue, $requiredCompatibility); + $matching = $changed = $missing = 0; + $examples = []; + + foreach ($this->eligibleWorkers($run, (string) $context['task_queue'], $context['required_build_id']) as $worker) { + $fingerprint = self::stringValue($worker->workflow_definition_fingerprints[$run->workflow_type] ?? null); + if ($fingerprint === null) { + $missing++; + } elseif ($fingerprint === $context['recorded_fingerprint']) { + $matching++; + } else { + $changed++; + } + if (count($examples) < 10) { + $examples[] = ['worker_id' => $worker->worker_id, 'fingerprint' => $fingerprint]; + } + } + + $eligible = $matching + $changed + $missing; + $status = match (true) { + ! $context['worker_pinned'] => 'not_worker_pinned', + ! $context['pinning_enabled'] => 'pinning_disabled', + $context['recorded_fingerprint'] === null => 'recorded_definition_missing', + $matching > 0 => 'matching_definition_available', + $changed > 0 => 'replay_required', + $missing > 0 => 'worker_definition_missing', + default => 'no_eligible_worker', + }; + $runnable = ! $run->status->isTerminal() && $task instanceof WorkflowTask; + $blocked = $runnable && $context['worker_pinned'] && $context['pinning_enabled'] + && in_array($status, ['recorded_definition_missing', 'worker_definition_missing', 'no_eligible_worker'], true); + + return [...$context, 'status' => $status, 'blocked' => $blocked, 'runnable_task' => $runnable, + 'eligible_worker_count' => $eligible, 'matching_worker_count' => $matching, + 'changed_worker_count' => $changed, 'missing_fingerprint_count' => $missing, + 'workers' => $examples, 'workers_truncated' => $eligible > count($examples), + 'resolution' => $blocked + ? 'Register a fresh worker for the original namespace, task queue, workflow type, build and replay capabilities with its definition fingerprint. Do not rewrite the run history.' + : ($status === 'replay_required' + ? 'Current workers may replay the original history. Use versioned branches for command changes; incompatible replay still refuses.' + : 'Recorded build, namespace, type and replay-capability guards remain in force.'), + ]; + } + + /** Bounded examples of due ready runs, never a repair or a payload read. */ + public function forQueue(string $namespace, string $taskQueue): array + { + $runs = WorkflowRun::query()->where('namespace', $namespace)->where('queue', $taskQueue) + ->whereIn('id', WorkflowTask::query()->select('workflow_run_id') + ->where('task_type', TaskType::Workflow->value)->where('status', TaskStatus::Ready->value) + ->where('queue', $taskQueue) + ->where(static fn ($query) => $query->whereNull('available_at')->orWhere('available_at', '<=', now()->format('Y-m-d H:i:s.u')))) + ->orderBy('id')->limit(11)->get(); + $examples = []; + foreach ($runs->take(10) as $run) { + if (! $run->status->isTerminal()) { + $examples[] = ['workflow_id' => $run->workflow_instance_id, 'run_id' => $run->id, + ...$this->inspect($run, $taskQueue)]; + } + } + + return ['sample_limit' => 10, 'has_more_ready_runs' => $runs->count() > 10, + 'blocked_sample_count' => count(array_filter($examples, static fn (array $row): bool => $row['blocked'])), + 'runs' => $examples]; + } + + /** @return array */ + private function context(WorkflowRun $run, ?string $taskQueue, ?string $requiredCompatibility): array + { + $started = WorkflowHistoryEvent::query()->where('workflow_run_id', $run->id) + ->where('event_type', 'WorkflowStarted')->orderBy('sequence')->first(); + $recorded = self::stringValue($started?->payload['workflow_definition_fingerprint'] ?? null); + + return [ + 'pinning_enabled' => (bool) config('workflows.v2.compatibility.pin_to_recorded_fingerprint', true), + 'worker_pinned' => ($started?->payload['workflow_definition_fingerprint_source'] ?? null) === 'worker', + 'recorded_fingerprint' => $recorded, + 'workflow_type' => $run->workflow_type, + 'task_queue' => $taskQueue ?? $run->queue, + 'required_build_id' => $requiredCompatibility ?? self::stringValue($run->compatibility), + ]; + } + + /** @return Generator */ + private function eligibleWorkers( + WorkflowRun $run, + string $taskQueue, + ?string $requiredCompatibility, + ): Generator { + $workers = WorkerRegistration::query()->where('namespace', $run->namespace) + ->where('task_queue', $taskQueue)->where('status', 'active'); + if ($requiredCompatibility !== null) { + $workers->where('build_id', $requiredCompatibility); + } + + foreach ($workers->cursor(['namespace', 'worker_id', 'task_queue', 'build_id', 'status', 'last_heartbeat_at', + 'workflow_definition_fingerprints', 'supported_workflow_types', 'max_concurrent_workflow_tasks', + 'capabilities', 'process_metrics']) as $worker) { + if (! in_array($run->workflow_type, $worker->supported_workflow_types ?? [], true) + || ($worker->max_concurrent_workflow_tasks !== null && (int) $worker->max_concurrent_workflow_tasks < 1)) { + continue; + } + if (WorkflowReplayEligibility::allows($worker, $run, WorkflowReplayEligibility::recordedProtocol($worker))) { + yield $worker; + } + } + } + + private static function stringValue(mixed $value): ?string + { + return is_string($value) && trim($value) !== '' ? trim($value) : null; + } +} diff --git a/app/Support/WorkflowReplayEligibility.php b/app/Support/WorkflowReplayEligibility.php new file mode 100644 index 000000000..172dc4b0f --- /dev/null +++ b/app/Support/WorkflowReplayEligibility.php @@ -0,0 +1,40 @@ +namespace !== $run->namespace || ! WorkerPollFence::isFresh($worker)) { + return false; + } + + $capabilities = is_array($worker->capabilities) ? $worker->capabilities : []; + + if (is_string($run->cancellation_request_command_id) + && ! CooperativeCancellationPolicy::supports($capabilities, $protocolVersion)) { + return false; + } + if (is_string($run->cancellation_request_command_id) + && NexusCooperativeCancellationPolicy::requiresCapabilityForReplay($run) + && ! NexusCooperativeCancellationPolicy::supports($capabilities, $protocolVersion)) { + return false; + } + + return PreparedLocalActivityPolicy::canReplayRun($run->id, $capabilities, $protocolVersion) + && WorkflowMetadataCapabilityPolicy::canReplayRun($run->id, $capabilities, $protocolVersion); + } + + public static function recordedProtocol(WorkerRegistration $worker): ?string + { + $version = $worker->process_metrics[self::PROTOCOL_METRIC] ?? null; + + return is_string($version) && $version !== '' ? $version : null; + } +} diff --git a/app/Support/WorkflowRunDiagnostics.php b/app/Support/WorkflowRunDiagnostics.php index 6f6490561..810d3f1be 100644 --- a/app/Support/WorkflowRunDiagnostics.php +++ b/app/Support/WorkflowRunDiagnostics.php @@ -41,6 +41,8 @@ class WorkflowRunDiagnostics private const LAST_EVENT_PAYLOAD_KEY_LIMIT = 20; + public function __construct(private readonly WorkflowDefinitionRouting $definitionRouting) {} + /** * @return array */ @@ -84,6 +86,7 @@ public function forRun(string $namespace, WorkflowRun $run, bool $includeLastEve 'recent_failures_truncated' => $failureRows->count() > self::FAILURE_LIMIT, 'latest_workflow_task_failure' => $latestWorkflowTaskFailure, 'compatibility' => $this->compatibility($namespace, $run, $summary, $taskQueue), + 'definition_routing' => $this->definitionRouting->inspect($run), 'cancellation_cascade_supported' => class_exists(CancellationCascadeView::class), 'cancellation_cascade' => class_exists(CancellationCascadeView::class) && $run->namespace === $namespace ? CancellationCascadeView::forRun($run) @@ -95,6 +98,16 @@ public function forRun(string $namespace, WorkflowRun $run, bool $includeLastEve ...$this->pendingWorkflowRoutingFindings($namespace, $run, $taskRows->all(), $taskQueue), ]; + if ($payload['definition_routing']['blocked']) { + $payload['findings'][] = [ + 'severity' => 'warning', 'code' => 'workflow_definition_routing_blocked', + 'message' => 'A ready workflow task has no eligible definition route.', + 'routing_status' => $payload['definition_routing']['status'], + 'expected_resolution' => $payload['definition_routing']['resolution'], + ]; + $payload['diagnostic_status'] = 'needs_attention'; + } + return $payload; } diff --git a/app/Support/WorkflowTaskPoller.php b/app/Support/WorkflowTaskPoller.php index 6c21da0b8..4cc917342 100644 --- a/app/Support/WorkflowTaskPoller.php +++ b/app/Support/WorkflowTaskPoller.php @@ -12,7 +12,6 @@ use Workflow\V2\Enums\TaskStatus; use Workflow\V2\Enums\TaskType; use Workflow\V2\Jobs\RunTimerTask; -use Workflow\V2\Models\WorkflowHistoryEvent; use Workflow\V2\Models\WorkflowRun; use Workflow\V2\Models\WorkflowSignal; use Workflow\V2\Models\WorkflowTask; @@ -42,6 +41,7 @@ public function __construct( private readonly PollRequestLeaseBinding $pollLeaseBindings, private readonly WorkflowUpdateValidationTaskBroker $updateValidationTasks, private readonly WorkflowTaskKindSelector $taskKindSelector, + private readonly WorkflowDefinitionRouting $definitionRouting, ) {} /** @@ -1041,9 +1041,11 @@ private function claimReadyTask( } if (! $this->matchesRecordedWorkflowDefinition( + $namespace, + $leaseOwner, $runId, - $readyTask['workflow_type'] ?? null, - $workflowDefinitionFingerprints, + $taskQueue, + $effectiveCompatibility, )) { continue; } @@ -2325,63 +2327,25 @@ private function workerCanReplayRun( ->where('worker_id', $workerId) ->first(); - if (! $worker instanceof WorkerRegistration || ! WorkerPollFence::isFresh($worker)) { - return false; - } - - $capabilities = is_array($worker->capabilities) - ? array_values(array_filter( - $worker->capabilities, - static fn (mixed $capability): bool => is_string($capability) && trim($capability) !== '', - )) - : []; - $run = WorkflowRun::query()->where('namespace', $namespace)->find($runId); - if ($run instanceof WorkflowRun && is_string($run->cancellation_request_command_id) - && ! CooperativeCancellationPolicy::supports($capabilities, $protocolVersion)) { - return false; - } - if ($run instanceof WorkflowRun && is_string($run->cancellation_request_command_id) - && NexusCooperativeCancellationPolicy::requiresCapabilityForReplay($run) - && ! NexusCooperativeCancellationPolicy::supports($capabilities, $protocolVersion)) { - return false; - } - return PreparedLocalActivityPolicy::canReplayRun($runId, $capabilities, $protocolVersion) - && WorkflowMetadataCapabilityPolicy::canReplayRun( - $runId, - $capabilities, - $protocolVersion, - ); + return $worker instanceof WorkerRegistration && $run instanceof WorkflowRun + && WorkflowReplayEligibility::allows($worker, $run, $protocolVersion); } - /** @param array $workflowDefinitionFingerprints */ private function matchesRecordedWorkflowDefinition( + string $namespace, + string $workerId, string $runId, - mixed $workflowType, - array $workflowDefinitionFingerprints, + string $taskQueue, + ?string $requiredCompatibility, ): bool { - $started = WorkflowHistoryEvent::query() - ->where('workflow_run_id', $runId) - ->where('event_type', 'WorkflowStarted') - ->orderBy('sequence') - ->first(); - - if (($started?->payload['workflow_definition_fingerprint_source'] ?? null) !== 'worker') { - return true; - } - - $recorded = $this->nonEmptyString($started?->payload['workflow_definition_fingerprint'] ?? null); - - if ($recorded === null) { - return false; - } - - $advertised = is_string($workflowType) - ? $this->nonEmptyString($workflowDefinitionFingerprints[$workflowType] ?? null) - : null; + $run = WorkflowRun::query()->where('namespace', $namespace)->find($runId); + $worker = WorkerRegistration::query()->where('namespace', $namespace) + ->where('worker_id', $workerId)->first(); - return $advertised !== null && hash_equals($recorded, $advertised); + return $run instanceof WorkflowRun && $worker instanceof WorkerRegistration + && $this->definitionRouting->forWorker($run, $worker, $taskQueue, $requiredCompatibility)['allowed']; } /** diff --git a/config/dw-contract.php b/config/dw-contract.php index f9f23786c..2e8abf2f3 100644 --- a/config/dw-contract.php +++ b/config/dw-contract.php @@ -889,7 +889,7 @@ 'legacy' => 'WORKFLOW_V2_COMPATIBILITY_HEARTBEAT_TTL', ], 'DW_V2_PIN_TO_RECORDED_FINGERPRINT' => [ - 'description' => 'When true (default), in-flight runs resolve their workflow class from the fingerprint recorded at WorkflowStarted.', + 'description' => 'Prefer an eligible worker matching the recorded definition. If none remains, current code must validate strict replay. False removes only this source preference, not build pins or replay checks.', 'default' => 'true', 'since' => '2.0.0', 'legacy' => 'WORKFLOW_V2_PIN_TO_RECORDED_FINGERPRINT', diff --git a/docs/server-reference.md b/docs/server-reference.md index 8d836aa79..d793ee967 100644 --- a/docs/server-reference.md +++ b/docs/server-reference.md @@ -418,6 +418,24 @@ Workers that omit fingerprints during re-registration cannot clear previously stored fingerprints for workflow types they still advertise; the server keeps the stored value until a new worker id is used. +Existing runs retain their original definition fingerprint, build pin, history +and deadlines. By default, the Server prefers a fresh eligible worker with the +recorded definition. When that definition is no longer available in the run's +namespace, queue, workflow type and build, a replacement definition can claim +the original task and must pass the SDK's strict replay checks. Use versioned +branches such as `get_version` for changes to durable commands. A different +source fingerprint alone is not proof of incompatible replay. + +`DW_V2_PIN_TO_RECORDED_FINGERPRINT=false` removes the matching-source preference. +It does not remove build pins, namespace or queue isolation, workflow-type and +capability checks, immutable registration identity, lease fences or strict replay. +Run detail, run debug, repair responses and queue detail expose +`definition_routing`. `replay_required` is an allowed fallback, while +`blocked=true` means a due ready workflow task has no eligible definition route. +Repair can fix transport state; it cannot manufacture a compatible worker or +rewrite the original definition. Queue observations contain at most ten run +examples and explicitly report whether more ready runs exist. + ### 5. Poll for Workflow Tasks The server holds the connection open (long-poll) until a task is ready or @@ -1715,7 +1733,7 @@ inside the package's `config/workflows.php` via | `DW_V2_SUPPORTED_COMPATIBILITIES` | (unset) | Comma-separated marker list the worker accepts, or `*` for any. | | `DW_V2_COMPATIBILITY_NAMESPACE` | (unset) | Compatibility namespace for independent fleets sharing one database. | | `DW_V2_COMPATIBILITY_HEARTBEAT_TTL` | `30` | Seconds a worker-compatibility heartbeat remains valid. | -| `DW_V2_PIN_TO_RECORDED_FINGERPRINT` | `true` | Resolve in-flight runs from the fingerprint recorded at WorkflowStarted. | +| `DW_V2_PIN_TO_RECORDED_FINGERPRINT` | `true` | Prefer an eligible recorded-definition worker; otherwise require strict replay by current code. False removes this preference, not build pins or replay checks. | | `DW_V2_CONTINUE_AS_NEW_EVENT_THRESHOLD` | `10000` | History event count at which the package signals continue-as-new. | | `DW_V2_CONTINUE_AS_NEW_SIZE_BYTES_THRESHOLD` | `5242880` | Serialized-history byte count at which the package signals continue-as-new. | | `DW_V2_HISTORY_EVENT_WARNING_THRESHOLD` | `8000` | Event count at which history pressure becomes `approaching`. | diff --git a/tests/Feature/WorkflowDefinitionEvolutionTest.php b/tests/Feature/WorkflowDefinitionEvolutionTest.php index 93ff05acc..3bbd0ac77 100644 --- a/tests/Feature/WorkflowDefinitionEvolutionTest.php +++ b/tests/Feature/WorkflowDefinitionEvolutionTest.php @@ -2,11 +2,16 @@ namespace Tests\Feature; +use App\Models\WorkerRegistration; +use App\Support\WorkflowReplayEligibility; use Illuminate\Foundation\Testing\RefreshDatabase; use Illuminate\Support\Facades\Queue; +use Illuminate\Testing\TestResponse; +use PHPUnit\Framework\Attributes\DataProvider; use Tests\Feature\Concerns\ServerTestHelpers; use Tests\TestCase; use Workflow\V2\Enums\TaskStatus; +use Workflow\V2\Models\WorkflowCommand; use Workflow\V2\Models\WorkflowHistoryEvent; use Workflow\V2\Models\WorkflowRun; use Workflow\V2\Models\WorkflowTask; @@ -33,7 +38,7 @@ protected function setUp(): void public function test_default_replacement_can_claim_when_the_recorded_definition_is_gone(): void { [$run, $task, $started] = $this->original(); - $original = [$started->getRawOriginal('payload'), $run->run_deadline_at, $run->execution_deadline_at]; + $original = [$started->getRawOriginal('payload'), $run->getRawOriginal('run_deadline_at'), $run->getRawOriginal('execution_deadline_at')]; $this->deleteJson('/api/workers/original-worker', [], $this->apiHeaders())->assertOk(); $this->worker('replacement-worker', 'definition-v2'); @@ -42,7 +47,7 @@ public function test_default_replacement_can_claim_when_the_recorded_definition_ ->assertJsonPath('task.task_id', $task->id); self::assertSame($original, [ - $started->fresh()->getRawOriginal('payload'), $run->fresh()->run_deadline_at, $run->fresh()->execution_deadline_at, + $started->fresh()->getRawOriginal('payload'), $run->fresh()->getRawOriginal('run_deadline_at'), $run->fresh()->getRawOriginal('execution_deadline_at'), ]); self::assertSame(1, $task->fresh()->attempt_count); self::assertSame(0, $task->fresh()->repair_count); @@ -72,6 +77,169 @@ public function test_default_keeps_the_matching_registered_peer_preference(): vo $this->poll('original-worker')->assertOk()->assertJsonPath('task.task_id', $task->id); } + #[DataProvider('ineligibleRecordedWorkers')] + public function test_an_ineligible_recorded_peer_does_not_strand_a_replacement(array $attributes): void + { + [, $task] = $this->original(); + WorkerRegistration::query()->where('worker_id', 'original-worker')->firstOrFail() + ->forceFill($attributes)->save(); + $this->worker('replacement-worker', 'definition-v2'); + + $this->poll('replacement-worker')->assertOk()->assertJsonPath('task.task_id', $task->id); + self::assertSame(1, $task->fresh()->attempt_count); + self::assertSame(0, $task->fresh()->repair_count); + } + + public static function ineligibleRecordedWorkers(): array + { + return [ + 'stale heartbeat' => [['last_heartbeat_at' => '2000-01-01 00:00:00']], + 'missing heartbeat' => [['last_heartbeat_at' => null]], + 'draining' => [['status' => 'draining']], + 'other namespace' => [['namespace' => 'other']], + 'other queue' => [['task_queue' => 'other']], + 'other type' => [['supported_workflow_types' => ['other']]], + 'no workflow capacity' => [['max_concurrent_workflow_tasks' => 0]], + 'missing definition' => [['workflow_definition_fingerprints' => []]], + ]; + } + + public function test_busy_matching_peer_retains_preference_without_inventing_slot_authority(): void + { + [, $task] = $this->original(); + WorkerRegistration::query()->where('worker_id', 'original-worker')->update(['available_workflow_slots' => 0]); + $this->worker('replacement-worker', 'definition-v2'); + + $this->poll('replacement-worker')->assertOk()->assertJsonPath('task', null); + self::assertSame(TaskStatus::Ready, $task->fresh()->status); + self::assertSame(0, $task->fresh()->attempt_count); + } + + public function test_fallback_does_not_remove_the_original_build_pin(): void + { + [$run, $task] = $this->original(); + $run->forceFill(['compatibility' => 'build-v1'])->save(); + $task->forceFill(['compatibility' => 'build-v1'])->save(); + $this->deleteJson('/api/workers/original-worker', [], $this->apiHeaders())->assertOk(); + $this->worker('replacement-worker', 'definition-v2', ['build_id' => 'build-v2']); + + $this->poll('replacement-worker', ['build_id' => 'build-v2'])->assertOk()->assertJsonPath('task', null); + self::assertSame(TaskStatus::Ready, $task->fresh()->status); + self::assertSame(0, $task->fresh()->attempt_count); + self::assertSame('build-v1', $run->fresh()->compatibility); + } + + public function test_a_matching_peer_on_another_build_cannot_suppress_a_pinned_replacement(): void + { + [$run, $task] = $this->original(); + $run->forceFill(['compatibility' => 'build-v1'])->save(); + $task->forceFill(['compatibility' => 'build-v1'])->save(); + WorkerRegistration::query()->where('worker_id', 'original-worker')->update(['build_id' => 'build-v2']); + $this->worker('replacement-worker', 'definition-v2', ['build_id' => 'build-v1']); + + $this->poll('replacement-worker', ['build_id' => 'build-v1'])->assertOk()->assertJsonPath('task.task_id', $task->id); + self::assertSame('build-v1', $run->fresh()->compatibility); + } + + public function test_matching_peer_must_support_the_recorded_replay_protocol(): void + { + [$run, $task] = $this->original(); + $worker = WorkerRegistration::query()->where('worker_id', 'original-worker')->firstOrFail(); + $worker->forceFill(['process_metrics' => [WorkflowReplayEligibility::PROTOCOL_METRIC => '1.16']])->save(); + WorkflowHistoryEvent::query()->create([ + 'workflow_run_id' => $run->id, + 'sequence' => WorkflowHistoryEvent::query()->where('workflow_run_id', $run->id)->max('sequence') + 1, + 'event_type' => 'TimerScheduled', + 'payload' => ['condition_wait_occurrence_id' => 'original-condition-occurrence'], + ]); + $this->worker('replacement-worker', 'definition-v2'); + + $this->poll('replacement-worker')->assertOk()->assertJsonPath('task.task_id', $task->id); + } + + public function test_client_process_metrics_cannot_spoof_the_saved_protocol(): void + { + $this->worker('original-worker', 'definition-v1', [ + 'process_metrics' => [WorkflowReplayEligibility::PROTOCOL_METRIC => '999.0', 'memory_bytes' => 123], + ]); + $worker = WorkerRegistration::query()->where('worker_id', 'original-worker')->firstOrFail(); + self::assertSame($this->workerHeaders()['X-Durable-Workflow-Protocol-Version'], WorkflowReplayEligibility::recordedProtocol($worker)); + self::assertSame(123, $worker->process_metrics['memory_bytes']); + + $this->postJson('/api/worker/heartbeat', [ + 'worker_id' => 'original-worker', 'process_metrics' => [WorkflowReplayEligibility::PROTOCOL_METRIC => '999.0', 'memory_bytes' => 456], + ], $this->workerHeaders())->assertOk(); + self::assertSame($this->workerHeaders()['X-Durable-Workflow-Protocol-Version'], WorkflowReplayEligibility::recordedProtocol($worker->fresh())); + self::assertSame(456, $worker->fresh()->process_metrics['memory_bytes']); + } + + public function test_run_debug_queue_and_repair_distinguish_fallback_from_a_real_block(): void + { + [$run, $task] = $this->original(); + $this->deleteJson('/api/workers/original-worker', [], $this->apiHeaders())->assertOk(); + $this->worker('replacement-worker', 'definition-v2'); + $this->getJson('/api/workflows/definition-original', $this->apiHeaders())->assertOk() + ->assertJsonPath('definition_routing.status', 'replay_required') + ->assertJsonPath('definition_routing.blocked', false) + ->assertJsonPath('definition_routing.recorded_fingerprint', 'definition-v1'); + $this->getJson('/api/workflows/definition-original/runs/'.$run->id, $this->apiHeaders())->assertOk() + ->assertJsonPath('definition_routing.status', 'replay_required'); + $this->getJson('/api/workflows/definition-original/debug', $this->apiHeaders())->assertOk() + ->assertJsonPath('definition_routing.blocked', false) + ->assertJsonMissing(['code' => 'workflow_definition_routing_blocked']); + + WorkerRegistration::query()->where('worker_id', 'replacement-worker')->firstOrFail() + ->forceFill(['workflow_definition_fingerprints' => []])->save(); + $this->getJson('/api/workflows/definition-original', $this->apiHeaders())->assertOk() + ->assertJsonPath('definition_routing.status', 'worker_definition_missing') + ->assertJsonPath('definition_routing.blocked', true); + $this->getJson('/api/workflows/definition-original/debug', $this->apiHeaders())->assertOk() + ->assertJsonPath('diagnostic_status', 'needs_attention') + ->assertJsonFragment(['code' => 'workflow_definition_routing_blocked']); + $this->getJson('/api/task-queues/'.self::QUEUE, $this->apiHeaders())->assertOk() + ->assertJsonPath('definition_routing.blocked_sample_count', 1) + ->assertJsonPath('definition_routing.runs.0.run_id', $run->id) + ->assertJsonPath('repair.needs_attention', true); + $repair = $this->postJson('/api/workflows/definition-original/repair', [ + 'request_id' => 'definition-routing-repair', + ], $this->apiHeaders())->assertOk() + ->assertJsonPath('definition_routing.status', 'worker_definition_missing') + ->assertJsonPath('definition_routing.blocked', true); + $command = WorkflowCommand::query()->findOrFail($repair->json('command_id')); + self::assertSame($command->commandReason(), $repair->json('reason')); + self::assertSame($command->outcome->value, $repair->json('outcome')); + self::assertSame(0, $task->fresh()->attempt_count); + self::assertSame(0, $task->fresh()->repair_count); + + $this->createNamespace('other'); + $this->getJson('/api/task-queues/'.self::QUEUE, $this->apiHeaders('other'))->assertOk() + ->assertJsonPath('definition_routing.runs', []); + } + + #[DataProvider('notRunnableTasks')] + public function test_definition_drift_is_not_a_ready_task_block_when_delivery_is_not_due(string $case): void + { + [$run, $task] = $this->original(); + $this->deleteJson('/api/workers/original-worker', [], $this->apiHeaders())->assertOk(); + if ($case === 'terminal') { + $run->forceFill(['status' => 'completed', 'closed_at' => now()])->save(); + } elseif ($case === 'leased') { + $task->forceFill(['status' => TaskStatus::Leased, 'lease_owner' => 'original-worker'])->save(); + } else { + $task->forceFill(['available_at' => now()->addMinutes(1)])->save(); + } + $this->getJson('/api/workflows/definition-original', $this->apiHeaders())->assertOk() + ->assertJsonPath('definition_routing.runnable_task', false) + ->assertJsonPath('definition_routing.blocked', false); + $this->getJson('/api/workflows/definition-original/debug', $this->apiHeaders())->assertOk() + ->assertJsonMissing(['code' => 'workflow_definition_routing_blocked']); + } + + public static function notRunnableTasks(): array + { + return [['terminal'], ['leased'], ['future']]; + } + private function original(): array { $this->worker('original-worker', 'definition-v1'); @@ -89,7 +257,7 @@ private function original(): array return [$run, $task, $started]; } - private function worker(string $id, string $fingerprint): void + private function worker(string $id, string $fingerprint, array $overrides = []): void { $this->postJson('/api/worker/register', [ 'worker_id' => $id, 'task_queue' => self::QUEUE, 'runtime' => 'python', 'sdk_version' => '2.6.1', @@ -97,13 +265,15 @@ private function worker(string $id, string $fingerprint): void 'supported_workflow_types' => [self::TYPE], 'workflow_definition_fingerprints' => [self::TYPE => $fingerprint], 'workflow_command_contracts' => [self::TYPE => ['signals' => [], 'queries' => [], 'updates' => [], 'update_validators' => []]], + ...$overrides, ], $this->workerHeaders())->assertCreated(); } - private function poll(string $id): \Illuminate\Testing\TestResponse + private function poll(string $id, array $overrides = []): TestResponse { return $this->postJson('/api/worker/workflow-tasks/poll', [ 'worker_id' => $id, 'task_queue' => self::QUEUE, 'timeout_seconds' => 0, + ...$overrides, ], $this->workerHeaders()); } } From cd7d110c724ba0a45c03041267c5879981e5983e Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 09:21:27 +0000 Subject: [PATCH 15/37] Convert each PHP release layout to native's instead of chasing it Co-Authored-By: Claude Opus 5.5 --- docs/rust-server-upgrade-takeover.md | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/docs/rust-server-upgrade-takeover.md b/docs/rust-server-upgrade-takeover.md index 52a8c03e2..ca8ddd6fa 100644 --- a/docs/rust-server-upgrade-takeover.md +++ b/docs/rust-server-upgrade-takeover.md @@ -103,13 +103,24 @@ marker exists. Native's frozen physical layout predates the current PHP release, so step 3 alone does not yield a database native accepts. PHP 2.8.0 adds its own -`workflow_worker_registration_incarnations` table, a `call_deadline_at` -column on `workflow_service_calls` and a changed `workflow_worker_registrations` -(`rust/upgrade/php-2.8.0/`). Before takeover is implemented, native's schema -moves to the layout of the release takeover starts from, and native reads -PHP's incarnation table instead of keeping a second one. A fresh native -database then has the same PHP layout, and takeover only adds native relations -and converts data. +`workflow_worker_registration_incarnations` table, a `registration_token` +column on `workflow_worker_registrations` and a `call_deadline_at` column on +`workflow_service_calls` (`rust/upgrade/php-2.8.0/`). Server 2.9.7, with Core +2.7.2, also adds `workflow_tasks.queue_delivery_reference`. Native keeps its +own layout, and step 4 converts each qualified PHP release's layout to it +with a recorded recipe: + +- PHP's registration incarnations and tokens are copied into native's + incarnation table, so a running SDK worker's fenced deregistration still + matches. The PHP table and column are then removed. +- `queue_delivery_reference` describes PHP's database queue delivery, which + native does not use. It is removed after its values are checked against + the queue inspection's timer bindings. +- Service calls are not part of native yet, so a source with service calls + is refused before any change, and `call_deadline_at` is removed only from + an empty table. + +A release whose layout has no recipe is refused by step 1. ## Tests From 57de9ec39b4107f32d2a7bcb6b070a4158cf3dc1 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 09:28:33 +0000 Subject: [PATCH 16/37] Bind routing diagnostics to the canonical selected or current run --- app/Support/ServerWorkflowControlPlane.php | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/app/Support/ServerWorkflowControlPlane.php b/app/Support/ServerWorkflowControlPlane.php index 3094a9476..caa2da4fe 100644 --- a/app/Support/ServerWorkflowControlPlane.php +++ b/app/Support/ServerWorkflowControlPlane.php @@ -323,10 +323,11 @@ private function withDefinitionRouting(string $instanceId, array $options, array return $result; } - $runId = data_get($result, 'run.run_id') ?? $result['workflow_run_id'] ?? $options['run_id'] ?? null; - $run = WorkflowRun::query()->where('namespace', $namespace)->where('workflow_instance_id', $instanceId) - ->when(is_string($runId), static fn ($query) => $query->whereKey($runId)) - ->orderByDesc('run_number')->first(); + $runId = data_get($result, 'run.run_id') ?? $result['resolved_run_id'] ?? $result['run_id'] + ?? $result['workflow_run_id'] ?? $options['run_id'] ?? null; + $run = is_string($runId) + ? NamespaceWorkflowScope::run($namespace, $instanceId, $runId) + : NamespaceWorkflowScope::currentRun($namespace, $instanceId); if ($run instanceof WorkflowRun) { $result['definition_routing'] = $this->definitionRouting->inspect($run); } From a8e324a548f7af06142c54756b4d29fd045bdc37 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 09:54:15 +0000 Subject: [PATCH 17/37] Exercise configured PostgreSQL bound queries through transaction poolers --- tests/Feature/PostgresPoolerTest.php | 83 ++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 tests/Feature/PostgresPoolerTest.php diff --git a/tests/Feature/PostgresPoolerTest.php b/tests/Feature/PostgresPoolerTest.php new file mode 100644 index 000000000..8be5876e7 --- /dev/null +++ b/tests/Feature/PostgresPoolerTest.php @@ -0,0 +1,83 @@ +markTestSkipped('Requires the isolated DW_POOLER_TEST_HOST endpoint.'); + } + $base = config('database.connections.pgsql'); + foreach (['pooler_first', 'pooler_second'] as $name) { + config(['database.connections.'.$name => [...$base, + 'host' => getenv('DW_POOLER_TEST_HOST'), + 'port' => getenv('DW_POOLER_TEST_PORT') ?: '6432', + 'database' => getenv('DW_POOLER_TEST_DATABASE') ?: 'dw_pooler_probe', + 'username' => getenv('DW_POOLER_TEST_USER') ?: 'pooler_probe', + 'password' => getenv('DW_POOLER_TEST_PASSWORD') ?: '', + ]]); + } + } + + protected function tearDown(): void + { + foreach (['pooler_first', 'pooler_second'] as $name) { + DB::purge($name); + } + parent::tearDown(); + } + + public function test_clients_keep_their_bound_values_and_types_when_backend_sessions_change(): void + { + $first = DB::connection('pooler_first'); + $second = DB::connection('pooler_second'); + + // Occupy two server sessions using simple protocol commands first. + // The probe itself then exercises Laravel's configured PDO strategy. + $first->getPdo()->exec('BEGIN'); + $first->getPdo()->exec('SELECT 1'); + $second->getPdo()->exec('BEGIN'); + $second->getPdo()->exec('SELECT 1'); + $first->getPdo()->exec('ROLLBACK'); + $second->getPdo()->exec('ROLLBACK'); + + for ($iteration = 0; $iteration < 20; $iteration++) { + $marker = 'first-'.$iteration.'-'.bin2hex(random_bytes(8)); + $row = $first->selectOne('SELECT CAST(? AS TEXT) AS marker, CAST(? AS BIGINT) AS number, CAST(? AS TEXT) AS absent', + [$marker, '9223372036854775807', null]); + self::assertSame($marker, $row->marker); + self::assertSame('9223372036854775807', (string) $row->number); + self::assertNull($row->absent); + + $other = 'second-'.$iteration.'-Київ'; + $row = $second->selectOne('SELECT CAST(? AS TEXT) AS marker', [$other]); + self::assertSame($other, $row->marker); + } + } + + public function test_bound_statements_do_not_abort_a_multi_query_transaction(): void + { + $connection = DB::connection('pooler_first'); + $connection->beginTransaction(); + try { + foreach (['first', 'second', 'third'] as $value) { + self::assertSame($value, $connection->selectOne('SELECT CAST(? AS TEXT) AS marker', [$value])->marker); + self::assertSame(2, (int) $connection->selectOne('SELECT 1 + 1 AS result')->result); + } + } finally { + $connection->rollBack(); + } + self::assertSame('after-rollback', $connection->selectOne('SELECT CAST(? AS TEXT) AS marker', ['after-rollback'])->marker); + } +} From 0447dd5bc6208602329d31552916fdae1fd2deb9 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 10:01:37 +0000 Subject: [PATCH 18/37] Reproduce readiness passing a connected database with broken bound reads --- tests/Feature/HealthControllerTest.php | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tests/Feature/HealthControllerTest.php b/tests/Feature/HealthControllerTest.php index 469190843..cb0d629b8 100644 --- a/tests/Feature/HealthControllerTest.php +++ b/tests/Feature/HealthControllerTest.php @@ -11,6 +11,7 @@ use App\Support\ServerReadiness; use App\Support\ServerTopology; use Illuminate\Foundation\Testing\RefreshDatabase; +use Illuminate\Database\SQLiteConnection; use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Schema; use RuntimeException; @@ -25,6 +26,30 @@ class HealthControllerTest extends TestCase use RefreshDatabase; use ServerTestHelpers; + public function test_readiness_refuses_a_connected_database_that_cannot_execute_bound_reads(): void + { + $pdo = DB::connection()->getPdo(); + DB::extend('broken_bound_reads', static fn (array $config): SQLiteConnection => new class($pdo, ':memory:', '', $config) extends SQLiteConnection + { + public function select($query, $bindings = [], $useReadPdo = true) + { + if ($bindings !== []) { + throw new \PDOException('The connected database cannot execute bound statements.'); + } + + return parent::select($query, $bindings, $useReadPdo); + } + }); + config([ + 'database.connections.broken_bound_reads' => ['driver' => 'broken_bound_reads'], + 'database.default' => 'broken_bound_reads', + ]); + + $this->getJson('/api/ready') + ->assertStatus(503) + ->assertJsonPath('checks.database.status', 'unavailable'); + } + public function test_health_check_returns_serving_when_database_is_available(): void { $response = $this->getJson('/api/health'); From ba06b0f8fff4c4c8e32cd041bd58cc7b35b27199 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 10:05:27 +0000 Subject: [PATCH 19/37] Use native unnamed PostgreSQL binding and probe primary query readiness --- app/Support/ServerReadiness.php | 19 ++++++++++++++----- config/database.php | 5 +++++ tests/Feature/HealthControllerTest.php | 6 ++++-- 3 files changed, 23 insertions(+), 7 deletions(-) diff --git a/app/Support/ServerReadiness.php b/app/Support/ServerReadiness.php index 43a4ce091..93bd425fb 100644 --- a/app/Support/ServerReadiness.php +++ b/app/Support/ServerReadiness.php @@ -40,7 +40,7 @@ public function snapshot(): array : $this->cacheCheck(); $checks = [ - 'database' => $this->databaseCheck(), + 'database' => $this->databaseCheck(probeQuery: true), 'migrations' => $this->migrationCheck(), 'queue' => $queue, 'default_namespace' => $this->defaultNamespaceCheck(), @@ -93,16 +93,25 @@ public function bootstrapStatus(): array ])); } - private function databaseCheck(): array + private function databaseCheck(bool $probeQuery = false): array { try { - DB::connection()->getPdo(); + $connection = DB::connection(); + $connection->getPdo(); + if ($probeQuery) { + $row = $connection->selectOne('SELECT ? AS readiness_probe', [1], useReadPdo: false); + if ((string) ($row->readiness_probe ?? '') !== '1') { + throw new \RuntimeException('The database did not return the readiness probe value.'); + } + } return [ 'status' => 'ok', - 'check_scope' => 'connection_only', + 'check_scope' => $probeQuery ? 'bound_read_query' : 'connection_only', 'write_capacity_verified' => false, - 'message' => 'The database connection is available. Write capacity has not been verified.', + 'message' => $probeQuery + ? 'The primary database connection can execute a bound read query. Write capacity has not been verified.' + : 'The database connection is available. Write capacity has not been verified.', ]; } catch (\Throwable $exception) { return [ diff --git a/config/database.php b/config/database.php index 6fafd2094..c80ae54bd 100644 --- a/config/database.php +++ b/config/database.php @@ -98,6 +98,11 @@ 'prefix_indexes' => true, 'search_path' => 'public', 'sslmode' => 'prefer', + // Keep native parameter binding without session-scoped statement + // names, so transaction poolers can safely change backend sessions. + 'options' => extension_loaded('pdo_pgsql') ? [ + (PHP_VERSION_ID >= 80500 ? Pdo\Pgsql::ATTR_DISABLE_PREPARES : PDO::PGSQL_ATTR_DISABLE_PREPARES) => true, + ] : [], ], 'sqlsrv' => [ diff --git a/tests/Feature/HealthControllerTest.php b/tests/Feature/HealthControllerTest.php index cb0d629b8..361b763e1 100644 --- a/tests/Feature/HealthControllerTest.php +++ b/tests/Feature/HealthControllerTest.php @@ -31,13 +31,13 @@ public function test_readiness_refuses_a_connected_database_that_cannot_execute_ $pdo = DB::connection()->getPdo(); DB::extend('broken_bound_reads', static fn (array $config): SQLiteConnection => new class($pdo, ':memory:', '', $config) extends SQLiteConnection { - public function select($query, $bindings = [], $useReadPdo = true) + public function select($query, $bindings = [], $useReadPdo = true, array $fetchUsing = []) { if ($bindings !== []) { throw new \PDOException('The connected database cannot execute bound statements.'); } - return parent::select($query, $bindings, $useReadPdo); + return parent::select($query, $bindings, $useReadPdo, $fetchUsing); } }); config([ @@ -147,6 +147,8 @@ public function test_readiness_check_returns_ready_when_bootstrap_state_is_avail $response->assertOk() ->assertJsonPath('status', 'ready') ->assertJsonPath('checks.database.status', 'ok') + ->assertJsonPath('checks.database.check_scope', 'bound_read_query') + ->assertJsonPath('checks.database.write_capacity_verified', false) ->assertJsonPath('checks.migrations.status', 'ok') ->assertJsonPath('checks.migrations.operator_surface.available', true) ->assertJsonPath('checks.migrations.readiness_contract.version', 1) From 2e59f1b38275581d58e0aeb86e4ff8fba858342b Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 10:13:55 +0000 Subject: [PATCH 20/37] Qualify PostgreSQL transaction pools in ordinary PHP CI --- .github/workflows/phpunit-feature.yml | 21 +++++++++++++ app/Support/ServerReadiness.php | 4 ++- config/database.php | 8 +++-- docs/server-reference.md | 31 +++++++++++++++++++ tests/Feature/HealthControllerTest.php | 5 ++-- tests/Feature/PostgresPoolerTest.php | 41 +++++++++++++++++++++----- 6 files changed, 96 insertions(+), 14 deletions(-) diff --git a/.github/workflows/phpunit-feature.yml b/.github/workflows/phpunit-feature.yml index 16b58a369..d403c73ed 100644 --- a/.github/workflows/phpunit-feature.yml +++ b/.github/workflows/phpunit-feature.yml @@ -220,6 +220,21 @@ jobs: done [ "$health" = healthy ] done + for mode in session transaction0 transaction200; do + pool_mode=transaction + prepared_statements=0 + [ "$mode" != session ] || pool_mode=session + [ "$mode" != transaction200 ] || prepared_statements=200 + docker run --detach \ + --name "dw-server-pooler-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }}-$mode" \ + --network "dw-server-feature-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }}" \ + --memory 128m --memory-swap 128m \ + -e DB_HOST="dw-server-backup-pgsql-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }}" \ + -e DB_NAME=postgres -e DB_USER=postgres -e DB_PASSWORD=pooler-disposable-password \ + -e AUTH_TYPE=trust -e "POOL_MODE=$pool_mode" -e "MAX_PREPARED_STATEMENTS=$prepared_statements" \ + -e DEFAULT_POOL_SIZE=2 -e MIN_POOL_SIZE=2 -e SERVER_ROUND_ROBIN=1 \ + edoburu/pgbouncer@sha256:b17551c776ef7e5769ef80b956d20f85e2fd25dd8912d31d58f782aad495b711 + done tar \ --exclude=vendor \ --exclude='*/vendor' \ @@ -236,6 +251,9 @@ jobs: -e DW_TEST_COMPLETION_MYSQL_HOST="dw-server-backup-mysql-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }}" \ -e DW_TEST_BACKUP_PGSQL_HOST="dw-server-backup-pgsql-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }}" \ -e DW_TEST_COMPLETION_PGSQL_HOST="dw-server-backup-pgsql-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }}" \ + -e DW_POOLER_TEST_HOST="dw-server-backup-pgsql-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }}" \ + -e DW_POOLER_TEST_PORT=5432 -e DW_POOLER_TEST_DATABASE=postgres -e DW_POOLER_TEST_USER=postgres \ + -e DW_POOLER_TEST_PREFIX="dw-server-pooler-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }}" \ -w /app \ --entrypoint sh \ durable-workflow-server-corpus-validator \ @@ -283,6 +301,9 @@ jobs: - name: Clean up test services if: always() run: | + for mode in session transaction0 transaction200; do + docker rm -f "dw-server-pooler-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }}-$mode" >/dev/null 2>&1 || true + done docker rm -f \ "dw-server-s3-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }}" \ "dw-server-backup-mysql-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }}" \ diff --git a/app/Support/ServerReadiness.php b/app/Support/ServerReadiness.php index 93bd425fb..f11d9c7cc 100644 --- a/app/Support/ServerReadiness.php +++ b/app/Support/ServerReadiness.php @@ -95,10 +95,12 @@ public function bootstrapStatus(): array private function databaseCheck(bool $probeQuery = false): array { + $queryAttempted = false; try { $connection = DB::connection(); $connection->getPdo(); if ($probeQuery) { + $queryAttempted = true; $row = $connection->selectOne('SELECT ? AS readiness_probe', [1], useReadPdo: false); if ((string) ($row->readiness_probe ?? '') !== '1') { throw new \RuntimeException('The database did not return the readiness probe value.'); @@ -117,7 +119,7 @@ private function databaseCheck(bool $probeQuery = false): array return [ 'status' => 'unavailable', 'message' => $exception->getMessage(), - 'reason' => BackendUnavailable::is($exception) ? 'backend_unavailable' : null, + 'reason' => BackendUnavailable::is($exception) ? 'backend_unavailable' : ($queryAttempted ? 'database_query_failed' : null), ]; } } diff --git a/config/database.php b/config/database.php index c80ae54bd..aebce8f60 100644 --- a/config/database.php +++ b/config/database.php @@ -1,6 +1,8 @@ true, 'engine' => null, 'options' => extension_loaded('pdo_mysql') ? array_filter([ - (PHP_VERSION_ID >= 80500 ? Pdo\Mysql::ATTR_SSL_CA : PDO::MYSQL_ATTR_SSL_CA) => env('MYSQL_ATTR_SSL_CA'), + (PHP_VERSION_ID >= 80500 ? Mysql::ATTR_SSL_CA : PDO::MYSQL_ATTR_SSL_CA) => env('MYSQL_ATTR_SSL_CA'), ]) : [], ], @@ -81,7 +83,7 @@ 'strict' => true, 'engine' => null, 'options' => extension_loaded('pdo_mysql') ? array_filter([ - (PHP_VERSION_ID >= 80500 ? Pdo\Mysql::ATTR_SSL_CA : PDO::MYSQL_ATTR_SSL_CA) => env('MYSQL_ATTR_SSL_CA'), + (PHP_VERSION_ID >= 80500 ? Mysql::ATTR_SSL_CA : PDO::MYSQL_ATTR_SSL_CA) => env('MYSQL_ATTR_SSL_CA'), ]) : [], ], @@ -101,7 +103,7 @@ // Keep native parameter binding without session-scoped statement // names, so transaction poolers can safely change backend sessions. 'options' => extension_loaded('pdo_pgsql') ? [ - (PHP_VERSION_ID >= 80500 ? Pdo\Pgsql::ATTR_DISABLE_PREPARES : PDO::PGSQL_ATTR_DISABLE_PREPARES) => true, + (PHP_VERSION_ID >= 80500 ? Pgsql::ATTR_DISABLE_PREPARES : PDO::PGSQL_ATTR_DISABLE_PREPARES) => true, ] : [], ], diff --git a/docs/server-reference.md b/docs/server-reference.md index 8d836aa79..7c6ca9c11 100644 --- a/docs/server-reference.md +++ b/docs/server-reference.md @@ -1599,6 +1599,37 @@ overlay or with `kubectl scale`. ### Configuration +#### PostgreSQL connection pooling + +The PostgreSQL connection uses native bound parameters without session-scoped +prepared-statement names. This lets PgBouncer transaction pooling change the +physical database session between transactions. Direct and session-pooled +connections use the same binding path. Statement pooling cannot support the +Server's multi-statement transactions. + +The PHP feature job runs the same alternating-client and rollback controls +against direct PostgreSQL, session pooling, and transaction pooling with +`max_prepared_statements=0` and `200`. The transaction controls require actual +backend PID changes. To run against an isolated existing endpoint: + +```bash +DW_POOLER_TEST_HOST=pooler DW_POOLER_TEST_PORT=6432 \ +DW_POOLER_TEST_DATABASE=isolated_test DW_POOLER_TEST_USER=test_user \ +vendor/bin/phpunit tests/Feature/PostgresPoolerTest.php --fail-on-skipped +``` + +These controls do not validate custom session settings, temporary tables, +advisory-lock policies, or arbitrary proxy configurations. Qualify the exact +published image, pooler version, authentication, schema and TLS settings used +by your deployment before routing production traffic. + +`/api/ready` executes a bound read on the primary database connection and +reports `checks.database.check_scope=bound_read_query`. A failed query reports +`database_query_failed`, or `backend_unavailable` when the backend is known to +be unavailable. This remains a read probe: `write_capacity_verified=false`. +Use an actual workflow canary to verify writes and the complete worker path. +The runtime's lightweight bootstrap guard remains connection-only. + All operator-facing configuration is via `DW_*` environment variables. `config/dw-contract.php` is the authoritative machine-checkable contract; CI (`tests/Unit/EnvContractTest.php`) diffs it against `.env.example`, diff --git a/tests/Feature/HealthControllerTest.php b/tests/Feature/HealthControllerTest.php index 361b763e1..d40ef79ff 100644 --- a/tests/Feature/HealthControllerTest.php +++ b/tests/Feature/HealthControllerTest.php @@ -10,8 +10,8 @@ use App\Support\RedisReadinessProcess; use App\Support\ServerReadiness; use App\Support\ServerTopology; -use Illuminate\Foundation\Testing\RefreshDatabase; use Illuminate\Database\SQLiteConnection; +use Illuminate\Foundation\Testing\RefreshDatabase; use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Schema; use RuntimeException; @@ -47,7 +47,8 @@ public function select($query, $bindings = [], $useReadPdo = true, array $fetchU $this->getJson('/api/ready') ->assertStatus(503) - ->assertJsonPath('checks.database.status', 'unavailable'); + ->assertJsonPath('checks.database.status', 'unavailable') + ->assertJsonPath('checks.database.reason', 'database_query_failed'); } public function test_health_check_returns_serving_when_database_is_available(): void diff --git a/tests/Feature/PostgresPoolerTest.php b/tests/Feature/PostgresPoolerTest.php index 8be5876e7..b04fbe2b4 100644 --- a/tests/Feature/PostgresPoolerTest.php +++ b/tests/Feature/PostgresPoolerTest.php @@ -3,6 +3,7 @@ namespace Tests\Feature; use Illuminate\Support\Facades\DB; +use PHPUnit\Framework\Attributes\DataProvider; use Tests\TestCase; /** @@ -12,17 +13,29 @@ */ class PostgresPoolerTest extends TestCase { - protected function setUp(): void + public static function endpoints(): array { - parent::setUp(); - if (! is_string(getenv('DW_POOLER_TEST_HOST')) || getenv('DW_POOLER_TEST_HOST') === '') { + $endpoints = ['configured' => [getenv('DW_POOLER_TEST_HOST') ?: null, (int) (getenv('DW_POOLER_TEST_PORT') ?: 6432), getenv('DW_POOLER_TEST_EXPECT_SWITCH') === '1']]; + $prefix = getenv('DW_POOLER_TEST_PREFIX'); + if (is_string($prefix) && $prefix !== '') { + foreach (['session', 'transaction0', 'transaction200'] as $mode) { + $endpoints[$mode] = [$prefix.'-'.$mode, 5432, $mode !== 'session']; + } + } + + return $endpoints; + } + + private function configureEndpoint(?string $host, int $port): void + { + if ($host === null) { $this->markTestSkipped('Requires the isolated DW_POOLER_TEST_HOST endpoint.'); } $base = config('database.connections.pgsql'); foreach (['pooler_first', 'pooler_second'] as $name) { config(['database.connections.'.$name => [...$base, - 'host' => getenv('DW_POOLER_TEST_HOST'), - 'port' => getenv('DW_POOLER_TEST_PORT') ?: '6432', + 'host' => $host, + 'port' => $port, 'database' => getenv('DW_POOLER_TEST_DATABASE') ?: 'dw_pooler_probe', 'username' => getenv('DW_POOLER_TEST_USER') ?: 'pooler_probe', 'password' => getenv('DW_POOLER_TEST_PASSWORD') ?: '', @@ -38,8 +51,10 @@ protected function tearDown(): void parent::tearDown(); } - public function test_clients_keep_their_bound_values_and_types_when_backend_sessions_change(): void + #[DataProvider('endpoints')] + public function test_clients_keep_their_bound_values_and_types_when_backend_sessions_change(?string $host, int $port, bool $expectSwitch): void { + $this->configureEndpoint($host, $port); $first = DB::connection('pooler_first'); $second = DB::connection('pooler_second'); @@ -52,10 +67,12 @@ public function test_clients_keep_their_bound_values_and_types_when_backend_sess $first->getPdo()->exec('ROLLBACK'); $second->getPdo()->exec('ROLLBACK'); + $backends = []; for ($iteration = 0; $iteration < 20; $iteration++) { $marker = 'first-'.$iteration.'-'.bin2hex(random_bytes(8)); - $row = $first->selectOne('SELECT CAST(? AS TEXT) AS marker, CAST(? AS BIGINT) AS number, CAST(? AS TEXT) AS absent', + $row = $first->selectOne('SELECT CAST(? AS TEXT) AS marker, CAST(? AS BIGINT) AS number, CAST(? AS TEXT) AS absent, pg_backend_pid() AS backend', [$marker, '9223372036854775807', null]); + $backends[] = $row->backend; self::assertSame($marker, $row->marker); self::assertSame('9223372036854775807', (string) $row->number); self::assertNull($row->absent); @@ -63,11 +80,19 @@ public function test_clients_keep_their_bound_values_and_types_when_backend_sess $other = 'second-'.$iteration.'-Київ'; $row = $second->selectOne('SELECT CAST(? AS TEXT) AS marker', [$other]); self::assertSame($other, $row->marker); + // An odd number of commands per iteration makes the next first + // client's query use another backend in the round-robin pool. + self::assertSame(1, (int) $first->selectOne('SELECT 1 AS value')->value); + } + if ($expectSwitch) { + self::assertGreaterThan(1, count(array_unique($backends)), 'The control must change this client\'s physical backend.'); } } - public function test_bound_statements_do_not_abort_a_multi_query_transaction(): void + #[DataProvider('endpoints')] + public function test_bound_statements_do_not_abort_a_multi_query_transaction(?string $host, int $port, bool $expectSwitch): void { + $this->configureEndpoint($host, $port); $connection = DB::connection('pooler_first'); $connection->beginTransaction(); try { From 75d6f3801c8a00bcef016045afe4ecc8d6e942b3 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 10:34:49 +0000 Subject: [PATCH 21/37] Clarify replay routing decisions and preserve explicit metrics clearing --- app/Http/Controllers/Api/WorkerController.php | 2 +- app/Support/WorkflowDefinitionRouting.php | 33 +++++++++---------- app/Support/WorkflowReplayEligibility.php | 9 ++--- app/Support/WorkflowTaskPoller.php | 3 -- .../WorkflowDefinitionEvolutionTest.php | 12 +++++++ 5 files changed, 32 insertions(+), 27 deletions(-) diff --git a/app/Http/Controllers/Api/WorkerController.php b/app/Http/Controllers/Api/WorkerController.php index a90ca6ccf..4da9a14eb 100644 --- a/app/Http/Controllers/Api/WorkerController.php +++ b/app/Http/Controllers/Api/WorkerController.php @@ -940,7 +940,7 @@ public function heartbeat(Request $request): JsonResponse $update['process_metrics'] = $this->normalizeProcessMetrics($validated['process_metrics']); } $update['process_metrics'] = [ - ...($update['process_metrics'] ?? $worker->process_metrics ?? []), + ...(array_key_exists('process_metrics', $update) ? ($update['process_metrics'] ?? []) : ($worker->process_metrics ?? [])), WorkflowReplayEligibility::PROTOCOL_METRIC => WorkerProtocol::requestVersion($request), ]; diff --git a/app/Support/WorkflowDefinitionRouting.php b/app/Support/WorkflowDefinitionRouting.php index cb1f264fa..2719ee015 100644 --- a/app/Support/WorkflowDefinitionRouting.php +++ b/app/Support/WorkflowDefinitionRouting.php @@ -25,18 +25,16 @@ public function forWorker( $advertised = self::stringValue($worker->workflow_definition_fingerprints[$run->workflow_type] ?? null); $decision['advertised_fingerprint'] = $advertised; - if (! $decision['worker_pinned']) { - return [...$decision, 'allowed' => true, 'status' => 'not_worker_pinned']; - } - if (! $decision['pinning_enabled']) { - return [...$decision, 'allowed' => true, 'status' => 'pinning_disabled']; - } - if ($recorded === null || $advertised === null) { - return [...$decision, 'allowed' => false, 'status' => $recorded === null - ? 'recorded_definition_missing' : 'worker_definition_missing']; - } - if (hash_equals($recorded, $advertised)) { - return [...$decision, 'allowed' => true, 'status' => 'matching_definition']; + $immediate = match (true) { + ! $decision['worker_pinned'] => ['allowed' => true, 'status' => 'not_worker_pinned'], + ! $decision['pinning_enabled'] => ['allowed' => true, 'status' => 'pinning_disabled'], + $recorded === null => ['allowed' => false, 'status' => 'recorded_definition_missing'], + $advertised === null => ['allowed' => false, 'status' => 'worker_definition_missing'], + hash_equals($recorded, $advertised) => ['allowed' => true, 'status' => 'matching_definition'], + default => null, + }; + if ($immediate !== null) { + return [...$decision, ...$immediate]; } foreach ($this->eligibleWorkers($run, $decision['task_queue'], $decision['required_build_id']) as $peer) { @@ -95,16 +93,17 @@ public function inspect(WorkflowRun $run, ?string $taskQueue = null, ?string $re $runnable = ! $run->status->isTerminal() && $task instanceof WorkflowTask; $blocked = $runnable && $context['worker_pinned'] && $context['pinning_enabled'] && in_array($status, ['recorded_definition_missing', 'worker_definition_missing', 'no_eligible_worker'], true); + $resolution = match (true) { + $blocked => 'Register a fresh worker for the original namespace, task queue, workflow type, build and replay capabilities with its definition fingerprint. Do not rewrite the run history.', + $status === 'replay_required' => 'Current workers may replay the original history. Use versioned branches for command changes; incompatible replay still refuses.', + default => 'Recorded build, namespace, type and replay-capability guards remain in force.', + }; return [...$context, 'status' => $status, 'blocked' => $blocked, 'runnable_task' => $runnable, 'eligible_worker_count' => $eligible, 'matching_worker_count' => $matching, 'changed_worker_count' => $changed, 'missing_fingerprint_count' => $missing, 'workers' => $examples, 'workers_truncated' => $eligible > count($examples), - 'resolution' => $blocked - ? 'Register a fresh worker for the original namespace, task queue, workflow type, build and replay capabilities with its definition fingerprint. Do not rewrite the run history.' - : ($status === 'replay_required' - ? 'Current workers may replay the original history. Use versioned branches for command changes; incompatible replay still refuses.' - : 'Recorded build, namespace, type and replay-capability guards remain in force.'), + 'resolution' => $resolution, ]; } diff --git a/app/Support/WorkflowReplayEligibility.php b/app/Support/WorkflowReplayEligibility.php index 172dc4b0f..da4744b2d 100644 --- a/app/Support/WorkflowReplayEligibility.php +++ b/app/Support/WorkflowReplayEligibility.php @@ -18,12 +18,9 @@ public static function allows(WorkerRegistration $worker, WorkflowRun $run, ?str $capabilities = is_array($worker->capabilities) ? $worker->capabilities : []; if (is_string($run->cancellation_request_command_id) - && ! CooperativeCancellationPolicy::supports($capabilities, $protocolVersion)) { - return false; - } - if (is_string($run->cancellation_request_command_id) - && NexusCooperativeCancellationPolicy::requiresCapabilityForReplay($run) - && ! NexusCooperativeCancellationPolicy::supports($capabilities, $protocolVersion)) { + && (! CooperativeCancellationPolicy::supports($capabilities, $protocolVersion) + || (NexusCooperativeCancellationPolicy::requiresCapabilityForReplay($run) + && ! NexusCooperativeCancellationPolicy::supports($capabilities, $protocolVersion)))) { return false; } diff --git a/app/Support/WorkflowTaskPoller.php b/app/Support/WorkflowTaskPoller.php index 4cc917342..f21fb6877 100644 --- a/app/Support/WorkflowTaskPoller.php +++ b/app/Support/WorkflowTaskPoller.php @@ -749,7 +749,6 @@ function () use ( historyPageSize: $historyPageSize, acceptHistoryEncoding: $acceptHistoryEncoding, supportedWorkflowTypes: $supportedWorkflowTypes, - workflowDefinitionFingerprints: $workflowDefinitionFingerprints, workerPollFence: $workerPollFence, ), ), @@ -798,7 +797,6 @@ function () use ( historyPageSize: $historyPageSize, acceptHistoryEncoding: $acceptHistoryEncoding, supportedWorkflowTypes: $supportedWorkflowTypes, - workflowDefinitionFingerprints: $workflowDefinitionFingerprints, workerPollFence: $workerPollFence, ), ), @@ -968,7 +966,6 @@ private function claimReadyTask( ?int $historyPageSize = null, ?string $acceptHistoryEncoding = null, array $supportedWorkflowTypes = [], - array $workflowDefinitionFingerprints = [], array $workerPollFence = [], ): ?array { $readyTasks = $this->pollReplayEligibleReadyTasks( diff --git a/tests/Feature/WorkflowDefinitionEvolutionTest.php b/tests/Feature/WorkflowDefinitionEvolutionTest.php index 3bbd0ac77..f8398e18b 100644 --- a/tests/Feature/WorkflowDefinitionEvolutionTest.php +++ b/tests/Feature/WorkflowDefinitionEvolutionTest.php @@ -173,6 +173,18 @@ public function test_client_process_metrics_cannot_spoof_the_saved_protocol(): v self::assertSame(456, $worker->fresh()->process_metrics['memory_bytes']); } + public function test_explicit_null_clears_client_metrics_without_clearing_server_protocol(): void + { + $this->worker('original-worker', 'definition-v1', ['process_metrics' => ['memory_bytes' => 123]]); + $this->postJson('/api/worker/heartbeat', [ + 'worker_id' => 'original-worker', 'process_metrics' => null, + ], $this->workerHeaders())->assertOk(); + $worker = WorkerRegistration::query()->where('worker_id', 'original-worker')->firstOrFail(); + self::assertSame([ + WorkflowReplayEligibility::PROTOCOL_METRIC => $this->workerHeaders()['X-Durable-Workflow-Protocol-Version'], + ], $worker->process_metrics); + } + public function test_run_debug_queue_and_repair_distinguish_fallback_from_a_real_block(): void { [$run, $task] = $this->original(); From d867e0f43955e591874b5ea9da28964f7578781b Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 10:39:48 +0000 Subject: [PATCH 22/37] Print failed Rust tests in the failing step's own log The parity jobs write Rust test output to evidence files, so a failed cargo test left only its exit code in the step log. An ERR trap now prints the panics and the failed test names from those files. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/server-parity.yml | 3 +++ .../conformance/server-parity/rust-test-failures.sh | 13 +++++++++++++ 2 files changed, 16 insertions(+) create mode 100755 scripts/conformance/server-parity/rust-test-failures.sh diff --git a/.github/workflows/server-parity.yml b/.github/workflows/server-parity.yml index c4bb53b6c..38d3bae9d 100644 --- a/.github/workflows/server-parity.yml +++ b/.github/workflows/server-parity.yml @@ -377,6 +377,7 @@ jobs: - name: Bootstrap independent published PHP and native MySQL databases run: | set -euo pipefail + trap 'bash scripts/conformance/server-parity/rust-test-failures.sh' ERR mkdir -p cargo-home rust-target parity-evidence mysql-state chmod 0777 parity-evidence docker network create "$RESOURCE_SCOPE" @@ -856,6 +857,7 @@ jobs: POSTGRES_IMAGE: ${{ matrix.image }} run: | set -euo pipefail + trap 'bash scripts/conformance/server-parity/rust-test-failures.sh' ERR mkdir -p cargo-home rust-target parity-evidence pg-state chmod 0777 parity-evidence docker network create "$RESOURCE_SCOPE" @@ -1364,6 +1366,7 @@ jobs: - name: Execute official Rust codec and cross-decode with published PHP run: | set -euo pipefail + trap 'bash scripts/conformance/server-parity/rust-test-failures.sh' ERR mkdir -p cargo-home rust-target parity-evidence # Compiling the library tests peaks near 2.5 GB; a 2 GB limit OOM-killed rustc (exit 101). docker run --rm --user="$(id -u):$(id -g)" --cpus=2 --memory=4g --memory-swap=4g \ diff --git a/scripts/conformance/server-parity/rust-test-failures.sh b/scripts/conformance/server-parity/rust-test-failures.sh new file mode 100755 index 000000000..6e83f9e9c --- /dev/null +++ b/scripts/conformance/server-parity/rust-test-failures.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# Print the failures in this job's Rust test output, so a failed test step is +# diagnosable from its own log. Reads parity-evidence; changes nothing. +set -uo pipefail +for output in parity-evidence/*tests*.txt; do + [[ -f $output ]] && grep -q '^test result: FAILED' "$output" || continue + echo "::group::Rust test failures in ${output#parity-evidence/}" + grep -E -A3 'panicked at' "$output" | cut -c1-600 | head -n 120 + # The closing summary names every failed test. + sed -n '/^failures:$/,$p' "$output" | grep -E '^failures:$|^ [A-Za-z0-9_:]+$|^test result:' | tail -n 60 + echo "::endgroup::" +done +exit 0 From bac6c80cb59379b532fd459118ac9620188ec963 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 10:45:15 +0000 Subject: [PATCH 23/37] Describe queue delivery references as the lookup metadata they are PHP grants queue_delivery_reference no task authority, and the queue inspection already binds every queued job to its task row, so the conversion removes the column and reports how many references it held. Co-Authored-By: Claude Opus 5.5 --- docs/rust-server-upgrade-takeover.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/rust-server-upgrade-takeover.md b/docs/rust-server-upgrade-takeover.md index ca8ddd6fa..3f03af109 100644 --- a/docs/rust-server-upgrade-takeover.md +++ b/docs/rust-server-upgrade-takeover.md @@ -113,9 +113,11 @@ with a recorded recipe: - PHP's registration incarnations and tokens are copied into native's incarnation table, so a running SDK worker's fenced deregistration still matches. The PHP table and column are then removed. -- `queue_delivery_reference` describes PHP's database queue delivery, which - native does not use. It is removed after its values are checked against - the queue inspection's timer bindings. +- `queue_delivery_reference` is optional lookup metadata for PHP's database + queue delivery, and PHP itself grants it no task authority. Native does not + use PHP's queue, and the queue inspection has already bound every queued + job to its durable task row, so the column is removed and the takeover + report counts the references it held. - Service calls are not part of native yet, so a source with service calls is refused before any change, and `call_deadline_at` is removed only from an empty table. From 6cc5d3d56957c7be6f5a69698daf041d12f70c0c Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 11:02:03 +0000 Subject: [PATCH 24/37] Match the design to the SQLite and PostgreSQL takeovers as built PostgreSQL locks every table in ACCESS EXCLUSIVE mode after native's bootstrap key, since the later DDL needs that mode anyway. On SQLite and PostgreSQL the marker commits with everything else, so an interrupted takeover is simply rerun; only MySQL and MariaDB need the step journal. Co-Authored-By: Claude Opus 5.5 --- docs/rust-server-upgrade-takeover.md | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/docs/rust-server-upgrade-takeover.md b/docs/rust-server-upgrade-takeover.md index 3f03af109..0aa1f8211 100644 --- a/docs/rust-server-upgrade-takeover.md +++ b/docs/rust-server-upgrade-takeover.md @@ -66,7 +66,7 @@ with any other writer. | Backend | Exclusion | Atomicity | | --- | --- | --- | | SQLite | `BEGIN EXCLUSIVE` on the database file | Fingerprint, marker, native relations and conversions commit in one transaction. | -| PostgreSQL | `LOCK TABLE … IN EXCLUSIVE MODE` on every relation, then the native advisory key | DDL is transactional, so everything commits in one transaction. | +| PostgreSQL | Native's bootstrap advisory key, then `LOCK TABLE … IN ACCESS EXCLUSIVE MODE` on every table, with a short lock wait. The later DDL needs that mode anyway, so no lock is upgraded while another session waits. | DDL is transactional, so everything commits in one transaction. | | MySQL and MariaDB | To be qualified: `LOCK TABLES … WRITE` on every relation, or a check that no other session uses the database, repeated after the marker is written | DDL commits implicitly, so the marker records a step journal (below). | On SQLite and PostgreSQL, a PHP connection that slipped past the stop blocks on @@ -75,18 +75,19 @@ leaves a mixed state. ## Marker and interruption -The marker is written first in each takeover transaction, with state -`takeover`, the fingerprint and the step list. Native startup refuses state -`takeover`. Each step is idempotent and records its completion in the marker. -Rerunning `upgrade-takeover` with the same fingerprint resumes after the last -recorded step; a different fingerprint is refused. The final step sets state -`ready` and the native schema version. - -On SQLite and PostgreSQL an interruption rolls the whole transaction back, so -the database is either unchanged PHP or fully native. On MySQL and MariaDB the -step journal makes each committed DDL step visible, and resumption continues -from it. The PHP fence refuses the database from the first step, because the -marker exists. +On SQLite and PostgreSQL the marker commits with everything else, so an +interruption leaves the database unchanged PHP and nothing needs a journal. +The operator reruns `upgrade-takeover` with the same fingerprint. A killed +SQLite takeover leaves a rollback journal; read-only inspection refuses it, +and the rerun lets SQLite roll it back before the fingerprint check. + +MySQL and MariaDB commit DDL implicitly, so there the marker is written first, +with state `takeover`, the fingerprint and the step list. Native startup +refuses state `takeover`. Each step is idempotent and records its completion +in the marker. Rerunning `upgrade-takeover` with the same fingerprint resumes +after the last recorded step; a different fingerprint is refused. The final +step sets state `ready` and the native schema version. The PHP fence refuses +the database from the first step, because the marker exists. ## Steps From d94b1180e24576f33dae03c64bb9adfed11ffaa8 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 11:03:51 +0000 Subject: [PATCH 25/37] Call the Rust test failure summary through a named trap function The Docker workflow isolation policy only accepts repository helpers in statically known positions, as the container log tails already use. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/server-parity.yml | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/.github/workflows/server-parity.yml b/.github/workflows/server-parity.yml index 38d3bae9d..31788a264 100644 --- a/.github/workflows/server-parity.yml +++ b/.github/workflows/server-parity.yml @@ -377,7 +377,11 @@ jobs: - name: Bootstrap independent published PHP and native MySQL databases run: | set -euo pipefail - trap 'bash scripts/conformance/server-parity/rust-test-failures.sh' ERR + # Failed Rust tests are printed from their evidence files into this step's log. + rust_test_failures() { + bash scripts/conformance/server-parity/rust-test-failures.sh + } + trap rust_test_failures ERR mkdir -p cargo-home rust-target parity-evidence mysql-state chmod 0777 parity-evidence docker network create "$RESOURCE_SCOPE" @@ -857,7 +861,11 @@ jobs: POSTGRES_IMAGE: ${{ matrix.image }} run: | set -euo pipefail - trap 'bash scripts/conformance/server-parity/rust-test-failures.sh' ERR + # Failed Rust tests are printed from their evidence files into this step's log. + rust_test_failures() { + bash scripts/conformance/server-parity/rust-test-failures.sh + } + trap rust_test_failures ERR mkdir -p cargo-home rust-target parity-evidence pg-state chmod 0777 parity-evidence docker network create "$RESOURCE_SCOPE" @@ -1366,7 +1374,11 @@ jobs: - name: Execute official Rust codec and cross-decode with published PHP run: | set -euo pipefail - trap 'bash scripts/conformance/server-parity/rust-test-failures.sh' ERR + # Failed Rust tests are printed from their evidence files into this step's log. + rust_test_failures() { + bash scripts/conformance/server-parity/rust-test-failures.sh + } + trap rust_test_failures ERR mkdir -p cargo-home rust-target parity-evidence # Compiling the library tests peaks near 2.5 GB; a 2 GB limit OOM-killed rustc (exit 101). docker run --rm --user="$(id -u):$(id -g)" --cpus=2 --memory=4g --memory-swap=4g \ From e6e349cafa881d59047d3ddc7943dca45326da71 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 11:21:31 +0000 Subject: [PATCH 26/37] fix: restore readiness probe fixture transaction ownership --- app/Support/ServerReadiness.php | 9 +++++-- config/database.php | 24 ++++++++++++------- .../Feature/DatabaseStorageExhaustedTest.php | 2 +- tests/Feature/HealthControllerTest.php | 16 +++++++++---- 4 files changed, 35 insertions(+), 16 deletions(-) diff --git a/app/Support/ServerReadiness.php b/app/Support/ServerReadiness.php index f11d9c7cc..9dbedf537 100644 --- a/app/Support/ServerReadiness.php +++ b/app/Support/ServerReadiness.php @@ -103,7 +103,7 @@ private function databaseCheck(bool $probeQuery = false): array $queryAttempted = true; $row = $connection->selectOne('SELECT ? AS readiness_probe', [1], useReadPdo: false); if ((string) ($row->readiness_probe ?? '') !== '1') { - throw new \RuntimeException('The database did not return the readiness probe value.'); + throw new \UnexpectedValueException('The database did not return the readiness probe value.'); } } @@ -116,10 +116,15 @@ private function databaseCheck(bool $probeQuery = false): array : 'The database connection is available. Write capacity has not been verified.', ]; } catch (\Throwable $exception) { + $reason = $queryAttempted ? 'database_query_failed' : null; + if (BackendUnavailable::is($exception)) { + $reason = 'backend_unavailable'; + } + return [ 'status' => 'unavailable', 'message' => $exception->getMessage(), - 'reason' => BackendUnavailable::is($exception) ? 'backend_unavailable' : ($queryAttempted ? 'database_query_failed' : null), + 'reason' => $reason, ]; } } diff --git a/config/database.php b/config/database.php index aebce8f60..fd1f198af 100644 --- a/config/database.php +++ b/config/database.php @@ -4,6 +4,18 @@ use Pdo\Mysql; use Pdo\Pgsql; +$mysqlOptions = []; +if (extension_loaded('pdo_mysql')) { + $sslCaAttribute = PHP_VERSION_ID >= 80500 ? Mysql::ATTR_SSL_CA : PDO::MYSQL_ATTR_SSL_CA; + $mysqlOptions = array_filter([$sslCaAttribute => env('MYSQL_ATTR_SSL_CA')]); +} + +$pgsqlOptions = []; +if (extension_loaded('pdo_pgsql')) { + $disablePreparesAttribute = PHP_VERSION_ID >= 80500 ? Pgsql::ATTR_DISABLE_PREPARES : PDO::PGSQL_ATTR_DISABLE_PREPARES; + $pgsqlOptions = [$disablePreparesAttribute => true]; +} + return [ /* @@ -62,9 +74,7 @@ 'prefix_indexes' => true, 'strict' => true, 'engine' => null, - 'options' => extension_loaded('pdo_mysql') ? array_filter([ - (PHP_VERSION_ID >= 80500 ? Mysql::ATTR_SSL_CA : PDO::MYSQL_ATTR_SSL_CA) => env('MYSQL_ATTR_SSL_CA'), - ]) : [], + 'options' => $mysqlOptions, ], 'mariadb' => [ @@ -82,9 +92,7 @@ 'prefix_indexes' => true, 'strict' => true, 'engine' => null, - 'options' => extension_loaded('pdo_mysql') ? array_filter([ - (PHP_VERSION_ID >= 80500 ? Mysql::ATTR_SSL_CA : PDO::MYSQL_ATTR_SSL_CA) => env('MYSQL_ATTR_SSL_CA'), - ]) : [], + 'options' => $mysqlOptions, ], 'pgsql' => [ @@ -102,9 +110,7 @@ 'sslmode' => 'prefer', // Keep native parameter binding without session-scoped statement // names, so transaction poolers can safely change backend sessions. - 'options' => extension_loaded('pdo_pgsql') ? [ - (PHP_VERSION_ID >= 80500 ? Pgsql::ATTR_DISABLE_PREPARES : PDO::PGSQL_ATTR_DISABLE_PREPARES) => true, - ] : [], + 'options' => $pgsqlOptions, ], 'sqlsrv' => [ diff --git a/tests/Feature/DatabaseStorageExhaustedTest.php b/tests/Feature/DatabaseStorageExhaustedTest.php index f17749ade..6b75c7bce 100644 --- a/tests/Feature/DatabaseStorageExhaustedTest.php +++ b/tests/Feature/DatabaseStorageExhaustedTest.php @@ -93,7 +93,7 @@ public function test_database_readiness_explains_that_write_capacity_is_not_veri $this->getJson('/api/ready') ->assertOk() ->assertJsonPath('checks.database.status', 'ok') - ->assertJsonPath('checks.database.check_scope', 'connection_only') + ->assertJsonPath('checks.database.check_scope', 'bound_read_query') ->assertJsonPath('checks.database.write_capacity_verified', false); } diff --git a/tests/Feature/HealthControllerTest.php b/tests/Feature/HealthControllerTest.php index d40ef79ff..ac6c13c62 100644 --- a/tests/Feature/HealthControllerTest.php +++ b/tests/Feature/HealthControllerTest.php @@ -28,6 +28,7 @@ class HealthControllerTest extends TestCase public function test_readiness_refuses_a_connected_database_that_cannot_execute_bound_reads(): void { + $originalDefault = config('database.default'); $pdo = DB::connection()->getPdo(); DB::extend('broken_bound_reads', static fn (array $config): SQLiteConnection => new class($pdo, ':memory:', '', $config) extends SQLiteConnection { @@ -45,10 +46,17 @@ public function select($query, $bindings = [], $useReadPdo = true, array $fetchU 'database.default' => 'broken_bound_reads', ]); - $this->getJson('/api/ready') - ->assertStatus(503) - ->assertJsonPath('checks.database.status', 'unavailable') - ->assertJsonPath('checks.database.reason', 'database_query_failed'); + try { + $this->getJson('/api/ready') + ->assertStatus(503) + ->assertJsonPath('checks.database.status', 'unavailable') + ->assertJsonPath('checks.database.reason', 'database_query_failed'); + } finally { + // RefreshDatabase must roll back its original connection, not + // this wrapper around the same PDO. In-memory suites reuse it. + config(['database.default' => $originalDefault]); + DB::purge('broken_bound_reads'); + } } public function test_health_check_returns_serving_when_database_is_available(): void From fe2b26e6f64e2b4b5ad7d4e14f6ee4b0a9dceaf2 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 11:45:56 +0000 Subject: [PATCH 27/37] Retry lock-pressure refusals inside the probes' recorders Guzzle wraps a handler stack in reverse, so HandlerStack::unshift put the retry outermost and each recorder logged the refused attempt as well as the retried reply. The patch deployment contract then rejected a published PHP activity-poll refusal it did not expect. The retry is now pushed after each recorder, which makes it innermost, so recorders see only the final reply, as the middleware intended. Co-Authored-By: Claude Opus 5.5 --- .../conformance/server-parity/cooperative-probe.php | 2 +- .../server-parity/lock-pressure-middleware.php | 10 ++++++---- scripts/conformance/server-parity/namespace-probe.php | 2 +- .../server-parity/patch-deployment-probe.php | 2 +- scripts/conformance/server-parity/probe.php | 2 +- 5 files changed, 10 insertions(+), 8 deletions(-) diff --git a/scripts/conformance/server-parity/cooperative-probe.php b/scripts/conformance/server-parity/cooperative-probe.php index b856b308c..3a52e3ee4 100644 --- a/scripts/conformance/server-parity/cooperative-probe.php +++ b/scripts/conformance/server-parity/cooperative-probe.php @@ -26,7 +26,6 @@ function cooperativeObservedTransport(array &$traffic): Psr18Transport { $stack = HandlerStack::create(); - $stack->unshift(LockPressurePollRetries::middleware(), 'lock_pressure_poll_retry'); $stack->push(static function (callable $handler) use (&$traffic): callable { return static function (RequestInterface $request, array $options) use ($handler, &$traffic) { return $handler($request, $options)->then(static function (ResponseInterface $response) use ($request, &$traffic): ResponseInterface { @@ -44,6 +43,7 @@ function cooperativeObservedTransport(array &$traffic): Psr18Transport }); }; }); + $stack->push(LockPressurePollRetries::middleware(), 'lock_pressure_poll_retry'); return new Psr18Transport(client: new GuzzleClient(['http_errors' => false, 'handler' => $stack])); } diff --git a/scripts/conformance/server-parity/lock-pressure-middleware.php b/scripts/conformance/server-parity/lock-pressure-middleware.php index 11cf1d5ef..1ced25c8d 100644 --- a/scripts/conformance/server-parity/lock-pressure-middleware.php +++ b/scripts/conformance/server-parity/lock-pressure-middleware.php @@ -19,9 +19,11 @@ * (see lockPressureOperationRetryDelay and lockPressurePollRetryDelay for the * exact conditions). This Guzzle middleware retries only that refusal, below the SDK transport, so * the transport keeps every capability the SDK checks (bounded requests, - * payload transfer). It stops within the request's own timeout. Push it - * innermost (HandlerStack::unshift) so recording middleware sees only the final - * reply and observations stay comparable across targets. Retries are reported + * payload transfer). It stops within the request's own timeout. Push it after + * any recording middleware (HandlerStack::push), which makes it innermost: + * Guzzle wraps the stack in reverse, so unshift would put it outermost and let + * recorders see each refused attempt. Recorders then see only the final reply + * and observations stay comparable across targets. Retries are reported * once on stderr when the process exits. */ final class LockPressurePollRetries @@ -80,7 +82,7 @@ public static function middleware(): callable public static function transport(): Psr18Transport { $stack = HandlerStack::create(); - $stack->unshift(self::middleware(), 'lock_pressure_poll_retry'); + $stack->push(self::middleware(), 'lock_pressure_poll_retry'); return new Psr18Transport(client: new GuzzleClient(['http_errors' => false, 'handler' => $stack])); } diff --git a/scripts/conformance/server-parity/namespace-probe.php b/scripts/conformance/server-parity/namespace-probe.php index 4f756671d..f6d1e0fa1 100644 --- a/scripts/conformance/server-parity/namespace-probe.php +++ b/scripts/conformance/server-parity/namespace-probe.php @@ -46,7 +46,6 @@ function namespaceHttpRun(Client $client, string $workflowId, string $runId): ar function namespaceTransport(array &$receipts, ?Closure $beforeCompletion = null): Psr18Transport { $stack = HandlerStack::create(); - $stack->unshift(LockPressurePollRetries::middleware(), 'lock_pressure_poll_retry'); $stack->push(static function (callable $handler) use (&$receipts, $beforeCompletion): callable { return static function (RequestInterface $request, array $options) use ($handler, &$receipts, $beforeCompletion) { $path = $request->getUri()->getPath(); @@ -68,6 +67,7 @@ function namespaceTransport(array &$receipts, ?Closure $beforeCompletion = null) }); }; }); + $stack->push(LockPressurePollRetries::middleware(), 'lock_pressure_poll_retry'); return new Psr18Transport(client: new NamespaceHttp(['http_errors' => false, 'handler' => $stack, 'connect_timeout' => 1, 'timeout' => 5])); diff --git a/scripts/conformance/server-parity/patch-deployment-probe.php b/scripts/conformance/server-parity/patch-deployment-probe.php index 204b073c4..030b34773 100644 --- a/scripts/conformance/server-parity/patch-deployment-probe.php +++ b/scripts/conformance/server-parity/patch-deployment-probe.php @@ -101,7 +101,6 @@ function patchHttpPhase(array $fixture, array $options): array $worker = $handle = null; $requests = $diagnostics = []; $stack = HandlerStack::create(); - $stack->unshift(LockPressurePollRetries::middleware(), 'lock_pressure_poll_retry'); $stack->push(static function (callable $handler) use (&$worker, &$requests, $phase, $fixture): callable { return static function (RequestInterface $request, array $transportOptions) use ($handler, &$worker, &$requests, $phase, $fixture) { return $handler($request, $transportOptions)->then(static function (ResponseInterface $response) use ($request, &$worker, &$requests, $phase, $fixture): ResponseInterface { @@ -128,6 +127,7 @@ function patchHttpPhase(array $fixture, array $options): array }); }; }); + $stack->push(LockPressurePollRetries::middleware(), 'lock_pressure_poll_retry'); $client = parityClient($options['url'], namespace: 'default', token: getenv('DW_PARITY_TOKEN') ?: null, transport: new Psr18Transport(client: new GuzzleClient(['http_errors' => false, 'handler' => $stack]))); if ($phase === 'replacement') { diff --git a/scripts/conformance/server-parity/probe.php b/scripts/conformance/server-parity/probe.php index 9c078e331..060d26745 100644 --- a/scripts/conformance/server-parity/probe.php +++ b/scripts/conformance/server-parity/probe.php @@ -71,7 +71,6 @@ function observedChildTransport(array &$polls, array &$completions, array &$acti // Observe real published transport I/O. No response or request is changed, // and authentication headers are never retained. $stack = HandlerStack::create(); - $stack->unshift(LockPressurePollRetries::middleware(), 'lock_pressure_poll_retry'); $stack->push(static function (callable $handler) use (&$polls, &$completions, &$activityPolls, &$activityOutcomes, $afterCompletion, &$scheduleReceipts): callable { return static function (RequestInterface $request, array $options) use ($handler, &$polls, &$completions, &$activityPolls, &$activityOutcomes, $afterCompletion, &$scheduleReceipts) { return $handler($request, $options)->then(static function (ResponseInterface $response) use ($request, &$polls, &$completions, &$activityPolls, &$activityOutcomes, $afterCompletion, &$scheduleReceipts): ResponseInterface { @@ -108,6 +107,7 @@ function observedChildTransport(array &$polls, array &$completions, array &$acti }); }; }); + $stack->push(LockPressurePollRetries::middleware(), 'lock_pressure_poll_retry'); return new Psr18Transport(client: new GuzzleClient(['http_errors' => false, 'handler' => $stack])); } From 4d8a28bd77f4e1a91e42c7ff804321afc7df4032 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 12:16:02 +0000 Subject: [PATCH 28/37] chore: prepare Server 2.9.8 pooler compatibility release --- composer.json | 2 +- composer.lock | 2 +- docker-compose.dedicated-matching.yml | 4 ++-- docker-compose.memo-rolling.yml | 4 ++-- docker-compose.published.yml | 4 ++-- docker-compose.small-cluster.yml | 2 +- docker-compose.yml | 8 ++++---- k8s/README.md | 10 +++++----- k8s/helm/durable-workflow/Chart.yaml | 8 ++++---- k8s/helm/durable-workflow/README.md | 2 +- .../durable-workflow/ci/existing-secrets-values.yaml | 2 +- .../durable-workflow/ci/ingress-and-hpa-values.yaml | 2 +- .../durable-workflow/ci/inline-secrets-values.yaml | 2 +- k8s/helm/durable-workflow/templates/_helpers.tpl | 2 +- k8s/helm/durable-workflow/values.yaml | 2 +- k8s/helm/examples/values-dev.yaml | 2 +- .../examples/values-external-secrets-operator.yaml | 2 +- .../examples/values-production-existing-secrets.yaml | 2 +- k8s/migration-job.yaml | 2 +- k8s/scheduler-cronjob.yaml | 2 +- k8s/secret.yaml | 2 +- k8s/server-deployment.yaml | 2 +- k8s/worker-deployment.yaml | 2 +- resources/release/source-release.json | 4 ++-- scripts/k8s-kind-smoke.sh | 2 +- 25 files changed, 39 insertions(+), 39 deletions(-) diff --git a/composer.json b/composer.json index 00e3ccfba..a31e9251c 100644 --- a/composer.json +++ b/composer.json @@ -48,7 +48,7 @@ }, "extra": { "durable-workflow": { - "product-train": "2.9.7" + "product-train": "2.9.8" }, "laravel": { "dont-discover": [] diff --git a/composer.lock b/composer.lock index 4f4553fcb..2e274b226 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "aa78b1f467750bfc7db005c71eb290e7", + "content-hash": "f3629921342639b58b34196e3ede3051", "packages": [ { "name": "apache/avro", diff --git a/docker-compose.dedicated-matching.yml b/docker-compose.dedicated-matching.yml index 071822046..099c8775d 100644 --- a/docker-compose.dedicated-matching.yml +++ b/docker-compose.dedicated-matching.yml @@ -32,13 +32,13 @@ name: durable-workflow-server # daemon reports `shape: dedicated`. # Generated by scripts/ci/sync-source-release.mjs. Do not edit the fallback. -x-server-image: &server-image ${DW_SERVER_IMAGE:-durableworkflow/server:${DW_SERVER_TAG:-2.9.7}} +x-server-image: &server-image ${DW_SERVER_IMAGE:-durableworkflow/server:${DW_SERVER_TAG:-2.9.8}} x-server-environment: &server-environment APP_NAME: "Durable Workflow Server" APP_ENV: ${APP_ENV:-local} DW_SERVER_KEY: ${DW_SERVER_KEY:-} - APP_VERSION: ${APP_VERSION:-${DW_SERVER_TAG:-2.9.7}} + APP_VERSION: ${APP_VERSION:-${DW_SERVER_TAG:-2.9.8}} APP_DEBUG: ${APP_DEBUG:-false} DB_CONNECTION: mysql DB_HOST: mysql diff --git a/docker-compose.memo-rolling.yml b/docker-compose.memo-rolling.yml index 0fc31e22c..6bb030f70 100644 --- a/docker-compose.memo-rolling.yml +++ b/docker-compose.memo-rolling.yml @@ -49,14 +49,14 @@ services: command: ["server-bootstrap"] environment: <<: *runtime-environment - APP_VERSION: ${APP_VERSION:-2.9.7} + APP_VERSION: ${APP_VERSION:-2.9.8} successor: image: ${DW_MEMO_SUCCESSOR_IMAGE:-durable-workflow/server-memo-rolling:local} ports: !override [] environment: <<: *runtime-environment - APP_VERSION: ${APP_VERSION:-2.9.7} + APP_VERSION: ${APP_VERSION:-2.9.8} DW_SERVER_ID: memo-successor DW_SERVER_TOPOLOGY_SHAPE: standalone_server DW_SERVER_PROCESS_CLASS: server_http_node diff --git a/docker-compose.published.yml b/docker-compose.published.yml index e6fed642f..e7acdad03 100644 --- a/docker-compose.published.yml +++ b/docker-compose.published.yml @@ -1,13 +1,13 @@ name: durable-workflow-server # Generated by scripts/ci/sync-source-release.mjs. Do not edit the fallback. -x-server-image: &server-image ${DW_SERVER_IMAGE:-durableworkflow/server:${DW_SERVER_TAG:-2.9.7}} +x-server-image: &server-image ${DW_SERVER_IMAGE:-durableworkflow/server:${DW_SERVER_TAG:-2.9.8}} x-server-environment: &server-environment APP_NAME: "Durable Workflow Server" APP_ENV: ${APP_ENV:-local} DW_SERVER_KEY: ${DW_SERVER_KEY:-} - APP_VERSION: ${APP_VERSION:-${DW_SERVER_TAG:-2.9.7}} + APP_VERSION: ${APP_VERSION:-${DW_SERVER_TAG:-2.9.8}} APP_DEBUG: ${APP_DEBUG:-false} LOG_CHANNEL: ${LOG_CHANNEL:-stderr} LOG_LEVEL: ${LOG_LEVEL:-info} diff --git a/docker-compose.small-cluster.yml b/docker-compose.small-cluster.yml index b5a994c31..3f3715b02 100644 --- a/docker-compose.small-cluster.yml +++ b/docker-compose.small-cluster.yml @@ -12,7 +12,7 @@ x-server-build: &server-build x-server-environment: &server-environment APP_NAME: "Durable Workflow Server" APP_ENV: testing - APP_VERSION: ${APP_VERSION:-2.9.7} + APP_VERSION: ${APP_VERSION:-2.9.8} APP_DEBUG: "false" DW_SERVER_KEY: ${DW_SERVER_KEY:-base64:5Zt4nUhlCm3DD0nLXZJQdHiwPfb56yGo9gNV/g3jYbY=} DB_CONNECTION: ${DW_SMALL_CLUSTER_DB:-mysql} diff --git a/docker-compose.yml b/docker-compose.yml index 5d587e3bb..6f18015e1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,7 +15,7 @@ services: DW_SERVER_KEY: "${DW_SERVER_KEY:-}" DW_SERVER_TOPOLOGY_SHAPE: standalone_server DW_SERVER_PROCESS_CLASS: server_http_node - APP_VERSION: "${APP_VERSION:-2.9.7}" + APP_VERSION: "${APP_VERSION:-2.9.8}" APP_DEBUG: "false" DB_CONNECTION: mysql DB_HOST: mysql @@ -62,7 +62,7 @@ services: APP_NAME: "Durable Workflow Server" APP_ENV: local DW_SERVER_KEY: "${DW_SERVER_KEY:-}" - APP_VERSION: "${APP_VERSION:-2.9.7}" + APP_VERSION: "${APP_VERSION:-2.9.8}" APP_DEBUG: "false" DB_CONNECTION: mysql DB_HOST: mysql @@ -124,7 +124,7 @@ services: DW_SERVER_KEY: "${DW_SERVER_KEY:-}" DW_SERVER_TOPOLOGY_SHAPE: standalone_server DW_SERVER_PROCESS_CLASS: worker_node - APP_VERSION: "${APP_VERSION:-2.9.7}" + APP_VERSION: "${APP_VERSION:-2.9.8}" DB_CONNECTION: mysql DB_HOST: mysql DB_PORT: 3306 @@ -178,7 +178,7 @@ services: DW_SERVER_KEY: "${DW_SERVER_KEY:-}" DW_SERVER_TOPOLOGY_SHAPE: standalone_server DW_SERVER_PROCESS_CLASS: scheduler_node - APP_VERSION: "${APP_VERSION:-2.9.7}" + APP_VERSION: "${APP_VERSION:-2.9.8}" DB_CONNECTION: mysql DB_HOST: mysql DB_PORT: 3306 diff --git a/k8s/README.md b/k8s/README.md index cf7c9a540..2375dfd6e 100644 --- a/k8s/README.md +++ b/k8s/README.md @@ -13,7 +13,7 @@ The checked-in manifests are synchronized with the repository's stable source release and pin its Docker Hub tag: ```text -durableworkflow/server:2.9.7 +durableworkflow/server:2.9.8 ``` Before production use, patch every workload image to the exact published tag or @@ -21,15 +21,15 @@ digest you intend to run: ```bash kubectl set image -n durable-workflow deploy/durable-workflow-server \ - server=durableworkflow/server:2.9.7 + server=durableworkflow/server:2.9.8 kubectl set image -n durable-workflow deploy/durable-workflow-worker \ - worker=durableworkflow/server:2.9.7 + worker=durableworkflow/server:2.9.8 kubectl set image -n durable-workflow cronjob/durable-workflow-scheduler \ - scheduler=durableworkflow/server:2.9.7 + scheduler=durableworkflow/server:2.9.8 ``` GitHub Container Registry publishes the same release line at -`ghcr.io/durable-workflow/server:2.9.7`. Digest pinning is preferred for strict +`ghcr.io/durable-workflow/server:2.9.8`. Digest pinning is preferred for strict change control. The manifests expect you to provide: diff --git a/k8s/helm/durable-workflow/Chart.yaml b/k8s/helm/durable-workflow/Chart.yaml index 3c1884699..5801c1bc4 100644 --- a/k8s/helm/durable-workflow/Chart.yaml +++ b/k8s/helm/durable-workflow/Chart.yaml @@ -5,11 +5,11 @@ type: application # The chart's own semver version. Bumped on every chart release; treated as # independent of the server image version (appVersion). Breaking-change rules # for this version live in docs/helm-upgrading.md alongside the chart. -version: 0.1.183 +version: 0.1.184 # The immutable Durable Workflow Server identity this chart release packages. # The onboarding default in values.yaml and appVersion are generated from the # checked-in source release record. -appVersion: "2.9.7" +appVersion: "2.9.8" kubeVersion: ">=1.27.0-0" home: https://durable-workflow.github.io/docs/2.0/deployment sources: @@ -30,10 +30,10 @@ annotations: # exact commit that most recently changed the packaged chart. org.opencontainers.image.source: https://github.com/durable-workflow/server dev.durable-workflow.source-revision: "unreleased" - dev.durable-workflow.image-reference: "docker.io/durableworkflow/server:2.9.7" + dev.durable-workflow.image-reference: "docker.io/durableworkflow/server:2.9.8" artifacthub.io/license: MIT artifacthub.io/category: integration-delivery # Free-form changelog for the current chart release shown by Artifact Hub. artifacthub.io/changes: | - kind: fixed - description: Preserve accepted signals at terminal workflow boundaries and exclude intentionally pruned run summaries from projection drift. + description: Use unnamed bound PostgreSQL statements with transaction poolers and verify a bound database read during readiness checks. diff --git a/k8s/helm/durable-workflow/README.md b/k8s/helm/durable-workflow/README.md index e286e2c20..69d84284d 100644 --- a/k8s/helm/durable-workflow/README.md +++ b/k8s/helm/durable-workflow/README.md @@ -63,7 +63,7 @@ helm install durable-workflow ./k8s/helm/durable-workflow \ ```yaml image: - tag: "2.9.7" + tag: "2.9.8" # Pin a digest in production: # digest: "sha256:abc123..." # memoPayloadStorage: "raw-json-v1" # Required for a digest or custom image. diff --git a/k8s/helm/durable-workflow/ci/existing-secrets-values.yaml b/k8s/helm/durable-workflow/ci/existing-secrets-values.yaml index 2aa2fb07c..03daee1a3 100644 --- a/k8s/helm/durable-workflow/ci/existing-secrets-values.yaml +++ b/k8s/helm/durable-workflow/ci/existing-secrets-values.yaml @@ -1,7 +1,7 @@ # CI fixture: GitOps / externally-managed-secret path. The chart consumes # existing Secrets and renders no Secret resources of its own. image: - tag: "2.9.7" + tag: "2.9.8" externalDatabase: connection: pgsql diff --git a/k8s/helm/durable-workflow/ci/ingress-and-hpa-values.yaml b/k8s/helm/durable-workflow/ci/ingress-and-hpa-values.yaml index bfc1fa4a5..298204fb9 100644 --- a/k8s/helm/durable-workflow/ci/ingress-and-hpa-values.yaml +++ b/k8s/helm/durable-workflow/ci/ingress-and-hpa-values.yaml @@ -1,6 +1,6 @@ # CI fixture: ingress + autoscaling enabled. Exercises optional templates. image: - tag: "2.9.7" + tag: "2.9.8" externalDatabase: connection: mysql diff --git a/k8s/helm/durable-workflow/ci/inline-secrets-values.yaml b/k8s/helm/durable-workflow/ci/inline-secrets-values.yaml index ab0467cf2..2e88b700b 100644 --- a/k8s/helm/durable-workflow/ci/inline-secrets-values.yaml +++ b/k8s/helm/durable-workflow/ci/inline-secrets-values.yaml @@ -2,7 +2,7 @@ # chart's render path is exercised end-to-end. Real deployments should use # existingSecret instead. image: - tag: "2.9.7" + tag: "2.9.8" externalDatabase: connection: mysql diff --git a/k8s/helm/durable-workflow/templates/_helpers.tpl b/k8s/helm/durable-workflow/templates/_helpers.tpl index d09b528d2..ef7394b29 100644 --- a/k8s/helm/durable-workflow/templates/_helpers.tpl +++ b/k8s/helm/durable-workflow/templates/_helpers.tpl @@ -88,7 +88,7 @@ resolved by an explicit capability declaration or an existing workload marker. {{- define "durable-workflow.memoPayloadStorageForImage" -}} {{- $image := toString . -}} {{- $normalized := regexReplaceAll "^index\\.docker\\.io/" $image "docker.io/" -}} -{{- if eq $normalized "docker.io/durableworkflow/server:2.9.7" -}} +{{- if eq $normalized "docker.io/durableworkflow/server:2.9.8" -}} dual-v1 {{- else if regexMatch "^docker\\.io/durableworkflow/server:2\\.0\\.0-rc\\.[0-9]+$" $normalized -}} {{- $releaseCandidate := atoi (regexFind "[0-9]+$" $normalized) -}} diff --git a/k8s/helm/durable-workflow/values.yaml b/k8s/helm/durable-workflow/values.yaml index 33e0ae404..e4f51aa9a 100644 --- a/k8s/helm/durable-workflow/values.yaml +++ b/k8s/helm/durable-workflow/values.yaml @@ -21,7 +21,7 @@ image: registry: docker.io repository: durableworkflow/server # Generated by scripts/ci/sync-source-release.mjs. Do not edit this default. - tag: "2.9.7" + tag: "2.9.8" # Optional digest pin. When set, takes precedence over tag for change control. # Example: "sha256:abc123..." digest: "" diff --git a/k8s/helm/examples/values-dev.yaml b/k8s/helm/examples/values-dev.yaml index 3a413fe4a..b20fdfdf3 100644 --- a/k8s/helm/examples/values-dev.yaml +++ b/k8s/helm/examples/values-dev.yaml @@ -3,7 +3,7 @@ # shape in production. image: - tag: "2.9.7" + tag: "2.9.8" externalDatabase: connection: mysql diff --git a/k8s/helm/examples/values-external-secrets-operator.yaml b/k8s/helm/examples/values-external-secrets-operator.yaml index 881442606..b251debb7 100644 --- a/k8s/helm/examples/values-external-secrets-operator.yaml +++ b/k8s/helm/examples/values-external-secrets-operator.yaml @@ -5,7 +5,7 @@ # concern. image: - tag: "2.9.7" + tag: "2.9.8" externalDatabase: connection: pgsql diff --git a/k8s/helm/examples/values-production-existing-secrets.yaml b/k8s/helm/examples/values-production-existing-secrets.yaml index 248b16bb3..0f3cb833d 100644 --- a/k8s/helm/examples/values-production-existing-secrets.yaml +++ b/k8s/helm/examples/values-production-existing-secrets.yaml @@ -10,7 +10,7 @@ image: repository: durable-workflow/server # Pin a digest in production for change-control auditability. digest: "" # e.g. "sha256:abc123..." - tag: "2.9.7" + tag: "2.9.8" externalDatabase: connection: pgsql diff --git a/k8s/migration-job.yaml b/k8s/migration-job.yaml index 6d6e2ed71..e5e21c004 100644 --- a/k8s/migration-job.yaml +++ b/k8s/migration-job.yaml @@ -13,7 +13,7 @@ spec: restartPolicy: OnFailure containers: - name: migrate - image: durableworkflow/server:2.9.7 + image: durableworkflow/server:2.9.8 command: ["server-entrypoint"] args: ["server-bootstrap"] envFrom: diff --git a/k8s/scheduler-cronjob.yaml b/k8s/scheduler-cronjob.yaml index fd8f44b8c..18aaba0dd 100644 --- a/k8s/scheduler-cronjob.yaml +++ b/k8s/scheduler-cronjob.yaml @@ -24,7 +24,7 @@ spec: restartPolicy: Never containers: - name: scheduler - image: durableworkflow/server:2.9.7 + image: durableworkflow/server:2.9.8 command: ["server-entrypoint"] args: - sh diff --git a/k8s/secret.yaml b/k8s/secret.yaml index 303e2bf60..c0ac4bfeb 100644 --- a/k8s/secret.yaml +++ b/k8s/secret.yaml @@ -12,7 +12,7 @@ metadata: app.kubernetes.io/name: durable-workflow data: APP_NAME: "Durable Workflow Server" - APP_VERSION: "2.9.7" + APP_VERSION: "2.9.8" APP_ENV: production APP_DEBUG: "false" DB_CONNECTION: mysql diff --git a/k8s/server-deployment.yaml b/k8s/server-deployment.yaml index 5b7a8b68f..e486ec44a 100644 --- a/k8s/server-deployment.yaml +++ b/k8s/server-deployment.yaml @@ -23,7 +23,7 @@ spec: spec: containers: - name: server - image: durableworkflow/server:2.9.7 + image: durableworkflow/server:2.9.8 ports: - containerPort: 8080 name: http diff --git a/k8s/worker-deployment.yaml b/k8s/worker-deployment.yaml index 0e933e8b5..2137c6c99 100644 --- a/k8s/worker-deployment.yaml +++ b/k8s/worker-deployment.yaml @@ -19,7 +19,7 @@ spec: spec: containers: - name: worker - image: durableworkflow/server:2.9.7 + image: durableworkflow/server:2.9.8 command: ["server-entrypoint"] args: ["php", "artisan", "queue:work", "--sleep=1", "--tries=3", "--max-time=3600"] envFrom: diff --git a/resources/release/source-release.json b/resources/release/source-release.json index 6397bb46f..ffb24ceeb 100644 --- a/resources/release/source-release.json +++ b/resources/release/source-release.json @@ -1,9 +1,9 @@ { "schema": "durable-workflow.server.source-release/v1", "server": { - "version": "2.9.7" + "version": "2.9.8" }, "helm_chart": { - "version": "0.1.183" + "version": "0.1.184" } } diff --git a/scripts/k8s-kind-smoke.sh b/scripts/k8s-kind-smoke.sh index b936d210e..9ce085d87 100755 --- a/scripts/k8s-kind-smoke.sh +++ b/scripts/k8s-kind-smoke.sh @@ -7,7 +7,7 @@ cluster="${K8S_SMOKE_CLUSTER:-durable-workflow-server-smoke}" image="${K8S_SMOKE_IMAGE:-durableworkflow/server:k8s-smoke}" # Generated by scripts/ci/sync-source-release.mjs so the smoke replaces the # same default shipped by the public manifests. -manifest_image="durableworkflow/server:2.9.7" +manifest_image="durableworkflow/server:2.9.8" kind_node_image="${K8S_SMOKE_KIND_NODE_IMAGE:-kindest/node:v1.29.4}" artifact_dir="${K8S_SMOKE_ARTIFACT_DIR:-/tmp/durable-workflow-k8s-kind-smoke-artifacts}" rendered_dir="${artifact_dir}/rendered-manifests" From 924e45f4280f7287712df4caf9faa350a5b0e2a6 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 12:42:19 +0000 Subject: [PATCH 29/37] Describe the MySQL journal and the SQLite WAL preparation as built MySQL and MariaDB use native's bootstrap lock and a session check rather than LOCK TABLES, which refuses to create or drop tables it does not hold. Their journal is a rust-takeover row in native's marker table. A SQLite source in PHP's default WAL mode is first switched to the rollback journal with upgrade-prepare-sqlite. Co-Authored-By: Claude Opus 5.5 --- docs/rust-server-upgrade-takeover.md | 32 +++++++++++++++++++--------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/docs/rust-server-upgrade-takeover.md b/docs/rust-server-upgrade-takeover.md index 0aa1f8211..dfa5d7003 100644 --- a/docs/rust-server-upgrade-takeover.md +++ b/docs/rust-server-upgrade-takeover.md @@ -67,7 +67,7 @@ with any other writer. | --- | --- | --- | | SQLite | `BEGIN EXCLUSIVE` on the database file | Fingerprint, marker, native relations and conversions commit in one transaction. | | PostgreSQL | Native's bootstrap advisory key, then `LOCK TABLE … IN ACCESS EXCLUSIVE MODE` on every table, with a short lock wait. The later DDL needs that mode anyway, so no lock is upgraded while another session waits. | DDL is transactional, so everything commits in one transaction. | -| MySQL and MariaDB | To be qualified: `LOCK TABLES … WRITE` on every relation, or a check that no other session uses the database, repeated after the marker is written | DDL commits implicitly, so the marker records a step journal (below). | +| MySQL and MariaDB | Native's bootstrap lock and a check that no other session uses the database (which needs the `PROCESS` privilege), repeated after the marker is written. `LOCK TABLES` cannot be used: under it MySQL refuses to create or drop any table it does not hold. | DDL commits implicitly, so the marker records a step journal (below). | On SQLite and PostgreSQL, a PHP connection that slipped past the stop blocks on these locks, and a write it committed earlier changes the fingerprint. Neither @@ -81,13 +81,27 @@ The operator reruns `upgrade-takeover` with the same fingerprint. A killed SQLite takeover leaves a rollback journal; read-only inspection refuses it, and the rerun lets SQLite roll it back before the fingerprint check. -MySQL and MariaDB commit DDL implicitly, so there the marker is written first, -with state `takeover`, the fingerprint and the step list. Native startup -refuses state `takeover`. Each step is idempotent and records its completion -in the marker. Rerunning `upgrade-takeover` with the same fingerprint resumes -after the last recorded step; a different fingerprint is refused. The final -step sets state `ready` and the native schema version. The PHP fence refuses -the database from the first step, because the marker exists. +MySQL and MariaDB commit DDL implicitly, so there native's marker table is +written first, with one row `('rust-takeover:', , +)`. Native startup refuses it with +`native_takeover_incomplete`. The sessions and the fingerprint are checked +again with the marker set aside, then each step runs and records its +completion in that row; row changes commit with their step, and DDL steps +check what is left to do. Rerunning `upgrade-takeover` with the same +fingerprint resumes after the last recorded step; a different fingerprint is +refused. The ledger and native's final marker `('rust-development', 4, +)` commit together. The PHP fence refuses the database from the +first step, because the marker table exists. + +## SQLite sources in WAL mode + +PHP Server keeps SQLite in WAL mode, and a read-only SQLite open can still +create WAL and shared-memory sidecars, so inspection, fingerprint and takeover +require the rollback journal. With PHP stopped, `upgrade-prepare-sqlite` opens +the file as its only connection and switches it to the rollback journal. +SQLite first checkpoints every committed WAL frame into the file, so the stored +rows are unchanged. The command refuses while another process holds the file, +and runs before the fingerprint and the backup. ## Steps @@ -142,8 +156,6 @@ acknowledged signal and an expired lease. The tests check: ## Open questions -- Whether MySQL and MariaDB can create the native relations while - `LOCK TABLES` holds every existing one, or need the session check instead. - Whether the declared column types print identically after a restore into another minor server version. Until that is shown, the restore check uses the same server version as the source. From a25fa493542f2441e0f96fc6ffdd45d379394fd0 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 13:33:36 +0000 Subject: [PATCH 30/37] Keep the activity deadline probe's own poll retry accounting This probe retries lock-pressure poll refusals itself and records each retry, while a recording proxy outside the client logs every refused poll. Retrying below the SDK hid those retries from the probe, so the contract found a refused poll without its recorded retry. The probe keeps its plain client. Co-Authored-By: Claude Opus 5.5 --- scripts/conformance/server-parity/activity-deadline-probe.php | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/conformance/server-parity/activity-deadline-probe.php b/scripts/conformance/server-parity/activity-deadline-probe.php index 4728f6264..b4bd61dfe 100644 --- a/scripts/conformance/server-parity/activity-deadline-probe.php +++ b/scripts/conformance/server-parity/activity-deadline-probe.php @@ -3,7 +3,6 @@ declare(strict_types=1); use DurableWorkflow\Client; -require_once __DIR__.'/lock-pressure-middleware.php'; require_once __DIR__.'/namespace-probe.php'; require_once __DIR__.'/activity-deadline-poll.php'; @@ -35,7 +34,7 @@ function activityDeadlineHttpObservation(array $fixture, array $options): array if (!preg_match('#^http://127\.0\.0\.1:[0-9]+$#', $url)) { throw new RuntimeException('Activity deadline recorder did not bind its local port'); } - $client = parityClient($url, namespace:'default', token:getenv('DW_PARITY_TOKEN') ?: null); + $client = new Client($url, namespace:'default', token:getenv('DW_PARITY_TOKEN') ?: null); $checkBudget = static function () use ($deadline): void { if (microtime(true) >= $deadline) { throw new RuntimeException('Original activity deadline observation expired'); From fa7695fd38810f7329fa6b94a907c6deb4ce6635 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 13:45:02 +0000 Subject: [PATCH 31/37] Resume a PHP-accepted signal after takeover and refuse unsupported features The upgrade source author now also writes a third original: a workflow that opens a signal wait, then receives its declared signal under published PHP, whose resume task stays unclaimed. After the takeover, native must deliver that signal from PHP's rows: the resumed task keeps the original history and the signal's PHP-written arguments, and the workflow completes. The resume accepts the three originals in any order. The payload inspection now also checks the codec-tagged payloads of workflow commands, signal records and updates. Takeover refuses a source holding rows of a published PHP feature native does not implement yet (runtime credentials, message streams, service definitions, update validation tasks, worker sessions, build ID rollouts and search attribute definitions), rather than leaving them unread. Co-Authored-By: Claude Opus 5.5 --- rust/src/runtime/upgrade/payload_scan.rs | 11 +++-- rust/src/runtime/upgrade/takeover.rs | 40 ++++++++++++++++++- rust/src/runtime/upgrade_tests.rs | 15 ++++++- .../server-parity/upgrade-source-author.php | 36 ++++++++++++----- .../server-parity/upgrade-source-profile.sh | 14 +++---- .../upgrade-source-sqlite-representations.php | 6 +-- .../server-parity/upgrade-takeover-resume.php | 26 +++++++++--- 7 files changed, 118 insertions(+), 30 deletions(-) diff --git a/rust/src/runtime/upgrade/payload_scan.rs b/rust/src/runtime/upgrade/payload_scan.rs index 555eea3b2..412c2b91e 100644 --- a/rust/src/runtime/upgrade/payload_scan.rs +++ b/rust/src/runtime/upgrade/payload_scan.rs @@ -72,11 +72,15 @@ macro_rules! column { }; } -const COLUMNS: [Column; 4] = [ +const COLUMNS: [Column; 8] = [ column!("workflow_runs", "arguments", "payload_codec"), column!("workflow_runs", "output", "output_payload_codec"), column!("activity_executions", "arguments", "payload_codec"), column!("activity_executions", "result", "payload_codec"), + column!("workflow_commands", "payload", "payload_codec"), + column!("workflow_signal_records", "arguments", "payload_codec"), + column!("workflow_updates", "arguments", "payload_codec"), + column!("workflow_updates", "result", "payload_codec"), ]; #[derive(Default, Serialize)] @@ -218,9 +222,10 @@ where "references_metadata_supported":references_supported, "diagnostics":diagnostics, "diagnostics_omitted":omitted, "inline_values_verified":inline_supported, "nested_payloads_verified":false, - // Published migrations also give these tables codec-tagged payloads. + // These tables also hold codec-tagged payloads; takeover refuses a + // source with any of their rows, since native lacks the features. "other_root_payload_columns_verified":false, - "unscanned_payload_codec_tables":["workflow_commands","workflow_signal_records","workflow_updates"], + "unscanned_payload_codec_tables":["workflow_durable_stream_items","workflow_inbound_stream_items","workflow_update_validation_tasks"], "inline_validation_limits":{"encoded_frame_bytes":65536,"traversal_depth":64, "traversal_visits":16384,"scalar_allocation_bytes":apache_avro::util::max_allocation_bytes(apache_avro::util::DEFAULT_MAX_ALLOCATION_BYTES)}, "provider_bytes_verified":false, "registry_binding_verified":false, diff --git a/rust/src/runtime/upgrade/takeover.rs b/rust/src/runtime/upgrade/takeover.rs index 2eb10c25d..0d9188b83 100644 --- a/rust/src/runtime/upgrade/takeover.rs +++ b/rust/src/runtime/upgrade/takeover.rs @@ -218,10 +218,46 @@ fn report(backend: &str, layout: &str, digest: &Value, inventory: &Value, conver }) } +/// Published PHP features native does not implement yet. A source holding any +/// of their rows is refused, rather than taken over without them. Derived +/// projections and PHP's own queue and poll bookkeeping are left in place. +const UNSUPPORTED_FEATURES: [(&str, &[&str]); 7] = [ + ("upgrade_source_runtime_credentials_unsupported", &["runtime_credentials"]), + ( + "upgrade_source_message_streams_unsupported", + &[ + "workflow_messages", + "workflow_durable_streams", + "workflow_durable_stream_items", + "workflow_inbound_streams", + "workflow_inbound_stream_items", + ], + ), + ( + "upgrade_source_services_unsupported", + &["workflow_services", "workflow_service_endpoints", "workflow_service_operations"], + ), + ("upgrade_source_update_validation_unsupported", &["workflow_update_validation_tasks"]), + ("upgrade_source_worker_sessions_unsupported", &["workflow_worker_sessions"]), + ("upgrade_source_build_id_rollouts_unsupported", &["workflow_worker_build_id_rollouts"]), + ( + "upgrade_source_search_attribute_definitions_unsupported", + &["search_attribute_definitions"], + ), +]; + /// Takeover needs every inspection result it relies on, and refuses what -/// native cannot represent yet: external payloads and service calls. -/// Returns the qualified source layout. +/// native cannot represent yet: external payloads, service calls and the +/// other unsupported features above. Returns the qualified source layout. fn qualified(inventory: &Value) -> Result<&str> { + for (reason, tables) in UNSUPPORTED_FEATURES { + if tables + .iter() + .any(|table| inventory["table_rows"][*table].as_i64().unwrap_or(0) != 0) + { + return Err(refuse(StatusCode::CONFLICT, reason)); + } + } if inventory["queue_inspection"]["status"] != "known_subset_bound" { return Err(refuse( StatusCode::CONFLICT, diff --git a/rust/src/runtime/upgrade_tests.rs b/rust/src/runtime/upgrade_tests.rs index 7dee18909..35e563a5a 100644 --- a/rust/src/runtime/upgrade_tests.rs +++ b/rust/src/runtime/upgrade_tests.rs @@ -406,7 +406,11 @@ async fn payload_scan_validates_inline_pages_and_reports_late_bad_values_without assert_eq!(report["payload_inspection"]["other_root_payload_columns_verified"], false); assert_eq!( report["payload_inspection"]["unscanned_payload_codec_tables"], - json!(["workflow_commands", "workflow_signal_records", "workflow_updates"]) + json!([ + "workflow_durable_stream_items", + "workflow_inbound_stream_items", + "workflow_update_validation_tasks" + ]) ); assert_eq!(report["payload_inspection"]["diagnostics"], json!([])); for (table, column, count) in [ @@ -1042,6 +1046,15 @@ async fn takeover_refusals_leave_the_php_database_unchanged() { "INSERT INTO runtime_external_payloads(id,namespace,storage_uri,storage_uri_sha256,codec,sha256,size_bytes) VALUES ('p','default','s3://b/k','x','avro','y',1)", "upgrade_source_external_payloads_unsupported", ), + // Features native lacks: their rows would be left behind unread. + ( + "INSERT INTO runtime_credentials(id,subject,roles,tenant,token_prefix,token_hash) VALUES ('c','worker','[]','default','dw_','h')", + "upgrade_source_runtime_credentials_unsupported", + ), + ( + "INSERT INTO workflow_messages(workflow_instance_id,direction,channel,stream_key,sequence) VALUES ('w','inbound','c','k',1)", + "upgrade_source_message_streams_unsupported", + ), ] { let (_directory, path) = php297_layout().await; mutate(&path, statement).await; diff --git a/scripts/conformance/server-parity/upgrade-source-author.php b/scripts/conformance/server-parity/upgrade-source-author.php index cae370f95..c16237701 100644 --- a/scripts/conformance/server-parity/upgrade-source-author.php +++ b/scripts/conformance/server-parity/upgrade-source-author.php @@ -42,7 +42,12 @@ $worker = $prefix.'-worker'; $expect($request('POST', '/api/worker/register', ['worker_id' => $worker, 'task_queue' => $prefix, 'runtime' => 'php', 'sdk_version' => 'raw-upgrade-inspection-author', - 'supported_workflow_types' => ['parity.upgrade_activity', 'parity.upgrade_timer'], + 'supported_workflow_types' => ['parity.upgrade_activity', 'parity.upgrade_timer', 'parity.upgrade_signal'], + // The signal original declares its one signal, as an SDK worker would. + 'workflow_command_contracts' => ['parity.upgrade_signal' => ['signals' => ['go'], 'signal_contracts' => [[ + 'name' => 'go', 'parameters' => [['name' => 'value', 'position' => 0, 'type' => 'array', 'required' => true, + 'variadic' => false, 'allows_null' => false, 'default_available' => false]], + ]]]], 'supported_activity_types' => ['parity.upgrade_activity'], 'capabilities' => [], 'capability_manifest' => array_fill_keys(['local_activities', 'worker_sessions', 'sticky_execution'], ['supported' => false, 'minimum_protocol_version' => '1.18', 'reason' => 'raw_protocol_author']), @@ -54,7 +59,7 @@ $timerSeconds = (int) (getenv('DW_PARITY_UPGRADE_TIMER_SECONDS') ?: 600); $check($timerSeconds >= 30 && $timerSeconds <= 600, 'Unsupported original timer delay'); $originals = []; -foreach (['activity', 'timer'] as $kind) { +foreach (['activity', 'timer', 'signal'] as $kind) { $id = $prefix.'-'.$kind; $run = $expect($request('POST', '/api/workflows', ['workflow_id' => $id, 'workflow_type' => 'parity.upgrade_'.$kind, 'task_queue' => $prefix, 'input' => $envelope, @@ -63,11 +68,13 @@ 'started_at' => $started, 'original_deadline' => $deadline], JSON_THROW_ON_ERROR)."\n"); $task = $expect($request('POST', '/api/worker/workflow-tasks/poll', $poll), 200, $kind.' claim')['task']; $check($task['run_id'] === $run['run_id'], 'Original workflow claim changed'); - $command = $kind === 'activity' - ? ['type' => 'schedule_activity', 'activity_type' => 'parity.upgrade_activity', 'arguments' => $envelope, + $command = match ($kind) { + 'activity' => ['type' => 'schedule_activity', 'activity_type' => 'parity.upgrade_activity', 'arguments' => $envelope, 'retry_policy' => ['max_attempts' => 1, 'backoff_seconds' => []], - 'start_to_close_timeout' => 600, 'schedule_to_close_timeout' => 900] - : ['type' => 'start_timer', 'delay_seconds' => $timerSeconds]; + 'start_to_close_timeout' => 600, 'schedule_to_close_timeout' => 900], + 'timer' => ['type' => 'start_timer', 'delay_seconds' => $timerSeconds], + 'signal' => ['type' => 'open_signal_wait', 'signal_name' => 'go'], + }; $expect($request('POST', '/api/worker/workflow-tasks/'.$task['task_id'].'/complete', [ 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], 'commands' => [$command], @@ -77,10 +84,21 @@ $activity = $expect($request('POST', '/api/worker/activity-tasks/poll', $poll), 200, 'activity claim')['task']; $check($activity['run_id'] === $run['run_id'] && $activity['attempt_number'] === 1, 'Original activity changed'); } + if ($kind === 'signal') { + // PHP accepts the signal and makes its resume task, which stays + // unclaimed until native resumes it after the takeover. + $signalled = $request('POST', '/api/workflows/'.$id.'/signal/go', ['input' => $envelope]); + $check(in_array($signalled[0], [200, 202], true), 'signal: '.json_encode(['status' => $signalled[0], + 'reason' => $signalled[1]['reason'] ?? null, 'errors' => $signalled[1]['errors'] ?? null])); + } $history = $expect($request('GET', '/api/workflows/'.$id.'/runs/'.$run['run_id'].'/history'), 200, 'history')['events']; - $check(array_column($history, 'event_type') === ($kind === 'activity' - ? ['StartAccepted', 'WorkflowStarted', 'ActivityScheduled', 'ActivityStarted'] - : ['StartAccepted', 'WorkflowStarted', 'TimerScheduled']), 'Original pending history differs'); + fwrite(STDERR, json_encode(['phase' => 'history', 'kind' => $kind, 'events' => array_column($history, 'event_type')], + JSON_THROW_ON_ERROR)."\n"); + $check(array_column($history, 'event_type') === match ($kind) { + 'activity' => ['StartAccepted', 'WorkflowStarted', 'ActivityScheduled', 'ActivityStarted'], + 'timer' => ['StartAccepted', 'WorkflowStarted', 'TimerScheduled'], + 'signal' => ['StartAccepted', 'WorkflowStarted', 'SignalWaitOpened', 'SignalReceived'], + }, 'Original pending history differs'); $originals[$kind] = ['workflow_id' => $id, 'run_id' => $run['run_id'], 'workflow_task' => $task, 'activity_task' => $activity, 'history' => $history]; } diff --git a/scripts/conformance/server-parity/upgrade-source-profile.sh b/scripts/conformance/server-parity/upgrade-source-profile.sh index 3613cf5bd..e7a504c98 100755 --- a/scripts/conformance/server-parity/upgrade-source-profile.sh +++ b/scripts/conformance/server-parity/upgrade-source-profile.sh @@ -101,8 +101,8 @@ if test "$backend" = sqlite; then /source/scripts/conformance/server-parity/upgrade-source-sqlite-representations.php upgrade-source \ > "$evidence/representations-comparison.json" jq -e '.outcome=="pass" and .source_mutated==false and .php_objects_executed==false - and (.runs|length)==2 and (.tasks|length)==4 and (.activities|length)==1 - and (.history|length)==7 and (.queue_jobs|length)==1 + and (.runs|length)==3 and (.tasks|length)==6 and (.activities|length)==1 + and (.history|length)==11 and (.queue_jobs|length)==1 and .queue_jobs[0].canonical_timer_task_match==true and all(.runs[]; .typed_arguments_match==true) and .backup_verified==false and .stored_representation_compatibility_verified==false @@ -134,7 +134,7 @@ snapshot() { } snapshot > "$evidence/before.json" inspect_source > "$evidence/inspection.json" 2> "$evidence/inspection.stderr" -jq -e --arg layout "$layout" '.status=="inspected" and .source_layout==$layout and (.table_rows|length)==50 and .table_rows.workflow_runs==2 +jq -e --arg layout "$layout" '.status=="inspected" and .source_layout==$layout and (.table_rows|length)==50 and .table_rows.workflow_runs==3 and .table_rows.activity_executions==1 and .catalog_visibility=="configured_role" and .source_mutated==false and .takeover_allowed==false and .backup_verified==false and .php_quiescence_verified==false and .stored_representation_compatibility_verified==false' "$evidence/inspection.json" >/dev/null @@ -145,10 +145,10 @@ jq -e '.status=="inspected" and .queue_inspection.status=="known_subset_bound" and .source_mutated==false and .takeover_allowed==false and .backup_verified==false and .php_quiescence_verified==false and .stored_representation_compatibility_verified==false' "$evidence/queue-inspection.json" >/dev/null inspect_source upgrade-payload-inspect > "$evidence/payload-inspection.json" 2> "$evidence/payload-inspection.stderr" -jq -e '.status=="inspected" and (.payload_inspection.columns|length)==4 +jq -e '.status=="inspected" and (.payload_inspection.columns|length)==8 and .payload_inspection.references_metadata_supported==true and ([.payload_inspection.columns[].counts.php_v1_references]|add)==0 - and ([.payload_inspection.columns[].counts.avro_inline_values]|add)==3 + and ([.payload_inspection.columns[].counts.avro_inline_values]|add)==9 and ([.payload_inspection.columns[].counts.other_unvalidated]|add)==0 and .payload_inspection.inline_values_verified==true and .payload_inspection.inline_validation_limits=={encoded_frame_bytes:65536,traversal_depth:64,traversal_visits:16384,scalar_allocation_bytes:65536} @@ -166,7 +166,7 @@ fingerprint() { } fingerprint "$database" > "$evidence/fingerprint.json" 2> "$evidence/fingerprint.stderr" jq -e --arg layout "$layout" '.status=="fingerprinted" and .source_layout==$layout and (.fingerprint|test("^sha256:[0-9a-f]{64}$")) - and (.table_rows|length)==50 and .table_rows.workflow_runs==2 and .table_rows.jobs==1 + and (.table_rows|length)==50 and .table_rows.workflow_runs==3 and .table_rows.jobs==1 and .source_mutated==false and .takeover_allowed==false and .backup_verified==false and .php_quiescence_verified==false and .stored_representation_compatibility_verified==false' "$evidence/fingerprint.json" >/dev/null source_fingerprint=$(jq -r .fingerprint "$evidence/fingerprint.json") @@ -285,7 +285,7 @@ if (( ${#target[@]} )); then "/source/$evidence/originals.json" 150 > "$evidence/resume.json" 2> "$evidence/resume.stderr" jq -e '.outcome=="pass" and .activity.stale_attempt_status==409 and .activity.events[-1]=="WorkflowCompleted" and .timer.fired_after_seconds>=0 - and .timer.events[-1]=="WorkflowCompleted"' "$evidence/resume.json" >/dev/null + and .timer.events[-1]=="WorkflowCompleted" and .signal.events[-1]=="WorkflowCompleted"' "$evidence/resume.json" >/dev/null docker stop --signal SIGTERM --time 10 "$resource-native" >/dev/null if docker run --rm --name "$resource-control" --user=1000:1000 --cpus=1 --memory=512m --memory-swap=512m \ --network "$target_network" "${target[@]}" -e DW_AUTH_TOKEN=upgrade-fixture-token \ diff --git a/scripts/conformance/server-parity/upgrade-source-sqlite-representations.php b/scripts/conformance/server-parity/upgrade-source-sqlite-representations.php index da17fe171..b36f69b1d 100644 --- a/scripts/conformance/server-parity/upgrade-source-sqlite-representations.php +++ b/scripts/conformance/server-parity/upgrade-source-sqlite-representations.php @@ -20,11 +20,11 @@ if ($pdo->query('PRAGMA journal_mode')->fetchColumn() !== 'delete') { throw new RuntimeException('Audit requires the already stopped DELETE-journal fixture'); } -$runs = $pdo->query('SELECT * FROM workflow_runs ORDER BY id LIMIT 3')->fetchAll(PDO::FETCH_ASSOC); +$runs = $pdo->query('SELECT * FROM workflow_runs ORDER BY id LIMIT 4')->fetchAll(PDO::FETCH_ASSOC); $workflowIds = array_column($runs, 'workflow_instance_id'); sort($workflowIds); -if ($workflowIds !== [$prefix.'-activity', $prefix.'-timer']) { - throw new RuntimeException('This reader accepts only the two isolated source-author originals'); +if ($workflowIds !== [$prefix.'-activity', $prefix.'-signal', $prefix.'-timer']) { + throw new RuntimeException('This reader accepts only the three isolated source-author originals'); } $report = ['scope' => 'published_php_stored_representation_observation', 'runs' => [], 'table_rows' => [], 'tasks' => [], 'activities' => [], 'history' => [], 'queue_jobs' => [], 'cache_files' => []]; diff --git a/scripts/conformance/server-parity/upgrade-takeover-resume.php b/scripts/conformance/server-parity/upgrade-takeover-resume.php index bddd3db37..2b695dc8d 100644 --- a/scripts/conformance/server-parity/upgrade-takeover-resume.php +++ b/scripts/conformance/server-parity/upgrade-takeover-resume.php @@ -56,6 +56,7 @@ $outcome = []; $activity = $originals['activity']; $timer = $originals['timer']; +$signal = $originals['signal']; $complete = static function (array $claimed, string $operation) use ($expect, $request, $result): void { $expect($request('POST', '/api/worker/workflow-tasks/'.$claimed['task_id'].'/complete', ['lease_owner' => $claimed['lease_owner'], 'workflow_task_attempt' => $claimed['workflow_task_attempt'], @@ -77,20 +78,20 @@ $replayed = $expect($request('POST', '/api/worker/activity-tasks/'.$task['task_id'].'/complete', $completion), 200, 'activity completion replay'); $check(($replayed['recorded'] ?? null) === false, 'A replayed activity completion was recorded twice'); -// Each original workflow then gets one workflow task: the activity's at -// once and the timer's at its deadline. On a slow host the timer is already -// due, so either may be claimed first. +// Each original workflow then gets one workflow task: the activity's and the +// signal's at once, the timer's at its deadline. On a slow host the timer is +// already due, so they may be claimed in any order. $original = $activity['history']; $claims = []; $until = max($deadline, microtime(true)) + 30; -while (count($claims) < 2 && microtime(true) < $until) { +while (count($claims) < 3 && microtime(true) < $until) { $claimed = $expect($request('POST', '/api/worker/workflow-tasks/poll', ['worker_id' => $worker, 'task_queue' => $queue, 'timeout_seconds' => 5]), 200, 'workflow task claim')['task'] ?? null; if (! is_array($claimed)) { continue; } $run = $claimed['run_id']; - $check(in_array($run, [$activity['run_id'], $timer['run_id']], true) && ! isset($claims[$run]), + $check(in_array($run, [$activity['run_id'], $timer['run_id'], $signal['run_id']], true) && ! isset($claims[$run]), 'An unexpected workflow task was claimed'); $claims[$run] = true; if ($run === $activity['run_id']) { @@ -98,12 +99,23 @@ 'Original activity history changed in the resumed task'); $check(end($claimed['history_events'])['event_type'] === 'ActivityCompleted', 'Activity completion is missing from history'); $complete($claimed, 'resumed workflow completion'); + } elseif ($run === $signal['run_id']) { + // The signal PHP accepted is delivered from PHP's rows, with its arguments. + $check(array_slice($types($claimed['history_events']), 0, count($signal['history'])) === $types($signal['history']), + 'Original signal history changed in the resumed task'); + $received = array_values(array_filter($claimed['history_events'], + static fn (array $event): bool => $event['event_type'] === 'SignalReceived')); + $check(count($received) === 1 && json_encode($sorted($received[0]['payload']['arguments']), JSON_THROW_ON_ERROR) + === json_encode($sorted(end($signal['history'])['payload']['arguments']), JSON_THROW_ON_ERROR), + 'The PHP-accepted signal arguments differ under native'); + $complete($claimed, 'signal workflow completion'); } else { $check(end($claimed['history_events'])['event_type'] === 'TimerFired', 'Timer firing is missing from history'); $complete($claimed, 'timer workflow completion'); } } $check(isset($claims[$activity['run_id']]), 'The resumed workflow task was not claimed'); +$check(isset($claims[$signal['run_id']]), 'The signal workflow task was not claimed'); $check(isset($claims[$timer['run_id']]), 'The original timer did not fire under native'); // The workflow task PHP already completed is a stale attempt. @@ -124,6 +136,10 @@ $check($firedAt >= $deadline, 'The original timer fired before its deadline'); $check($keep($timer['history'], $timerHistory) && end($timerHistory)['event_type'] === 'WorkflowCompleted', 'The timer workflow did not complete on its original history'); +$signalHistory = $history($signal); +$check($keep($signal['history'], $signalHistory) && end($signalHistory)['event_type'] === 'WorkflowCompleted', + 'The signal workflow did not complete on its original history'); +$outcome['signal'] = ['run_id' => $signal['run_id'], 'events' => array_column($signalHistory, 'event_type')]; $outcome['timer'] = ['run_id' => $timer['run_id'], 'fire_at' => $fireAt, 'fired_after_seconds' => round($firedAt - $deadline, 3), 'events' => array_column($timerHistory, 'event_type')]; echo json_encode(['outcome' => 'pass', 'scope' => 'native_resumes_published_php_originals_after_takeover'] + $outcome, From 6978d60b137a26c70e946a1ad33a4dafa28c3c32 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 14:43:53 +0000 Subject: [PATCH 32/37] Redeliver a PHP lease that expires after the takeover The upgrade source author adds a fourth original whose worker stops after claiming its workflow task. PHP's lease on that task expires after the takeover. Native must then redeliver the same task as its next attempt, refuse the expired PHP attempt's completion, and complete the workflow on its original history. Part of #325. Co-Authored-By: Claude Opus 5.5 --- .../server-parity/upgrade-source-author.php | 20 ++++++--- .../server-parity/upgrade-source-profile.sh | 14 +++--- .../upgrade-source-sqlite-representations.php | 6 +-- .../server-parity/upgrade-takeover-resume.php | 44 ++++++++++++++++--- 4 files changed, 63 insertions(+), 21 deletions(-) diff --git a/scripts/conformance/server-parity/upgrade-source-author.php b/scripts/conformance/server-parity/upgrade-source-author.php index c16237701..fa85db118 100644 --- a/scripts/conformance/server-parity/upgrade-source-author.php +++ b/scripts/conformance/server-parity/upgrade-source-author.php @@ -42,7 +42,8 @@ $worker = $prefix.'-worker'; $expect($request('POST', '/api/worker/register', ['worker_id' => $worker, 'task_queue' => $prefix, 'runtime' => 'php', 'sdk_version' => 'raw-upgrade-inspection-author', - 'supported_workflow_types' => ['parity.upgrade_activity', 'parity.upgrade_timer', 'parity.upgrade_signal'], + 'supported_workflow_types' => ['parity.upgrade_activity', 'parity.upgrade_timer', 'parity.upgrade_signal', + 'parity.upgrade_lease'], // The signal original declares its one signal, as an SDK worker would. 'workflow_command_contracts' => ['parity.upgrade_signal' => ['signals' => ['go'], 'signal_contracts' => [[ 'name' => 'go', 'parameters' => [['name' => 'value', 'position' => 0, 'type' => 'array', 'required' => true, @@ -59,7 +60,8 @@ $timerSeconds = (int) (getenv('DW_PARITY_UPGRADE_TIMER_SECONDS') ?: 600); $check($timerSeconds >= 30 && $timerSeconds <= 600, 'Unsupported original timer delay'); $originals = []; -foreach (['activity', 'timer', 'signal'] as $kind) { +// The lease original claims before the signal original makes its resume task. +foreach (['activity', 'timer', 'lease', 'signal'] as $kind) { $id = $prefix.'-'.$kind; $run = $expect($request('POST', '/api/workflows', ['workflow_id' => $id, 'workflow_type' => 'parity.upgrade_'.$kind, 'task_queue' => $prefix, 'input' => $envelope, @@ -74,11 +76,16 @@ 'start_to_close_timeout' => 600, 'schedule_to_close_timeout' => 900], 'timer' => ['type' => 'start_timer', 'delay_seconds' => $timerSeconds], 'signal' => ['type' => 'open_signal_wait', 'signal_name' => 'go'], + // The lease original's worker stops after its claim. PHP's workflow + // task lease expires after the takeover, and native redelivers it. + 'lease' => null, }; - $expect($request('POST', '/api/worker/workflow-tasks/'.$task['task_id'].'/complete', [ - 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], - 'commands' => [$command], - ]), 200, $kind.' command'); + if ($command !== null) { + $expect($request('POST', '/api/worker/workflow-tasks/'.$task['task_id'].'/complete', [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'commands' => [$command], + ]), 200, $kind.' command'); + } $activity = null; if ($kind === 'activity') { $activity = $expect($request('POST', '/api/worker/activity-tasks/poll', $poll), 200, 'activity claim')['task']; @@ -98,6 +105,7 @@ 'activity' => ['StartAccepted', 'WorkflowStarted', 'ActivityScheduled', 'ActivityStarted'], 'timer' => ['StartAccepted', 'WorkflowStarted', 'TimerScheduled'], 'signal' => ['StartAccepted', 'WorkflowStarted', 'SignalWaitOpened', 'SignalReceived'], + 'lease' => ['StartAccepted', 'WorkflowStarted'], }, 'Original pending history differs'); $originals[$kind] = ['workflow_id' => $id, 'run_id' => $run['run_id'], 'workflow_task' => $task, 'activity_task' => $activity, 'history' => $history]; diff --git a/scripts/conformance/server-parity/upgrade-source-profile.sh b/scripts/conformance/server-parity/upgrade-source-profile.sh index e7a504c98..b5f340ef0 100755 --- a/scripts/conformance/server-parity/upgrade-source-profile.sh +++ b/scripts/conformance/server-parity/upgrade-source-profile.sh @@ -101,8 +101,8 @@ if test "$backend" = sqlite; then /source/scripts/conformance/server-parity/upgrade-source-sqlite-representations.php upgrade-source \ > "$evidence/representations-comparison.json" jq -e '.outcome=="pass" and .source_mutated==false and .php_objects_executed==false - and (.runs|length)==3 and (.tasks|length)==6 and (.activities|length)==1 - and (.history|length)==11 and (.queue_jobs|length)==1 + and (.runs|length)==4 and (.tasks|length)==7 and (.activities|length)==1 + and (.history|length)==13 and (.queue_jobs|length)==1 and .queue_jobs[0].canonical_timer_task_match==true and all(.runs[]; .typed_arguments_match==true) and .backup_verified==false and .stored_representation_compatibility_verified==false @@ -134,7 +134,7 @@ snapshot() { } snapshot > "$evidence/before.json" inspect_source > "$evidence/inspection.json" 2> "$evidence/inspection.stderr" -jq -e --arg layout "$layout" '.status=="inspected" and .source_layout==$layout and (.table_rows|length)==50 and .table_rows.workflow_runs==3 +jq -e --arg layout "$layout" '.status=="inspected" and .source_layout==$layout and (.table_rows|length)==50 and .table_rows.workflow_runs==4 and .table_rows.activity_executions==1 and .catalog_visibility=="configured_role" and .source_mutated==false and .takeover_allowed==false and .backup_verified==false and .php_quiescence_verified==false and .stored_representation_compatibility_verified==false' "$evidence/inspection.json" >/dev/null @@ -148,7 +148,7 @@ inspect_source upgrade-payload-inspect > "$evidence/payload-inspection.json" 2> jq -e '.status=="inspected" and (.payload_inspection.columns|length)==8 and .payload_inspection.references_metadata_supported==true and ([.payload_inspection.columns[].counts.php_v1_references]|add)==0 - and ([.payload_inspection.columns[].counts.avro_inline_values]|add)==9 + and ([.payload_inspection.columns[].counts.avro_inline_values]|add)==11 and ([.payload_inspection.columns[].counts.other_unvalidated]|add)==0 and .payload_inspection.inline_values_verified==true and .payload_inspection.inline_validation_limits=={encoded_frame_bytes:65536,traversal_depth:64,traversal_visits:16384,scalar_allocation_bytes:65536} @@ -166,7 +166,7 @@ fingerprint() { } fingerprint "$database" > "$evidence/fingerprint.json" 2> "$evidence/fingerprint.stderr" jq -e --arg layout "$layout" '.status=="fingerprinted" and .source_layout==$layout and (.fingerprint|test("^sha256:[0-9a-f]{64}$")) - and (.table_rows|length)==50 and .table_rows.workflow_runs==3 and .table_rows.jobs==1 + and (.table_rows|length)==50 and .table_rows.workflow_runs==4 and .table_rows.jobs==1 and .source_mutated==false and .takeover_allowed==false and .backup_verified==false and .php_quiescence_verified==false and .stored_representation_compatibility_verified==false' "$evidence/fingerprint.json" >/dev/null source_fingerprint=$(jq -r .fingerprint "$evidence/fingerprint.json") @@ -285,7 +285,9 @@ if (( ${#target[@]} )); then "/source/$evidence/originals.json" 150 > "$evidence/resume.json" 2> "$evidence/resume.stderr" jq -e '.outcome=="pass" and .activity.stale_attempt_status==409 and .activity.events[-1]=="WorkflowCompleted" and .timer.fired_after_seconds>=0 - and .timer.events[-1]=="WorkflowCompleted" and .signal.events[-1]=="WorkflowCompleted"' "$evidence/resume.json" >/dev/null + and .timer.events[-1]=="WorkflowCompleted" and .signal.events[-1]=="WorkflowCompleted" + and .lease.attempt==2 and .lease.redelivered_after_seconds>=0 and .lease.stale_attempt_status==409 + and .lease.events[-1]=="WorkflowCompleted"' "$evidence/resume.json" >/dev/null docker stop --signal SIGTERM --time 10 "$resource-native" >/dev/null if docker run --rm --name "$resource-control" --user=1000:1000 --cpus=1 --memory=512m --memory-swap=512m \ --network "$target_network" "${target[@]}" -e DW_AUTH_TOKEN=upgrade-fixture-token \ diff --git a/scripts/conformance/server-parity/upgrade-source-sqlite-representations.php b/scripts/conformance/server-parity/upgrade-source-sqlite-representations.php index b36f69b1d..09da717d5 100644 --- a/scripts/conformance/server-parity/upgrade-source-sqlite-representations.php +++ b/scripts/conformance/server-parity/upgrade-source-sqlite-representations.php @@ -20,11 +20,11 @@ if ($pdo->query('PRAGMA journal_mode')->fetchColumn() !== 'delete') { throw new RuntimeException('Audit requires the already stopped DELETE-journal fixture'); } -$runs = $pdo->query('SELECT * FROM workflow_runs ORDER BY id LIMIT 4')->fetchAll(PDO::FETCH_ASSOC); +$runs = $pdo->query('SELECT * FROM workflow_runs ORDER BY id LIMIT 5')->fetchAll(PDO::FETCH_ASSOC); $workflowIds = array_column($runs, 'workflow_instance_id'); sort($workflowIds); -if ($workflowIds !== [$prefix.'-activity', $prefix.'-signal', $prefix.'-timer']) { - throw new RuntimeException('This reader accepts only the three isolated source-author originals'); +if ($workflowIds !== [$prefix.'-activity', $prefix.'-lease', $prefix.'-signal', $prefix.'-timer']) { + throw new RuntimeException('This reader accepts only the four isolated source-author originals'); } $report = ['scope' => 'published_php_stored_representation_observation', 'runs' => [], 'table_rows' => [], 'tasks' => [], 'activities' => [], 'history' => [], 'queue_jobs' => [], 'cache_files' => []]; diff --git a/scripts/conformance/server-parity/upgrade-takeover-resume.php b/scripts/conformance/server-parity/upgrade-takeover-resume.php index 2b695dc8d..41f0281d5 100644 --- a/scripts/conformance/server-parity/upgrade-takeover-resume.php +++ b/scripts/conformance/server-parity/upgrade-takeover-resume.php @@ -57,7 +57,8 @@ $activity = $originals['activity']; $timer = $originals['timer']; $signal = $originals['signal']; -$complete = static function (array $claimed, string $operation) use ($expect, $request, $result): void { +$lease = $originals['lease']; +$complete =static function (array $claimed, string $operation) use ($expect, $request, $result): void { $expect($request('POST', '/api/worker/workflow-tasks/'.$claimed['task_id'].'/complete', ['lease_owner' => $claimed['lease_owner'], 'workflow_task_attempt' => $claimed['workflow_task_attempt'], 'commands' => [['type' => 'complete_workflow', 'result' => $result]]]), 200, $operation); @@ -70,6 +71,12 @@ $deadline = (float) (new DateTimeImmutable($fireAt))->format('U.u'); $check($deadline - microtime(true) <= (float) $timerBudget, 'The original timer deadline is outside the wait budget'); +// The workflow task PHP leased to a stopped worker keeps its PHP expiry. +$check(is_string($lease['workflow_task']['lease_expires_at'] ?? null), 'The original lease expiry is missing'); +$leaseExpiry = (float) (new DateTimeImmutable($lease['workflow_task']['lease_expires_at']))->format('U.u'); +$check($leaseExpiry - microtime(true) <= (float) $timerBudget, 'The original lease expiry is outside the wait budget'); +$check($keep($lease['history'], $history($lease)), 'Original lease history changed'); + // The activity leased under PHP completes with its original attempt. $task = $activity['activity_task']; $completion = ['activity_attempt_id' => $task['activity_attempt_id'], 'lease_owner' => $task['lease_owner'], 'result' => $result]; @@ -79,19 +86,21 @@ $check(($replayed['recorded'] ?? null) === false, 'A replayed activity completion was recorded twice'); // Each original workflow then gets one workflow task: the activity's and the -// signal's at once, the timer's at its deadline. On a slow host the timer is -// already due, so they may be claimed in any order. +// signal's at once, the timer's at its deadline and the lease's when PHP's +// lease expires. On a slow host some are already due, so they may be claimed +// in any order. $original = $activity['history']; $claims = []; -$until = max($deadline, microtime(true)) + 30; -while (count($claims) < 3 && microtime(true) < $until) { +$leaseOutcome = null; +$until = max($deadline, $leaseExpiry, microtime(true)) + 30; +while (count($claims) < 4 && microtime(true) < $until) { $claimed = $expect($request('POST', '/api/worker/workflow-tasks/poll', ['worker_id' => $worker, 'task_queue' => $queue, 'timeout_seconds' => 5]), 200, 'workflow task claim')['task'] ?? null; if (! is_array($claimed)) { continue; } $run = $claimed['run_id']; - $check(in_array($run, [$activity['run_id'], $timer['run_id'], $signal['run_id']], true) && ! isset($claims[$run]), + $check(in_array($run, [$activity['run_id'], $timer['run_id'], $signal['run_id'], $lease['run_id']], true) && ! isset($claims[$run]), 'An unexpected workflow task was claimed'); $claims[$run] = true; if ($run === $activity['run_id']) { @@ -109,6 +118,23 @@ === json_encode($sorted(end($signal['history'])['payload']['arguments']), JSON_THROW_ON_ERROR), 'The PHP-accepted signal arguments differ under native'); $complete($claimed, 'signal workflow completion'); + } elseif ($run === $lease['run_id']) { + // After PHP's lease expires, native redelivers the same task as its + // next attempt. The expired PHP attempt cannot complete over it. + $expired = $lease['workflow_task']; + $check($claimed['task_id'] === $expired['task_id'] + && $claimed['workflow_task_attempt'] === $expired['workflow_task_attempt'] + 1, + 'The expired PHP lease was not redelivered as its next attempt'); + $check(microtime(true) >= $leaseExpiry, 'Native redelivered a PHP lease before it expired'); + $check($types($claimed['history_events']) === $types($lease['history']), + 'Original lease history changed in the redelivered task'); + $stale = $request('POST', '/api/worker/workflow-tasks/'.$expired['task_id'].'/complete', ['lease_owner' => $expired['lease_owner'], + 'workflow_task_attempt' => $expired['workflow_task_attempt'], 'commands' => [['type' => 'complete_workflow', 'result' => $result]]]); + $check($stale[0] === 409, 'The expired PHP attempt completed over its redelivery'); + $leaseOutcome = ['attempt' => $claimed['workflow_task_attempt'], + 'redelivered_after_seconds' => round(microtime(true) - $leaseExpiry, 3), + 'stale_attempt_status' => $stale[0], 'stale_attempt_reason' => $stale[1]['reason'] ?? null]; + $complete($claimed, 'redelivered lease completion'); } else { $check(end($claimed['history_events'])['event_type'] === 'TimerFired', 'Timer firing is missing from history'); $complete($claimed, 'timer workflow completion'); @@ -117,6 +143,7 @@ $check(isset($claims[$activity['run_id']]), 'The resumed workflow task was not claimed'); $check(isset($claims[$signal['run_id']]), 'The signal workflow task was not claimed'); $check(isset($claims[$timer['run_id']]), 'The original timer did not fire under native'); +$check(isset($claims[$lease['run_id']]), 'The expired PHP lease was not redelivered'); // The workflow task PHP already completed is a stale attempt. $stale = $activity['workflow_task']; @@ -140,6 +167,11 @@ $check($keep($signal['history'], $signalHistory) && end($signalHistory)['event_type'] === 'WorkflowCompleted', 'The signal workflow did not complete on its original history'); $outcome['signal'] = ['run_id' => $signal['run_id'], 'events' => array_column($signalHistory, 'event_type')]; +$leaseHistory = $history($lease); +$check($keep($lease['history'], $leaseHistory) && end($leaseHistory)['event_type'] === 'WorkflowCompleted', + 'The redelivered lease workflow did not complete on its original history'); +$outcome['lease'] = ['run_id' => $lease['run_id'], 'lease_expires_at' => $lease['workflow_task']['lease_expires_at']] + + $leaseOutcome + ['events' => array_column($leaseHistory, 'event_type')]; $outcome['timer'] = ['run_id' => $timer['run_id'], 'fire_at' => $fireAt, 'fired_after_seconds' => round($firedAt - $deadline, 3), 'events' => array_column($timerHistory, 'event_type')]; echo json_encode(['outcome' => 'pass', 'scope' => 'native_resumes_published_php_originals_after_takeover'] + $outcome, From ebfcbff19970cd8324df0e6792bef6ed929925de Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 15:03:02 +0000 Subject: [PATCH 33/37] Fence entry-schema failures and require safe replacement workers --- app/Http/Controllers/Api/WorkerController.php | 1 + app/Support/WorkflowDefinitionRouting.php | 18 ++++- docs/server-reference.md | 7 +- .../WorkflowDefinitionEvolutionTest.php | 69 +++++++++++++++++++ 4 files changed, 91 insertions(+), 4 deletions(-) diff --git a/app/Http/Controllers/Api/WorkerController.php b/app/Http/Controllers/Api/WorkerController.php index 4da9a14eb..b192dc0f6 100644 --- a/app/Http/Controllers/Api/WorkerController.php +++ b/app/Http/Controllers/Api/WorkerController.php @@ -161,6 +161,7 @@ class WorkerController 'version_marker_kind_mismatch', 'version_marker_sequence_invalid', 'version_range_invalid', + 'workflow_entry_arguments_mismatch', 'workflow_nondeterministic', ]; diff --git a/app/Support/WorkflowDefinitionRouting.php b/app/Support/WorkflowDefinitionRouting.php index 2719ee015..3109f8bab 100644 --- a/app/Support/WorkflowDefinitionRouting.php +++ b/app/Support/WorkflowDefinitionRouting.php @@ -13,6 +13,8 @@ final class WorkflowDefinitionRouting { + public const ENTRY_VALIDATION_CAPABILITY = 'workflow_entry_arguments_validation'; + /** @return array */ public function forWorker( WorkflowRun $run, @@ -44,6 +46,10 @@ public function forWorker( } } + if (! in_array(self::ENTRY_VALIDATION_CAPABILITY, $worker->capabilities ?? [], true)) { + return [...$decision, 'allowed' => false, 'status' => 'entry_validation_unavailable']; + } + // Match the embedded resolver: prefer the recorded source if it is // available, otherwise let the current definition validate replay. // This never changes the run's original fingerprint or build pin. @@ -63,7 +69,7 @@ public function inspect(WorkflowRun $run, ?string $taskQueue = null, ?string $re $requiredCompatibility ??= TaskCompatibility::resolve($task, $run); } $context = $this->context($run, $taskQueue, $requiredCompatibility); - $matching = $changed = $missing = 0; + $matching = $changed = $missing = $unsafeReplacement = 0; $examples = []; foreach ($this->eligibleWorkers($run, (string) $context['task_queue'], $context['required_build_id']) as $worker) { @@ -74,6 +80,9 @@ public function inspect(WorkflowRun $run, ?string $taskQueue = null, ?string $re $matching++; } else { $changed++; + if (! in_array(self::ENTRY_VALIDATION_CAPABILITY, $worker->capabilities ?? [], true)) { + $unsafeReplacement++; + } } if (count($examples) < 10) { $examples[] = ['worker_id' => $worker->worker_id, 'fingerprint' => $fingerprint]; @@ -86,14 +95,16 @@ public function inspect(WorkflowRun $run, ?string $taskQueue = null, ?string $re ! $context['pinning_enabled'] => 'pinning_disabled', $context['recorded_fingerprint'] === null => 'recorded_definition_missing', $matching > 0 => 'matching_definition_available', - $changed > 0 => 'replay_required', + $changed > $unsafeReplacement => 'replay_required', + $unsafeReplacement > 0 => 'entry_validation_unavailable', $missing > 0 => 'worker_definition_missing', default => 'no_eligible_worker', }; $runnable = ! $run->status->isTerminal() && $task instanceof WorkflowTask; $blocked = $runnable && $context['worker_pinned'] && $context['pinning_enabled'] - && in_array($status, ['recorded_definition_missing', 'worker_definition_missing', 'no_eligible_worker'], true); + && in_array($status, ['recorded_definition_missing', 'worker_definition_missing', 'no_eligible_worker', 'entry_validation_unavailable'], true); $resolution = match (true) { + $blocked && $status === 'entry_validation_unavailable' => 'Upgrade the replacement worker SDK to an implementation advertising workflow_entry_arguments_validation, or restore the recorded definition. The current replacement cannot safely refuse incompatible entry input.', $blocked => 'Register a fresh worker for the original namespace, task queue, workflow type, build and replay capabilities with its definition fingerprint. Do not rewrite the run history.', $status === 'replay_required' => 'Current workers may replay the original history. Use versioned branches for command changes; incompatible replay still refuses.', default => 'Recorded build, namespace, type and replay-capability guards remain in force.', @@ -102,6 +113,7 @@ public function inspect(WorkflowRun $run, ?string $taskQueue = null, ?string $re return [...$context, 'status' => $status, 'blocked' => $blocked, 'runnable_task' => $runnable, 'eligible_worker_count' => $eligible, 'matching_worker_count' => $matching, 'changed_worker_count' => $changed, 'missing_fingerprint_count' => $missing, + 'entry_validation_unavailable_count' => $unsafeReplacement, 'workers' => $examples, 'workers_truncated' => $eligible > count($examples), 'resolution' => $resolution, ]; diff --git a/docs/server-reference.md b/docs/server-reference.md index 00af2598a..2c3fe5e44 100644 --- a/docs/server-reference.md +++ b/docs/server-reference.md @@ -422,7 +422,12 @@ Existing runs retain their original definition fingerprint, build pin, history and deadlines. By default, the Server prefers a fresh eligible worker with the recorded definition. When that definition is no longer available in the run's namespace, queue, workflow type and build, a replacement definition can claim -the original task and must pass the SDK's strict replay checks. Use versioned +the original task only when it advertises `workflow_entry_arguments_validation`, +and must pass the SDK's strict replay checks. This worker capability declares +pre-entry refusal without failing the business run; older worker implementations +do not gain it from their runtime or protocol version. `entry_validation_unavailable` +in `definition_routing` identifies a replacement that needs an SDK upgrade. +Use versioned branches such as `get_version` for changes to durable commands. A different source fingerprint alone is not proof of incompatible replay. diff --git a/tests/Feature/WorkflowDefinitionEvolutionTest.php b/tests/Feature/WorkflowDefinitionEvolutionTest.php index f8398e18b..52e973657 100644 --- a/tests/Feature/WorkflowDefinitionEvolutionTest.php +++ b/tests/Feature/WorkflowDefinitionEvolutionTest.php @@ -4,6 +4,7 @@ use App\Models\WorkerRegistration; use App\Support\WorkflowReplayEligibility; +use App\Support\WorkflowDefinitionRouting; use Illuminate\Foundation\Testing\RefreshDatabase; use Illuminate\Support\Facades\Queue; use Illuminate\Testing\TestResponse; @@ -64,6 +65,24 @@ public function test_explicit_opt_out_does_not_wait_for_a_matching_peer(): void ->assertJsonPath('task.task_id', $task->id); } + public function test_legacy_replacement_cannot_gain_unsafe_default_fallback(): void + { + [$run, $task] = $this->original(); + $this->deleteJson('/api/workers/original-worker', [], $this->apiHeaders())->assertOk(); + $this->worker('legacy-replacement', 'definition-v2', ['capabilities' => []]); + $this->poll('legacy-replacement')->assertOk()->assertJsonPath('task', null); + self::assertSame(0, $task->fresh()->attempt_count); + $this->getJson('/api/workflows/definition-original', $this->apiHeaders())->assertOk() + ->assertJsonPath('definition_routing.status', 'entry_validation_unavailable') + ->assertJsonPath('definition_routing.blocked', true) + ->assertJsonPath('definition_routing.entry_validation_unavailable_count', 1); + $this->worker('safe-replacement', 'definition-v2'); + $this->getJson('/api/workflows/definition-original', $this->apiHeaders())->assertOk() + ->assertJsonPath('definition_routing.status', 'replay_required') + ->assertJsonPath('definition_routing.blocked', false); + $this->poll('safe-replacement')->assertOk()->assertJsonPath('task.run_id', $run->id); + } + public function test_default_keeps_the_matching_registered_peer_preference(): void { [, $task] = $this->original(); @@ -252,6 +271,55 @@ public static function notRunnableTasks(): array return [['terminal'], ['leased'], ['future']]; } + #[DataProvider('entryFailureTypes')] + public function test_entry_binding_failure_preserves_the_original_until_supported_repair(string $failureType): void + { + [$run, $task, $started] = $this->original(); + $original = [$started->getRawOriginal('payload'), $run->getRawOriginal('run_deadline_at'), $run->getRawOriginal('execution_deadline_at')]; + $claim = $this->poll('original-worker')->assertOk(); + $failure = [ + 'lease_owner' => $claim->json('task.lease_owner'), + 'workflow_task_attempt' => $claim->json('task.workflow_task_attempt'), + 'failure' => [ + 'type' => $failureType, + 'reason' => 'workflow_entry_arguments_mismatch', + 'message' => 'The recorded input cannot bind to the deployed entry.', + ], + ]; + + $this->postJson('/api/worker/workflow-tasks/'.$task->id.'/fail', [ + ...$failure, 'lease_owner' => 'another-worker', + ], $this->workerHeaders())->assertConflict(); + self::assertSame(TaskStatus::Leased, $task->fresh()->status); + + $this->postJson('/api/worker/workflow-tasks/'.$task->id.'/fail', $failure, $this->workerHeaders()) + ->assertOk()->assertJsonPath('recorded', true)->assertJsonPath('next_task_id', null); + self::assertSame(TaskStatus::Failed, $task->fresh()->status); + self::assertTrue($task->fresh()->payload['replay_blocked'] ?? false); + self::assertSame('workflow_entry_arguments_mismatch', $task->fresh()->payload['failure_reason']); + self::assertSame(1, WorkflowTask::query()->where('workflow_run_id', $run->id)->count()); + self::assertNotContains($run->fresh()->status->value, ['failed', 'completed']); + $this->poll('original-worker')->assertOk()->assertJsonPath('task', null); + + $this->postJson('/api/workflows/definition-original/repair', [ + 'request_id' => 'entry-compatible-repair', + ], $this->apiHeaders())->assertOk(); + $this->poll('original-worker')->assertOk()->assertJsonPath('task.task_id', $task->id); + self::assertSame(1, $task->fresh()->repair_count); + self::assertSame($original, [ + $started->fresh()->getRawOriginal('payload'), $run->fresh()->getRawOriginal('run_deadline_at'), $run->fresh()->getRawOriginal('execution_deadline_at'), + ]); + } + + public static function entryFailureTypes(): array + { + return [ + 'PHP' => ['DurableWorkflow\\Exception\\WorkflowEntryArgumentsMismatch'], + 'Python' => ['WorkflowEntryArgumentsMismatchError'], + 'Rust' => ['WorkflowEntryArgumentsMismatch'], + ]; + } + private function original(): array { $this->worker('original-worker', 'definition-v1'); @@ -273,6 +341,7 @@ private function worker(string $id, string $fingerprint, array $overrides = []): { $this->postJson('/api/worker/register', [ 'worker_id' => $id, 'task_queue' => self::QUEUE, 'runtime' => 'python', 'sdk_version' => '2.6.1', + 'capabilities' => [WorkflowDefinitionRouting::ENTRY_VALIDATION_CAPABILITY], 'capability_manifest' => $this->portableWorkerAffinityRefusalManifest(), 'supported_workflow_types' => [self::TYPE], 'workflow_definition_fingerprints' => [self::TYPE => $fingerprint], From 85a81e67e2db0469a86831efe0171e8ce69dbdf4 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 15:24:26 +0000 Subject: [PATCH 34/37] Keep replacement safety decision in one final return --- app/Support/WorkflowDefinitionRouting.php | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/app/Support/WorkflowDefinitionRouting.php b/app/Support/WorkflowDefinitionRouting.php index 3109f8bab..0579f3ba0 100644 --- a/app/Support/WorkflowDefinitionRouting.php +++ b/app/Support/WorkflowDefinitionRouting.php @@ -46,14 +46,13 @@ public function forWorker( } } - if (! in_array(self::ENTRY_VALIDATION_CAPABILITY, $worker->capabilities ?? [], true)) { - return [...$decision, 'allowed' => false, 'status' => 'entry_validation_unavailable']; - } - // Match the embedded resolver: prefer the recorded source if it is // available, otherwise let the current definition validate replay. // This never changes the run's original fingerprint or build pin. - return [...$decision, 'allowed' => true, 'status' => 'replay_required']; + $entryValidation = in_array(self::ENTRY_VALIDATION_CAPABILITY, $worker->capabilities ?? [], true); + + return [...$decision, 'allowed' => $entryValidation, + 'status' => $entryValidation ? 'replay_required' : 'entry_validation_unavailable']; } /** @return array */ From 3a07c5585b2be7a2610f15250ea4d4dccf3b5bfe Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 15:40:45 +0000 Subject: [PATCH 35/37] Prepare Server 2.9.9 entry-safety release --- composer.json | 2 +- composer.lock | 2 +- docker-compose.dedicated-matching.yml | 4 ++-- docker-compose.memo-rolling.yml | 4 ++-- docker-compose.published.yml | 4 ++-- docker-compose.small-cluster.yml | 2 +- docker-compose.yml | 8 ++++---- k8s/README.md | 10 +++++----- k8s/helm/durable-workflow/Chart.yaml | 8 ++++---- k8s/helm/durable-workflow/README.md | 2 +- .../durable-workflow/ci/existing-secrets-values.yaml | 2 +- .../durable-workflow/ci/ingress-and-hpa-values.yaml | 2 +- .../durable-workflow/ci/inline-secrets-values.yaml | 2 +- k8s/helm/durable-workflow/templates/_helpers.tpl | 2 +- k8s/helm/durable-workflow/values.yaml | 2 +- k8s/helm/examples/values-dev.yaml | 2 +- .../examples/values-external-secrets-operator.yaml | 2 +- .../examples/values-production-existing-secrets.yaml | 2 +- k8s/migration-job.yaml | 2 +- k8s/scheduler-cronjob.yaml | 2 +- k8s/secret.yaml | 2 +- k8s/server-deployment.yaml | 2 +- k8s/worker-deployment.yaml | 2 +- resources/release/source-release.json | 4 ++-- scripts/k8s-kind-smoke.sh | 2 +- 25 files changed, 39 insertions(+), 39 deletions(-) diff --git a/composer.json b/composer.json index a31e9251c..2de1c9710 100644 --- a/composer.json +++ b/composer.json @@ -48,7 +48,7 @@ }, "extra": { "durable-workflow": { - "product-train": "2.9.8" + "product-train": "2.9.9" }, "laravel": { "dont-discover": [] diff --git a/composer.lock b/composer.lock index 2e274b226..6ffbeab65 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "f3629921342639b58b34196e3ede3051", + "content-hash": "1230e17c71101ee7340e52d0c1fe2e35", "packages": [ { "name": "apache/avro", diff --git a/docker-compose.dedicated-matching.yml b/docker-compose.dedicated-matching.yml index 099c8775d..48c69e0ab 100644 --- a/docker-compose.dedicated-matching.yml +++ b/docker-compose.dedicated-matching.yml @@ -32,13 +32,13 @@ name: durable-workflow-server # daemon reports `shape: dedicated`. # Generated by scripts/ci/sync-source-release.mjs. Do not edit the fallback. -x-server-image: &server-image ${DW_SERVER_IMAGE:-durableworkflow/server:${DW_SERVER_TAG:-2.9.8}} +x-server-image: &server-image ${DW_SERVER_IMAGE:-durableworkflow/server:${DW_SERVER_TAG:-2.9.9}} x-server-environment: &server-environment APP_NAME: "Durable Workflow Server" APP_ENV: ${APP_ENV:-local} DW_SERVER_KEY: ${DW_SERVER_KEY:-} - APP_VERSION: ${APP_VERSION:-${DW_SERVER_TAG:-2.9.8}} + APP_VERSION: ${APP_VERSION:-${DW_SERVER_TAG:-2.9.9}} APP_DEBUG: ${APP_DEBUG:-false} DB_CONNECTION: mysql DB_HOST: mysql diff --git a/docker-compose.memo-rolling.yml b/docker-compose.memo-rolling.yml index 6bb030f70..4c77c8946 100644 --- a/docker-compose.memo-rolling.yml +++ b/docker-compose.memo-rolling.yml @@ -49,14 +49,14 @@ services: command: ["server-bootstrap"] environment: <<: *runtime-environment - APP_VERSION: ${APP_VERSION:-2.9.8} + APP_VERSION: ${APP_VERSION:-2.9.9} successor: image: ${DW_MEMO_SUCCESSOR_IMAGE:-durable-workflow/server-memo-rolling:local} ports: !override [] environment: <<: *runtime-environment - APP_VERSION: ${APP_VERSION:-2.9.8} + APP_VERSION: ${APP_VERSION:-2.9.9} DW_SERVER_ID: memo-successor DW_SERVER_TOPOLOGY_SHAPE: standalone_server DW_SERVER_PROCESS_CLASS: server_http_node diff --git a/docker-compose.published.yml b/docker-compose.published.yml index e7acdad03..e5cc0b811 100644 --- a/docker-compose.published.yml +++ b/docker-compose.published.yml @@ -1,13 +1,13 @@ name: durable-workflow-server # Generated by scripts/ci/sync-source-release.mjs. Do not edit the fallback. -x-server-image: &server-image ${DW_SERVER_IMAGE:-durableworkflow/server:${DW_SERVER_TAG:-2.9.8}} +x-server-image: &server-image ${DW_SERVER_IMAGE:-durableworkflow/server:${DW_SERVER_TAG:-2.9.9}} x-server-environment: &server-environment APP_NAME: "Durable Workflow Server" APP_ENV: ${APP_ENV:-local} DW_SERVER_KEY: ${DW_SERVER_KEY:-} - APP_VERSION: ${APP_VERSION:-${DW_SERVER_TAG:-2.9.8}} + APP_VERSION: ${APP_VERSION:-${DW_SERVER_TAG:-2.9.9}} APP_DEBUG: ${APP_DEBUG:-false} LOG_CHANNEL: ${LOG_CHANNEL:-stderr} LOG_LEVEL: ${LOG_LEVEL:-info} diff --git a/docker-compose.small-cluster.yml b/docker-compose.small-cluster.yml index 3f3715b02..a6a242017 100644 --- a/docker-compose.small-cluster.yml +++ b/docker-compose.small-cluster.yml @@ -12,7 +12,7 @@ x-server-build: &server-build x-server-environment: &server-environment APP_NAME: "Durable Workflow Server" APP_ENV: testing - APP_VERSION: ${APP_VERSION:-2.9.8} + APP_VERSION: ${APP_VERSION:-2.9.9} APP_DEBUG: "false" DW_SERVER_KEY: ${DW_SERVER_KEY:-base64:5Zt4nUhlCm3DD0nLXZJQdHiwPfb56yGo9gNV/g3jYbY=} DB_CONNECTION: ${DW_SMALL_CLUSTER_DB:-mysql} diff --git a/docker-compose.yml b/docker-compose.yml index 6f18015e1..95d538a38 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,7 +15,7 @@ services: DW_SERVER_KEY: "${DW_SERVER_KEY:-}" DW_SERVER_TOPOLOGY_SHAPE: standalone_server DW_SERVER_PROCESS_CLASS: server_http_node - APP_VERSION: "${APP_VERSION:-2.9.8}" + APP_VERSION: "${APP_VERSION:-2.9.9}" APP_DEBUG: "false" DB_CONNECTION: mysql DB_HOST: mysql @@ -62,7 +62,7 @@ services: APP_NAME: "Durable Workflow Server" APP_ENV: local DW_SERVER_KEY: "${DW_SERVER_KEY:-}" - APP_VERSION: "${APP_VERSION:-2.9.8}" + APP_VERSION: "${APP_VERSION:-2.9.9}" APP_DEBUG: "false" DB_CONNECTION: mysql DB_HOST: mysql @@ -124,7 +124,7 @@ services: DW_SERVER_KEY: "${DW_SERVER_KEY:-}" DW_SERVER_TOPOLOGY_SHAPE: standalone_server DW_SERVER_PROCESS_CLASS: worker_node - APP_VERSION: "${APP_VERSION:-2.9.8}" + APP_VERSION: "${APP_VERSION:-2.9.9}" DB_CONNECTION: mysql DB_HOST: mysql DB_PORT: 3306 @@ -178,7 +178,7 @@ services: DW_SERVER_KEY: "${DW_SERVER_KEY:-}" DW_SERVER_TOPOLOGY_SHAPE: standalone_server DW_SERVER_PROCESS_CLASS: scheduler_node - APP_VERSION: "${APP_VERSION:-2.9.8}" + APP_VERSION: "${APP_VERSION:-2.9.9}" DB_CONNECTION: mysql DB_HOST: mysql DB_PORT: 3306 diff --git a/k8s/README.md b/k8s/README.md index 2375dfd6e..e1a7dd001 100644 --- a/k8s/README.md +++ b/k8s/README.md @@ -13,7 +13,7 @@ The checked-in manifests are synchronized with the repository's stable source release and pin its Docker Hub tag: ```text -durableworkflow/server:2.9.8 +durableworkflow/server:2.9.9 ``` Before production use, patch every workload image to the exact published tag or @@ -21,15 +21,15 @@ digest you intend to run: ```bash kubectl set image -n durable-workflow deploy/durable-workflow-server \ - server=durableworkflow/server:2.9.8 + server=durableworkflow/server:2.9.9 kubectl set image -n durable-workflow deploy/durable-workflow-worker \ - worker=durableworkflow/server:2.9.8 + worker=durableworkflow/server:2.9.9 kubectl set image -n durable-workflow cronjob/durable-workflow-scheduler \ - scheduler=durableworkflow/server:2.9.8 + scheduler=durableworkflow/server:2.9.9 ``` GitHub Container Registry publishes the same release line at -`ghcr.io/durable-workflow/server:2.9.8`. Digest pinning is preferred for strict +`ghcr.io/durable-workflow/server:2.9.9`. Digest pinning is preferred for strict change control. The manifests expect you to provide: diff --git a/k8s/helm/durable-workflow/Chart.yaml b/k8s/helm/durable-workflow/Chart.yaml index 5801c1bc4..321c99ce0 100644 --- a/k8s/helm/durable-workflow/Chart.yaml +++ b/k8s/helm/durable-workflow/Chart.yaml @@ -5,11 +5,11 @@ type: application # The chart's own semver version. Bumped on every chart release; treated as # independent of the server image version (appVersion). Breaking-change rules # for this version live in docs/helm-upgrading.md alongside the chart. -version: 0.1.184 +version: 0.1.185 # The immutable Durable Workflow Server identity this chart release packages. # The onboarding default in values.yaml and appVersion are generated from the # checked-in source release record. -appVersion: "2.9.8" +appVersion: "2.9.9" kubeVersion: ">=1.27.0-0" home: https://durable-workflow.github.io/docs/2.0/deployment sources: @@ -30,10 +30,10 @@ annotations: # exact commit that most recently changed the packaged chart. org.opencontainers.image.source: https://github.com/durable-workflow/server dev.durable-workflow.source-revision: "unreleased" - dev.durable-workflow.image-reference: "docker.io/durableworkflow/server:2.9.8" + dev.durable-workflow.image-reference: "docker.io/durableworkflow/server:2.9.9" artifacthub.io/license: MIT artifacthub.io/category: integration-delivery # Free-form changelog for the current chart release shown by Artifact Hub. artifacthub.io/changes: | - kind: fixed - description: Use unnamed bound PostgreSQL statements with transaction poolers and verify a bound database read during readiness checks. + description: Allow compatible workflow-code replacement with validated workers and preserve incompatible entry failures for supported same-run repair. diff --git a/k8s/helm/durable-workflow/README.md b/k8s/helm/durable-workflow/README.md index 69d84284d..42c39a15c 100644 --- a/k8s/helm/durable-workflow/README.md +++ b/k8s/helm/durable-workflow/README.md @@ -63,7 +63,7 @@ helm install durable-workflow ./k8s/helm/durable-workflow \ ```yaml image: - tag: "2.9.8" + tag: "2.9.9" # Pin a digest in production: # digest: "sha256:abc123..." # memoPayloadStorage: "raw-json-v1" # Required for a digest or custom image. diff --git a/k8s/helm/durable-workflow/ci/existing-secrets-values.yaml b/k8s/helm/durable-workflow/ci/existing-secrets-values.yaml index 03daee1a3..11f807c49 100644 --- a/k8s/helm/durable-workflow/ci/existing-secrets-values.yaml +++ b/k8s/helm/durable-workflow/ci/existing-secrets-values.yaml @@ -1,7 +1,7 @@ # CI fixture: GitOps / externally-managed-secret path. The chart consumes # existing Secrets and renders no Secret resources of its own. image: - tag: "2.9.8" + tag: "2.9.9" externalDatabase: connection: pgsql diff --git a/k8s/helm/durable-workflow/ci/ingress-and-hpa-values.yaml b/k8s/helm/durable-workflow/ci/ingress-and-hpa-values.yaml index 298204fb9..70d9fa26c 100644 --- a/k8s/helm/durable-workflow/ci/ingress-and-hpa-values.yaml +++ b/k8s/helm/durable-workflow/ci/ingress-and-hpa-values.yaml @@ -1,6 +1,6 @@ # CI fixture: ingress + autoscaling enabled. Exercises optional templates. image: - tag: "2.9.8" + tag: "2.9.9" externalDatabase: connection: mysql diff --git a/k8s/helm/durable-workflow/ci/inline-secrets-values.yaml b/k8s/helm/durable-workflow/ci/inline-secrets-values.yaml index 2e88b700b..8160b1c33 100644 --- a/k8s/helm/durable-workflow/ci/inline-secrets-values.yaml +++ b/k8s/helm/durable-workflow/ci/inline-secrets-values.yaml @@ -2,7 +2,7 @@ # chart's render path is exercised end-to-end. Real deployments should use # existingSecret instead. image: - tag: "2.9.8" + tag: "2.9.9" externalDatabase: connection: mysql diff --git a/k8s/helm/durable-workflow/templates/_helpers.tpl b/k8s/helm/durable-workflow/templates/_helpers.tpl index ef7394b29..fb7c05772 100644 --- a/k8s/helm/durable-workflow/templates/_helpers.tpl +++ b/k8s/helm/durable-workflow/templates/_helpers.tpl @@ -88,7 +88,7 @@ resolved by an explicit capability declaration or an existing workload marker. {{- define "durable-workflow.memoPayloadStorageForImage" -}} {{- $image := toString . -}} {{- $normalized := regexReplaceAll "^index\\.docker\\.io/" $image "docker.io/" -}} -{{- if eq $normalized "docker.io/durableworkflow/server:2.9.8" -}} +{{- if eq $normalized "docker.io/durableworkflow/server:2.9.9" -}} dual-v1 {{- else if regexMatch "^docker\\.io/durableworkflow/server:2\\.0\\.0-rc\\.[0-9]+$" $normalized -}} {{- $releaseCandidate := atoi (regexFind "[0-9]+$" $normalized) -}} diff --git a/k8s/helm/durable-workflow/values.yaml b/k8s/helm/durable-workflow/values.yaml index e4f51aa9a..3aac505f6 100644 --- a/k8s/helm/durable-workflow/values.yaml +++ b/k8s/helm/durable-workflow/values.yaml @@ -21,7 +21,7 @@ image: registry: docker.io repository: durableworkflow/server # Generated by scripts/ci/sync-source-release.mjs. Do not edit this default. - tag: "2.9.8" + tag: "2.9.9" # Optional digest pin. When set, takes precedence over tag for change control. # Example: "sha256:abc123..." digest: "" diff --git a/k8s/helm/examples/values-dev.yaml b/k8s/helm/examples/values-dev.yaml index b20fdfdf3..871391cbe 100644 --- a/k8s/helm/examples/values-dev.yaml +++ b/k8s/helm/examples/values-dev.yaml @@ -3,7 +3,7 @@ # shape in production. image: - tag: "2.9.8" + tag: "2.9.9" externalDatabase: connection: mysql diff --git a/k8s/helm/examples/values-external-secrets-operator.yaml b/k8s/helm/examples/values-external-secrets-operator.yaml index b251debb7..2bbfd25c3 100644 --- a/k8s/helm/examples/values-external-secrets-operator.yaml +++ b/k8s/helm/examples/values-external-secrets-operator.yaml @@ -5,7 +5,7 @@ # concern. image: - tag: "2.9.8" + tag: "2.9.9" externalDatabase: connection: pgsql diff --git a/k8s/helm/examples/values-production-existing-secrets.yaml b/k8s/helm/examples/values-production-existing-secrets.yaml index 0f3cb833d..e1faeeff0 100644 --- a/k8s/helm/examples/values-production-existing-secrets.yaml +++ b/k8s/helm/examples/values-production-existing-secrets.yaml @@ -10,7 +10,7 @@ image: repository: durable-workflow/server # Pin a digest in production for change-control auditability. digest: "" # e.g. "sha256:abc123..." - tag: "2.9.8" + tag: "2.9.9" externalDatabase: connection: pgsql diff --git a/k8s/migration-job.yaml b/k8s/migration-job.yaml index e5e21c004..a514b2e47 100644 --- a/k8s/migration-job.yaml +++ b/k8s/migration-job.yaml @@ -13,7 +13,7 @@ spec: restartPolicy: OnFailure containers: - name: migrate - image: durableworkflow/server:2.9.8 + image: durableworkflow/server:2.9.9 command: ["server-entrypoint"] args: ["server-bootstrap"] envFrom: diff --git a/k8s/scheduler-cronjob.yaml b/k8s/scheduler-cronjob.yaml index 18aaba0dd..6017ed7c3 100644 --- a/k8s/scheduler-cronjob.yaml +++ b/k8s/scheduler-cronjob.yaml @@ -24,7 +24,7 @@ spec: restartPolicy: Never containers: - name: scheduler - image: durableworkflow/server:2.9.8 + image: durableworkflow/server:2.9.9 command: ["server-entrypoint"] args: - sh diff --git a/k8s/secret.yaml b/k8s/secret.yaml index c0ac4bfeb..753b17c9e 100644 --- a/k8s/secret.yaml +++ b/k8s/secret.yaml @@ -12,7 +12,7 @@ metadata: app.kubernetes.io/name: durable-workflow data: APP_NAME: "Durable Workflow Server" - APP_VERSION: "2.9.8" + APP_VERSION: "2.9.9" APP_ENV: production APP_DEBUG: "false" DB_CONNECTION: mysql diff --git a/k8s/server-deployment.yaml b/k8s/server-deployment.yaml index e486ec44a..15c71b386 100644 --- a/k8s/server-deployment.yaml +++ b/k8s/server-deployment.yaml @@ -23,7 +23,7 @@ spec: spec: containers: - name: server - image: durableworkflow/server:2.9.8 + image: durableworkflow/server:2.9.9 ports: - containerPort: 8080 name: http diff --git a/k8s/worker-deployment.yaml b/k8s/worker-deployment.yaml index 2137c6c99..910009def 100644 --- a/k8s/worker-deployment.yaml +++ b/k8s/worker-deployment.yaml @@ -19,7 +19,7 @@ spec: spec: containers: - name: worker - image: durableworkflow/server:2.9.8 + image: durableworkflow/server:2.9.9 command: ["server-entrypoint"] args: ["php", "artisan", "queue:work", "--sleep=1", "--tries=3", "--max-time=3600"] envFrom: diff --git a/resources/release/source-release.json b/resources/release/source-release.json index ffb24ceeb..234add484 100644 --- a/resources/release/source-release.json +++ b/resources/release/source-release.json @@ -1,9 +1,9 @@ { "schema": "durable-workflow.server.source-release/v1", "server": { - "version": "2.9.8" + "version": "2.9.9" }, "helm_chart": { - "version": "0.1.184" + "version": "0.1.185" } } diff --git a/scripts/k8s-kind-smoke.sh b/scripts/k8s-kind-smoke.sh index 9ce085d87..9d49762dd 100755 --- a/scripts/k8s-kind-smoke.sh +++ b/scripts/k8s-kind-smoke.sh @@ -7,7 +7,7 @@ cluster="${K8S_SMOKE_CLUSTER:-durable-workflow-server-smoke}" image="${K8S_SMOKE_IMAGE:-durableworkflow/server:k8s-smoke}" # Generated by scripts/ci/sync-source-release.mjs so the smoke replaces the # same default shipped by the public manifests. -manifest_image="durableworkflow/server:2.9.8" +manifest_image="durableworkflow/server:2.9.9" kind_node_image="${K8S_SMOKE_KIND_NODE_IMAGE:-kindest/node:v1.29.4}" artifact_dir="${K8S_SMOKE_ARTIFACT_DIR:-/tmp/durable-workflow-k8s-kind-smoke-artifacts}" rendered_dir="${artifact_dir}/rendered-manifests" From f4f78d0d97a336208c6744c5731bab5b3886c7bc Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 15:51:41 +0000 Subject: [PATCH 36/37] Record two repeated matched PHP/native pairs after #516 Two alternating PHP/native pairs on a second dedicated host repeat the morning's matched comparison on main after #516. Native meets or beats PHP in every window, with no drift within any run. PHP's own capacity at 1/s differs between the two hosts, which the observations record. Part of #325. Co-Authored-By: Claude Opus 5.5 --- benchmarks/server-port/v3/README.md | 4 +- .../server-port/v3/repeated-2026-10-20.md | 101 ++ .../summary-repeated-native-1-2026-10-20.json | 911 +++++++++++++ .../summary-repeated-native-2-2026-10-20.json | 911 +++++++++++++ .../v3/summary-repeated-php-1-2026-10-20.json | 1124 +++++++++++++++++ .../v3/summary-repeated-php-2-2026-10-20.json | 1124 +++++++++++++++++ 6 files changed, 4174 insertions(+), 1 deletion(-) create mode 100644 benchmarks/server-port/v3/repeated-2026-10-20.md create mode 100644 benchmarks/server-port/v3/summary-repeated-native-1-2026-10-20.json create mode 100644 benchmarks/server-port/v3/summary-repeated-native-2-2026-10-20.json create mode 100644 benchmarks/server-port/v3/summary-repeated-php-1-2026-10-20.json create mode 100644 benchmarks/server-port/v3/summary-repeated-php-2-2026-10-20.json diff --git a/benchmarks/server-port/v3/README.md b/benchmarks/server-port/v3/README.md index c99656586..f74ef55e1 100644 --- a/benchmarks/server-port/v3/README.md +++ b/benchmarks/server-port/v3/README.md @@ -5,7 +5,9 @@ run's native source, optimized binary and machine facts filled in. The workload, artifacts, limits and probe hashes are unchanged. The [October 20 observations](observations-2026-10-20.md) are one PHP/native pair on a disposable VM, run with `scripts/perf/run-server-port-matched.sh` -and summarized with `scripts/perf/summarize-server-port-matched.py`. +and summarized with `scripts/perf/summarize-server-port-matched.py`. The +[repeated observations](repeated-2026-10-20.md) are two alternating pairs +on a second host, after #516. To repeat it on a fresh dedicated host with Docker and Compose: diff --git a/benchmarks/server-port/v3/repeated-2026-10-20.md b/benchmarks/server-port/v3/repeated-2026-10-20.md new file mode 100644 index 000000000..f5d1eebed --- /dev/null +++ b/benchmarks/server-port/v3/repeated-2026-10-20.md @@ -0,0 +1,101 @@ +# Repeated matched observations, October 20 + +Two alternating PHP/native pairs on one dedicated experiment host, after #516 +landed on main. They repeat the [morning pair](observations-2026-10-20.md) +with the same workload, artifacts, limits and commands. These are development +observations on one host and one database family, not a release claim. + +## Conditions + +- **Source.** Server main `8957ba27a` (#516 merged), native tree `55aec4d8`. + The optimized binary was built with `cargo build --release --locked --bin + durable-workflow-server`, SHA-256 `edc283f1…6e2cc`. The PHP references, + PHP SDK, MySQL and Redis artifacts are the profile's, unchanged. +- **Host.** A fresh Vultr `vhp-4c-8gb-amd` VM: 4 vCPU AMD EPYC-Rome, 7.7 GiB, + Ubuntu 24.04.5 with kernel 6.8, Docker 29.1.3 and Compose 2.40.3, no swap. + No other workload ran. It ran from 13:15 to 15:48 UTC, was deleted at the + end and its deletion was verified, at about $0.20. +- **Procedure.** A native smoke check, then PHP, native, PHP and native full + runs in that order. Each run uses its own Compose project and fresh + database. Each run is the morning procedure: warmup, three 120 s windows at + 0.25, 0.5 and 1 workflow/s, the 0.25/s windows repeated after as a drift + check, and mixed idle polls. The four runs' summaries are in + `summary-repeated-{php,native}-{1,2}-2026-10-20.json`. + +All four runs finished with status 0. Every window's workflows verified, with +nothing pending. No container restarted or was OOM-killed. + +## Offered work and latency + +Each cell gives the range of the three per-window values. + +| Run / rate | In-window completions | p50 s | p95 s | p99 s | +| --- | --- | --- | --- | --- | +| PHP 1 / 0.25 before | 30/30/30 | 1.129–1.188 | 1.459–1.620 | 1.515–1.652 | +| PHP 1 / 0.5 | 60/60/60 | 1.129–1.215 | 1.517–1.545 | 1.633–1.675 | +| PHP 1 / 1 | 69/68/66 of 120 | 44.782–48.343 | 64.771–67.389 | 66.026–69.027 | +| PHP 1 / 0.25 after | 30/30/30 | 1.195–1.226 | 1.568–1.732 | 1.686–1.818 | +| Native 1 / 0.25 before | 30/30/30 | 0.815–1.171 | 1.485–1.785 | 1.867–2.202 | +| Native 1 / 0.5 | 60/60/60 | 0.806–0.862 | 1.361–1.434 | 1.490–1.730 | +| Native 1 / 1 | 120/120/120 | 0.642–0.688 | 1.139–2.111 | 1.604–2.545 | +| Native 1 / 0.25 after | 30/30/30 | 1.061–1.139 | 1.691–1.887 | 1.916–2.042 | +| PHP 2 / 0.25 before | 30/30/30 | 1.236–1.306 | 1.649–1.680 | 1.697–1.818 | +| PHP 2 / 0.5 | 60/60/60 | 1.179–1.268 | 1.557–1.635 | 1.694–1.867 | +| PHP 2 / 1 | 62/67/65 of 120 | 48.434–55.744 | 65.491–72.870 | 66.325–74.189 | +| PHP 2 / 0.25 after | 30/30/30 | 1.164–1.258 | 1.441–1.643 | 1.531–1.731 | +| Native 2 / 0.25 before | 30/30/30 | 0.942–0.989 | 1.655–1.876 | 1.860–1.944 | +| Native 2 / 0.5 | 60/60/60 | 0.874–0.985 | 1.380–1.555 | 1.618–1.918 | +| Native 2 / 1 | 120/119/120 | 0.675–0.715 | 1.121–1.567 | 1.759–2.945 | +| Native 2 / 0.25 after | 30/30/30 | 0.951–1.049 | 1.503–1.822 | 1.716–1.958 | + +PHP still does not sustain 1/s. Its in-window completions at 1/s fell from +93–96 this morning to 62–69 here, and its 1/s median latency grew from 15–18 s +to 45–56 s. Its CPU per workflow also rose about a quarter on this host, as the +next table shows. Native completed all 120 in both pairs here and this +morning. The two PHP runs agree with each other, and so do the two native runs. +So the change is between hosts, not drift within this one. + +## CPU and memory + +Core-seconds per verified workflow, and the highest container memory sampled. + +| Run / rate | Runtime core-s/wf | MySQL core-s/wf | Runtime max MiB | +| --- | --- | --- | --- | +| PHP 1 / 0.25 before | 2.574–2.755 | 0.662–0.713 | 192.0 | +| PHP 1 / 1 | 1.599–1.638 | 0.554–0.566 | 201.2 | +| PHP 2 / 0.25 before | 2.737–2.829 | 0.710–0.748 | 203.0 | +| PHP 2 / 1 | 1.603–1.676 | 0.549–0.573 | 213.1 | +| Native 1 / 0.25 before | 0.239–0.247 | 0.489–0.506 | 6.3 | +| Native 1 / 1 | 0.102–0.104 | 0.241–0.245 | 7.0 | +| Native 2 / 0.25 before | 0.245–0.256 | 0.503–0.518 | 6.9 | +| Native 2 / 1 | 0.110–0.116 | 0.264–0.275 | 6.8 | + +At 1/s, native uses about a fifteenth of PHP's runtime CPU per workflow and +about half of its database CPU. Its peak memory is about 7 MiB against PHP's +190–215 MiB. + +## Idle mixed long polls + +Runtime core-seconds per idle window, three windows each. + +| Run | 6 polls | 12 polls | 24 polls | Health failures | +| --- | --- | --- | --- | --- | +| PHP 1 | 6.117–6.877 | 7.279–8.642 | 9.160–9.564 | 0, 1, 2 per window | +| Native 1 | 0.645–0.683 | 0.790–0.917 | 1.206–1.256 | none | +| PHP 2 | 5.699–6.651 | 7.803–8.198 | 9.408–9.695 | 0, 1, 2–3 per window | +| Native 2 | 0.684–0.755 | 0.822–0.922 | 1.189–1.241 | none | + +PHP answers part of its 12- and 24-poll windows with backpressure, and its +health checks fail in them. Native answers every poll empty, as expected. + +## What this shows and what it does not + +On this profile, native meets or beats PHP on throughput, CPU, memory and idle +long-poll cost in every window of two repeated pairs. The before and after +windows show no drift within any run. + +It does not show the result on PostgreSQL or SQLite, on other machine types, +across nodes, or under the CPU-heavy deadline profile. The host-to-host change +in PHP's capacity also says one host is not enough for a release claim. The +driver records host load and swap but not CPU steal. Sampling steal in +`host-state.tsv` is the next change to the driver. diff --git a/benchmarks/server-port/v3/summary-repeated-native-1-2026-10-20.json b/benchmarks/server-port/v3/summary-repeated-native-1-2026-10-20.json new file mode 100644 index 000000000..89b505fd0 --- /dev/null +++ b/benchmarks/server-port/v3/summary-repeated-native-1-2026-10-20.json @@ -0,0 +1,911 @@ +{ + "identity": { + "implementation": "native", + "compose_project": "dw-port-native-20261020t141652", + "profile": "/home/linuxuser/profile.json", + "profile_sha256": "5e7946a6a506e391510e9222021e60c80d184742b030ee6e2978ab0c0b1e8ac5", + "source_revision": "8957ba27aa24b79ec2ea59e0c37b5ce662d434ab", + "native_binary_sha256": "edc283f130ff5c802fe5a80d56730cc9fe377c01c9c90340e08a68eb0866e2cc", + "smoke": false, + "idle_only": false, + "machine": { + "kernel": "6.8.0-146-generic", + "logical_cpus": 4, + "physical_memory_bytes": 8320786432, + "cpu_model": "AMD EPYC-Rome Processor", + "docker": "29.1.3", + "cgroup_driver": "systemd", + "docker_storage_driver": "overlayfs", + "docker_root_filesystem": "ext4" + } + }, + "windows": { + "setup": { + "seconds": 5.4, + "health": { + "replies": 8, + "failures": 0, + "slowest_ok_seconds": 0.302 + }, + "cpu_core_seconds": { + "mysql": 0.876176, + "native-server": 0.408746, + "probe": 0.0, + "sdk-worker": 0.039165 + }, + "max_memory_bytes": { + "mysql": 527196160, + "native-server": 4235264, + "probe": 16654336, + "sdk-worker": 19292160 + }, + "max_runtime_memory_bytes": 4235264, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 2, + "accepted_starts": 2, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 2, + "pending_at_deadline": 0, + "closed_during_offer_window": 2, + "latency_p50_seconds": 0.2922959327697754, + "latency_p95_seconds": 0.5931649208068848, + "latency_p99_seconds": 0.5931649208068848, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.2044, + "mysql_core_seconds_per_workflow": 0.4381 + }, + "warmup": { + "seconds": 59.3, + "health": { + "replies": 88, + "failures": 0, + "slowest_ok_seconds": 0.476 + }, + "cpu_core_seconds": { + "mysql": 7.408028, + "native-server": 3.565426, + "probe": 0.383084, + "sdk-worker": 0.367419 + }, + "max_memory_bytes": { + "mysql": 540975104, + "native-server": 6533120, + "probe": 16683008, + "sdk-worker": 19447808 + }, + "max_runtime_memory_bytes": 6533120, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 15, + "accepted_starts": 15, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 15, + "pending_at_deadline": 0, + "closed_during_offer_window": 15, + "latency_p50_seconds": 1.2313079833984375, + "latency_p95_seconds": 2.112252950668335, + "latency_p99_seconds": 2.112252950668335, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.2377, + "mysql_core_seconds_per_workflow": 0.4939 + }, + "low-before-1": { + "seconds": 120.5, + "health": { + "replies": 181, + "failures": 0, + "slowest_ok_seconds": 0.431 + }, + "cpu_core_seconds": { + "mysql": 14.753484, + "native-server": 7.199888, + "probe": 0.365237, + "sdk-worker": 0.774945 + }, + "max_memory_bytes": { + "mysql": 556244992, + "native-server": 6656000, + "probe": 16691200, + "sdk-worker": 21241856 + }, + "max_runtime_memory_bytes": 6656000, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 30, + "accepted_starts": 30, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 30, + "pending_at_deadline": 0, + "closed_during_offer_window": 30, + "latency_p50_seconds": 0.8146710395812988, + "latency_p95_seconds": 1.784688949584961, + "latency_p99_seconds": 1.8666729927062988, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.24, + "mysql_core_seconds_per_workflow": 0.4918 + }, + "low-before-2": { + "seconds": 119.5, + "health": { + "replies": 178, + "failures": 0, + "slowest_ok_seconds": 0.435 + }, + "cpu_core_seconds": { + "mysql": 15.181942, + "native-server": 7.39961, + "probe": 0.023146, + "sdk-worker": 0.804998 + }, + "max_memory_bytes": { + "mysql": 569720832, + "native-server": 5312512, + "probe": 16683008, + "sdk-worker": 23449600 + }, + "max_runtime_memory_bytes": 5312512, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 30, + "accepted_starts": 30, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 30, + "pending_at_deadline": 0, + "closed_during_offer_window": 30, + "latency_p50_seconds": 1.170957088470459, + "latency_p95_seconds": 1.6039619445800781, + "latency_p99_seconds": 2.2019379138946533, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.2467, + "mysql_core_seconds_per_workflow": 0.5061 + }, + "low-before-3": { + "seconds": 120.1, + "health": { + "replies": 183, + "failures": 0, + "slowest_ok_seconds": 0.342 + }, + "cpu_core_seconds": { + "mysql": 14.658095, + "native-server": 7.169432, + "probe": -0.017477, + "sdk-worker": 0.760215 + }, + "max_memory_bytes": { + "mysql": 580112384, + "native-server": 5115904, + "probe": 16936960, + "sdk-worker": 24776704 + }, + "max_runtime_memory_bytes": 5115904, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 30, + "accepted_starts": 30, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 30, + "pending_at_deadline": 0, + "closed_during_offer_window": 30, + "latency_p50_seconds": 0.8877580165863037, + "latency_p95_seconds": 1.4847869873046875, + "latency_p99_seconds": 2.122774839401245, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.239, + "mysql_core_seconds_per_workflow": 0.4886 + }, + "rate-0.5-1": { + "seconds": 120.2, + "health": { + "replies": 174, + "failures": 0, + "slowest_ok_seconds": 0.527 + }, + "cpu_core_seconds": { + "mysql": 19.666724, + "native-server": 8.952809, + "probe": 0.261888, + "sdk-worker": 1.170249 + }, + "max_memory_bytes": { + "mysql": 604008448, + "native-server": 7004160, + "probe": 16703488, + "sdk-worker": 28123136 + }, + "max_runtime_memory_bytes": 7004160, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.5, + "planned_starts": 60, + "accepted_starts": 60, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 60, + "pending_at_deadline": 0, + "closed_during_offer_window": 60, + "latency_p50_seconds": 0.8057119846343994, + "latency_p95_seconds": 1.4087231159210205, + "latency_p99_seconds": 1.4897749423980713, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.1492, + "mysql_core_seconds_per_workflow": 0.3278 + }, + "rate-0.5-2": { + "seconds": 120.6, + "health": { + "replies": 169, + "failures": 0, + "slowest_ok_seconds": 0.606 + }, + "cpu_core_seconds": { + "mysql": 20.18183, + "native-server": 9.157594, + "probe": -0.046921, + "sdk-worker": 1.216679 + }, + "max_memory_bytes": { + "mysql": 626311168, + "native-server": 6995968, + "probe": 16932864, + "sdk-worker": 30769152 + }, + "max_runtime_memory_bytes": 6995968, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.5, + "planned_starts": 60, + "accepted_starts": 60, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 60, + "pending_at_deadline": 0, + "closed_during_offer_window": 60, + "latency_p50_seconds": 0.8616900444030762, + "latency_p95_seconds": 1.3606820106506348, + "latency_p99_seconds": 1.7296950817108154, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.1526, + "mysql_core_seconds_per_workflow": 0.3364 + }, + "rate-0.5-3": { + "seconds": 119.8, + "health": { + "replies": 168, + "failures": 0, + "slowest_ok_seconds": 0.595 + }, + "cpu_core_seconds": { + "mysql": 19.68832, + "native-server": 8.950379, + "probe": -0.012808, + "sdk-worker": 1.201273 + }, + "max_memory_bytes": { + "mysql": 647774208, + "native-server": 7008256, + "probe": 16949248, + "sdk-worker": 33390592 + }, + "max_runtime_memory_bytes": 7008256, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.5, + "planned_starts": 60, + "accepted_starts": 60, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 60, + "pending_at_deadline": 0, + "closed_during_offer_window": 60, + "latency_p50_seconds": 0.8583979606628418, + "latency_p95_seconds": 1.4343540668487549, + "latency_p99_seconds": 1.5413579940795898, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.1492, + "mysql_core_seconds_per_workflow": 0.3281 + }, + "rate-1-1": { + "seconds": 121.3, + "health": { + "replies": 158, + "failures": 0, + "slowest_ok_seconds": 0.763 + }, + "cpu_core_seconds": { + "mysql": 28.940648, + "native-server": 12.366946, + "probe": 0.301992, + "sdk-worker": 2.071431 + }, + "max_memory_bytes": { + "mysql": 691490816, + "native-server": 7041024, + "probe": 16936960, + "sdk-worker": 36601856 + }, + "max_runtime_memory_bytes": 7041024, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 1, + "planned_starts": 120, + "accepted_starts": 120, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 120, + "pending_at_deadline": 0, + "closed_during_offer_window": 120, + "latency_p50_seconds": 0.642268180847168, + "latency_p95_seconds": 1.1391160488128662, + "latency_p99_seconds": 1.6041810512542725, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.1031, + "mysql_core_seconds_per_workflow": 0.2412 + }, + "rate-1-2": { + "seconds": 121.3, + "health": { + "replies": 156, + "failures": 0, + "slowest_ok_seconds": 0.764 + }, + "cpu_core_seconds": { + "mysql": 29.034662, + "native-server": 12.278764, + "probe": -0.090032, + "sdk-worker": 2.080174 + }, + "max_memory_bytes": { + "mysql": 735203328, + "native-server": 7110656, + "probe": 18374656, + "sdk-worker": 36524032 + }, + "max_runtime_memory_bytes": 7110656, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 1, + "planned_starts": 120, + "accepted_starts": 120, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 120, + "pending_at_deadline": 0, + "closed_during_offer_window": 120, + "latency_p50_seconds": 0.6831719875335693, + "latency_p95_seconds": 2.1111860275268555, + "latency_p99_seconds": 2.5451819896698, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.1023, + "mysql_core_seconds_per_workflow": 0.242 + }, + "rate-1-3": { + "seconds": 121.6, + "health": { + "replies": 156, + "failures": 0, + "slowest_ok_seconds": 0.717 + }, + "cpu_core_seconds": { + "mysql": 29.387882, + "native-server": 12.440495, + "probe": 0.039508, + "sdk-worker": 2.065757 + }, + "max_memory_bytes": { + "mysql": 777027584, + "native-server": 7311360, + "probe": 16703488, + "sdk-worker": 37605376 + }, + "max_runtime_memory_bytes": 7311360, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 1, + "planned_starts": 120, + "accepted_starts": 120, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 120, + "pending_at_deadline": 0, + "closed_during_offer_window": 120, + "latency_p50_seconds": 0.6876001358032227, + "latency_p95_seconds": 1.6906061172485352, + "latency_p99_seconds": 2.5086309909820557, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.1037, + "mysql_core_seconds_per_workflow": 0.2449 + }, + "low-after-1": { + "seconds": 120.1, + "health": { + "replies": 180, + "failures": 0, + "slowest_ok_seconds": 0.387 + }, + "cpu_core_seconds": { + "mysql": 15.418894, + "native-server": 7.249945, + "probe": -0.469773, + "sdk-worker": 0.772241 + }, + "max_memory_bytes": { + "mysql": 788262912, + "native-server": 6823936, + "probe": 16687104, + "sdk-worker": 37871616 + }, + "max_runtime_memory_bytes": 6823936, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 30, + "accepted_starts": 30, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 30, + "pending_at_deadline": 0, + "closed_during_offer_window": 30, + "latency_p50_seconds": 1.120251178741455, + "latency_p95_seconds": 1.8871901035308838, + "latency_p99_seconds": 1.9591500759124756, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.2417, + "mysql_core_seconds_per_workflow": 0.514 + }, + "low-after-2": { + "seconds": 120.2, + "health": { + "replies": 180, + "failures": 0, + "slowest_ok_seconds": 0.429 + }, + "cpu_core_seconds": { + "mysql": 15.669081, + "native-server": 7.285336, + "probe": 0.020311, + "sdk-worker": 0.811745 + }, + "max_memory_bytes": { + "mysql": 794497024, + "native-server": 7094272, + "probe": 16699392, + "sdk-worker": 38146048 + }, + "max_runtime_memory_bytes": 7094272, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 30, + "accepted_starts": 30, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 30, + "pending_at_deadline": 0, + "closed_during_offer_window": 30, + "latency_p50_seconds": 1.0607280731201172, + "latency_p95_seconds": 1.8657190799713135, + "latency_p99_seconds": 1.9156861305236816, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.2428, + "mysql_core_seconds_per_workflow": 0.5223 + }, + "low-after-3": { + "seconds": 120.2, + "health": { + "replies": 180, + "failures": 0, + "slowest_ok_seconds": 0.388 + }, + "cpu_core_seconds": { + "mysql": 15.794167, + "native-server": 7.405617, + "probe": 0.017688, + "sdk-worker": 0.800382 + }, + "max_memory_bytes": { + "mysql": 800604160, + "native-server": 6619136, + "probe": 16941056, + "sdk-worker": 38514688 + }, + "max_runtime_memory_bytes": 6619136, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 30, + "accepted_starts": 30, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 30, + "pending_at_deadline": 0, + "closed_during_offer_window": 30, + "latency_p50_seconds": 1.1393530368804932, + "latency_p95_seconds": 1.6913490295410156, + "latency_p99_seconds": 2.042344093322754, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.2469, + "mysql_core_seconds_per_workflow": 0.5265 + } + }, + "idle": { + "idle-6-1": { + "seconds": 12.3, + "health": { + "replies": 20, + "failures": 0, + "slowest_ok_seconds": 0.104 + }, + "cpu_core_seconds": { + "mysql": 1.271815, + "native-server": 0.678925, + "probe": -0.55062, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 801001472, + "native-server": 5025792, + "probe": 31444992, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 5025792, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "requested_polls": 6, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "empty": 6 + } + }, + "idle-6-2": { + "seconds": 11.2, + "health": { + "replies": 18, + "failures": 0, + "slowest_ok_seconds": 0.104 + }, + "cpu_core_seconds": { + "mysql": 1.277346, + "native-server": 0.682507, + "probe": -0.004253, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 804032512, + "native-server": 6746112, + "probe": 31248384, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 6746112, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "requested_polls": 6, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "empty": 6 + } + }, + "idle-6-3": { + "seconds": 11.3, + "health": { + "replies": 17, + "failures": 0, + "slowest_ok_seconds": 0.105 + }, + "cpu_core_seconds": { + "mysql": 1.187882, + "native-server": 0.644606, + "probe": 0.024741, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 802537472, + "native-server": 6971392, + "probe": 31236096, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 6971392, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "requested_polls": 6, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "empty": 6 + } + }, + "idle-12-1": { + "seconds": 11.5, + "health": { + "replies": 17, + "failures": 0, + "slowest_ok_seconds": 0.105 + }, + "cpu_core_seconds": { + "mysql": 1.518828, + "native-server": 0.80815, + "probe": 0.14792, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 801198080, + "native-server": 5578752, + "probe": 45658112, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 5578752, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "requested_polls": 12, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "empty": 12 + } + }, + "idle-12-2": { + "seconds": 11.3, + "health": { + "replies": 18, + "failures": 0, + "slowest_ok_seconds": 0.016 + }, + "cpu_core_seconds": { + "mysql": 1.467164, + "native-server": 0.789659, + "probe": -0.105134, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 801349632, + "native-server": 5386240, + "probe": 44597248, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 5386240, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "requested_polls": 12, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "empty": 12 + } + }, + "idle-12-3": { + "seconds": 11.5, + "health": { + "replies": 16, + "failures": 0, + "slowest_ok_seconds": 0.104 + }, + "cpu_core_seconds": { + "mysql": 1.661435, + "native-server": 0.916948, + "probe": 0.13414, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 801509376, + "native-server": 5398528, + "probe": 43933696, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 5398528, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "requested_polls": 12, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "empty": 12 + } + }, + "idle-24-1": { + "seconds": 11.9, + "health": { + "replies": 18, + "failures": 0, + "slowest_ok_seconds": 0.019 + }, + "cpu_core_seconds": { + "mysql": 2.294075, + "native-server": 1.256186, + "probe": 0.249966, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 801611776, + "native-server": 5935104, + "probe": 68399104, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 5935104, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "requested_polls": 24, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "empty": 24 + } + }, + "idle-24-2": { + "seconds": 12.2, + "health": { + "replies": 18, + "failures": 0, + "slowest_ok_seconds": 0.017 + }, + "cpu_core_seconds": { + "mysql": 2.315425, + "native-server": 1.232444, + "probe": -0.043559, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 801931264, + "native-server": 5935104, + "probe": 67612672, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 5935104, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "requested_polls": 24, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "empty": 24 + } + }, + "idle-24-3": { + "seconds": 12.2, + "health": { + "replies": 19, + "failures": 0, + "slowest_ok_seconds": 0.019 + }, + "cpu_core_seconds": { + "mysql": 2.217726, + "native-server": 1.206354, + "probe": 0.083076, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 801779712, + "native-server": 6131712, + "probe": 73183232, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 6131712, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "requested_polls": 24, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "empty": 24 + } + } + } +} diff --git a/benchmarks/server-port/v3/summary-repeated-native-2-2026-10-20.json b/benchmarks/server-port/v3/summary-repeated-native-2-2026-10-20.json new file mode 100644 index 000000000..36aa1e61c --- /dev/null +++ b/benchmarks/server-port/v3/summary-repeated-native-2-2026-10-20.json @@ -0,0 +1,911 @@ +{ + "identity": { + "implementation": "native", + "compose_project": "dw-port-native-20261020t151845", + "profile": "/home/linuxuser/profile.json", + "profile_sha256": "5e7946a6a506e391510e9222021e60c80d184742b030ee6e2978ab0c0b1e8ac5", + "source_revision": "8957ba27aa24b79ec2ea59e0c37b5ce662d434ab", + "native_binary_sha256": "edc283f130ff5c802fe5a80d56730cc9fe377c01c9c90340e08a68eb0866e2cc", + "smoke": false, + "idle_only": false, + "machine": { + "kernel": "6.8.0-146-generic", + "logical_cpus": 4, + "physical_memory_bytes": 8320786432, + "cpu_model": "AMD EPYC-Rome Processor", + "docker": "29.1.3", + "cgroup_driver": "systemd", + "docker_storage_driver": "overlayfs", + "docker_root_filesystem": "ext4" + } + }, + "windows": { + "setup": { + "seconds": 5.5, + "health": { + "replies": 8, + "failures": 0, + "slowest_ok_seconds": 0.185 + }, + "cpu_core_seconds": { + "mysql": 0.78864, + "native-server": 0.406774, + "probe": 0.006179, + "sdk-worker": 0.046886 + }, + "max_memory_bytes": { + "mysql": 529104896, + "native-server": 4411392, + "probe": 16662528, + "sdk-worker": 18796544 + }, + "max_runtime_memory_bytes": 4411392, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 2, + "accepted_starts": 2, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 2, + "pending_at_deadline": 0, + "closed_during_offer_window": 2, + "latency_p50_seconds": 0.264415979385376, + "latency_p95_seconds": 0.4201939105987549, + "latency_p99_seconds": 0.4201939105987549, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.2034, + "mysql_core_seconds_per_workflow": 0.3943 + }, + "warmup": { + "seconds": 60.4, + "health": { + "replies": 90, + "failures": 0, + "slowest_ok_seconds": 0.434 + }, + "cpu_core_seconds": { + "mysql": 7.903876, + "native-server": 3.848973, + "probe": 0.431635, + "sdk-worker": 0.430859 + }, + "max_memory_bytes": { + "mysql": 544555008, + "native-server": 6356992, + "probe": 16945152, + "sdk-worker": 18952192 + }, + "max_runtime_memory_bytes": 6356992, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 15, + "accepted_starts": 15, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 15, + "pending_at_deadline": 0, + "closed_during_offer_window": 15, + "latency_p50_seconds": 0.7263832092285156, + "latency_p95_seconds": 1.335334062576294, + "latency_p99_seconds": 1.335334062576294, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.2566, + "mysql_core_seconds_per_workflow": 0.5269 + }, + "low-before-1": { + "seconds": 120.3, + "health": { + "replies": 180, + "failures": 0, + "slowest_ok_seconds": 0.344 + }, + "cpu_core_seconds": { + "mysql": 15.542938, + "native-server": 7.685158, + "probe": 0.347865, + "sdk-worker": 0.825953 + }, + "max_memory_bytes": { + "mysql": 557219840, + "native-server": 7204864, + "probe": 18534400, + "sdk-worker": 20742144 + }, + "max_runtime_memory_bytes": 7204864, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 30, + "accepted_starts": 30, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 30, + "pending_at_deadline": 0, + "closed_during_offer_window": 30, + "latency_p50_seconds": 0.9674599170684814, + "latency_p95_seconds": 1.6804499626159668, + "latency_p99_seconds": 1.8725829124450684, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.2562, + "mysql_core_seconds_per_workflow": 0.5181 + }, + "low-before-2": { + "seconds": 119.7, + "health": { + "replies": 180, + "failures": 0, + "slowest_ok_seconds": 0.428 + }, + "cpu_core_seconds": { + "mysql": 15.094367, + "native-server": 7.400587, + "probe": 0.013393, + "sdk-worker": 0.78088 + }, + "max_memory_bytes": { + "mysql": 569982976, + "native-server": 5160960, + "probe": 16703488, + "sdk-worker": 24485888 + }, + "max_runtime_memory_bytes": 5160960, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 30, + "accepted_starts": 30, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 30, + "pending_at_deadline": 0, + "closed_during_offer_window": 30, + "latency_p50_seconds": 0.989130973815918, + "latency_p95_seconds": 1.6551799774169922, + "latency_p99_seconds": 1.8601720333099365, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.2467, + "mysql_core_seconds_per_workflow": 0.5031 + }, + "low-before-3": { + "seconds": 120.0, + "health": { + "replies": 178, + "failures": 0, + "slowest_ok_seconds": 0.519 + }, + "cpu_core_seconds": { + "mysql": 15.215566, + "native-server": 7.359013, + "probe": -0.015717, + "sdk-worker": 0.806679 + }, + "max_memory_bytes": { + "mysql": 581537792, + "native-server": 5169152, + "probe": 16691200, + "sdk-worker": 24260608 + }, + "max_runtime_memory_bytes": 5169152, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 30, + "accepted_starts": 30, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 30, + "pending_at_deadline": 0, + "closed_during_offer_window": 30, + "latency_p50_seconds": 0.9421291351318359, + "latency_p95_seconds": 1.8761250972747803, + "latency_p99_seconds": 1.94413423538208, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.2453, + "mysql_core_seconds_per_workflow": 0.5072 + }, + "rate-0.5-1": { + "seconds": 119.9, + "health": { + "replies": 170, + "failures": 0, + "slowest_ok_seconds": 0.513 + }, + "cpu_core_seconds": { + "mysql": 21.17952, + "native-server": 9.658587, + "probe": 0.251825, + "sdk-worker": 1.308499 + }, + "max_memory_bytes": { + "mysql": 604405760, + "native-server": 6815744, + "probe": 16932864, + "sdk-worker": 27607040 + }, + "max_runtime_memory_bytes": 6815744, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.5, + "planned_starts": 60, + "accepted_starts": 60, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 60, + "pending_at_deadline": 0, + "closed_during_offer_window": 60, + "latency_p50_seconds": 0.874359130859375, + "latency_p95_seconds": 1.5553700923919678, + "latency_p99_seconds": 1.6183700561523438, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.161, + "mysql_core_seconds_per_workflow": 0.353 + }, + "rate-0.5-2": { + "seconds": 120.4, + "health": { + "replies": 166, + "failures": 0, + "slowest_ok_seconds": 0.548 + }, + "cpu_core_seconds": { + "mysql": 21.947495, + "native-server": 9.851621, + "probe": 0.012017, + "sdk-worker": 1.336684 + }, + "max_memory_bytes": { + "mysql": 628731904, + "native-server": 6828032, + "probe": 18669568, + "sdk-worker": 30281728 + }, + "max_runtime_memory_bytes": 6828032, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.5, + "planned_starts": 60, + "accepted_starts": 60, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 60, + "pending_at_deadline": 0, + "closed_during_offer_window": 60, + "latency_p50_seconds": 0.8867721557617188, + "latency_p95_seconds": 1.3797781467437744, + "latency_p99_seconds": 1.711742877960205, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.1642, + "mysql_core_seconds_per_workflow": 0.3658 + }, + "rate-0.5-3": { + "seconds": 120.6, + "health": { + "replies": 165, + "failures": 0, + "slowest_ok_seconds": 0.518 + }, + "cpu_core_seconds": { + "mysql": 24.106008, + "native-server": 10.781074, + "probe": 0.075858, + "sdk-worker": 1.460856 + }, + "max_memory_bytes": { + "mysql": 649342976, + "native-server": 7081984, + "probe": 16957440, + "sdk-worker": 32976896 + }, + "max_runtime_memory_bytes": 7081984, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.5, + "planned_starts": 60, + "accepted_starts": 60, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 60, + "pending_at_deadline": 0, + "closed_during_offer_window": 60, + "latency_p50_seconds": 0.9854328632354736, + "latency_p95_seconds": 1.5133790969848633, + "latency_p99_seconds": 1.9184279441833496, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.1797, + "mysql_core_seconds_per_workflow": 0.4018 + }, + "rate-1-1": { + "seconds": 121.0, + "health": { + "replies": 150, + "failures": 0, + "slowest_ok_seconds": 0.719 + }, + "cpu_core_seconds": { + "mysql": 33.022403, + "native-server": 13.945428, + "probe": 0.224814, + "sdk-worker": 2.466644 + }, + "max_memory_bytes": { + "mysql": 692928512, + "native-server": 7155712, + "probe": 16941056, + "sdk-worker": 36147200 + }, + "max_runtime_memory_bytes": 7155712, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 1, + "planned_starts": 120, + "accepted_starts": 120, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 120, + "pending_at_deadline": 0, + "closed_during_offer_window": 120, + "latency_p50_seconds": 0.7154889106750488, + "latency_p95_seconds": 1.5674889087677002, + "latency_p99_seconds": 2.94549298286438, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.1162, + "mysql_core_seconds_per_workflow": 0.2752 + }, + "rate-1-2": { + "seconds": 122.0, + "health": { + "replies": 153, + "failures": 0, + "slowest_ok_seconds": 0.86 + }, + "cpu_core_seconds": { + "mysql": 32.588333, + "native-server": 13.690904, + "probe": -0.024711, + "sdk-worker": 2.364287 + }, + "max_memory_bytes": { + "mysql": 736538624, + "native-server": 7127040, + "probe": 16945152, + "sdk-worker": 36696064 + }, + "max_runtime_memory_bytes": 7127040, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 1, + "planned_starts": 120, + "accepted_starts": 120, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 120, + "pending_at_deadline": 0, + "closed_during_offer_window": 119, + "latency_p50_seconds": 0.6948199272155762, + "latency_p95_seconds": 1.394827127456665, + "latency_p99_seconds": 1.7588160037994385, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.1141, + "mysql_core_seconds_per_workflow": 0.2716 + }, + "rate-1-3": { + "seconds": 121.1, + "health": { + "replies": 150, + "failures": 0, + "slowest_ok_seconds": 0.728 + }, + "cpu_core_seconds": { + "mysql": 31.711915, + "native-server": 13.246497, + "probe": -0.056814, + "sdk-worker": 2.334332 + }, + "max_memory_bytes": { + "mysql": 778280960, + "native-server": 7151616, + "probe": 16691200, + "sdk-worker": 37113856 + }, + "max_runtime_memory_bytes": 7151616, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 1, + "planned_starts": 120, + "accepted_starts": 120, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 120, + "pending_at_deadline": 0, + "closed_during_offer_window": 120, + "latency_p50_seconds": 0.6748061180114746, + "latency_p95_seconds": 1.1207969188690186, + "latency_p99_seconds": 2.0817408561706543, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.1104, + "mysql_core_seconds_per_workflow": 0.2643 + }, + "low-after-1": { + "seconds": 119.9, + "health": { + "replies": 174, + "failures": 0, + "slowest_ok_seconds": 0.48 + }, + "cpu_core_seconds": { + "mysql": 18.335757, + "native-server": 8.539634, + "probe": -0.419468, + "sdk-worker": 0.943311 + }, + "max_memory_bytes": { + "mysql": 789663744, + "native-server": 7131136, + "probe": 16699392, + "sdk-worker": 37392384 + }, + "max_runtime_memory_bytes": 7131136, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 30, + "accepted_starts": 30, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 30, + "pending_at_deadline": 0, + "closed_during_offer_window": 30, + "latency_p50_seconds": 1.0487730503082275, + "latency_p95_seconds": 1.6618030071258545, + "latency_p99_seconds": 1.9577748775482178, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.2847, + "mysql_core_seconds_per_workflow": 0.6112 + }, + "low-after-2": { + "seconds": 120.2, + "health": { + "replies": 176, + "failures": 0, + "slowest_ok_seconds": 0.483 + }, + "cpu_core_seconds": { + "mysql": 17.32578, + "native-server": 8.043092, + "probe": -0.002517, + "sdk-worker": 0.906545 + }, + "max_memory_bytes": { + "mysql": 795537408, + "native-server": 7127040, + "probe": 16687104, + "sdk-worker": 37650432 + }, + "max_runtime_memory_bytes": 7127040, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 30, + "accepted_starts": 30, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 30, + "pending_at_deadline": 0, + "closed_during_offer_window": 30, + "latency_p50_seconds": 0.989509105682373, + "latency_p95_seconds": 1.8215291500091553, + "latency_p99_seconds": 1.8415191173553467, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.2681, + "mysql_core_seconds_per_workflow": 0.5775 + }, + "low-after-3": { + "seconds": 119.1, + "health": { + "replies": 174, + "failures": 0, + "slowest_ok_seconds": 0.356 + }, + "cpu_core_seconds": { + "mysql": 17.427466, + "native-server": 8.173384, + "probe": -0.029393, + "sdk-worker": 0.915135 + }, + "max_memory_bytes": { + "mysql": 801325056, + "native-server": 7168000, + "probe": 18608128, + "sdk-worker": 39112704 + }, + "max_runtime_memory_bytes": 7168000, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 30, + "accepted_starts": 30, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 30, + "pending_at_deadline": 0, + "closed_during_offer_window": 30, + "latency_p50_seconds": 0.9510529041290283, + "latency_p95_seconds": 1.502997875213623, + "latency_p99_seconds": 1.7160539627075195, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.2724, + "mysql_core_seconds_per_workflow": 0.5809 + } + }, + "idle": { + "idle-6-1": { + "seconds": 12.4, + "health": { + "replies": 18, + "failures": 0, + "slowest_ok_seconds": 0.601 + }, + "cpu_core_seconds": { + "mysql": 1.388882, + "native-server": 0.694483, + "probe": -0.556331, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 801333248, + "native-server": 6881280, + "probe": 31330304, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 6881280, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "requested_polls": 6, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "empty": 6 + } + }, + "idle-6-2": { + "seconds": 11.2, + "health": { + "replies": 17, + "failures": 0, + "slowest_ok_seconds": 0.107 + }, + "cpu_core_seconds": { + "mysql": 1.305827, + "native-server": 0.755374, + "probe": -0.025734, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 801746944, + "native-server": 5115904, + "probe": 30846976, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 5115904, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "requested_polls": 6, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "empty": 6 + } + }, + "idle-6-3": { + "seconds": 11.4, + "health": { + "replies": 18, + "failures": 0, + "slowest_ok_seconds": 0.09 + }, + "cpu_core_seconds": { + "mysql": 1.233457, + "native-server": 0.683825, + "probe": 0.043972, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 801890304, + "native-server": 5021696, + "probe": 31555584, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 5021696, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "requested_polls": 6, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "empty": 6 + } + }, + "idle-12-1": { + "seconds": 11.8, + "health": { + "replies": 19, + "failures": 0, + "slowest_ok_seconds": 0.101 + }, + "cpu_core_seconds": { + "mysql": 1.61893, + "native-server": 0.822211, + "probe": 0.205678, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 802017280, + "native-server": 5345280, + "probe": 45867008, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 5345280, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "requested_polls": 12, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "empty": 12 + } + }, + "idle-12-2": { + "seconds": 11.4, + "health": { + "replies": 18, + "failures": 0, + "slowest_ok_seconds": 0.105 + }, + "cpu_core_seconds": { + "mysql": 1.645461, + "native-server": 0.921932, + "probe": -0.173109, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 802181120, + "native-server": 5292032, + "probe": 42070016, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 5292032, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "requested_polls": 12, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "empty": 12 + } + }, + "idle-12-3": { + "seconds": 11.5, + "health": { + "replies": 18, + "failures": 0, + "slowest_ok_seconds": 0.098 + }, + "cpu_core_seconds": { + "mysql": 1.590614, + "native-server": 0.873969, + "probe": 0.067627, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 802652160, + "native-server": 5328896, + "probe": 42647552, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 5328896, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "requested_polls": 12, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "empty": 12 + } + }, + "idle-24-1": { + "seconds": 12.2, + "health": { + "replies": 20, + "failures": 0, + "slowest_ok_seconds": 0.102 + }, + "cpu_core_seconds": { + "mysql": 2.273613, + "native-server": 1.241196, + "probe": 0.300321, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 802402304, + "native-server": 7852032, + "probe": 70094848, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 7852032, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "requested_polls": 24, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "empty": 24 + } + }, + "idle-24-2": { + "seconds": 11.7, + "health": { + "replies": 17, + "failures": 0, + "slowest_ok_seconds": 0.107 + }, + "cpu_core_seconds": { + "mysql": 2.244408, + "native-server": 1.18888, + "probe": -0.185103, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 802770944, + "native-server": 8011776, + "probe": 58925056, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 8011776, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "requested_polls": 24, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "empty": 24 + } + }, + "idle-24-3": { + "seconds": 12.2, + "health": { + "replies": 18, + "failures": 0, + "slowest_ok_seconds": 0.101 + }, + "cpu_core_seconds": { + "mysql": 2.191166, + "native-server": 1.235365, + "probe": 0.126906, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 802361344, + "native-server": 5984256, + "probe": 69505024, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 5984256, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "requested_polls": 24, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "empty": 24 + } + } + } +} diff --git a/benchmarks/server-port/v3/summary-repeated-php-1-2026-10-20.json b/benchmarks/server-port/v3/summary-repeated-php-1-2026-10-20.json new file mode 100644 index 000000000..62de913d8 --- /dev/null +++ b/benchmarks/server-port/v3/summary-repeated-php-1-2026-10-20.json @@ -0,0 +1,1124 @@ +{ + "identity": { + "implementation": "php", + "compose_project": "dw-port-php-20261020t134400", + "profile": "/home/linuxuser/profile.json", + "profile_sha256": "5e7946a6a506e391510e9222021e60c80d184742b030ee6e2978ab0c0b1e8ac5", + "source_revision": "8957ba27aa24b79ec2ea59e0c37b5ce662d434ab", + "native_binary_sha256": "edc283f130ff5c802fe5a80d56730cc9fe377c01c9c90340e08a68eb0866e2cc", + "smoke": false, + "idle_only": false, + "machine": { + "kernel": "6.8.0-146-generic", + "logical_cpus": 4, + "physical_memory_bytes": 8320786432, + "cpu_model": "AMD EPYC-Rome Processor", + "docker": "29.1.3", + "cgroup_driver": "systemd", + "docker_storage_driver": "overlayfs", + "docker_root_filesystem": "ext4" + } + }, + "windows": { + "setup": { + "seconds": 6.6, + "health": { + "replies": 8, + "failures": 0, + "slowest_ok_seconds": 0.874 + }, + "cpu_core_seconds": { + "mysql": 1.504809, + "php-scheduler": 1.649437, + "php-server": 5.056903, + "php-worker": 0.023939, + "probe": 0.007121, + "redis": 0.182108, + "sdk-worker": 0.052484 + }, + "max_memory_bytes": { + "mysql": 569229312, + "php-scheduler": 43175936, + "php-server": 91193344, + "php-worker": 49610752, + "probe": 19148800, + "redis": 3719168, + "sdk-worker": 20959232 + }, + "max_runtime_memory_bytes": 187621376, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 2, + "accepted_starts": 2, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 2, + "pending_at_deadline": 0, + "closed_during_offer_window": 1, + "latency_p50_seconds": 1.3568880558013916, + "latency_p95_seconds": 2.585994005203247, + "latency_p99_seconds": 2.585994005203247, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 3.4562, + "mysql_core_seconds_per_workflow": 0.7524 + }, + "warmup": { + "seconds": 59.2, + "health": { + "replies": 78, + "failures": 0, + "slowest_ok_seconds": 0.863 + }, + "cpu_core_seconds": { + "mysql": 9.797015, + "php-scheduler": 6.664555, + "php-server": 30.112366, + "php-worker": 0.394338, + "probe": 0.365943, + "redis": 1.656685, + "sdk-worker": 0.485625 + }, + "max_memory_bytes": { + "mysql": 585056256, + "php-scheduler": 48877568, + "php-server": 97693696, + "php-worker": 49614848, + "probe": 19161088, + "redis": 4448256, + "sdk-worker": 22761472 + }, + "max_runtime_memory_bytes": 195813376, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 15, + "accepted_starts": 15, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 15, + "pending_at_deadline": 0, + "closed_during_offer_window": 15, + "latency_p50_seconds": 1.0513579845428467, + "latency_p95_seconds": 1.5864241123199463, + "latency_p99_seconds": 1.5864241123199463, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 2.5885, + "mysql_core_seconds_per_workflow": 0.6531 + }, + "low-before-1": { + "seconds": 120.2, + "health": { + "replies": 158, + "failures": 0, + "slowest_ok_seconds": 0.584 + }, + "cpu_core_seconds": { + "mysql": 20.040341, + "php-scheduler": 13.497456, + "php-server": 60.623072, + "php-worker": 0.798438, + "probe": 0.37686, + "redis": 3.363789, + "sdk-worker": 0.944666 + }, + "max_memory_bytes": { + "mysql": 605970432, + "php-scheduler": 52334592, + "php-server": 109006848, + "php-worker": 50028544, + "probe": 20021248, + "redis": 6221824, + "sdk-worker": 25751552 + }, + "max_runtime_memory_bytes": 198287360, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 30, + "accepted_starts": 30, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 30, + "pending_at_deadline": 0, + "closed_during_offer_window": 30, + "latency_p50_seconds": 1.128861904144287, + "latency_p95_seconds": 1.4585249423980713, + "latency_p99_seconds": 1.537226915359497, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 2.6094, + "mysql_core_seconds_per_workflow": 0.668 + }, + "low-before-2": { + "seconds": 120.2, + "health": { + "replies": 158, + "failures": 0, + "slowest_ok_seconds": 0.702 + }, + "cpu_core_seconds": { + "mysql": 19.857788, + "php-scheduler": 12.698215, + "php-server": 60.391955, + "php-worker": 0.785859, + "probe": 0.01672, + "redis": 3.357225, + "sdk-worker": 0.950209 + }, + "max_memory_bytes": { + "mysql": 621260800, + "php-scheduler": 51154944, + "php-server": 104452096, + "php-worker": 50839552, + "probe": 19791872, + "redis": 6172672, + "sdk-worker": 28938240 + }, + "max_runtime_memory_bytes": 195870720, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 30, + "accepted_starts": 30, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 30, + "pending_at_deadline": 0, + "closed_during_offer_window": 30, + "latency_p50_seconds": 1.1518518924713135, + "latency_p95_seconds": 1.4961800575256348, + "latency_p99_seconds": 1.5145728588104248, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 2.5744, + "mysql_core_seconds_per_workflow": 0.6619 + }, + "low-before-3": { + "seconds": 120.2, + "health": { + "replies": 156, + "failures": 0, + "slowest_ok_seconds": 0.858 + }, + "cpu_core_seconds": { + "mysql": 21.376865, + "php-scheduler": 14.672424, + "php-server": 63.623704, + "php-worker": 0.823863, + "probe": 0.05507, + "redis": 3.531226, + "sdk-worker": 1.027329 + }, + "max_memory_bytes": { + "mysql": 638160896, + "php-scheduler": 51429376, + "php-server": 112340992, + "php-worker": 51687424, + "probe": 21409792, + "redis": 6946816, + "sdk-worker": 31318016 + }, + "max_runtime_memory_bytes": 201371648, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 30, + "accepted_starts": 30, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 30, + "pending_at_deadline": 0, + "closed_during_offer_window": 30, + "latency_p50_seconds": 1.187798023223877, + "latency_p95_seconds": 1.6202080249786377, + "latency_p99_seconds": 1.651698112487793, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 2.755, + "mysql_core_seconds_per_workflow": 0.7126 + }, + "rate-0.5-1": { + "seconds": 122.4, + "health": { + "replies": 154, + "failures": 0, + "slowest_ok_seconds": 0.787 + }, + "cpu_core_seconds": { + "mysql": 34.159995, + "php-scheduler": 14.081556, + "php-server": 90.804063, + "php-worker": 0.827854, + "probe": 0.356474, + "redis": 4.042836, + "sdk-worker": 1.722026 + }, + "max_memory_bytes": { + "mysql": 672313344, + "php-scheduler": 50196480, + "php-server": 106352640, + "php-worker": 53936128, + "probe": 20967424, + "redis": 7548928, + "sdk-worker": 35209216 + }, + "max_runtime_memory_bytes": 204369920, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.5, + "planned_starts": 60, + "accepted_starts": 60, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 60, + "pending_at_deadline": 0, + "closed_during_offer_window": 60, + "latency_p50_seconds": 1.1291561126708984, + "latency_p95_seconds": 1.526949167251587, + "latency_p99_seconds": 1.633439064025879, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 1.8293, + "mysql_core_seconds_per_workflow": 0.5693 + }, + "rate-0.5-2": { + "seconds": 122.9, + "health": { + "replies": 151, + "failures": 0, + "slowest_ok_seconds": 0.809 + }, + "cpu_core_seconds": { + "mysql": 34.1152, + "php-scheduler": 13.440402, + "php-server": 90.610046, + "php-worker": 0.799173, + "probe": 0.01692, + "redis": 4.091798, + "sdk-worker": 1.702909 + }, + "max_memory_bytes": { + "mysql": 702459904, + "php-scheduler": 49590272, + "php-server": 110915584, + "php-worker": 52420608, + "probe": 20811776, + "redis": 6348800, + "sdk-worker": 38711296 + }, + "max_runtime_memory_bytes": 202567680, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.5, + "planned_starts": 60, + "accepted_starts": 60, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 60, + "pending_at_deadline": 0, + "closed_during_offer_window": 60, + "latency_p50_seconds": 1.1356868743896484, + "latency_p95_seconds": 1.5167322158813477, + "latency_p99_seconds": 1.6396760940551758, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 1.8157, + "mysql_core_seconds_per_workflow": 0.5686 + }, + "rate-0.5-3": { + "seconds": 122.1, + "health": { + "replies": 147, + "failures": 0, + "slowest_ok_seconds": 0.868 + }, + "cpu_core_seconds": { + "mysql": 35.826468, + "php-scheduler": 14.224871, + "php-server": 93.479474, + "php-worker": 0.827446, + "probe": -0.005189, + "redis": 4.158363, + "sdk-worker": 1.809416 + }, + "max_memory_bytes": { + "mysql": 733597696, + "php-scheduler": 51208192, + "php-server": 120877056, + "php-worker": 52404224, + "probe": 19980288, + "redis": 7733248, + "sdk-worker": 38957056 + }, + "max_runtime_memory_bytes": 226398208, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.5, + "planned_starts": 60, + "accepted_starts": 60, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 60, + "pending_at_deadline": 0, + "closed_during_offer_window": 60, + "latency_p50_seconds": 1.2154319286346436, + "latency_p95_seconds": 1.5453448295593262, + "latency_p99_seconds": 1.6753089427947998, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 1.8782, + "mysql_core_seconds_per_workflow": 0.5971 + }, + "rate-1-1": { + "seconds": 176.8, + "health": { + "replies": 204, + "failures": 0, + "slowest_ok_seconds": 0.884 + }, + "cpu_core_seconds": { + "mysql": 66.727563, + "php-scheduler": 21.013864, + "php-server": 164.107583, + "php-worker": 1.188324, + "probe": 0.830886, + "redis": 6.423366, + "sdk-worker": 3.396652 + }, + "max_memory_bytes": { + "mysql": 781090816, + "php-scheduler": 51212288, + "php-server": 118976512, + "php-worker": 52404224, + "probe": 19783680, + "redis": 8462336, + "sdk-worker": 39272448 + }, + "max_runtime_memory_bytes": 207253504, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 1, + "planned_starts": 120, + "accepted_starts": 120, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 120, + "pending_at_deadline": 0, + "closed_during_offer_window": 69, + "latency_p50_seconds": 44.781501054763794, + "latency_p95_seconds": 64.77137804031372, + "latency_p99_seconds": 66.02623295783997, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 1.6061, + "mysql_core_seconds_per_workflow": 0.5561 + }, + "rate-1-2": { + "seconds": 178.3, + "health": { + "replies": 210, + "failures": 0, + "slowest_ok_seconds": 0.787 + }, + "cpu_core_seconds": { + "mysql": 66.481232, + "php-scheduler": 20.010903, + "php-server": 164.224564, + "php-worker": 1.228025, + "probe": 0.038332, + "redis": 6.447414, + "sdk-worker": 3.377742 + }, + "max_memory_bytes": { + "mysql": 818790400, + "php-scheduler": 48791552, + "php-server": 114503680, + "php-worker": 52760576, + "probe": 20025344, + "redis": 8364032, + "sdk-worker": 40349696 + }, + "max_runtime_memory_bytes": 210096128, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 1, + "planned_starts": 120, + "accepted_starts": 120, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 120, + "pending_at_deadline": 0, + "closed_during_offer_window": 68, + "latency_p50_seconds": 46.07006311416626, + "latency_p95_seconds": 65.88363909721375, + "latency_p99_seconds": 67.44143986701965, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 1.5993, + "mysql_core_seconds_per_workflow": 0.554 + }, + "rate-1-3": { + "seconds": 181.3, + "health": { + "replies": 208, + "failures": 0, + "slowest_ok_seconds": 0.875 + }, + "cpu_core_seconds": { + "mysql": 67.915662, + "php-scheduler": 20.903777, + "php-server": 167.68432, + "php-worker": 1.2719, + "probe": 0.061674, + "redis": 6.673205, + "sdk-worker": 3.479508 + }, + "max_memory_bytes": { + "mysql": 856571904, + "php-scheduler": 52236288, + "php-server": 118317056, + "php-worker": 54628352, + "probe": 19775488, + "redis": 8306688, + "sdk-worker": 41897984 + }, + "max_runtime_memory_bytes": 210964480, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 1, + "planned_starts": 120, + "accepted_starts": 120, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 120, + "pending_at_deadline": 0, + "closed_during_offer_window": 66, + "latency_p50_seconds": 48.342694997787476, + "latency_p95_seconds": 67.38858103752136, + "latency_p99_seconds": 69.02710199356079, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 1.6378, + "mysql_core_seconds_per_workflow": 0.566 + }, + "low-after-1": { + "seconds": 119.4, + "health": { + "replies": 152, + "failures": 0, + "slowest_ok_seconds": 0.782 + }, + "cpu_core_seconds": { + "mysql": 22.179925, + "php-scheduler": 14.484918, + "php-server": 64.695478, + "php-worker": 0.829711, + "probe": -1.265806, + "redis": 3.54183, + "sdk-worker": 1.127742 + }, + "max_memory_bytes": { + "mysql": 866603008, + "php-scheduler": 47493120, + "php-server": 114991104, + "php-worker": 54050816, + "probe": 19767296, + "redis": 7155712, + "sdk-worker": 41742336 + }, + "max_runtime_memory_bytes": 201912320, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 30, + "accepted_starts": 30, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 30, + "pending_at_deadline": 0, + "closed_during_offer_window": 30, + "latency_p50_seconds": 1.2256431579589844, + "latency_p95_seconds": 1.61228609085083, + "latency_p99_seconds": 1.7355849742889404, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 2.7851, + "mysql_core_seconds_per_workflow": 0.7393 + }, + "low-after-2": { + "seconds": 120.2, + "health": { + "replies": 154, + "failures": 0, + "slowest_ok_seconds": 0.683 + }, + "cpu_core_seconds": { + "mysql": 21.854782, + "php-scheduler": 13.381766, + "php-server": 63.860201, + "php-worker": 0.806617, + "probe": -0.048811, + "redis": 3.524795, + "sdk-worker": 1.107556 + }, + "max_memory_bytes": { + "mysql": 875786240, + "php-scheduler": 52068352, + "php-server": 111415296, + "php-worker": 52908032, + "probe": 19775488, + "redis": 6717440, + "sdk-worker": 41689088 + }, + "max_runtime_memory_bytes": 197271552, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 30, + "accepted_starts": 30, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 30, + "pending_at_deadline": 0, + "closed_during_offer_window": 30, + "latency_p50_seconds": 1.197011947631836, + "latency_p95_seconds": 1.7318499088287354, + "latency_p99_seconds": 1.8175239562988281, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 2.7191, + "mysql_core_seconds_per_workflow": 0.7285 + }, + "low-after-3": { + "seconds": 120.0, + "health": { + "replies": 156, + "failures": 0, + "slowest_ok_seconds": 0.686 + }, + "cpu_core_seconds": { + "mysql": 21.31017, + "php-scheduler": 13.8905, + "php-server": 62.803105, + "php-worker": 0.825999, + "probe": 0.016519, + "redis": 3.496282, + "sdk-worker": 1.113688 + }, + "max_memory_bytes": { + "mysql": 885342208, + "php-scheduler": 47980544, + "php-server": 122150912, + "php-worker": 52764672, + "probe": 20742144, + "redis": 7487488, + "sdk-worker": 43237376 + }, + "max_runtime_memory_bytes": 195055616, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 30, + "accepted_starts": 30, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 30, + "pending_at_deadline": 0, + "closed_during_offer_window": 30, + "latency_p50_seconds": 1.1954889297485352, + "latency_p95_seconds": 1.56825590133667, + "latency_p99_seconds": 1.68644118309021, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 2.7005, + "mysql_core_seconds_per_workflow": 0.7103 + } + }, + "idle": { + "idle-6-1": { + "seconds": 12.9, + "health": { + "replies": 16, + "failures": 0, + "slowest_ok_seconds": 1.431 + }, + "cpu_core_seconds": { + "mysql": 1.119762, + "php-scheduler": 1.189401, + "php-server": 4.417996, + "php-worker": 0.084415, + "probe": -0.557917, + "redis": 0.425557, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 886943744, + "php-scheduler": 43503616, + "php-server": 117972992, + "php-worker": 54337536, + "probe": 31399936, + "redis": 4829184, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 205033472, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "requested_polls": 6, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "empty": 6 + } + }, + "idle-6-2": { + "seconds": 13.0, + "health": { + "replies": 16, + "failures": 0, + "slowest_ok_seconds": 1.217 + }, + "cpu_core_seconds": { + "mysql": 1.085944, + "php-scheduler": 2.023353, + "php-server": 4.351547, + "php-worker": 0.100568, + "probe": -0.046043, + "redis": 0.401379, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 887758848, + "php-scheduler": 43229184, + "php-server": 104980480, + "php-worker": 52760576, + "probe": 30400512, + "redis": 4653056, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 190066688, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "requested_polls": 6, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "empty": 6 + } + }, + "idle-6-3": { + "seconds": 13.1, + "health": { + "replies": 16, + "failures": 0, + "slowest_ok_seconds": 0.549 + }, + "cpu_core_seconds": { + "mysql": 1.005154, + "php-scheduler": 1.77791, + "php-server": 4.344363, + "php-worker": 0.079783, + "probe": -0.002653, + "redis": 0.39114, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 887844864, + "php-scheduler": 42561536, + "php-server": 101949440, + "php-worker": 52764672, + "probe": 31424512, + "redis": 4689920, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 183525376, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "requested_polls": 6, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "empty": 6 + } + }, + "idle-12-1": { + "seconds": 15.3, + "health": { + "replies": 16, + "failures": 1, + "slowest_ok_seconds": 1.301 + }, + "cpu_core_seconds": { + "mysql": 1.450981, + "php-scheduler": 1.344831, + "php-server": 5.344161, + "php-worker": 0.089381, + "probe": 0.11171, + "redis": 0.501114, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 925487104, + "php-scheduler": 46772224, + "php-server": 139833344, + "php-worker": 52760576, + "probe": 46059520, + "redis": 4550656, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 198746112, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "requested_polls": 12, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "backpressure": 4, + "empty": 8 + } + }, + "idle-12-2": { + "seconds": 16.1, + "health": { + "replies": 16, + "failures": 1, + "slowest_ok_seconds": 1.708 + }, + "cpu_core_seconds": { + "mysql": 1.617205, + "php-scheduler": 2.118904, + "php-server": 5.849974, + "php-worker": 0.090799, + "probe": 0.101958, + "redis": 0.582651, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 925876224, + "php-scheduler": 36306944, + "php-server": 131977216, + "php-worker": 52764672, + "probe": 46710784, + "redis": 4497408, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 218853376, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "requested_polls": 12, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "empty": 8, + "backpressure": 4 + } + }, + "idle-12-3": { + "seconds": 16.1, + "health": { + "replies": 17, + "failures": 1, + "slowest_ok_seconds": 1.476 + }, + "cpu_core_seconds": { + "mysql": 1.445036, + "php-scheduler": 1.548843, + "php-server": 5.645729, + "php-worker": 0.130054, + "probe": 0.086832, + "redis": 0.520426, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 924483584, + "php-scheduler": 44150784, + "php-server": 134873088, + "php-worker": 52760576, + "probe": 46292992, + "redis": 4390912, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 232247296, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "requested_polls": 12, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "empty": 8, + "backpressure": 4 + } + }, + "idle-24-1": { + "seconds": 17.2, + "health": { + "replies": 18, + "failures": 2, + "slowest_ok_seconds": 0.896 + }, + "cpu_core_seconds": { + "mysql": 1.803645, + "php-scheduler": 2.060201, + "php-server": 6.451095, + "php-worker": 0.092034, + "probe": 0.219455, + "redis": 0.55685, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 931160064, + "php-scheduler": 51216384, + "php-server": 132136960, + "php-worker": 52756480, + "probe": 67342336, + "redis": 4497408, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 224530432, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "requested_polls": 24, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "backpressure": 16, + "empty": 8 + } + }, + "idle-24-2": { + "seconds": 17.1, + "health": { + "replies": 17, + "failures": 2, + "slowest_ok_seconds": 1.676 + }, + "cpu_core_seconds": { + "mysql": 1.906241, + "php-scheduler": 1.901284, + "php-server": 6.692275, + "php-worker": 0.122455, + "probe": -0.035132, + "redis": 0.579389, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 936239104, + "php-scheduler": 35033088, + "php-server": 152723456, + "php-worker": 52756480, + "probe": 69390336, + "redis": 5758976, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 217575424, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "requested_polls": 24, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "backpressure": 16, + "empty": 8 + } + }, + "idle-24-3": { + "seconds": 17.2, + "health": { + "replies": 16, + "failures": 2, + "slowest_ok_seconds": 0.916 + }, + "cpu_core_seconds": { + "mysql": 1.92142, + "php-scheduler": 2.029289, + "php-server": 6.846726, + "php-worker": 0.124736, + "probe": 0.083039, + "redis": 0.563508, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 937267200, + "php-scheduler": 43589632, + "php-server": 168722432, + "php-worker": 52764672, + "probe": 72994816, + "redis": 4243456, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 246112256, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "requested_polls": 24, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "backpressure": 16, + "empty": 8 + } + } + } +} diff --git a/benchmarks/server-port/v3/summary-repeated-php-2-2026-10-20.json b/benchmarks/server-port/v3/summary-repeated-php-2-2026-10-20.json new file mode 100644 index 000000000..5d054322e --- /dev/null +++ b/benchmarks/server-port/v3/summary-repeated-php-2-2026-10-20.json @@ -0,0 +1,1124 @@ +{ + "identity": { + "implementation": "php", + "compose_project": "dw-port-php-20261020t144545", + "profile": "/home/linuxuser/profile.json", + "profile_sha256": "5e7946a6a506e391510e9222021e60c80d184742b030ee6e2978ab0c0b1e8ac5", + "source_revision": "8957ba27aa24b79ec2ea59e0c37b5ce662d434ab", + "native_binary_sha256": "edc283f130ff5c802fe5a80d56730cc9fe377c01c9c90340e08a68eb0866e2cc", + "smoke": false, + "idle_only": false, + "machine": { + "kernel": "6.8.0-146-generic", + "logical_cpus": 4, + "physical_memory_bytes": 8320786432, + "cpu_model": "AMD EPYC-Rome Processor", + "docker": "29.1.3", + "cgroup_driver": "systemd", + "docker_storage_driver": "overlayfs", + "docker_root_filesystem": "ext4" + } + }, + "windows": { + "setup": { + "seconds": 6.1, + "health": { + "replies": 8, + "failures": 0, + "slowest_ok_seconds": 0.403 + }, + "cpu_core_seconds": { + "mysql": 1.170159, + "php-scheduler": 1.375, + "php-server": 3.530644, + "php-worker": 0.017897, + "probe": 0.007011, + "redis": 0.170855, + "sdk-worker": 0.042902 + }, + "max_memory_bytes": { + "mysql": 572542976, + "php-scheduler": 43524096, + "php-server": 88588288, + "php-worker": 49451008, + "probe": 19144704, + "redis": 3624960, + "sdk-worker": 20951040 + }, + "max_runtime_memory_bytes": 182153216, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 2, + "accepted_starts": 2, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 2, + "pending_at_deadline": 0, + "closed_during_offer_window": 2, + "latency_p50_seconds": 0.8652470111846924, + "latency_p95_seconds": 1.8545799255371094, + "latency_p99_seconds": 1.8545799255371094, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 2.5472, + "mysql_core_seconds_per_workflow": 0.5851 + }, + "warmup": { + "seconds": 60.8, + "health": { + "replies": 78, + "failures": 0, + "slowest_ok_seconds": 0.723 + }, + "cpu_core_seconds": { + "mysql": 10.668871, + "php-scheduler": 6.668423, + "php-server": 32.386096, + "php-worker": 0.41917, + "probe": 0.372215, + "redis": 1.763566, + "sdk-worker": 0.500587 + }, + "max_memory_bytes": { + "mysql": 593006592, + "php-scheduler": 44797952, + "php-server": 96911360, + "php-worker": 50581504, + "probe": 21094400, + "redis": 5603328, + "sdk-worker": 22888448 + }, + "max_runtime_memory_bytes": 189968384, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 15, + "accepted_starts": 15, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 15, + "pending_at_deadline": 0, + "closed_during_offer_window": 15, + "latency_p50_seconds": 1.2237191200256348, + "latency_p95_seconds": 1.5795209407806396, + "latency_p99_seconds": 1.5795209407806396, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 2.7492, + "mysql_core_seconds_per_workflow": 0.7113 + }, + "low-before-1": { + "seconds": 120.3, + "health": { + "replies": 154, + "failures": 0, + "slowest_ok_seconds": 0.713 + }, + "cpu_core_seconds": { + "mysql": 21.296271, + "php-scheduler": 13.985756, + "php-server": 63.837221, + "php-worker": 0.833778, + "probe": 0.438814, + "redis": 3.467972, + "sdk-worker": 1.000538 + }, + "max_memory_bytes": { + "mysql": 614154240, + "php-scheduler": 47267840, + "php-server": 114008064, + "php-worker": 50053120, + "probe": 20639744, + "redis": 6180864, + "sdk-worker": 25747456 + }, + "max_runtime_memory_bytes": 199602176, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 30, + "accepted_starts": 30, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 30, + "pending_at_deadline": 0, + "closed_during_offer_window": 30, + "latency_p50_seconds": 1.2357759475708008, + "latency_p95_seconds": 1.679650068283081, + "latency_p99_seconds": 1.8180029392242432, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 2.7375, + "mysql_core_seconds_per_workflow": 0.7099 + }, + "low-before-2": { + "seconds": 120.4, + "health": { + "replies": 152, + "failures": 0, + "slowest_ok_seconds": 0.695 + }, + "cpu_core_seconds": { + "mysql": 21.78718, + "php-scheduler": 13.668353, + "php-server": 64.700526, + "php-worker": 0.852338, + "probe": 0.01564, + "redis": 3.596517, + "sdk-worker": 1.001945 + }, + "max_memory_bytes": { + "mysql": 631521280, + "php-scheduler": 52035584, + "php-server": 120164352, + "php-worker": 50896896, + "probe": 19759104, + "redis": 6782976, + "sdk-worker": 28925952 + }, + "max_runtime_memory_bytes": 212869120, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 30, + "accepted_starts": 30, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 30, + "pending_at_deadline": 0, + "closed_during_offer_window": 30, + "latency_p50_seconds": 1.2468769550323486, + "latency_p95_seconds": 1.6789839267730713, + "latency_p99_seconds": 1.6973340511322021, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 2.7606, + "mysql_core_seconds_per_workflow": 0.7262 + }, + "low-before-3": { + "seconds": 120.7, + "health": { + "replies": 152, + "failures": 0, + "slowest_ok_seconds": 0.806 + }, + "cpu_core_seconds": { + "mysql": 22.43567, + "php-scheduler": 14.76506, + "php-server": 65.670336, + "php-worker": 0.866131, + "probe": -0.000866, + "redis": 3.581031, + "sdk-worker": 1.043077 + }, + "max_memory_bytes": { + "mysql": 647872512, + "php-scheduler": 49594368, + "php-server": 113401856, + "php-worker": 52457472, + "probe": 19767296, + "redis": 6266880, + "sdk-worker": 30670848 + }, + "max_runtime_memory_bytes": 201875456, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 30, + "accepted_starts": 30, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 30, + "pending_at_deadline": 0, + "closed_during_offer_window": 30, + "latency_p50_seconds": 1.3055198192596436, + "latency_p95_seconds": 1.649399995803833, + "latency_p99_seconds": 1.7223989963531494, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 2.8294, + "mysql_core_seconds_per_workflow": 0.7479 + }, + "rate-0.5-1": { + "seconds": 122.9, + "health": { + "replies": 150, + "failures": 0, + "slowest_ok_seconds": 0.825 + }, + "cpu_core_seconds": { + "mysql": 35.34101, + "php-scheduler": 13.665336, + "php-server": 93.721084, + "php-worker": 0.846103, + "probe": 0.383153, + "redis": 4.218765, + "sdk-worker": 1.686126 + }, + "max_memory_bytes": { + "mysql": 681295872, + "php-scheduler": 49901568, + "php-server": 117063680, + "php-worker": 54022144, + "probe": 20889600, + "redis": 6488064, + "sdk-worker": 35143680 + }, + "max_runtime_memory_bytes": 207396864, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.5, + "planned_starts": 60, + "accepted_starts": 60, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 60, + "pending_at_deadline": 0, + "closed_during_offer_window": 60, + "latency_p50_seconds": 1.1792030334472656, + "latency_p95_seconds": 1.609131097793579, + "latency_p99_seconds": 1.6938180923461914, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 1.8742, + "mysql_core_seconds_per_workflow": 0.589 + }, + "rate-0.5-2": { + "seconds": 122.1, + "health": { + "replies": 146, + "failures": 0, + "slowest_ok_seconds": 0.793 + }, + "cpu_core_seconds": { + "mysql": 36.534184, + "php-scheduler": 14.546593, + "php-server": 96.016586, + "php-worker": 0.927426, + "probe": 0.072128, + "redis": 4.226208, + "sdk-worker": 1.752287 + }, + "max_memory_bytes": { + "mysql": 713097216, + "php-scheduler": 48873472, + "php-server": 99962880, + "php-worker": 52498432, + "probe": 20951040, + "redis": 8265728, + "sdk-worker": 39034880 + }, + "max_runtime_memory_bytes": 201519104, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.5, + "planned_starts": 60, + "accepted_starts": 60, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 60, + "pending_at_deadline": 0, + "closed_during_offer_window": 60, + "latency_p50_seconds": 1.2057840824127197, + "latency_p95_seconds": 1.6348719596862793, + "latency_p99_seconds": 1.7296020984649658, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 1.9286, + "mysql_core_seconds_per_workflow": 0.6089 + }, + "rate-0.5-3": { + "seconds": 121.9, + "health": { + "replies": 144, + "failures": 0, + "slowest_ok_seconds": 0.812 + }, + "cpu_core_seconds": { + "mysql": 36.688681, + "php-scheduler": 14.632643, + "php-server": 95.642259, + "php-worker": 0.866217, + "probe": -0.083048, + "redis": 4.215405, + "sdk-worker": 1.7989 + }, + "max_memory_bytes": { + "mysql": 742236160, + "php-scheduler": 50565120, + "php-server": 118951936, + "php-worker": 54308864, + "probe": 21495808, + "redis": 7311360, + "sdk-worker": 39960576 + }, + "max_runtime_memory_bytes": 204169216, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.5, + "planned_starts": 60, + "accepted_starts": 60, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 60, + "pending_at_deadline": 0, + "closed_during_offer_window": 60, + "latency_p50_seconds": 1.2680859565734863, + "latency_p95_seconds": 1.557013988494873, + "latency_p99_seconds": 1.8666460514068604, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 1.9226, + "mysql_core_seconds_per_workflow": 0.6115 + }, + "rate-1-1": { + "seconds": 185.9, + "health": { + "replies": 212, + "failures": 0, + "slowest_ok_seconds": 0.911 + }, + "cpu_core_seconds": { + "mysql": 68.790216, + "php-scheduler": 22.00776, + "php-server": 171.008503, + "php-worker": 1.330057, + "probe": 0.87524, + "redis": 6.809588, + "sdk-worker": 3.413805 + }, + "max_memory_bytes": { + "mysql": 788774912, + "php-scheduler": 52244480, + "php-server": 118599680, + "php-worker": 52498432, + "probe": 19771392, + "redis": 7471104, + "sdk-worker": 40652800 + }, + "max_runtime_memory_bytes": 214515712, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 1, + "planned_starts": 120, + "accepted_starts": 120, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 120, + "pending_at_deadline": 0, + "closed_during_offer_window": 62, + "latency_p50_seconds": 55.74392104148865, + "latency_p95_seconds": 72.86986804008484, + "latency_p99_seconds": 74.18884301185608, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 1.6763, + "mysql_core_seconds_per_workflow": 0.5733 + }, + "rate-1-2": { + "seconds": 178.4, + "health": { + "replies": 204, + "failures": 0, + "slowest_ok_seconds": 0.834 + }, + "cpu_core_seconds": { + "mysql": 65.858819, + "php-scheduler": 20.794849, + "php-server": 163.897122, + "php-worker": 1.237972, + "probe": -0.027616, + "redis": 6.473824, + "sdk-worker": 3.309925 + }, + "max_memory_bytes": { + "mysql": 827719680, + "php-scheduler": 51093504, + "php-server": 119918592, + "php-worker": 52858880, + "probe": 21254144, + "redis": 9441280, + "sdk-worker": 41283584 + }, + "max_runtime_memory_bytes": 209801216, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 1, + "planned_starts": 120, + "accepted_starts": 120, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 120, + "pending_at_deadline": 0, + "closed_during_offer_window": 67, + "latency_p50_seconds": 48.43381905555725, + "latency_p95_seconds": 65.49115705490112, + "latency_p99_seconds": 66.32482099533081, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 1.6034, + "mysql_core_seconds_per_workflow": 0.5488 + }, + "rate-1-3": { + "seconds": 181.6, + "health": { + "replies": 208, + "failures": 0, + "slowest_ok_seconds": 0.805 + }, + "cpu_core_seconds": { + "mysql": 67.678031, + "php-scheduler": 21.939475, + "php-server": 167.652277, + "php-worker": 1.27479, + "probe": 0.059808, + "redis": 6.565551, + "sdk-worker": 3.383119 + }, + "max_memory_bytes": { + "mysql": 864178176, + "php-scheduler": 50556928, + "php-server": 119033856, + "php-worker": 52858880, + "probe": 20824064, + "redis": 8126464, + "sdk-worker": 41496576 + }, + "max_runtime_memory_bytes": 223461376, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 1, + "planned_starts": 120, + "accepted_starts": 120, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 120, + "pending_at_deadline": 0, + "closed_during_offer_window": 65, + "latency_p50_seconds": 49.91317009925842, + "latency_p95_seconds": 69.5694899559021, + "latency_p99_seconds": 71.26597094535828, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 1.6453, + "mysql_core_seconds_per_workflow": 0.564 + }, + "low-after-1": { + "seconds": 119.3, + "health": { + "replies": 152, + "failures": 0, + "slowest_ok_seconds": 0.826 + }, + "cpu_core_seconds": { + "mysql": 21.861028, + "php-scheduler": 13.144639, + "php-server": 64.432165, + "php-worker": 0.842851, + "probe": -1.237331, + "redis": 3.57914, + "sdk-worker": 1.100568 + }, + "max_memory_bytes": { + "mysql": 873193472, + "php-scheduler": 51814400, + "php-server": 104218624, + "php-worker": 52879360, + "probe": 19767296, + "redis": 8040448, + "sdk-worker": 40128512 + }, + "max_runtime_memory_bytes": 200564736, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 30, + "accepted_starts": 30, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 30, + "pending_at_deadline": 0, + "closed_during_offer_window": 30, + "latency_p50_seconds": 1.2581219673156738, + "latency_p95_seconds": 1.5643978118896484, + "latency_p99_seconds": 1.730724811553955, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 2.7333, + "mysql_core_seconds_per_workflow": 0.7287 + }, + "low-after-2": { + "seconds": 120.4, + "health": { + "replies": 156, + "failures": 0, + "slowest_ok_seconds": 0.816 + }, + "cpu_core_seconds": { + "mysql": 21.271663, + "php-scheduler": 14.092062, + "php-server": 63.232411, + "php-worker": 0.818295, + "probe": -0.057113, + "redis": 3.492146, + "sdk-worker": 1.08394 + }, + "max_memory_bytes": { + "mysql": 881983488, + "php-scheduler": 50245632, + "php-server": 112705536, + "php-worker": 52858880, + "probe": 19750912, + "redis": 7168000, + "sdk-worker": 40996864 + }, + "max_runtime_memory_bytes": 209444864, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 30, + "accepted_starts": 30, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 30, + "pending_at_deadline": 0, + "closed_during_offer_window": 30, + "latency_p50_seconds": 1.1640570163726807, + "latency_p95_seconds": 1.6432580947875977, + "latency_p99_seconds": 1.6643199920654297, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 2.7212, + "mysql_core_seconds_per_workflow": 0.7091 + }, + "low-after-3": { + "seconds": 120.0, + "health": { + "replies": 152, + "failures": 0, + "slowest_ok_seconds": 0.699 + }, + "cpu_core_seconds": { + "mysql": 21.127632, + "php-scheduler": 14.040819, + "php-server": 62.560674, + "php-worker": 0.81184, + "probe": -0.023795, + "redis": 3.422797, + "sdk-worker": 1.044698 + }, + "max_memory_bytes": { + "mysql": 891281408, + "php-scheduler": 49897472, + "php-server": 114806784, + "php-worker": 54472704, + "probe": 20803584, + "redis": 6688768, + "sdk-worker": 41263104 + }, + "max_runtime_memory_bytes": 207220736, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 30, + "accepted_starts": 30, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 30, + "pending_at_deadline": 0, + "closed_during_offer_window": 30, + "latency_p50_seconds": 1.232442855834961, + "latency_p95_seconds": 1.440964937210083, + "latency_p99_seconds": 1.5305070877075195, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 2.6945, + "mysql_core_seconds_per_workflow": 0.7043 + } + }, + "idle": { + "idle-6-1": { + "seconds": 13.7, + "health": { + "replies": 16, + "failures": 0, + "slowest_ok_seconds": 1.375 + }, + "cpu_core_seconds": { + "mysql": 1.104864, + "php-scheduler": 1.495123, + "php-server": 4.535996, + "php-worker": 0.126425, + "probe": -0.518748, + "redis": 0.493259, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 893804544, + "php-scheduler": 48877568, + "php-server": 104476672, + "php-worker": 52858880, + "probe": 31391744, + "redis": 6451200, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 208138240, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "requested_polls": 6, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "empty": 6 + } + }, + "idle-6-2": { + "seconds": 13.0, + "health": { + "replies": 17, + "failures": 0, + "slowest_ok_seconds": 0.516 + }, + "cpu_core_seconds": { + "mysql": 1.089478, + "php-scheduler": 0.78164, + "php-server": 4.397601, + "php-worker": 0.078596, + "probe": -0.063075, + "redis": 0.441157, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 894009344, + "php-scheduler": 34197504, + "php-server": 124526592, + "php-worker": 52858880, + "probe": 31281152, + "redis": 6389760, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 183857152, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "requested_polls": 6, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "empty": 6 + } + }, + "idle-6-3": { + "seconds": 13.0, + "health": { + "replies": 16, + "failures": 0, + "slowest_ok_seconds": 1.093 + }, + "cpu_core_seconds": { + "mysql": 1.008821, + "php-scheduler": 1.749659, + "php-server": 3.779992, + "php-worker": 0.081498, + "probe": 0.004168, + "redis": 0.373502, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 894189568, + "php-scheduler": 43732992, + "php-server": 121151488, + "php-worker": 52854784, + "probe": 30355456, + "redis": 4386816, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 201003008, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "requested_polls": 6, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "empty": 6 + } + }, + "idle-12-1": { + "seconds": 15.6, + "health": { + "replies": 16, + "failures": 1, + "slowest_ok_seconds": 1.104 + }, + "cpu_core_seconds": { + "mysql": 1.584993, + "php-scheduler": 2.108578, + "php-server": 5.409933, + "php-worker": 0.128848, + "probe": 0.099836, + "redis": 0.543311, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 932577280, + "php-scheduler": 50565120, + "php-server": 124182528, + "php-worker": 52858880, + "probe": 45785088, + "redis": 4153344, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 213549056, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "requested_polls": 12, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "backpressure": 4, + "empty": 8 + } + }, + "idle-12-2": { + "seconds": 16.1, + "health": { + "replies": 16, + "failures": 1, + "slowest_ok_seconds": 1.15 + }, + "cpu_core_seconds": { + "mysql": 1.471128, + "php-scheduler": 1.61049, + "php-server": 5.549401, + "php-worker": 0.099235, + "probe": 0.076546, + "redis": 0.543377, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 934100992, + "php-scheduler": 36921344, + "php-server": 132620288, + "php-worker": 52854784, + "probe": 45920256, + "redis": 5201920, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 196657152, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "requested_polls": 12, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "backpressure": 4, + "empty": 8 + } + }, + "idle-12-3": { + "seconds": 15.9, + "health": { + "replies": 16, + "failures": 1, + "slowest_ok_seconds": 1.554 + }, + "cpu_core_seconds": { + "mysql": 1.400935, + "php-scheduler": 2.024423, + "php-server": 5.508323, + "php-worker": 0.121851, + "probe": 0.015692, + "redis": 0.543894, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 935186432, + "php-scheduler": 44457984, + "php-server": 126164992, + "php-worker": 52858880, + "probe": 46223360, + "redis": 5644288, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 204046336, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "requested_polls": 12, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "backpressure": 4, + "empty": 8 + } + }, + "idle-24-1": { + "seconds": 17.3, + "health": { + "replies": 18, + "failures": 2, + "slowest_ok_seconds": 0.844 + }, + "cpu_core_seconds": { + "mysql": 1.972297, + "php-scheduler": 1.896292, + "php-server": 6.795804, + "php-worker": 0.128665, + "probe": 0.437206, + "redis": 0.587129, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 941764608, + "php-scheduler": 52084736, + "php-server": 137228288, + "php-worker": 52858880, + "probe": 77619200, + "redis": 4280320, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 198823936, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "requested_polls": 24, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "backpressure": 16, + "empty": 8 + } + }, + "idle-24-2": { + "seconds": 17.2, + "health": { + "replies": 16, + "failures": 2, + "slowest_ok_seconds": 0.937 + }, + "cpu_core_seconds": { + "mysql": 2.037846, + "php-scheduler": 2.212841, + "php-server": 6.790381, + "php-worker": 0.097124, + "probe": -0.146052, + "redis": 0.594822, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 947671040, + "php-scheduler": 39317504, + "php-server": 157556736, + "php-worker": 52858880, + "probe": 66371584, + "redis": 4366336, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 235614208, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "requested_polls": 24, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "empty": 8, + "backpressure": 16 + } + }, + "idle-24-3": { + "seconds": 17.3, + "health": { + "replies": 15, + "failures": 3, + "slowest_ok_seconds": 0.441 + }, + "cpu_core_seconds": { + "mysql": 2.059356, + "php-scheduler": 1.905943, + "php-server": 6.98372, + "php-worker": 0.123942, + "probe": -0.076262, + "redis": 0.59964, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 947970048, + "php-scheduler": 50569216, + "php-server": 147697664, + "php-worker": 52854784, + "probe": 61833216, + "redis": 4112384, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 206655488, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "requested_polls": 24, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "backpressure": 16, + "empty": 8 + } + } + } +} From 70f610bef99f1bf1171cba9838f3d0d5a7089198 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 17:27:18 +0000 Subject: [PATCH 37/37] Compare restart history without the public principal; scope the SQLite 2.9.7 cohort Public history now lifts each command's principal onto its event, and worker task history has none, so the recovered terminal resumption compares the fields both views share. The SQLite PHP 2.9.7 upgrade cohort steps set the job's resource scope like their neighbours; the cleanup step failed without it. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/server-parity.yml | 2 ++ scripts/conformance/server-parity/restart.php | 9 ++++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/server-parity.yml b/.github/workflows/server-parity.yml index 681aa9e1b..699b59641 100644 --- a/.github/workflows/server-parity.yml +++ b/.github/workflows/server-parity.yml @@ -1671,6 +1671,7 @@ jobs: - name: Back up, take over and resume a published PHP 2.9.7 upgrade source timeout-minutes: 8 env: + RESOURCE_SCOPE: dw-parity-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }} DW_PARITY_UPGRADE_LABEL: php297 DW_PARITY_UPGRADE_EXPECTED_LAYOUT: php-2.9.7 DW_PARITY_UPGRADE_PHP_IMAGE: ghcr.io/durable-workflow/server@sha256:30c34c5f3256a812fbccbe505a068309b58b0c14d01ffff0465d88604cb36d50 @@ -1679,6 +1680,7 @@ jobs: if: always() timeout-minutes: 3 env: + RESOURCE_SCOPE: dw-parity-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }} DW_PARITY_UPGRADE_LABEL: php297 DW_PARITY_UPGRADE_EXPECTED_LAYOUT: php-2.9.7 DW_PARITY_UPGRADE_PHP_IMAGE: ghcr.io/durable-workflow/server@sha256:30c34c5f3256a812fbccbe505a068309b58b0c14d01ffff0465d88604cb36d50 diff --git a/scripts/conformance/server-parity/restart.php b/scripts/conformance/server-parity/restart.php index a72f94979..5ded7dc5a 100644 --- a/scripts/conformance/server-parity/restart.php +++ b/scripts/conformance/server-parity/restart.php @@ -53,6 +53,13 @@ function restartControlHistory(array $events): array 'event_type' => $event['event_type'], 'timestamp' => $event['recorded_at'], 'payload' => $event['payload']], $events); } +// Public history lifts each command's principal onto its event; worker task +// history has no principal. Compare the fields both views share. +function restartWithoutPrincipal(array $events): array +{ + return array_map(static fn (array $event): array => array_diff_key($event, ['principal' => true]), $events); +} + function restartSignalWorkflow(WorkflowContext $context, array $input): array { $context->waitCondition(fn (): bool => count($context->signals('payload')) > 0, 'restart.payload'); @@ -336,7 +343,7 @@ function restartTerminalWorkflow(WorkflowContext $context, array $input): array $terminalClaim = $client->pollWorkflowTask('restart-terminal-new', 'restart-terminal-v1', 0); assertRestart(is_array($terminalClaim) && $terminalClaim['task_id'] === $terminal['failure_receipt']['next_task_id'] && $terminalClaim['run_id'] === $terminal['run_id'] - && sameRestartValue(restartControlHistory($terminalClaim['history_events']), $terminal['history']) + && sameRestartValue(restartControlHistory($terminalClaim['history_events']), restartWithoutPrincipal($terminal['history'])) && $terminalClaim['history_events'][4]['workflow_task_id'] === $terminalActivity['task_id'], 'Recovered terminal resumption lost original task/run/history.'); $terminalReplay = (new Replayer($client->payloadCodec()))->replay(restartTerminalWorkflow(...), $terminalClaim['history_events'], $client->payloadCodec()->decodeEnvelope($terminalClaim['arguments']), 'restart-terminal-v1', $terminalClaim);