From 3fdee04faeb7f5c97b7f4d72bc28ea061b415c23 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Wed, 21 Oct 2026 00:42:42 +0000 Subject: [PATCH 1/7] Serve operator metrics and health natively GET /api/system/operator-metrics and GET /api/system/health follow Workflow 2.5.5's OperatorMetrics and HealthCheck over the shared tables: PHP's counts and predicates, whole-second "now" comparisons, the twelve health checks and 503 on error, admin-only in any namespace. Policy sections describe native, and the timeline, timer and lineage projections native reads from history are reported as not maintained. New tasks now carry PHP's creation and dispatch timestamps, and run summaries classify a task's transport (lease expired, dispatch failed, claim failed, dispatch overdue) as PHP's projector does instead of declining to summarize. An operator-metrics parity fixture checks seven stages of one run against published PHP 2.5.14. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/server-parity.yml | 4 +- docs/rust-server-port.md | 46 + rust/src/runtime/active_summary.rs | 144 ++- rust/src/runtime/backend.rs | 9 + rust/src/runtime/execution.rs | 2 + rust/src/runtime/history_budget.rs | 5 + rust/src/runtime/http.rs | 29 + rust/src/runtime/mod.rs | 2 + rust/src/runtime/operator_metrics.rs | 973 ++++++++++++++++++ rust/src/runtime/store.rs | 14 +- rust/src/runtime/system_health.rs | 483 +++++++++ rust/tests/execution.rs | 2 + rust/tests/execution/operator_metrics.rs | 180 ++++ .../run_observation_active_summaries.rs | 62 +- .../conformance/server-parity/contract.mjs | 3 + .../operator-metrics-contract.mjs | 153 +++ .../server-parity/operator-metrics-probe.php | 96 ++ scripts/conformance/server-parity/probe.php | 6 + .../ServerParity/operator-metrics.json | 21 + ...erverParityOperatorMetricsContractTest.mjs | 147 +++ 20 files changed, 2348 insertions(+), 33 deletions(-) create mode 100644 rust/src/runtime/operator_metrics.rs create mode 100644 rust/src/runtime/system_health.rs create mode 100644 rust/tests/execution/operator_metrics.rs create mode 100644 scripts/conformance/server-parity/operator-metrics-contract.mjs create mode 100644 scripts/conformance/server-parity/operator-metrics-probe.php create mode 100644 tests/Fixtures/ServerParity/operator-metrics.json create mode 100644 tests/Unit/ServerParityOperatorMetricsContractTest.mjs diff --git a/.github/workflows/server-parity.yml b/.github/workflows/server-parity.yml index 0a01b6373..91ab2f11c 100644 --- a/.github/workflows/server-parity.yml +++ b/.github/workflows/server-parity.yml @@ -26,6 +26,7 @@ on: - tests/Unit/ServerParityWorkerDeregistrationContractTest.mjs - tests/Unit/ServerParityWorkerListingContractTest.mjs - tests/Unit/ServerParityWorkerReplacementContractTest.mjs + - tests/Unit/ServerParityOperatorMetricsContractTest.mjs - tests/Unit/ServerParityWaitingHistoryContractTest.mjs - tests/Unit/ServerParityTaskRetryContractTest.mjs - tests/Unit/ServerParityPublishedPatchContractTest.mjs @@ -65,6 +66,7 @@ on: - tests/Unit/ServerParityWorkerDeregistrationContractTest.mjs - tests/Unit/ServerParityWorkerListingContractTest.mjs - tests/Unit/ServerParityWorkerReplacementContractTest.mjs + - tests/Unit/ServerParityOperatorMetricsContractTest.mjs - tests/Unit/ServerParityWaitingHistoryContractTest.mjs - tests/Unit/ServerParityTaskRetryContractTest.mjs - tests/Unit/ServerParityPublishedPatchContractTest.mjs @@ -1424,7 +1426,7 @@ jobs: -e COMPOSER_HOME=/tmp/composer -v "$PWD:/source" -w /source "$image" \ install --working-dir=scripts/conformance/server-parity --no-interaction --no-progress docker run --rm --user=1000:1000 --entrypoint node -v "$PWD:/source:ro" -w /source \ - "$image" --test tests/Unit/ServerParityRunnerTest.mjs tests/Unit/ServerParityCooperativeContractTest.mjs tests/Unit/ServerParityChildCancellationContractTest.mjs tests/Unit/ServerParityScheduleContractTest.mjs tests/Unit/ServerParityVisibilityContractTest.mjs tests/Unit/ServerParityAdmissionContractTest.mjs tests/Unit/ServerParityRoleTokenContractTest.mjs tests/Unit/ServerParityWorkerDeregistrationContractTest.mjs tests/Unit/ServerParityWorkerListingContractTest.mjs tests/Unit/ServerParityWorkerReplacementContractTest.mjs tests/Unit/ServerParityPatchDeploymentContractTest.mjs tests/Unit/ServerParityWaitingHistoryContractTest.mjs tests/Unit/ServerParityTaskRetryContractTest.mjs tests/Unit/ServerParityPublishedPatchContractTest.mjs tests/Unit/ServerParityPublishedPatchTransportTest.mjs tests/Unit/ServerParityRecordFileTest.mjs > parity-evidence/comparator-tests.txt + "$image" --test tests/Unit/ServerParityRunnerTest.mjs tests/Unit/ServerParityCooperativeContractTest.mjs tests/Unit/ServerParityChildCancellationContractTest.mjs tests/Unit/ServerParityScheduleContractTest.mjs tests/Unit/ServerParityVisibilityContractTest.mjs tests/Unit/ServerParityAdmissionContractTest.mjs tests/Unit/ServerParityRoleTokenContractTest.mjs tests/Unit/ServerParityWorkerDeregistrationContractTest.mjs tests/Unit/ServerParityWorkerListingContractTest.mjs tests/Unit/ServerParityWorkerReplacementContractTest.mjs tests/Unit/ServerParityOperatorMetricsContractTest.mjs tests/Unit/ServerParityPatchDeploymentContractTest.mjs tests/Unit/ServerParityWaitingHistoryContractTest.mjs tests/Unit/ServerParityTaskRetryContractTest.mjs tests/Unit/ServerParityPublishedPatchContractTest.mjs tests/Unit/ServerParityPublishedPatchTransportTest.mjs tests/Unit/ServerParityRecordFileTest.mjs > parity-evidence/comparator-tests.txt docker run --rm --user=1000:1000 --cpus=1 --memory=256m --memory-swap=256m \ --network=none --entrypoint node -v "$PWD:/source:ro" -w /source \ "$image" --test tests/Unit/ServerParityCodecDeadlineLoadTest.mjs > parity-evidence/codec-load-tests.txt diff --git a/docs/rust-server-port.md b/docs/rust-server-port.md index 84290f3e9..07db75e6b 100644 --- a/docs/rust-server-port.md +++ b/docs/rust-server-port.md @@ -223,6 +223,52 @@ support is not evidence of multi-node safety. Newest checkpoints first. Each entry is exact to the source, tree and runs it names, including unsuccessful observations. +October 21, operator metrics and health: + +- **Source.** Native serves admin-only `GET /api/system/operator-metrics` and + `GET /api/system/health`, in the default or a named namespace, from the + rules of Workflow 2.5.5's `OperatorMetrics` and `HealthCheck`: + - the run, task, activity, backlog, start, history, schedule, worker and + command-contract counts, with PHP's predicates over the shared tables; + - the twelve health checks, their categories and messages, and 503 while a + check reports an error. + + Like Laravel, task comparisons with "now" bind whole seconds, while activity + deadlines and schedule times compare with microseconds. The policy + sections describe native: no repair loop, sticky routing or worker + sessions, in-process long-poll wake, and native's structural limits and + workflow task lease. `capacity_evidence` reports itself unavailable. +- **Decision: selected-run projections.** Native stores run waits in the + transaction that records history, so they cannot drift. It reads + timelines, timers and lineage from history, so those tables are reported as + not maintained rather than aligned. The health check reports native's wait + projections. PHP's same check also warns while its asynchronous timeline + projection catches up. +- **Task rows.** Native now stamps a new task's `created_at`, `updated_at`, + `last_dispatched_at` and `last_dispatch_attempt_at`, as PHP does in poll + mode. Run summaries classify a task's transport as PHP's + `RunSummaryProjector::taskLiveness` does: lease expired, dispatch failed, + claim failed (`_task_claim_failed`), dispatch overdue, then leased or + ready, with PHP's wait reasons and task-problem flag. Before this, native + declined to project a run whose tasks carried these columns. +- **Fixture.** `operator-metrics` drives a fresh namespace through seven + stages, from empty to a completed run with one activity retry. The stages + cover a waiting run, an overdue unclaimed task, and an activity scheduled, + running and retrying. Published PHP 2.5.14, embedded PHP (which declares it + inapplicable) and native pass the contract. In a local copy of the SQLite + parity step, 32 fixtures (the schedule and visibility fixtures need CI's + scheduler and CLI artifact) passed on all three targets, and the + comparison passed. +- **Known differences.** + - Summary `history_size_bytes` differs from PHP's (8 events: PHP 10,351 + bytes, native 7,961). + - PHP stamps a dispatch on some new tasks and not on an activity scheduled + from a workflow task completion, so `tasks_dispatched_last_minute` + differs. + - Native refuses worker deregistration in a named namespace. + - The operator dashboards, capacity evidence, and the repair, retention, + activity-timeout and payload-cleanup pass endpoints remain. + October 20, worker listing: - **Source.** Native serves `GET /api/workers` and `GET /api/workers/{id}`, diff --git a/rust/src/runtime/active_summary.rs b/rust/src/runtime/active_summary.rs index a014d4a19..a7cb0242f 100644 --- a/rust/src/runtime/active_summary.rs +++ b/rust/src/runtime/active_summary.rs @@ -65,13 +65,6 @@ where let run_id = DB::string(run, "id")?; if Self::scalar("SELECT COUNT(*) FROM workflow_run_waits WHERE workflow_run_id=$1 AND status='open' AND kind NOT IN ('condition','signal','timer','activity','child','update')") .bind(&run_id).fetch_one(&mut **tx).await? > 0 { return Ok(None); } - // These native task transitions do not author queue-driver metadata. - // Its presence needs the separately qualified transport classifier, - // including historical task problems and redispatch configuration. - if Self::scalar("SELECT COUNT(*) FROM workflow_tasks WHERE workflow_run_id=$1 AND (created_at IS NOT NULL OR last_dispatch_attempt_at IS NOT NULL OR last_claim_failed_at IS NOT NULL OR last_dispatched_at IS NOT NULL)") - .bind(&run_id).fetch_one(&mut **tx).await? > 0 { - return Ok(None); - } let at = now(); let mut summary = ActiveSummary { state: "repair_needed".into(), @@ -174,29 +167,68 @@ where let id = DB::string(task, "id")?; let status = DB::string(task, "status")?; let expires = DB::optional_instant(task, "lease_expires_at")?; - let expired = status == "leased" && expires.is_some_and(|v| v <= at); + let transport = TaskTransport::read::(task, &status, at)?; let label = label.unwrap_or(match kind.as_str() { "timer" => "Timer", "activity" => "Activity", _ => "Workflow", }); - summary.state = if expired { - "repair_needed".into() - } else { - format!("{kind}_task_{status}") + let json_time = |at: Option>| { + at.map_or_else( + || "an unknown time".to_owned(), + |at| at.format("%Y-%m-%dT%H:%M:%S%.6fZ").to_string(), + ) }; - summary.reason = if expired { - format!( - "{label} task {id} lease expired at {}.", - expires.unwrap().format("%Y-%m-%dT%H:%M:%S%.6fZ") + // PHP's RunSummaryProjector::taskLiveness, in its order. + (summary.state, summary.reason) = if transport.lease_expired { + ( + "repair_needed".into(), + format!("{label} task {id} lease expired at {}.", json_time(expires)), + ) + } else if let Some(error) = &transport.dispatch_failed { + ( + "repair_needed".into(), + format!( + "{label} task {id} could not be dispatched at {}. {error}", + json_time(transport.last_dispatch_attempt_at) + ), + ) + } else if let Some(error) = &transport.claim_failed { + ( + format!("{kind}_task_claim_failed"), + format!( + "{label} task {id} could not be claimed by a worker at {}. {error}", + json_time(transport.last_claim_failed_at) + ), + ) + } else if let Some(reference) = transport.dispatch_overdue { + ( + "repair_needed".into(), + format!( + "{label} task {id} is ready but has not been dispatched since {}.", + json_time(Some(reference)) + ), ) } else if status == "leased" { - format!("{label} task {id} is leased to a worker.") + ( + format!("{kind}_task_{status}"), + format!("{label} task {id} is leased to a worker."), + ) } else { - format!("{label} task {id} is ready to run.") + ( + format!("{kind}_task_{status}"), + format!("{label} task {id} is ready to run."), + ) }; - let wait_reason = if expired { + // PHP's wait reason checks the same states in another order. + let wait_reason = if transport.dispatch_failed.is_some() { + "Workflow task dispatch failed" + } else if transport.claim_failed.is_some() { + "Workflow task claim failed" + } else if transport.lease_expired { "Workflow task lease expired" + } else if transport.dispatch_overdue.is_some() { + "Workflow task ready but dispatch is overdue" } else if status == "leased" { "Workflow task leased to worker" } else { @@ -216,9 +248,9 @@ where summary.lease_expires_at = expires; summary.task_id = Some(id); summary.task_status = Some(status); - summary.problem |= expired && kind == "workflow"; + summary.problem |= kind == "workflow" && transport.problem(); + summary.repair_blocked = (summary.state != "repair_needed").then_some("repair_not_needed"); summary.task_type = Some(kind); - summary.repair_blocked = (!expired).then_some("repair_not_needed"); Ok(true) } @@ -238,3 +270,73 @@ where Ok(()) } } + +/// PHP's `TaskRepairPolicy` per-task transport predicates. +struct TaskTransport { + lease_expired: bool, + /// The trimmed dispatch error when the latest dispatch attempt failed. + dispatch_failed: Option, + /// The trimmed claim error when a worker could not claim the task. + claim_failed: Option, + /// When a ready task was last made available, if that is past the + /// redispatch window. + dispatch_overdue: Option>, + last_dispatch_attempt_at: Option>, + last_claim_failed_at: Option>, +} + +impl TaskTransport { + fn read(task: &DB::Row, status: &str, at: DateTime) -> Result { + let ready = status == "ready"; + let error = |field: &str| -> Result> { + Ok(DB::optional_string(task, field)? + .map(|error| error.trim().to_owned()) + .filter(|error| !error.is_empty())) + }; + let attempted = DB::optional_instant(task, "last_dispatch_attempt_at")?; + let dispatched = DB::optional_instant(task, "last_dispatched_at")?; + let claim_failed_at = DB::optional_instant(task, "last_claim_failed_at")?; + let dispatch_failed = match (ready, attempted) { + (true, Some(attempted)) + if dispatched.is_none_or(|dispatched| attempted > dispatched) => + { + error("last_dispatch_error")? + } + _ => None, + }; + let claim_failed = if ready && claim_failed_at.is_some() { + error("last_claim_error")? + } else { + None + }; + let available = DB::optional_instant(task, "available_at")?; + let reference = dispatched.or(DB::optional_instant(task, "created_at")?); + let cutoff = + at - chrono::Duration::seconds(super::operator_metrics::REDISPATCH_AFTER_SECONDS); + let dispatch_overdue = (ready + && available.is_none_or(|available| available <= at) + && dispatch_failed.is_none() + && claim_failed.is_none()) + .then_some(reference) + .flatten() + .filter(|reference| *reference <= cutoff); + Ok(Self { + lease_expired: status == "leased" + && DB::optional_instant(task, "lease_expires_at")? + .is_some_and(|expires| expires <= at), + dispatch_failed, + claim_failed, + dispatch_overdue, + last_dispatch_attempt_at: attempted, + last_claim_failed_at: claim_failed_at, + }) + } + + /// `workflowTaskProblemEvidence` for this task's transport state. + fn problem(&self) -> bool { + self.lease_expired + || self.dispatch_failed.is_some() + || self.claim_failed.is_some() + || self.dispatch_overdue.is_some() + } +} diff --git a/rust/src/runtime/backend.rs b/rust/src/runtime/backend.rs index 4be547cdd..b7bd93f3f 100644 --- a/rust/src/runtime/backend.rs +++ b/rust/src/runtime/backend.rs @@ -35,6 +35,9 @@ pub(super) trait Backend: Database { const REPAIR_COUNTER_MAX: i64 = i32::MAX as i64; const RANGE_REPAIR_TASK_SQL: &'static str = "SELECT * FROM workflow_tasks WHERE workflow_run_id=$1 AND namespace=$2 AND task_type='workflow' AND status='failed' AND payload->>'replay_blocked'='true' ORDER BY CASE WHEN available_at IS NULL THEN 1 ELSE 0 END,available_at,created_at,id LIMIT 1"; const ACTIVITY_OPEN_TASK_SQL: &'static str = "SELECT * FROM workflow_tasks WHERE workflow_run_id=$1 AND task_type='activity' AND status IN ('ready','leased') AND payload->>'activity_execution_id'=$2 ORDER BY id LIMIT 1"; + // Local activities in the namespace, as PHP's operator metrics count them. + const LOCAL_ACTIVITY_EXECUTIONS: &'static str = "SELECT status FROM activity_executions WHERE workflow_run_id IN (SELECT id FROM workflow_runs WHERE namespace=$1) AND activity_options->>'execution_mode'='local'"; + const LOCAL_ACTIVITY_ATTEMPTS: &'static str = "SELECT p.status FROM activity_attempts p JOIN activity_executions a ON a.id=p.activity_execution_id WHERE p.workflow_run_id IN (SELECT id FROM workflow_runs WHERE namespace=$1) AND a.activity_options->>'execution_mode'='local'"; const QUERY_CACHE_SCAN: &'static str = "SELECT \"key\",value,expiration FROM dw_query_cache WHERE \"key\" LIKE 'dw-native-query:%' AND value::jsonb->>'namespace'=$1 ORDER BY \"key\" LIMIT 65"; const QUERY_CACHE_PENDING_FOR_RUN: &'static str = "SELECT value::jsonb->>'query_admitted_at' AS query_admitted_at,value::jsonb->>'query_deadline_at' AS query_deadline_at,value::jsonb->>'receipt_expires_at' AS receipt_expires_at FROM dw_query_cache WHERE \"key\" LIKE 'dw-native-query:%' AND expiration>$2 AND value::jsonb->>'run_id'=$1 AND value::jsonb->>'status' IN ('pending','leased') ORDER BY \"key\" LIMIT 65"; const QUERY_CACHE_GET: &'static str = @@ -148,6 +151,8 @@ ORDER BY t.available_order,t.id LIMIT 100 const REPAIR_COUNTER_MAX: i64 = i64::MAX; const RANGE_REPAIR_TASK_SQL: &'static str = "SELECT * FROM workflow_tasks WHERE workflow_run_id=$1 AND namespace=$2 AND task_type='workflow' AND status='failed' AND json_extract(payload,'$.replay_blocked')=1 ORDER BY CASE WHEN available_at IS NULL THEN 1 ELSE 0 END,available_at,created_at,id LIMIT 1"; const ACTIVITY_OPEN_TASK_SQL: &'static str = "SELECT * FROM workflow_tasks WHERE workflow_run_id=$1 AND task_type='activity' AND status IN ('ready','leased') AND json_extract(payload,'$.activity_execution_id')=$2 ORDER BY id LIMIT 1"; + const LOCAL_ACTIVITY_EXECUTIONS: &'static str = "SELECT status FROM activity_executions WHERE workflow_run_id IN (SELECT id FROM workflow_runs WHERE namespace=$1) AND json_extract(activity_options,'$.execution_mode')='local'"; + const LOCAL_ACTIVITY_ATTEMPTS: &'static str = "SELECT p.status FROM activity_attempts p JOIN activity_executions a ON a.id=p.activity_execution_id WHERE p.workflow_run_id IN (SELECT id FROM workflow_runs WHERE namespace=$1) AND json_extract(a.activity_options,'$.execution_mode')='local'"; const VISIBILITY_PAGE: &'static str = super::visibility::SQLITE_VISIBILITY_PAGE; type EncodedTime = String; type EncodedOptionalTime = Option; @@ -379,6 +384,8 @@ impl Backend for MySql { const REPAIR_COUNTER_MAX: i64 = u32::MAX as i64; const RANGE_REPAIR_TASK_SQL: &'static str = "SELECT * FROM workflow_tasks WHERE workflow_run_id=? AND namespace=? AND task_type='workflow' AND status='failed' AND JSON_UNQUOTE(JSON_EXTRACT(payload,'$.replay_blocked'))='true' ORDER BY CASE WHEN available_at IS NULL THEN 1 ELSE 0 END,available_at,created_at,id LIMIT 1"; const ACTIVITY_OPEN_TASK_SQL: &'static str = "SELECT * FROM workflow_tasks WHERE workflow_run_id=? AND task_type='activity' AND status IN ('ready','leased') AND JSON_UNQUOTE(JSON_EXTRACT(payload,'$.activity_execution_id'))=? ORDER BY id LIMIT 1"; + const LOCAL_ACTIVITY_EXECUTIONS: &'static str = "SELECT status FROM activity_executions WHERE workflow_run_id IN (SELECT id FROM workflow_runs WHERE namespace=?) AND JSON_UNQUOTE(JSON_EXTRACT(activity_options,'$.execution_mode'))='local'"; + const LOCAL_ACTIVITY_ATTEMPTS: &'static str = "SELECT p.status FROM activity_attempts p JOIN activity_executions a ON a.id=p.activity_execution_id WHERE p.workflow_run_id IN (SELECT id FROM workflow_runs WHERE namespace=?) AND JSON_UNQUOTE(JSON_EXTRACT(a.activity_options,'$.execution_mode'))='local'"; type EncodedTime = DateTime; type EncodedOptionalTime = Option>; type EncodedDocument = Json; @@ -407,6 +414,8 @@ impl Backend for MySql { || sql == Self::RANGE_REPAIR_TASK_SQL || sql == Self::SELECTION_RESOLUTION_SQL || sql == Self::ACTIVITY_OPEN_TASK_SQL + || sql == Self::LOCAL_ACTIVITY_EXECUTIONS + || sql == Self::LOCAL_ACTIVITY_ATTEMPTS || sql == Self::ACTIVITY_RESULT_EVENT_SQL || sql == Self::OBSERVATION_FAILURE_EVENT || sql == Self::POLL_WAKE_QUERIES_SQL diff --git a/rust/src/runtime/execution.rs b/rust/src/runtime/execution.rs index 3a54ecc0a..52092015e 100644 --- a/rust/src/runtime/execution.rs +++ b/rust/src/runtime/execution.rs @@ -287,6 +287,8 @@ impl Runtime { delegate!(namespace_exists(name: &str) -> Result); delegate!(list_namespaces() -> Result); delegate!(describe_namespace(name: &str) -> Result); + delegate!(operator_metrics(config: &super::operator_metrics::MetricsConfig) -> Result); + delegate!(system_health(config: &super::operator_metrics::MetricsConfig) -> Result<(bool, Value)>); delegate!(create_namespace(body: Value) -> Result); delegate!(set_external_policy(name: &str, body: Value) -> Result); delegate!(update_namespace(name: &str, body: Value) -> Result); diff --git a/rust/src/runtime/history_budget.rs b/rust/src/runtime/history_budget.rs index c5e9232a9..9c00d5d55 100644 --- a/rust/src/runtime/history_budget.rs +++ b/rust/src/runtime/history_budget.rs @@ -80,6 +80,11 @@ impl Config { Ok(config) } + /// The hard event, size and fan-out thresholds, then their warning thresholds. + pub(super) fn thresholds(&self) -> [i64; 6] { + self.thresholds + } + pub(super) fn summarize(self, count: i64, bytes: i64, fan_out: i64) -> Budget { let values = [count.max(0), bytes.max(0), fan_out.max(0)]; let labels = ["event_count", "size_bytes", "fan_out"]; diff --git a/rust/src/runtime/http.rs b/rust/src/runtime/http.rs index 37288d86a..5ff323e45 100644 --- a/rust/src/runtime/http.rs +++ b/rust/src/runtime/http.rs @@ -155,6 +155,8 @@ pub fn router(runtime: Runtime) -> Router { "/api/worker/registrations/{worker_id}/deregister", post(deregister_fenced), ) + .route("/api/system/operator-metrics", get(operator_metrics)) + .route("/api/system/health", get(system_health)) .route("/api/workers", get(list_workers)) .route( "/api/workers/{worker_id}", @@ -285,6 +287,8 @@ async fn guard(State(runtime): State, mut request: Request, next: Next) || path == "/api/namespaces" && request.method() == Method::POST // Updating a namespace or its external storage needs an admin. || path.starts_with("/api/namespaces/") && request.method() == Method::PUT + // PHP's system routes are admin-only. + || path.starts_with("/api/system/") { &["admin"] } else { @@ -423,6 +427,8 @@ fn named_namespace_operation(method: &str, path: &str) -> bool { || path .strip_prefix("/api/workers/") .is_some_and(|id| !id.is_empty() && !id.contains('/')) + || path == "/api/system/operator-metrics" + || path == "/api/system/health" { return method == "GET"; } @@ -952,6 +958,29 @@ async fn worker_heartbeat( ) -> Result> { runtime.worker_heartbeat(body).await.map(Json) } +fn metrics_config(runtime: &Runtime) -> super::operator_metrics::MetricsConfig { + super::operator_metrics::MetricsConfig { + // Native leases a claimed workflow task for 30 seconds. + workflow_task_lease_seconds: 30, + abandoned_upload_expiry_seconds: runtime.external.abandoned_seconds.max(1), + } +} + +async fn operator_metrics(Extension(runtime): Extension) -> Result> { + Ok(Json(runtime.operator_metrics(&metrics_config(&runtime)).await?)) +} + +/// PHP answers 503 while a check reports an error. +async fn system_health(Extension(runtime): Extension) -> Result { + let (unavailable, body) = runtime.system_health(&metrics_config(&runtime)).await?; + let status = if unavailable { + StatusCode::SERVICE_UNAVAILABLE + } else { + StatusCode::OK + }; + Ok((status, Json(body)).into_response()) +} + async fn list_workers( Extension(runtime): Extension, Query(query): Query>, diff --git a/rust/src/runtime/mod.rs b/rust/src/runtime/mod.rs index 2e8893020..9f9176004 100644 --- a/rust/src/runtime/mod.rs +++ b/rust/src/runtime/mod.rs @@ -34,6 +34,7 @@ mod memo_limits; mod memos; pub mod mysql; mod namespaces; +mod operator_metrics; mod poll_readiness; pub mod postgres; mod previews; @@ -51,6 +52,7 @@ mod selection_summary; mod selections; mod sqlite; mod store; +mod system_health; mod task_failures; mod task_repair; mod task_repair_http; diff --git a/rust/src/runtime/operator_metrics.rs b/rust/src/runtime/operator_metrics.rs new file mode 100644 index 000000000..067f7ea4e --- /dev/null +++ b/rust/src/runtime/operator_metrics.rs @@ -0,0 +1,973 @@ +//! The operator metrics document of one namespace, as published PHP Server +//! 2.5.14 (Workflow 2.5.5 `OperatorMetrics::snapshot`) builds it for +//! `GET /api/system/operator-metrics`. +//! +//! The count sections read the same durable tables with PHP's predicates. +//! As Laravel binds a bare timestamp in whole seconds, task comparisons with +//! "now" use the current second, so a task made available within it still +//! counts as delayed; activity deadlines and schedule times compare with +//! microseconds, as PHP formats them. The policy sections describe this +//! runtime, and projections native derives from history instead of storing +//! are reported as not maintained. +use super::{ + Result, + backend::Backend, + store::{Store, now}, +}; +use chrono::{DateTime, Duration, SecondsFormat, Timelike, Utc}; +use serde_json::{Value, json}; +use sqlx::{ColumnIndex, Decode, Encode, Executor, IntoArguments, Type}; + +/// PHP's `TaskRepairPolicy::redispatchAfterSeconds()` default: a ready task +/// still unclaimed this long after it was made available is overdue. +pub(super) const REDISPATCH_AFTER_SECONDS: i64 = 3; + +/// Laravel binds a bare timestamp as `Y-m-d H:i:s`. +fn whole_second(at: DateTime) -> DateTime { + at.with_nanosecond(0).unwrap_or(at) +} + +/// Carbon's `toJSON()`. +pub(super) fn time(at: Option>) -> Value { + json!(at.map(|at| at.to_rfc3339_opts(SecondsFormat::Micros, true))) +} + +/// `(int) $oldest->diffInMilliseconds($now)`, or 0 without a timestamp. +pub(super) fn age(at: Option>, now: DateTime) -> i64 { + at.map_or(0, |at| (now - at).num_milliseconds().max(0)) +} + +fn earliest(times: &[Option>]) -> Option> { + times.iter().flatten().min().copied() +} + +/// Rows of a table that belong to the namespace through their run. +macro_rules! scoped { + ($select:literal, $table:literal, $condition:expr) => { + concat!( + "SELECT ", + $select, + " FROM ", + $table, + " WHERE workflow_run_id IN (SELECT id FROM workflow_runs WHERE namespace=$1) AND ", + $condition + ) + }; +} +macro_rules! summaries { + ($condition:literal) => { + concat!( + "SELECT COUNT(*) FROM workflow_run_summaries WHERE namespace=$1 AND ", + $condition + ) + }; +} +// PHP's task predicates. $2 is the bound "now". +macro_rules! ready_due { + () => { + "status='ready' AND (available_at IS NULL OR available_at<=$2)" + }; +} +macro_rules! lease_expired { + () => { + "status='leased' AND lease_expires_at IS NOT NULL AND lease_expires_at<=$2" + }; +} +macro_rules! dispatch_failed { + () => { + "status='ready' AND last_dispatch_attempt_at IS NOT NULL AND last_dispatch_error IS NOT NULL AND last_dispatch_error!='' AND (last_dispatched_at IS NULL OR last_dispatch_attempt_at>last_dispatched_at)" + }; +} +macro_rules! claim_failed { + () => { + "status='ready' AND last_claim_failed_at IS NOT NULL AND last_claim_error IS NOT NULL AND last_claim_error!=''" + }; +} +// Ready and due with healthy dispatch and claim, and made available (its last +// dispatch, or its creation when never dispatched) before the cutoff $3. +macro_rules! dispatch_overdue { + () => { + "status='ready' AND (available_at IS NULL OR available_at<=$2) AND (last_dispatch_attempt_at IS NULL OR last_dispatch_error IS NULL OR last_dispatch_error='' OR (last_dispatched_at IS NOT NULL AND last_dispatch_attempt_at<=last_dispatched_at)) AND (last_claim_failed_at IS NULL OR last_claim_error IS NULL OR last_claim_error='') AND COALESCE(last_dispatched_at,created_at)<=$3" + }; +} + +impl Store +where + for<'c> &'c mut DB::Connection: Executor<'c, Database = DB>, + DB::Arguments: IntoArguments, + for<'q> &'q str: Encode<'q, DB> + Type, + for<'q> Option<&'q str>: Encode<'q, DB>, + for<'q> Option: Encode<'q, DB>, + for<'q> String: Encode<'q, DB> + Type, + for<'q> i64: Encode<'q, DB> + Type, + for<'r> i64: Decode<'r, DB>, + usize: ColumnIndex, +{ + /// A count over the namespace, with any further bound timestamps. + async fn tally(&self, sql: &'static str, times: &[DateTime]) -> Result { + let mut query = Self::scalar(sql).bind(self.namespace.as_ref()); + for at in times { + query = query.bind(DB::bind_time(*at)); + } + Ok(query.fetch_one(&self.pool).await?) + } + + /// The first row's `column`, or its `fallback` when that is null. + async fn first_time( + &self, + sql: &'static str, + times: &[DateTime], + column: &str, + fallback: Option<&str>, + ) -> Result>> { + let mut query = Self::query(sql).bind(self.namespace.as_ref()); + for at in times { + query = query.bind(DB::bind_time(*at)); + } + let Some(row) = query.fetch_optional(&self.pool).await? else { + return Ok(None); + }; + match DB::optional_instant(&row, column)? { + Some(at) => Ok(Some(at)), + None => fallback.map_or(Ok(None), |column| DB::optional_instant(&row, column)), + } + } + + async fn largest(&self, sql: &'static str) -> Result { + let row = Self::query(sql) + .bind(self.namespace.as_ref()) + .fetch_one(&self.pool) + .await?; + Ok(DB::optional_number(&row, "n")?.unwrap_or_default()) + } + + async fn statuses(&self, sql: &'static str) -> Result> { + Self::query(sql) + .bind(self.namespace.as_ref()) + .fetch_all(&self.pool) + .await? + .iter() + .map(|row| DB::string(row, "status")) + .collect() + } + + /// `OperatorMetrics::snapshot` for this namespace. + pub(super) async fn operator_metrics_snapshot(&self, config: &MetricsConfig) -> Result { + let now = now(); + Ok(json!({ + "generated_at": time(Some(now)), + "runs": self.run_metrics(now).await?, + "tasks": self.task_metrics(now).await?, + "activities": self.activity_metrics(now).await?, + "backlog": self.backlog_metrics(now).await?, + "repair": empty_repair_snapshot(), + "starts": self.start_metrics(now).await?, + "history": self.history_metrics().await?, + "command_contracts": self.command_contract_metrics().await?, + "projections": self.projection_metrics(now).await?, + "schedules": self.schedule_metrics(now).await?, + "workers": self.worker_metrics().await?, + "backend": backend_capabilities(now, DB::PHP_CONNECTION), + "structural_limits": structural_limits(DB::PHP_CONNECTION), + "update_wait": {"completion_timeout_seconds": 10, "poll_interval_milliseconds": 50}, + "repair_policy": repair_policy(), + "matching_role": matching_role(config), + "sticky_execution": sticky_execution(), + })) + } + + /// `GET /api/system/operator-metrics`: the snapshot with the sections the + /// server adds. + pub(super) async fn operator_metrics(&self, config: &MetricsConfig) -> Result { + let mut metrics = self.operator_metrics_snapshot(config).await?; + metrics["worker_sessions"] = + json!({"total": 0, "active": 0, "closed": 0, "expired": 0, "failed": 0, "orphaned": 0}); + metrics["capacity_evidence"] = + json!({"available": false, "reason": "native_capacity_evidence_unavailable"}); + metrics["runtime_external_payload_cleanup"] = + self.external_payload_cleanup_metrics(config).await?; + Ok(json!({"namespace": self.namespace.as_ref(), "operator_metrics": metrics})) + } + + /// `GET /api/system/health`, with PHP's HTTP status. + pub(super) async fn system_health(&self, config: &MetricsConfig) -> Result<(bool, Value)> { + let health = super::system_health::snapshot(&self.operator_metrics_snapshot(config).await?); + let retention = Self::query( + "SELECT retention_mode,retention_days FROM workflow_namespaces WHERE name=$1", + ) + .bind(self.namespace.as_ref()) + .fetch_optional(&self.pool) + .await?; + let (mode, days) = match &retention { + Some(row) => ( + DB::optional_string(row, "retention_mode")?, + DB::optional_number(row, "retention_days")?, + ), + None => (Some("bounded".to_owned()), Some(30)), + }; + let unavailable = health["status"] == "error"; + Ok(( + unavailable, + json!({"namespace": self.namespace.as_ref(), "retention_mode": mode, "retention_days": days, "health": health}), + )) + } + + /// PHP's `RunCommandContract::backfillStatus` over the namespace's runs + /// with a WorkflowStarted event, 200 runs at a time. Native has no PHP + /// class snapshot to backfill from, so none is available. + async fn command_contract_metrics(&self) -> Result { + let (mut needed, mut actionable) = (0_i64, 0_i64); + let mut after = String::new(); + loop { + let rows = Self::query("SELECT r.id,r.status,h.payload FROM workflow_runs r JOIN workflow_history_events h ON h.workflow_run_id=r.id AND h.event_type='WorkflowStarted' WHERE r.namespace=$1 AND r.id>$2 ORDER BY r.id,h.sequence LIMIT 200") + .bind(self.namespace.as_ref()) + .bind(after.as_str()) + .fetch_all(&self.pool) + .await?; + let Some(last) = rows.last() else { break }; + let last = DB::string(last, "id")?; + for row in &rows { + let id = DB::string(row, "id")?; + // The first WorkflowStarted event by sequence decides. + if id == after { + continue; + } + after = id; + let payload = DB::optional_document_row(row, "payload")?.unwrap_or(Value::Null); + if super::run_observation::command_contract(Some(&payload))["backfill_needed"] + != true + { + continue; + } + needed += 1; + if !super::run_observation_store::terminal(&DB::string(row, "status")?) { + actionable += 1; + } + } + after = last; + if rows.len() < 200 { + break; + } + } + Ok(json!({ + "backfill_needed_runs": needed, + "backfill_available_runs": 0, + "backfill_unavailable_runs": needed, + "actionable_backfill_needed_runs": actionable, + "actionable_backfill_available_runs": 0, + "actionable_backfill_unavailable_runs": actionable, + "closed_backfill_needed_runs": needed - actionable, + })) + } + + async fn run_metrics(&self, now: DateTime) -> Result { + let repair = self.first_time("SELECT updated_at,started_at FROM workflow_run_summaries WHERE namespace=$1 AND liveness_state='repair_needed' ORDER BY COALESCE(updated_at,started_at) LIMIT 1", + &[], "updated_at", Some("started_at")).await?; + let wait = self.first_time("SELECT wait_started_at FROM workflow_run_summaries WHERE namespace=$1 AND status_bucket='running' AND wait_started_at IS NOT NULL ORDER BY wait_started_at LIMIT 1", + &[], "wait_started_at", None).await?; + Ok(json!({ + "total": self.tally("SELECT COUNT(*) FROM workflow_run_summaries WHERE namespace=$1", &[]).await?, + "current": self.tally(summaries!("is_current_run=TRUE"), &[]).await?, + "running": self.tally(summaries!("status_bucket='running'"), &[]).await?, + "completed": self.tally(summaries!("status='completed'"), &[]).await?, + "failed": self.tally(summaries!("status='failed'"), &[]).await?, + "cancelled": self.tally(summaries!("status='cancelled'"), &[]).await?, + "terminated": self.tally(summaries!("status='terminated'"), &[]).await?, + "archived": self.tally(summaries!("archived_at IS NOT NULL"), &[]).await?, + "repair_needed": self.tally(summaries!("liveness_state='repair_needed'"), &[]).await?, + "oldest_repair_needed_at": time(repair), + "max_repair_needed_age_ms": age(repair, now), + "claim_failed": self.tally(summaries!("liveness_state LIKE '%_task_claim_failed'"), &[]).await?, + "compatibility_blocked": self.compatibility_blocked_runs().await?, + "waiting": self.tally(summaries!("status_bucket='running' AND wait_started_at IS NOT NULL"), &[]).await?, + "oldest_wait_started_at": time(wait), + "max_wait_age_ms": age(wait, now), + })) + } + + async fn compatibility_blocked_runs(&self) -> Result { + self.tally( + summaries!("liveness_state LIKE '%_task_waiting_for_compatible_worker'"), + &[], + ) + .await + } + + async fn task_metrics(&self, now: DateTime) -> Result { + let bound = whole_second(now); + let cutoff = whole_second(now - Duration::seconds(REDISPATCH_AFTER_SECONDS)); + let lease_expired_at = self + .first_time( + concat!( + scoped!("lease_expires_at", "workflow_tasks", lease_expired!()), + " ORDER BY lease_expires_at LIMIT 1" + ), + &[bound], + "lease_expires_at", + None, + ) + .await?; + let ready_due_at = self + .first_time( + concat!( + scoped!("available_at,created_at", "workflow_tasks", ready_due!()), + " ORDER BY COALESCE(available_at,created_at) LIMIT 1" + ), + &[bound], + "available_at", + Some("created_at"), + ) + .await?; + let overdue_since = self + .first_time( + concat!( + scoped!( + "last_dispatched_at,created_at", + "workflow_tasks", + dispatch_overdue!() + ), + " ORDER BY COALESCE(last_dispatched_at,created_at) LIMIT 1" + ), + &[bound, cutoff], + "last_dispatched_at", + Some("created_at"), + ) + .await?; + let claim_failed_at = self + .first_time( + concat!( + scoped!("last_claim_failed_at", "workflow_tasks", claim_failed!()), + " ORDER BY last_claim_failed_at LIMIT 1" + ), + &[], + "last_claim_failed_at", + None, + ) + .await?; + let dispatch_failed_at = self + .first_time( + concat!( + scoped!( + "last_dispatch_attempt_at", + "workflow_tasks", + dispatch_failed!() + ), + " ORDER BY last_dispatch_attempt_at LIMIT 1" + ), + &[], + "last_dispatch_attempt_at", + None, + ) + .await?; + let unhealthy = self.unhealthy_tasks(now).await?; + let unhealthy_at = earliest(&[ + dispatch_failed_at, + claim_failed_at, + overdue_since, + lease_expired_at, + ]); + Ok(json!({ + "open": self.tally(scoped!("COUNT(*)", "workflow_tasks", "status IN ('ready','leased')"), &[]).await?, + "ready": self.tally(scoped!("COUNT(*)", "workflow_tasks", "status='ready'"), &[]).await?, + "ready_due": self.tally(scoped!("COUNT(*)", "workflow_tasks", ready_due!()), &[bound]).await?, + "delayed": self.tally(scoped!("COUNT(*)", "workflow_tasks", "status='ready' AND available_at>$2"), &[bound]).await?, + "leased": self.tally(scoped!("COUNT(*)", "workflow_tasks", "status='leased'"), &[]).await?, + "dispatch_failed": unhealthy.dispatch_failed, + "claim_failed": unhealthy.claim_failed, + "dispatch_overdue": unhealthy.dispatch_overdue, + "lease_expired": unhealthy.lease_expired, + "oldest_lease_expired_at": time(lease_expired_at), + "max_lease_expired_age_ms": age(lease_expired_at, now), + "oldest_ready_due_at": time(ready_due_at), + "max_ready_due_age_ms": age(ready_due_at, now), + "oldest_dispatch_overdue_since": time(overdue_since), + "max_dispatch_overdue_age_ms": age(overdue_since, now), + "oldest_claim_failed_at": time(claim_failed_at), + "max_claim_failed_age_ms": age(claim_failed_at, now), + "oldest_dispatch_failed_at": time(dispatch_failed_at), + "max_dispatch_failed_age_ms": age(dispatch_failed_at, now), + "max_attempt_count": self.largest(scoped!("MAX(attempt_count) AS n", "workflow_tasks", "status IN ('ready','leased')")).await?, + "max_repair_count": self.largest(scoped!("MAX(repair_count) AS n", "workflow_tasks", "status IN ('ready','leased')")).await?, + "unhealthy": unhealthy.total(), + "oldest_unhealthy_at": time(unhealthy_at), + "max_unhealthy_age_ms": age(unhealthy_at, now), + })) + } + + async fn unhealthy_tasks(&self, now: DateTime) -> Result { + let bound = whole_second(now); + let cutoff = whole_second(now - Duration::seconds(REDISPATCH_AFTER_SECONDS)); + Ok(UnhealthyTasks { + dispatch_failed: self + .tally( + scoped!("COUNT(*)", "workflow_tasks", dispatch_failed!()), + &[], + ) + .await?, + claim_failed: self + .tally(scoped!("COUNT(*)", "workflow_tasks", claim_failed!()), &[]) + .await?, + dispatch_overdue: self + .tally( + scoped!("COUNT(*)", "workflow_tasks", dispatch_overdue!()), + &[bound, cutoff], + ) + .await?, + lease_expired: self + .tally( + scoped!("COUNT(*)", "workflow_tasks", lease_expired!()), + &[bound], + ) + .await?, + }) + } + + async fn activity_metrics(&self, now: DateTime) -> Result { + let retrying_at = self.first_time(scoped!("started_at", "activity_executions", "status='pending' AND attempt_count>0 AND started_at IS NOT NULL ORDER BY started_at LIMIT 1"), + &[], "started_at", None).await?; + let mut deadlines = Vec::new(); + for (sql, column) in [ + ( + scoped!( + "schedule_deadline_at", + "activity_executions", + "status='pending' AND schedule_deadline_at IS NOT NULL AND schedule_deadline_at<=$2 ORDER BY schedule_deadline_at LIMIT 1" + ), + "schedule_deadline_at", + ), + ( + scoped!( + "close_deadline_at", + "activity_executions", + "status='running' AND close_deadline_at IS NOT NULL AND close_deadline_at<=$2 ORDER BY close_deadline_at LIMIT 1" + ), + "close_deadline_at", + ), + ( + scoped!( + "schedule_to_close_deadline_at", + "activity_executions", + "status IN ('pending','running') AND schedule_to_close_deadline_at IS NOT NULL AND schedule_to_close_deadline_at<=$2 ORDER BY schedule_to_close_deadline_at LIMIT 1" + ), + "schedule_to_close_deadline_at", + ), + ( + scoped!( + "heartbeat_deadline_at", + "activity_executions", + "status='running' AND heartbeat_deadline_at IS NOT NULL AND heartbeat_deadline_at<=$2 ORDER BY heartbeat_deadline_at LIMIT 1" + ), + "heartbeat_deadline_at", + ), + ] { + deadlines.push(self.first_time(sql, &[now], column, None).await?); + } + let overdue_at = earliest(&deadlines); + let open = self + .tally( + scoped!( + "COUNT(*)", + "activity_executions", + "status IN ('pending','running')" + ), + &[], + ) + .await?; + let pending = self + .tally( + scoped!("COUNT(*)", "activity_executions", "status='pending'"), + &[], + ) + .await?; + let running = self + .tally( + scoped!("COUNT(*)", "activity_executions", "status='running'"), + &[], + ) + .await?; + let failed_attempts = self + .tally( + scoped!("COUNT(*)", "activity_attempts", "status='failed'"), + &[], + ) + .await?; + let local = self.statuses(DB::LOCAL_ACTIVITY_EXECUTIONS).await?; + let local_count = |status: &[&str]| { + local + .iter() + .filter(|s| status.contains(&s.as_str())) + .count() as i64 + }; + let (local_open, local_pending, local_running) = ( + local_count(&["pending", "running"]), + local_count(&["pending"]), + local_count(&["running"]), + ); + let local_attempts = self.statuses(DB::LOCAL_ACTIVITY_ATTEMPTS).await?; + let local_failed_attempts = local_attempts.iter().filter(|s| *s == "failed").count() as i64; + Ok(json!({ + "open": open, + "pending": pending, + "running": running, + "local": local.len(), + "local_open": local_open, + "local_pending": local_pending, + "local_running": local_running, + "local_attempts": local_attempts.len(), + "local_failed_attempts": local_failed_attempts, + "queued_open": (open - local_open).max(0), + "queued_pending": (pending - local_pending).max(0), + "queued_running": (running - local_running).max(0), + "queued_failed_attempts": (failed_attempts - local_failed_attempts).max(0), + "retrying": self.retrying_activities().await?, + "oldest_retrying_started_at": time(retrying_at), + "max_retrying_age_ms": age(retrying_at, now), + "timeout_overdue": self.tally(scoped!("COUNT(*)", "activity_executions", "status IN ('pending','running') AND ((status='pending' AND schedule_deadline_at IS NOT NULL AND schedule_deadline_at<=$2) OR (status='running' AND close_deadline_at IS NOT NULL AND close_deadline_at<=$3) OR (schedule_to_close_deadline_at IS NOT NULL AND schedule_to_close_deadline_at<=$4) OR (status='running' AND heartbeat_deadline_at IS NOT NULL AND heartbeat_deadline_at<=$5))"), &[now, now, now, now]).await?, + "oldest_timeout_overdue_at": time(overdue_at), + "max_timeout_overdue_age_ms": age(overdue_at, now), + "failed_attempts": failed_attempts, + "max_attempt_count": self.largest(scoped!("MAX(attempt_count) AS n", "activity_executions", "1=1")).await?, + })) + } + + async fn retrying_activities(&self) -> Result { + self.tally( + scoped!( + "COUNT(*)", + "activity_executions", + "status='pending' AND attempt_count>0" + ), + &[], + ) + .await + } + + async fn backlog_metrics(&self, now: DateTime) -> Result { + let bound = whole_second(now); + let minute = whole_second(now - Duration::minutes(1)); + let blocked_at = self.first_time("SELECT wait_started_at,next_task_at FROM workflow_run_summaries WHERE namespace=$1 AND liveness_state LIKE '%_task_waiting_for_compatible_worker' ORDER BY COALESCE(wait_started_at,next_task_at) LIMIT 1", + &[], "wait_started_at", Some("next_task_at")).await?; + Ok(json!({ + "runnable_tasks": self.tally(scoped!("COUNT(*)", "workflow_tasks", ready_due!()), &[bound]).await?, + "delayed_tasks": self.tally(scoped!("COUNT(*)", "workflow_tasks", "status='ready' AND available_at>$2"), &[bound]).await?, + "leased_tasks": self.tally(scoped!("COUNT(*)", "workflow_tasks", "status='leased'"), &[]).await?, + "tasks_added_last_minute": self.tally(scoped!("COUNT(*)", "workflow_tasks", "created_at>=$2"), &[minute]).await?, + "tasks_dispatched_last_minute": self.tally(scoped!("COUNT(*)", "workflow_tasks", "last_dispatched_at IS NOT NULL AND last_dispatched_at>=$2"), &[minute]).await?, + "retrying_activities": self.retrying_activities().await?, + "unhealthy_tasks": self.unhealthy_tasks(now).await?.total(), + "repair_needed_runs": self.tally(summaries!("liveness_state='repair_needed'"), &[]).await?, + "claim_failed_runs": self.tally(summaries!("liveness_state LIKE '%_task_claim_failed'"), &[]).await?, + "compatibility_blocked_runs": self.compatibility_blocked_runs().await?, + "oldest_compatibility_blocked_started_at": time(blocked_at), + "max_compatibility_blocked_age_ms": age(blocked_at, now), + })) + } + + async fn start_metrics(&self, now: DateTime) -> Result { + let bound = whole_second(now); + let accepted = self.first_time("SELECT accepted_at FROM workflow_commands WHERE workflow_run_id IN (SELECT id FROM workflow_runs WHERE namespace=$1 AND status='pending') AND command_type='start' AND status='accepted' AND outcome='started_new' AND accepted_at IS NOT NULL ORDER BY accepted_at,id LIMIT 1", + &[], "accepted_at", None).await?; + let pending_at = match accepted { + Some(at) => Some(at), + None => self.first_time("SELECT started_at FROM workflow_run_summaries WHERE namespace=$1 AND status='pending' AND started_at IS NOT NULL ORDER BY started_at,id LIMIT 1", + &[], "started_at", None).await?, + }; + Ok(json!({ + "pending_runs": self.tally(summaries!("status='pending'"), &[]).await?, + "pending_commands": self.tally("SELECT COUNT(*) FROM workflow_commands WHERE workflow_run_id IN (SELECT id FROM workflow_runs WHERE namespace=$1 AND status='pending') AND command_type='start' AND status='accepted' AND outcome='started_new'", &[]).await?, + "ready_tasks": self.tally("SELECT COUNT(*) FROM workflow_tasks WHERE workflow_run_id IN (SELECT id FROM workflow_runs WHERE namespace=$1 AND status='pending') AND task_type='workflow' AND status='ready' AND (available_at IS NULL OR available_at<=$2)", &[bound]).await?, + "oldest_pending_start_at": time(pending_at), + "max_pending_ms": age(pending_at, now), + })) + } + + async fn history_metrics(&self) -> Result { + let [ + events, + bytes, + fan_out, + events_warning, + bytes_warning, + fan_out_warning, + ] = self.history_budget.thresholds(); + Ok(json!({ + "continue_as_new_recommended_runs": self.tally(summaries!("continue_as_new_recommended=TRUE"), &[]).await?, + "approaching_budget_runs": self.tally(summaries!("history_budget_pressure='approaching'"), &[]).await?, + "events": self.tally(scoped!("COUNT(*)", "workflow_history_events", "1=1"), &[]).await?, + // PHP counts orphaned events only without a namespace. + "history_orphan_total": 0, + "max_event_count": self.largest("SELECT MAX(history_event_count) AS n FROM workflow_run_summaries WHERE namespace=$1").await?, + "max_size_bytes": self.largest("SELECT MAX(history_size_bytes) AS n FROM workflow_run_summaries WHERE namespace=$1").await?, + "max_fan_out": self.largest("SELECT MAX(history_fan_out) AS n FROM workflow_run_summaries WHERE namespace=$1").await?, + "event_threshold": events, + "size_bytes_threshold": bytes, + "fan_out_threshold": fan_out, + "event_warning_threshold": events_warning, + "size_bytes_warning_threshold": bytes_warning, + "fan_out_warning_threshold": fan_out_warning, + })) + } + + async fn projection_metrics(&self, now: DateTime) -> Result { + let runs = self + .tally("SELECT COUNT(*) FROM workflow_runs WHERE namespace=$1", &[]) + .await?; + let summaries = self + .tally( + "SELECT COUNT(*) FROM workflow_run_summaries WHERE namespace=$1", + &[], + ) + .await?; + let missing = self.tally("SELECT COUNT(*) FROM workflow_runs r WHERE r.namespace=$1 AND NOT EXISTS (SELECT 1 FROM workflow_run_summaries s WHERE s.id=r.id)", &[]).await?; + let missing_at = self.first_time("SELECT started_at,created_at FROM workflow_runs r WHERE r.namespace=$1 AND NOT EXISTS (SELECT 1 FROM workflow_run_summaries s WHERE s.id=r.id) ORDER BY COALESCE(started_at,created_at) LIMIT 1", + &[], "started_at", Some("created_at")).await?; + let summary_times = Self::query("SELECT MIN(updated_at) AS oldest,MAX(updated_at) AS newest FROM workflow_run_summaries WHERE namespace=$1") + .bind(self.namespace.as_ref()).fetch_one(&self.pool).await?; + let with_open_waits = self + .tally(summaries!("open_wait_id IS NOT NULL"), &[]) + .await?; + let projected_wait_runs = self + .tally( + scoped!( + "COUNT(DISTINCT workflow_run_id)", + "workflow_run_waits", + "1=1" + ), + &[], + ) + .await?; + let missing_open_waits = self.tally("SELECT COUNT(*) FROM workflow_run_summaries s WHERE s.namespace=$1 AND s.open_wait_id IS NOT NULL AND NOT EXISTS (SELECT 1 FROM workflow_run_waits w WHERE w.workflow_run_id=s.id AND w.wait_id=s.open_wait_id)", &[]).await?; + let wait_times = Self::query("SELECT MIN(updated_at) AS oldest,MAX(updated_at) AS newest FROM workflow_run_waits WHERE workflow_run_id IN (SELECT id FROM workflow_runs WHERE namespace=$1)") + .bind(self.namespace.as_ref()).fetch_one(&self.pool).await?; + let not_maintained = |source: &str| { + json!({ + "maintained": false, + "reason": "native_derives_from_history", + "source": source, + "runs": runs, + "needs_rebuild": 0, + }) + }; + Ok(json!({ + "run_summaries": { + "runs": runs, + "summaries": summaries, + "missing": missing, + "orphaned": 0, + "stale": 0, + "schema_outdated": self.tally(summaries!("(projection_schema_version IS NULL OR projection_schema_version<2)"), &[]).await?, + "needs_rebuild": missing, + "oldest_updated_at": time(DB::optional_instant(&summary_times, "oldest")?), + "newest_updated_at": time(DB::optional_instant(&summary_times, "newest")?), + "oldest_missing_run_started_at": time(missing_at), + "max_missing_run_age_ms": age(missing_at, now), + }, + // Native writes a run's waits in the transaction that records its + // history, so the waits cannot drift from it: every run with waits is + // projected, and only a deleted run could leave rows orphaned. + "run_waits": { + "runs": runs, + "rows": self.tally(scoped!("COUNT(*)", "workflow_run_waits", "1=1"), &[]).await?, + "projected_runs": projected_wait_runs, + "runs_with_waits": projected_wait_runs, + "projected_runs_with_waits": projected_wait_runs, + "missing_runs_with_waits": 0, + "stale_projected_runs": 0, + "summaries_with_open_waits": with_open_waits, + "projected_current_open_waits": (with_open_waits - missing_open_waits).max(0), + "missing_current_open_waits": missing_open_waits, + "orphaned": 0, + "needs_rebuild": 0, + "oldest_updated_at": time(DB::optional_instant(&wait_times, "oldest")?), + "newest_updated_at": time(DB::optional_instant(&wait_times, "newest")?), + }, + "run_timeline_entries": not_maintained("workflow_history_events"), + "run_timer_entries": not_maintained("workflow_history_events"), + "run_lineage_entries": not_maintained("workflow_history_events"), + })) + } + + async fn schedule_metrics(&self, now: DateTime) -> Result { + let missed_at = self.first_time("SELECT next_fire_at FROM workflow_schedules WHERE namespace=$1 AND status='active' AND next_fire_at IS NOT NULL AND next_fire_at<=$2 ORDER BY next_fire_at LIMIT 1", + &[now], "next_fire_at", None).await?; + let mut fires = 0; + let mut failures = 0; + for row in Self::query("SELECT fires_count,failures_count FROM workflow_schedules WHERE namespace=$1 AND status='active'") + .bind(self.namespace.as_ref()).fetch_all(&self.pool).await? + { + fires += DB::optional_number(&row, "fires_count")?.unwrap_or_default(); + failures += DB::optional_number(&row, "failures_count")?.unwrap_or_default(); + } + Ok(json!({ + "active": self.tally("SELECT COUNT(*) FROM workflow_schedules WHERE namespace=$1 AND status='active'", &[]).await?, + "paused": self.tally("SELECT COUNT(*) FROM workflow_schedules WHERE namespace=$1 AND status='paused'", &[]).await?, + "missed": self.tally("SELECT COUNT(*) FROM workflow_schedules WHERE namespace=$1 AND status='active' AND next_fire_at IS NOT NULL AND next_fire_at<=$2", &[now]).await?, + "oldest_overdue_at": time(missed_at), + "max_overdue_ms": age(missed_at, now), + "fires_total": fires, + "failures_total": failures, + })) + } + + /// PHP's worker compatibility fleet: workers whose last registration or + /// heartbeat is within the compatibility heartbeat TTL. Native records + /// these on the worker registration itself. + async fn worker_metrics(&self) -> Result { + let fresh_after = now() - Duration::seconds(COMPATIBILITY_HEARTBEAT_TTL_SECONDS); + let rows = Self::query("SELECT worker_id,task_queue,last_heartbeat_at FROM workflow_worker_registrations WHERE namespace=$1 AND status='active' AND last_heartbeat_at>$2 ORDER BY worker_id") + .bind(self.namespace.as_ref()).bind(DB::bind_time(fresh_after)).fetch_all(&self.pool).await?; + let mut fleet = Vec::new(); + for row in &rows { + let recorded = DB::optional_instant(row, "last_heartbeat_at")?; + fleet.push(json!({ + "worker_id": DB::string(row, "worker_id")?, + "namespace": self.namespace.as_ref(), + "host": null, + "process_id": null, + "connection": null, + "queue": DB::optional_string(row, "task_queue")?, + "supported": [], + "supports_required": true, + "recorded_at": time(recorded), + "expires_at": time(recorded.map(|at| at + Duration::seconds(COMPATIBILITY_HEARTBEAT_TTL_SECONDS))), + "source": "worker_registration", + })); + } + Ok(json!({ + "compatibility_namespace": self.namespace.as_ref(), + "required_compatibility": null, + "active_workers": fleet.len(), + "active_worker_scopes": fleet.len(), + "active_workers_supporting_required": fleet.len(), + "fleet": fleet, + })) + } + + /// PHP's `RuntimeExternalPayloadCleanupMetrics`: the expired registrations + /// and the namespace's recorded cleanup passes. + async fn external_payload_cleanup_metrics(&self, config: &MetricsConfig) -> Result { + let bound = whole_second(now()); + let stats = + Self::query("SELECT * FROM runtime_external_payload_cleanup_stats WHERE namespace=$1") + .bind(self.namespace.as_ref()) + .fetch_optional(&self.pool) + .await?; + let number = |field: &str| -> Result { + Ok(match &stats { + Some(row) => DB::optional_number(row, field)?.unwrap_or_default(), + None => 0, + }) + }; + let instant = |field: &str| -> Result { + Ok(time(match &stats { + Some(row) => DB::optional_instant(row, field)?, + None => None, + })) + }; + let status = match &stats { + Some(row) => DB::optional_string(row, "last_pass_status")?, + None => None, + }; + Ok(json!({ + "schema": "durable-workflow.v2.runtime-external-payload-cleanup-metrics.v1", + "backlog": { + "expired_unclaimed": self.tally("SELECT COUNT(*) FROM runtime_external_payloads WHERE namespace=$1 AND retained_at IS NULL AND expires_at IS NOT NULL AND expires_at<=$2", &[bound]).await?, + "failed_registration_objects": self.tally("SELECT COUNT(*) FROM runtime_external_payloads WHERE namespace=$1 AND retained_at IS NULL AND expires_at IS NOT NULL AND expires_at<=$2 AND upload_status='writing'", &[bound]).await?, + }, + "bounds": {"abandoned_upload_expiry_seconds": config.abandoned_upload_expiry_seconds, "default_batch_size": 100, "maximum_batch_size": 1000}, + "totals": { + "passes": number("passes_total")?, + "deleted_references": number("deleted_references_total")?, + "deleted_backing_objects": number("deleted_backing_objects_total")?, + "shared_objects_preserved": number("shared_objects_preserved_total")?, + "blocked_outcomes": number("blocked_outcomes_total")?, + "storage_driver_failures": number("storage_driver_failures_total")?, + }, + "last_pass": { + "status": status, + "processed": number("last_processed")?, + "deleted_references": number("last_deleted_references")?, + "deleted_backing_objects": number("last_deleted_backing_objects")?, + "shared_objects_preserved": number("last_shared_objects_preserved")?, + "blocked_outcomes": number("last_blocked_outcomes")?, + "storage_driver_failures": number("last_storage_driver_failures")?, + "completed_at": instant("last_completed_at")?, + "last_storage_failure_at": instant("last_storage_failure_at")?, + }, + "identity_policy": {"provider_credentials_exposed": false, "provider_locations_exposed": false, "reusable_reference_identities_exposed": false}, + })) + } +} + +struct UnhealthyTasks { + dispatch_failed: i64, + claim_failed: i64, + dispatch_overdue: i64, + lease_expired: i64, +} + +impl UnhealthyTasks { + fn total(&self) -> i64 { + self.dispatch_failed + self.claim_failed + self.dispatch_overdue + self.lease_expired + } +} + +/// PHP's `workflows.v2.compatibility.heartbeat_ttl_seconds` default. +const COMPATIBILITY_HEARTBEAT_TTL_SECONDS: i64 = 30; + +/// Runtime settings the document reports. +pub(crate) struct MetricsConfig { + pub(crate) workflow_task_lease_seconds: i64, + pub(crate) abandoned_upload_expiry_seconds: i64, +} + +fn empty_repair_snapshot() -> Value { + json!({ + "existing_task_candidates": 0, + "missing_task_candidates": 0, + "total_candidates": 0, + "scan_limit": 25, + "scan_strategy": "scope_fair_round_robin", + "selected_existing_task_candidates": 0, + "selected_missing_task_candidates": 0, + "selected_total_candidates": 0, + "existing_task_scan_limit_reached": false, + "missing_task_scan_limit_reached": false, + "scan_pressure": false, + "oldest_task_candidate_created_at": null, + "oldest_missing_run_started_at": null, + "max_task_candidate_age_ms": 0, + "max_missing_run_age_ms": 0, + "scopes": [], + }) +} + +fn structural_limits(connection: &str) -> Value { + let configured = json!({ + "pending_activity_count": 2000, + "pending_child_count": 1000, + "pending_timer_count": 2000, + "pending_signal_count": 5000, + "pending_update_count": 500, + "command_batch_size": 1000, + "payload_size_bytes": 2_097_152, + "memo_size_bytes": 262_144, + "search_attribute_size_bytes": 40960, + "history_transaction_size": 5000, + "warning_threshold_percent": 80, + }); + let sqlite = connection == "sqlite"; + let mut effective = configured.clone(); + let mut adjustments = json!({}); + let mut issues = Vec::new(); + if sqlite { + effective["concurrent_write_safety"] = json!("limited"); + adjustments["concurrent_write_safety"] = json!("limited"); + issues.push(json!({"component": "structural_limits", "severity": "info", "code": "sqlite_concurrency_note", + "message": "SQLite serializes writes; high pending-count limits may cause lock contention under concurrent worker load."})); + } + json!({"configured": configured, "backend_adjustments": adjustments, "effective": effective, "issues": issues}) +} + +/// What this runtime needs from its backends, in PHP's capability shape. +fn backend_capabilities(now: DateTime, connection: &str) -> Value { + let limits = structural_limits(connection); + let issues = limits["issues"].clone(); + let severity = if issues.as_array().is_some_and(|issues| !issues.is_empty()) { + "info" + } else { + "ok" + }; + json!({ + "generated_at": time(Some(now)), + "runtime": "rust", + "supported": true, + "database": { + "connection": connection, + "driver": connection, + "supported": true, + "capabilities": {"transactions": true, "after_commit_callbacks": false, "durable_ordering": true, "row_locks": connection != "sqlite"}, + "issues": [], + }, + "queue": { + "connection": null, + "driver": "database_poll", + "supported": true, + "capabilities": {"async_delivery": false, "delayed_delivery": true, "requires_worker": false, "max_delay_seconds": null, "after_commit_option": false}, + "issues": [], + }, + "cache": {"store": null, "driver": "none", "supported": true, "capabilities": {"atomic_locks": false}, "issues": []}, + "codec": {"configured": "avro", "configured_canonical": "avro", "canonical": "avro", "universal": true, "configured_universal": true, "supported": true, "issues": []}, + "structural_limits": limits, + "severity": severity, + "issues": issues, + }) +} + +/// Native claims ready tasks on each poll; it runs no repair loop. +fn repair_policy() -> Value { + json!({ + "redispatch_after_seconds": REDISPATCH_AFTER_SECONDS, + "loop_throttle_seconds": null, + "scan_limit": 25, + "scan_strategy": "claim_time_lease_recovery", + "failure_backoff_max_seconds": null, + "failure_backoff_strategy": null, + }) +} + +fn matching_role(config: &MetricsConfig) -> Value { + json!({ + "queue_wake_enabled": true, + "shape": "in_server", + "wake_owner": "server_process", + "task_dispatch_mode": "poll", + "partition_primitives": ["queue", "namespace"], + "backpressure_model": "lease_ownership", + "discovery_limits": { + "poll_batch_cap": 100, + "availability_ceiling_seconds": null, + "wake_signal_ttl_seconds": null, + "workflow_task_lease_seconds": config.workflow_task_lease_seconds, + "activity_task_lease_seconds": 300, + }, + }) +} + +/// Native routes no task to a sticky worker. +fn sticky_execution() -> Value { + json!({ + "enabled": false, + "ttl_seconds": 0, + "active_sticky_runs": 0, + "ready_sticky_tasks": 0, + "leased_sticky_tasks": 0, + "hit_expected_last_minute": 0, + "miss_last_minute": 0, + "forced_cold_replay_last_minute": 0, + "cold_replay_last_minute": 0, + "hit_rate_last_minute": 0.0, + "miss_rate_last_minute": 0.0, + "capacity_pressure_tasks": 0, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn comparisons_bind_whole_seconds_and_ages_truncate() { + let at = DateTime::parse_from_rfc3339("2026-10-20T23:06:14.208471Z") + .unwrap() + .with_timezone(&Utc); + assert_eq!(whole_second(at).to_rfc3339(), "2026-10-20T23:06:14+00:00"); + assert_eq!(time(Some(at)), json!("2026-10-20T23:06:14.208471Z")); + assert_eq!(age(Some(at), at + Duration::microseconds(1_999)), 1); + assert_eq!(age(None, at), 0); + assert_eq!( + earliest(&[None, Some(at), Some(at - Duration::seconds(1))]), + Some(at - Duration::seconds(1)) + ); + } +} diff --git a/rust/src/runtime/store.rs b/rust/src/runtime/store.rs index 6059cc28c..db2778998 100644 --- a/rust/src/runtime/store.rs +++ b/rust/src/runtime/store.rs @@ -1526,11 +1526,14 @@ where payload["workflow_task_retry_of"] = json!(task_id); payload["workflow_task_retry_after_error"] = json!(message); let next = id(); + // As PHP in poll mode, a task is dispatched to pollers when it is created. + let created = now(); // Copy routing in SQL so JSON/null representations remain // native to each backend. A new lease increments this carried // attempt count rather than restarting it at one. - Self::query("INSERT INTO workflow_tasks(id,workflow_run_id,namespace,task_type,status,attempt_count,available_at,payload,connection,queue,compatibility,priority,fairness_key,fairness_weight) SELECT $1,r.id,r.namespace,'workflow','ready',t.attempt_count,$2,$3,COALESCE(t.connection,r.connection),COALESCE(t.queue,r.queue),COALESCE(t.compatibility,r.compatibility),COALESCE(t.priority,r.priority,5),COALESCE(t.fairness_key,r.fairness_key),COALESCE(t.fairness_weight,r.fairness_weight,1) FROM workflow_tasks t JOIN workflow_runs r ON r.id=t.workflow_run_id WHERE t.id=$4") - .bind(&next).bind(DB::bind_time(now())).bind(DB::document(&payload)).bind(task_id).execute(&mut *tx).await?; + Self::query("INSERT INTO workflow_tasks(id,workflow_run_id,namespace,task_type,status,attempt_count,available_at,created_at,updated_at,last_dispatched_at,last_dispatch_attempt_at,payload,connection,queue,compatibility,priority,fairness_key,fairness_weight) SELECT $1,r.id,r.namespace,'workflow','ready',t.attempt_count,$2,$3,$4,$5,$6,$7,COALESCE(t.connection,r.connection),COALESCE(t.queue,r.queue),COALESCE(t.compatibility,r.compatibility),COALESCE(t.priority,r.priority,5),COALESCE(t.fairness_key,r.fairness_key),COALESCE(t.fairness_weight,r.fairness_weight,1) FROM workflow_tasks t JOIN workflow_runs r ON r.id=t.workflow_run_id WHERE t.id=$8") + .bind(&next).bind(DB::bind_time(created)).bind(DB::bind_time(created)).bind(DB::bind_time(created)).bind(DB::bind_time(created)).bind(DB::bind_time(created)) + .bind(DB::document(&payload)).bind(task_id).execute(&mut *tx).await?; next_task_id = Some(next); } tx.commit().await?; @@ -2539,8 +2542,11 @@ where .fetch_one(&mut **tx) .await?; let namespace = DB::string(&run, "namespace")?; - Self::query("INSERT INTO workflow_tasks(id,workflow_run_id,task_type,status,payload,queue,available_at,namespace) VALUES ($1,$2,$3,'ready',$4,$5,$6,$7)") - .bind(&task_id).bind(run_id).bind(kind).bind(DB::document(&payload)).bind(queue).bind(DB::bind_time(now())).bind(&namespace).execute(&mut **tx).await?; + // As PHP in poll mode, a task is dispatched to pollers when it is created. + let created = now(); + Self::query("INSERT INTO workflow_tasks(id,workflow_run_id,task_type,status,payload,queue,available_at,namespace,created_at,updated_at,last_dispatched_at,last_dispatch_attempt_at) VALUES ($1,$2,$3,'ready',$4,$5,$6,$7,$8,$9,$10,$11)") + .bind(&task_id).bind(run_id).bind(kind).bind(DB::document(&payload)).bind(queue).bind(DB::bind_time(created)).bind(&namespace) + .bind(DB::bind_time(created)).bind(DB::bind_time(created)).bind(DB::bind_time(created)).bind(DB::bind_time(created)).execute(&mut **tx).await?; Ok(task_id) } diff --git a/rust/src/runtime/system_health.rs b/rust/src/runtime/system_health.rs new file mode 100644 index 000000000..fd5a9354f --- /dev/null +++ b/rust/src/runtime/system_health.rs @@ -0,0 +1,483 @@ +//! The health document of one namespace, as published PHP Server 2.5.14 +//! (Workflow 2.5.5 `HealthCheck::snapshot`) derives it from the operator +//! metrics: the same twelve checks, categories, statuses and messages. Checks +//! about PHP-only machinery (projection tables native does not store, the +//! cache-backed long-poll wake) report what native does instead. +use serde_json::{Value, json}; + +const CORRECTNESS: &str = "correctness"; +const ACCELERATION: &str = "acceleration"; + +fn integer(value: &Value) -> i64 { + value.as_i64().unwrap_or_default() +} + +fn check(name: &str, status: &str, message: &str, category: &str, data: Value) -> Value { + json!({"name": name, "status": status, "category": category, "message": message, "data": data}) +} + +fn when(healthy: bool) -> &'static str { + if healthy { "ok" } else { "warning" } +} + +/// Copies the named fields, as PHP's checks echo them. +fn fields(source: &Value, names: &[(&str, &str)]) -> Value { + let mut data = serde_json::Map::new(); + for (target, field) in names { + data.insert((*target).to_owned(), source[*field].clone()); + } + Value::Object(data) +} + +fn backend_check(backend: &Value) -> Value { + let issues = backend["issues"].clone(); + let severity = backend["severity"].as_str().unwrap_or("ok"); + let (status, message) = match severity { + "error" => ( + "error", + "One or more configured v2 backend capabilities are unsupported.", + ), + "warning" => ( + "warning", + "One or more configured v2 backend capabilities are degraded but non-blocking.", + ), + _ => ( + "ok", + "The configured database, queue, cache, and codec backends satisfy the v2 capability contract.", + ), + }; + let count = issues.as_array().map_or(0, Vec::len); + check( + "backend_capabilities", + status, + message, + CORRECTNESS, + json!({"severity": severity, "issue_count": count, "issues": issues}), + ) +} + +fn run_summary_check(projection: &Value) -> Value { + let needs_rebuild = integer(&projection["needs_rebuild"]); + check( + "run_summary_projection", + when(needs_rebuild == 0), + if needs_rebuild == 0 { + "Run-summary projections are aligned with durable v2 runs." + } else { + "Run-summary projections are missing, stale, schema-outdated, or orphaned; rebuild them before trusting Waterline lists." + }, + CORRECTNESS, + json!({"needs_rebuild": needs_rebuild, "missing": integer(&projection["missing"]), "orphaned": integer(&projection["orphaned"]), + "stale": integer(&projection["stale"]), "schema_outdated": integer(&projection["schema_outdated"]), "projection_schema_version": 2}), + ) +} + +/// Native stores run waits; it reads timelines, timers and lineage from +/// history, so those cannot drift. +fn selected_run_check(projections: &Value) -> Value { + let waits = &projections["run_waits"]; + let needs_rebuild = integer(&waits["needs_rebuild"]); + check( + "selected_run_projections", + when(needs_rebuild == 0), + if needs_rebuild == 0 { + "Selected-run wait projections are aligned with durable v2 detail; timeline, timer, and lineage detail is read from history." + } else { + "Selected-run wait projections need rebuild before trusting Waterline detail." + }, + CORRECTNESS, + json!({"needs_rebuild": needs_rebuild, "run_waits_needs_rebuild": needs_rebuild, + "run_waits_missing_current_open_waits": integer(&waits["missing_current_open_waits"]), + "run_waits_orphaned": integer(&waits["orphaned"]), + "timeline_maintained": false, "timer_maintained": false, "lineage_maintained": false}), + ) +} + +fn history_check(history: &Value) -> Value { + let orphaned = integer(&history["history_orphan_total"]); + check( + "history_retention_invariant", + when(orphaned == 0), + if orphaned == 0 { + "Workflow history events all reference retained workflow runs." + } else { + "Workflow history events exist without retained workflow runs; retention cleanup must reconcile them." + }, + CORRECTNESS, + json!({"history_orphan_total": orphaned, "events": integer(&history["events"])}), + ) +} + +fn command_contract_check(metrics: &Value) -> Value { + let needed = integer(&metrics["backfill_needed_runs"]); + let actionable = integer(&metrics["actionable_backfill_needed_runs"]); + let message = if actionable > 0 { + "Some open runs need WorkflowStarted command-contract backfill before operators can trust command forms." + } else if needed > 0 { + "Historical WorkflowStarted command-contract gaps are retained as informational metrics; no open run depends on them." + } else { + "WorkflowStarted command-contract snapshots are complete." + }; + check( + "command_contract_snapshots", + when(actionable == 0), + message, + CORRECTNESS, + metrics.clone(), + ) +} + +fn task_transport_check(tasks: &Value, backlog: &Value) -> Value { + let unhealthy = integer(&tasks["unhealthy"]); + let mut data = fields( + tasks, + &[ + ("lease_expired_tasks", "lease_expired"), + ("oldest_lease_expired_at", "oldest_lease_expired_at"), + ("max_lease_expired_age_ms", "max_lease_expired_age_ms"), + ("ready_due_tasks", "ready_due"), + ("oldest_ready_due_at", "oldest_ready_due_at"), + ("max_ready_due_age_ms", "max_ready_due_age_ms"), + ("dispatch_overdue_tasks", "dispatch_overdue"), + ( + "oldest_dispatch_overdue_since", + "oldest_dispatch_overdue_since", + ), + ("max_dispatch_overdue_age_ms", "max_dispatch_overdue_age_ms"), + ("claim_failed_tasks", "claim_failed"), + ("oldest_claim_failed_at", "oldest_claim_failed_at"), + ("max_claim_failed_age_ms", "max_claim_failed_age_ms"), + ("dispatch_failed_tasks", "dispatch_failed"), + ("oldest_dispatch_failed_at", "oldest_dispatch_failed_at"), + ("max_dispatch_failed_age_ms", "max_dispatch_failed_age_ms"), + ("max_attempt_count", "max_attempt_count"), + ("max_repair_count", "max_repair_count"), + ], + ); + data["unhealthy_tasks"] = json!(unhealthy); + for (target, field) in [ + ("repair_needed_runs", "repair_needed_runs"), + ("claim_failed_runs", "claim_failed_runs"), + ("compatibility_blocked_runs", "compatibility_blocked_runs"), + ( + "oldest_compatibility_blocked_started_at", + "oldest_compatibility_blocked_started_at", + ), + ( + "max_compatibility_blocked_age_ms", + "max_compatibility_blocked_age_ms", + ), + ] { + data[target] = backlog[field].clone(); + } + check( + "task_transport", + when(unhealthy == 0), + if unhealthy == 0 { + "No unhealthy durable task transport state is currently projected." + } else { + "One or more durable tasks have unhealthy transport, claim, dispatch, or lease state." + }, + CORRECTNESS, + data, + ) +} + +fn activity_path_check(activities: &Value) -> Value { + let overdue = integer(&activities["timeout_overdue"]); + check( + "activity_path", + when(overdue == 0), + if overdue == 0 { + "No activity executions have an enforcement-overdue deadline; the activity timeout sweep is keeping up." + } else { + "One or more activity executions have a schedule-to-start, start-to-close, schedule-to-close, or heartbeat deadline past due without enforcement; the activity timeout sweep is lagging or stalled." + }, + CORRECTNESS, + fields( + activities, + &[ + ("timeout_overdue", "timeout_overdue"), + ("oldest_timeout_overdue_at", "oldest_timeout_overdue_at"), + ("max_timeout_overdue_age_ms", "max_timeout_overdue_age_ms"), + ("retrying", "retrying"), + ("oldest_retrying_started_at", "oldest_retrying_started_at"), + ("max_retrying_age_ms", "max_retrying_age_ms"), + ("failed_attempts", "failed_attempts"), + ("max_attempt_count", "max_attempt_count"), + ], + ), + ) +} + +fn routing_check(metrics: &Value) -> Value { + let (tasks, backlog, role, workers) = ( + &metrics["tasks"], + &metrics["backlog"], + &metrics["matching_role"], + &metrics["workers"], + ); + let blocked = integer(&backlog["compatibility_blocked_runs"]) > 0; + let overdue = integer(&tasks["dispatch_overdue"]); + let claim_failed = integer(&tasks["claim_failed"]); + let required = workers["required_compatibility"].as_str(); + let supporting = integer(&workers["active_workers_supporting_required"]); + let fleet_supports = required.is_none() || supporting > 0; + let signals = usize::from(blocked) + usize::from(overdue > 0) + usize::from(claim_failed > 0); + let message = match () { + _ if signals == 0 => { + "No routing drains, compatibility blocks, or uncleared claim failures are currently projected." + } + _ if signals > 1 => { + "Routing health is degraded: compatibility blocks, dispatch lag, or uncleared claim failures are visible in durable state." + } + _ if blocked && !fleet_supports => { + "One or more runs are blocked because no active worker heartbeat advertises the required compatibility marker." + } + _ if blocked => { + "One or more runs are ready but waiting for a compatible worker in the active fleet." + } + _ if overdue > 0 => { + "One or more ready tasks have waited past the redispatch window without a successful dispatch wake." + } + _ => "One or more ready tasks still carry an uncleared claim failure.", + }; + let mut data = fields( + backlog, + &[ + ("compatibility_blocked_runs", "compatibility_blocked_runs"), + ( + "oldest_compatibility_blocked_started_at", + "oldest_compatibility_blocked_started_at", + ), + ( + "max_compatibility_blocked_age_ms", + "max_compatibility_blocked_age_ms", + ), + ], + ); + for (target, field) in [ + ("dispatch_overdue_tasks", "dispatch_overdue"), + ( + "oldest_dispatch_overdue_since", + "oldest_dispatch_overdue_since", + ), + ("max_dispatch_overdue_age_ms", "max_dispatch_overdue_age_ms"), + ("claim_failed_tasks", "claim_failed"), + ("oldest_claim_failed_at", "oldest_claim_failed_at"), + ("max_claim_failed_age_ms", "max_claim_failed_age_ms"), + ] { + data[target] = tasks[field].clone(); + } + data["queue_wake_enabled"] = role["queue_wake_enabled"].clone(); + data["matching_shape"] = role["shape"].clone(); + data["wake_owner"] = role["wake_owner"].clone(); + data["task_dispatch_mode"] = role["task_dispatch_mode"].clone(); + data["active_worker_scopes"] = workers["active_worker_scopes"].clone(); + data["required_compatibility"] = workers["required_compatibility"].clone(); + data["active_workers"] = workers["active_workers"].clone(); + data["active_workers_supporting_required"] = json!(supporting); + data["fleet_supports_required"] = json!(fleet_supports); + check( + "routing_health", + when(signals == 0), + message, + CORRECTNESS, + data, + ) +} + +fn durable_resume_check(metrics: &Value) -> Value { + let (runs, repair) = (&metrics["runs"], &metrics["repair"]); + let needed = integer(&metrics["backlog"]["repair_needed_runs"]); + check( + "durable_resume_paths", + when(needed == 0), + if needed == 0 { + "Every open v2 run has a projected durable resume path." + } else { + "One or more open v2 runs are missing their durable next-resume source and need repair." + }, + CORRECTNESS, + json!({"repair_needed_runs": needed, "oldest_repair_needed_at": runs["oldest_repair_needed_at"], + "max_repair_needed_age_ms": runs["max_repair_needed_age_ms"], + "missing_task_candidates": repair["missing_task_candidates"], "selected_missing_task_candidates": repair["selected_missing_task_candidates"], + "oldest_missing_run_started_at": repair["oldest_missing_run_started_at"], "max_missing_run_age_ms": repair["max_missing_run_age_ms"], + "waiting_runs": runs["waiting"], "oldest_wait_started_at": runs["oldest_wait_started_at"], "max_wait_age_ms": runs["max_wait_age_ms"]}), + ) +} + +fn worker_compatibility_check(workers: &Value) -> Value { + let required = workers["required_compatibility"].as_str(); + let supporting = integer(&workers["active_workers_supporting_required"]); + let data = json!({"required_compatibility": required, "active_workers": integer(&workers["active_workers"]), + "active_worker_scopes": integer(&workers["active_worker_scopes"]), "active_workers_supporting_required": supporting, + "validation_mode": "warn"}); + if required.is_none() || supporting > 0 { + let message = if required.is_none() { + "No current v2 compatibility marker is required." + } else { + "At least one active worker heartbeat advertises the current v2 compatibility marker." + }; + return check("worker_compatibility", "ok", message, CORRECTNESS, data); + } + check( + "worker_compatibility", + "warning", + "No active worker heartbeat advertises the current v2 compatibility marker.", + CORRECTNESS, + data, + ) +} + +fn scheduler_check(schedules: &Value) -> Value { + let missed = integer(&schedules["missed"]); + check( + "scheduler_role", + when(missed == 0), + if missed == 0 { + "The scheduler tick is caught up to every active schedule in the namespace." + } else { + "One or more active schedules are past their next_fire_at; the scheduler tick has not yet caught up." + }, + CORRECTNESS, + schedules.clone(), + ) +} + +/// Native wakes long polls in its own process. Other nodes still discover +/// work by bounded database polling, which is the durable path. +fn long_poll_check() -> Value { + check( + "long_poll_wake_acceleration", + "ok", + "Wake acceleration runs in-process; dispatch discovery on this node benefits from sub-second signalling, and other nodes discover work by bounded polling.", + ACCELERATION, + json!({"multi_node": false, "backend": "in_process", "capable": false, "safe": true, + "reason": "In-process wake signals reach only this node; other nodes discover work by bounded polling."}), + ) +} + +fn status(checks: &[Value]) -> &'static str { + if checks.iter().any(|check| check["status"] == "error") { + "error" + } else if checks.iter().any(|check| check["status"] == "warning") { + "warning" + } else { + "ok" + } +} + +/// `HealthCheck::snapshot` for the metrics document. +pub(super) fn snapshot(metrics: &Value) -> Value { + let checks = vec![ + backend_check(&metrics["backend"]), + run_summary_check(&metrics["projections"]["run_summaries"]), + selected_run_check(&metrics["projections"]), + history_check(&metrics["history"]), + command_contract_check(&metrics["command_contracts"]), + task_transport_check(&metrics["tasks"], &metrics["backlog"]), + activity_path_check(&metrics["activities"]), + routing_check(metrics), + durable_resume_check(metrics), + worker_compatibility_check(&metrics["workers"]), + scheduler_check(&metrics["schedules"]), + long_poll_check(), + ]; + let overall = status(&checks); + let category = |name: &str| { + let entries: Vec = checks + .iter() + .filter(|check| check["category"] == name) + .cloned() + .collect(); + json!({"status": status(&entries), "check_count": entries.len()}) + }; + json!({ + "generated_at": metrics["generated_at"], + "status": overall, + "healthy": overall != "error", + "categories": {CORRECTNESS: category(CORRECTNESS), ACCELERATION: category(ACCELERATION)}, + "operator_metrics": metrics, + "structural_limits": metrics["structural_limits"], + "routing_drains": [], + "checks": checks, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn quiet() -> Value { + json!({"generated_at": "2026-10-20T23:06:14.000000Z", "backend": {"severity": "ok", "issues": []}, + "projections": {"run_summaries": {"needs_rebuild": 0}, "run_waits": {"needs_rebuild": 0}}, + "history": {"history_orphan_total": 0, "events": 0}, "command_contracts": {"backfill_needed_runs": 0, "actionable_backfill_needed_runs": 0}, + "tasks": {"unhealthy": 0, "dispatch_overdue": 0, "claim_failed": 0}, "backlog": {"repair_needed_runs": 0, "compatibility_blocked_runs": 0}, + "activities": {"timeout_overdue": 0}, "matching_role": {}, "workers": {"required_compatibility": null}, + "runs": {}, "repair": {}, "schedules": {"missed": 0}, "structural_limits": {}}) + } + + #[test] + fn checks_follow_php_names_categories_and_rules() { + let health = snapshot(&quiet()); + let names: Vec<_> = health["checks"] + .as_array() + .unwrap() + .iter() + .map(|check| check["name"].as_str().unwrap()) + .collect(); + assert_eq!( + names, + [ + "backend_capabilities", + "run_summary_projection", + "selected_run_projections", + "history_retention_invariant", + "command_contract_snapshots", + "task_transport", + "activity_path", + "routing_health", + "durable_resume_paths", + "worker_compatibility", + "scheduler_role", + "long_poll_wake_acceleration" + ] + ); + assert_eq!(health["status"], "ok"); + assert_eq!( + health["categories"], + json!({"correctness": {"status": "ok", "check_count": 11}, "acceleration": {"status": "ok", "check_count": 1}}) + ); + + let mut overdue = quiet(); + overdue["tasks"] = json!({"unhealthy": 1, "dispatch_overdue": 1, "claim_failed": 0}); + let health = snapshot(&overdue); + let statuses: Vec<_> = health["checks"] + .as_array() + .unwrap() + .iter() + .filter(|check| check["status"] != "ok") + .map(|check| check["name"].as_str().unwrap()) + .collect(); + assert_eq!(statuses, ["task_transport", "routing_health"]); + assert_eq!( + (health["status"].as_str(), health["healthy"].as_bool()), + (Some("warning"), Some(true)) + ); + assert_eq!( + health["checks"][7]["message"], + "One or more ready tasks have waited past the redispatch window without a successful dispatch wake." + ); + + let mut broken = quiet(); + broken["backend"] = json!({"severity": "error", "issues": [{"code": "x"}]}); + assert_eq!( + ( + snapshot(&broken)["status"].as_str(), + snapshot(&broken)["healthy"].as_bool() + ), + (Some("error"), Some(false)) + ); + } +} diff --git a/rust/tests/execution.rs b/rust/tests/execution.rs index 79bbb11f0..9f403c3f9 100644 --- a/rust/tests/execution.rs +++ b/rust/tests/execution.rs @@ -33,6 +33,8 @@ mod admission; #[path = "execution/role_tokens.rs"] mod role_tokens; +#[path = "execution/operator_metrics.rs"] +mod operator_metrics; #[path = "execution/namespaces.rs"] mod namespaces; diff --git a/rust/tests/execution/operator_metrics.rs b/rust/tests/execution/operator_metrics.rs new file mode 100644 index 000000000..9afc170a7 --- /dev/null +++ b/rust/tests/execution/operator_metrics.rs @@ -0,0 +1,180 @@ +use super::*; + +const METRICS: &str = "/api/system/operator-metrics"; +const HEALTH: &str = "/api/system/health"; + +async fn metrics(app: &Router) -> Value { + let (status, body) = request(app, "GET", METRICS, Value::Null).await; + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!(body["namespace"], "default"); + body["operator_metrics"].clone() +} + +fn warnings(health: &Value) -> Vec { + health["health"]["checks"] + .as_array() + .unwrap() + .iter() + .filter(|check| check["status"] != "ok") + .map(|check| check["name"].as_str().unwrap().to_owned()) + .collect() +} + +#[tokio::test] +async fn operator_metrics_and_health_count_a_waiting_run_as_published_php_does() { + let database = TestDatabase::new().await; + let runtime = database.open().await.unwrap(); + let app = router(runtime.clone()); + let empty = metrics(&app).await; + for section in [ + "runs", + "tasks", + "activities", + "backlog", + "repair", + "starts", + "history", + "command_contracts", + "projections", + "schedules", + "workers", + "backend", + "structural_limits", + "update_wait", + "repair_policy", + "matching_role", + "sticky_execution", + "worker_sessions", + "capacity_evidence", + "runtime_external_payload_cleanup", + ] { + assert!(empty[section].is_object(), "{section}"); + } + assert_eq!( + [&empty["runs"]["total"], &empty["tasks"]["open"]], + [&json!(0), &json!(0)] + ); + let (status, health) = request(&app, "GET", HEALTH, Value::Null).await; + assert_eq!(status, StatusCode::OK, "{health}"); + assert_eq!( + [ + &health["namespace"], + &health["retention_mode"], + &health["retention_days"] + ], + [&json!("default"), &json!("bounded"), &json!(30)] + ); + assert_eq!(health["health"]["checks"].as_array().unwrap().len(), 12); + assert_eq!(health["health"]["status"], "ok", "{health}"); + + // A run started before any worker declared its command contract. + start(&app, "metrics-run").await; + let waiting = metrics(&app).await; + assert_eq!( + [ + &waiting["runs"]["total"], + &waiting["runs"]["waiting"], + &waiting["tasks"]["open"], + &waiting["tasks"]["ready"], + &waiting["starts"]["pending_runs"], + &waiting["starts"]["pending_commands"], + &waiting["history"]["events"], + &waiting["command_contracts"]["actionable_backfill_needed_runs"], + &waiting["backlog"]["tasks_added_last_minute"] + ], + [ + &json!(1), + &json!(1), + &json!(1), + &json!(1), + &json!(1), + &json!(1), + &json!(2), + &json!(1), + &json!(1) + ], + "{waiting}" + ); + assert_eq!(waiting["tasks"]["dispatch_overdue"], 0); + assert!(waiting["runs"]["oldest_wait_started_at"].is_string()); + + // Past the redispatch window, the unclaimed task is overdue. + database.execute("UPDATE workflow_tasks SET available_at='2000-01-01 00:00:00',created_at='2000-01-01 00:00:00',last_dispatched_at='2000-01-01 00:00:00',last_dispatch_attempt_at='2000-01-01 00:00:00'").await; + let overdue = metrics(&app).await; + assert_eq!( + [ + &overdue["tasks"]["ready_due"], + &overdue["tasks"]["dispatch_overdue"], + &overdue["tasks"]["unhealthy"], + &overdue["backlog"]["unhealthy_tasks"], + &overdue["starts"]["ready_tasks"] + ], + [&json!(1), &json!(1), &json!(1), &json!(1), &json!(1)], + "{overdue}" + ); + assert!( + overdue["tasks"]["max_dispatch_overdue_age_ms"] + .as_i64() + .unwrap() + > 1_000 + ); + let (status, health) = request(&app, "GET", HEALTH, Value::Null).await; + assert_eq!(status, StatusCode::OK); + assert_eq!(health["health"]["status"], "warning"); + assert_eq!( + warnings(&health), + [ + "command_contract_snapshots", + "task_transport", + "routing_health" + ] + ); + runtime.close().await; + database.remove().await; +} + +#[tokio::test] +async fn operator_routes_are_admin_only_and_namespace_scoped() { + let database = TestDatabase::new().await; + let runtime = database.open().await.unwrap(); + let legacy = router(runtime.clone()); + namespaces::create(&legacy, "metrics-scope").await; + start(&legacy, "metrics-default-run").await; + let (status, scoped) = + namespaces::scoped(&legacy, "metrics-scope", "GET", METRICS, Value::Null).await; + assert_eq!(status, StatusCode::OK, "{scoped}"); + assert_eq!( + [ + &scoped["namespace"], + &scoped["operator_metrics"]["runs"]["total"] + ], + [&json!("metrics-scope"), &json!(0)] + ); + let app = router(runtime.clone().with_role_tokens( + Some("worker-token".into()), + Some("operator-token".into()), + Some("admin-token".into()), + )); + for path in [METRICS, HEALTH] { + for (token, allowed) in [ + ("operator-token", false), + ("worker-token", false), + ("admin-token", true), + ] { + let (status, _, body) = admission::admission_http( + &app, + "GET", + path, + false, + Some(token), + Some("2"), + Some("default"), + Value::Null, + ) + .await; + assert_eq!(status == StatusCode::OK, allowed, "{path} {token} {body}"); + } + } + runtime.close().await; + database.remove().await; +} diff --git a/rust/tests/execution/run_observation_active_summaries.rs b/rust/tests/execution/run_observation_active_summaries.rs index 4e8415f6b..96c69058b 100644 --- a/rust/tests/execution/run_observation_active_summaries.rs +++ b/rust/tests/execution/run_observation_active_summaries.rs @@ -149,24 +149,72 @@ async fn run_observation_active_summary_failure_rolls_back_claim_and_missing_rea } #[tokio::test] -async fn run_observation_active_summary_keeps_unqualified_transport_state_unavailable() { +async fn run_observation_active_summary_classifies_task_transport_as_published_php_does() { let database = TestDatabase::new().await; let runtime = database.open().await.unwrap(); let app = router(runtime.clone()); - register(&app, "default").await; + // The started run declares a signal, so a signal can project it again. + let registered = namespaces::scoped( + &app, + "default", + "POST", + "/api/worker/register", + json!({ + "worker_id":"active-summary-worker", "task_queue":"test", "runtime":"php", + "supported_workflow_types":["echo"], "supported_activity_types":[], + "workflow_command_contracts":{"echo":{"signals":["payload"],"signal_contracts":[{"name":"payload","parameters":[{"name":"value","position":0,"type":"array","required":true,"variadic":false,"default_available":false,"allows_null":false}]}]}}}), + ) + .await; + assert_eq!(registered.0, StatusCode::CREATED, "{}", registered.1); let run = namespaces::start_scoped(&app, "default", "summary-transport").await; assert_eq!( observe(&app, "default", &run).await["liveness_state"], "workflow_task_ready" ); - database.execute("UPDATE workflow_tasks SET created_at=available_at,last_dispatch_attempt_at=available_at,last_dispatch_error='retained transport failure'").await; + // A signal projects the run again while its workflow task stays ready. + let signal = || async { + let response = namespaces::scoped( + &app, + "default", + "POST", + "/api/workflows/summary-transport/signal/payload", + json!({"input":envelope(Payload::Array(vec![Payload::Array(vec![])]))}), + ) + .await; + assert!(response.0.is_success(), "{}", response.1); + }; + // A dispatch attempt after the last dispatch failed: the run needs repair. + database.execute("UPDATE workflow_tasks SET last_dispatched_at=NULL,last_dispatch_error=' rejected by the queue '").await; + signal().await; + let failed = observe(&app, "default", &run).await; + assert_eq!(failed["liveness_state"], "repair_needed", "{failed}"); + let reason = failed["liveness_reason"].as_str().unwrap(); + assert!( + reason.starts_with("Workflow task ") + && reason.contains(" could not be dispatched at 20") + && reason.ends_with(". rejected by the queue"), + "{reason}" + ); + // A worker that could not claim the task names the task type. + database.execute("UPDATE workflow_tasks SET last_dispatched_at=last_dispatch_attempt_at,last_dispatch_error=NULL,last_claim_failed_at=last_dispatch_attempt_at,last_claim_error='backend rejected the claim'").await; + signal().await; + let unclaimed = observe(&app, "default", &run).await; + assert_eq!( + unclaimed["liveness_state"], "workflow_task_claim_failed", + "{unclaimed}" + ); + assert!( + unclaimed["liveness_reason"] + .as_str() + .unwrap() + .ends_with(". backend rejected the claim") + ); + // A leased task carries no ready-task transport problem. let claimed = claim(&app, "default").await; assert_eq!(claimed.0, StatusCode::OK, "{}", claimed.1); assert!(claimed.1["task"].is_object()); - let unknown = observe(&app, "default", &run).await; - assert_eq!(unknown["summary_state"], "unavailable"); - assert!(unknown["liveness_state"].is_null()); - assert!(unknown["history_event_count"].is_null()); + let leased = observe(&app, "default", &run).await; + assert_eq!(leased["liveness_state"], "workflow_task_leased", "{leased}"); runtime.close().await; database.remove().await; } diff --git a/scripts/conformance/server-parity/contract.mjs b/scripts/conformance/server-parity/contract.mjs index d688dc6df..0012e3796 100644 --- a/scripts/conformance/server-parity/contract.mjs +++ b/scripts/conformance/server-parity/contract.mjs @@ -11,6 +11,7 @@ import {checkNamespaces} from './namespace-contract.mjs'; import {checkWorkerDeregistration} from './worker-deregistration-contract.mjs'; import {checkWorkerListing} from './worker-listing-contract.mjs'; import {checkWorkerReplacement} from './worker-replacement-contract.mjs'; +import {checkOperatorMetrics} from './operator-metrics-contract.mjs'; import {checkPatchDeployment, checkPublishedPatchArtifacts, checkPatchPackageObservation} from './patch-deployment-contract.mjs'; import {checkLegacyMarkerArtifacts, checkLegacyMarkerPackageObservation, checkLegacyMarkerDeployment} from './legacy-marker-contract.mjs'; import {checkWaitingHistory} from './waiting-history-contract.mjs'; @@ -133,6 +134,7 @@ export function checkObservation(fixture, observation, workflowId, artifacts) { const workerDeregistration = fixture.worker_deregistration ? checkWorkerDeregistration(fixture, observation, workflowId) : null; const workerListing = fixture.worker_listing ? checkWorkerListing(fixture, observation, workflowId) : null; const workerReplacement = fixture.worker_replacement ? checkWorkerReplacement(fixture, observation, workflowId) : null; + const operatorMetrics = fixture.operator_metrics ? checkOperatorMetrics(fixture, observation, workflowId) : null; const patchDeployment = fixture.patch_deployment ? (fixture.patch_deployment.legacy_marker_history ? checkLegacyMarkerDeployment(fixture, observation, workflowId) : checkPatchDeployment(fixture, observation, workflowId)) : null; const waitingHistory = fixture.waiting_for_history ? checkWaitingHistory(fixture, observation, workflowId) : null; @@ -672,6 +674,7 @@ export function checkObservation(fixture, observation, workflowId, artifacts) { ...(workerDeregistration ? {worker_deregistration: workerDeregistration} : {}), ...(workerListing ? {worker_listing: workerListing} : {}), ...(workerReplacement ? {worker_replacement: workerReplacement} : {}), + ...(operatorMetrics ? {operator_metrics: operatorMetrics} : {}), ...(patchDeployment ? {patch_deployment: patchDeployment} : {}), ...(waitingHistory ? {waiting_for_history: waitingHistory} : {}), ...(taskRetry ? {workflow_task_retry: taskRetry} : {}), diff --git a/scripts/conformance/server-parity/operator-metrics-contract.mjs b/scripts/conformance/server-parity/operator-metrics-contract.mjs new file mode 100644 index 000000000..168bf8e83 --- /dev/null +++ b/scripts/conformance/server-parity/operator-metrics-contract.mjs @@ -0,0 +1,153 @@ +import assert from 'node:assert/strict'; + +// Reviewed operator metrics and health of one namespace, as published PHP +// 2.5.14 reports them through a run's stages. Counts must match exactly, with +// two normalizations: +// - Laravel compares "now" in whole seconds, so a task made available within +// the current second counts as delayed. Ready-due and delayed tasks are +// compared as one total, and runnable and delayed backlog likewise. +// - `tasks_dispatched_last_minute` is left out: PHP stamps a dispatch on some +// new tasks and not on an activity scheduled by a workflow task completion. +// Elapsed ages and summary history byte sizes are not compared. Each +// `oldest_*` timestamp must be present exactly when its count is not zero. +// The selected-run projection check is not compared: PHP fills its timeline +// projections asynchronously, and native reads timelines from history. +const STAGES = ['empty', 'waiting', 'waiting_after_redispatch', 'activity_scheduled', 'activity_running', + 'activity_retrying', 'completed']; +const CHECKS = [ + ['backend_capabilities', 'correctness'], ['run_summary_projection', 'correctness'], + ['selected_run_projections', 'correctness'], ['history_retention_invariant', 'correctness'], + ['command_contract_snapshots', 'correctness'], ['task_transport', 'correctness'], ['activity_path', 'correctness'], + ['routing_health', 'correctness'], ['durable_resume_paths', 'correctness'], ['worker_compatibility', 'correctness'], + ['scheduler_role', 'correctness'], ['long_poll_wake_acceleration', 'acceleration'], +]; +const UNCOMPARED_CHECK = 'selected_run_projections'; +const TIMESTAMPS = { + runs: {oldest_repair_needed_at: 'repair_needed', oldest_wait_started_at: 'waiting'}, + tasks: {oldest_lease_expired_at: 'lease_expired', oldest_ready_due_at: 'ready_due', + oldest_dispatch_overdue_since: 'dispatch_overdue', oldest_claim_failed_at: 'claim_failed', + oldest_dispatch_failed_at: 'dispatch_failed', oldest_unhealthy_at: 'unhealthy'}, + activities: {oldest_retrying_started_at: 'retrying', oldest_timeout_overdue_at: 'timeout_overdue'}, + backlog: {oldest_compatibility_blocked_started_at: 'compatibility_blocked_runs'}, + starts: {oldest_pending_start_at: 'pending_runs'}, + schedules: {oldest_overdue_at: 'missed'}, +}; +const THRESHOLDS = {event_threshold: 10000, size_bytes_threshold: 5242880, fan_out_threshold: 200, + event_warning_threshold: 8000, size_bytes_warning_threshold: 4194304, fan_out_warning_threshold: 160}; +const IGNORED = {backlog: ['tasks_dispatched_last_minute'], history: ['max_size_bytes', ...Object.keys(THRESHOLDS)]}; + +const waitingRun = {total: 1, current: 1, running: 1, waiting: 1}; +const missingContract = {backfill_needed_runs: 1, backfill_unavailable_runs: 1, actionable_backfill_needed_runs: 1, + actionable_backfill_unavailable_runs: 1}; +const started = {starts: {pending_runs: 1, pending_commands: 1}}; +// Nonzero counts at each stage; every other count is zero. +const EXPECTED = { + empty: {}, + waiting: {runs: waitingRun, tasks: {open: 1, ready: 1, ready_due_or_delayed: 1}, + backlog: {runnable_or_delayed_tasks: 1, tasks_added_last_minute: 1}, ...started, + history: {events: 2, max_event_count: 2}, command_contracts: missingContract}, + waiting_after_redispatch: {runs: waitingRun, + tasks: {open: 1, ready: 1, ready_due_or_delayed: 1, dispatch_overdue: 1, unhealthy: 1}, + backlog: {runnable_or_delayed_tasks: 1, tasks_added_last_minute: 1, unhealthy_tasks: 1}, + ...started, + history: {events: 2, max_event_count: 2}, command_contracts: missingContract}, + activity_scheduled: {runs: waitingRun, tasks: {open: 1, ready: 1, ready_due_or_delayed: 1}, + activities: {open: 1, pending: 1, queued_open: 1, queued_pending: 1}, + backlog: {runnable_or_delayed_tasks: 1, tasks_added_last_minute: 2}, + history: {events: 3, max_event_count: 3}, command_contracts: missingContract}, + activity_running: {runs: waitingRun, tasks: {open: 1, leased: 1, max_attempt_count: 1}, + activities: {open: 1, running: 1, queued_open: 1, queued_running: 1, max_attempt_count: 1}, + backlog: {leased_tasks: 1, tasks_added_last_minute: 2}, + history: {events: 4, max_event_count: 4}, command_contracts: missingContract}, + activity_retrying: {runs: waitingRun, tasks: {open: 1, ready: 1, ready_due_or_delayed: 1, max_attempt_count: 1}, + activities: {open: 1, pending: 1, queued_open: 1, queued_pending: 1, queued_failed_attempts: 1, retrying: 1, + failed_attempts: 1, max_attempt_count: 1}, + backlog: {runnable_or_delayed_tasks: 1, tasks_added_last_minute: 3, retrying_activities: 1}, + history: {events: 5, max_event_count: 5}, command_contracts: missingContract}, + completed: {runs: {total: 1, current: 1, completed: 1}, + activities: {queued_failed_attempts: 1, failed_attempts: 1, max_attempt_count: 2}, + backlog: {tasks_added_last_minute: 4}, history: {events: 8, max_event_count: 8}, + command_contracts: {backfill_needed_runs: 1, backfill_unavailable_runs: 1, closed_backfill_needed_runs: 1}}, +}; +// Checks other than ok at each stage (the uncompared check aside). +const WARNINGS = { + empty: [], + waiting: ['command_contract_snapshots'], + waiting_after_redispatch: ['command_contract_snapshots', 'task_transport', 'routing_health'], + activity_scheduled: ['command_contract_snapshots'], + activity_running: ['command_contract_snapshots'], + activity_retrying: ['command_contract_snapshots'], + completed: [], +}; + +function normalized(sections) { + const counts = {}; + for (const [name, section] of Object.entries(sections)) { + const values = {...section.counts}; + for (const field of IGNORED[name] ?? []) delete values[field]; + counts[name] = values; + } + const {tasks, backlog} = counts; + tasks.ready_due_or_delayed = tasks.ready_due + tasks.delayed; + backlog.runnable_or_delayed_tasks = backlog.runnable_tasks + backlog.delayed_tasks; + for (const [section, fields] of [[tasks, ['ready_due', 'delayed']], [backlog, ['runnable_tasks', 'delayed_tasks']], + [counts.starts, ['ready_tasks']]]) { + for (const field of fields) delete section[field]; + } + return counts; +} + +function checkStage(name, stage, namespace) { + const label = `operator metrics stage ${name}`; + assert.deepStrictEqual([stage.metrics_status, stage.metrics_namespace, stage.health_namespace, stage.expected_namespace], + [200, namespace, namespace, namespace], `${label} namespace`); + assert.deepStrictEqual(stage.metrics_keys, ['namespace', 'operator_metrics'], `${label} document`); + for (const [section, pairs] of Object.entries(TIMESTAMPS)) { + for (const [field, count] of Object.entries(pairs)) { + assert.equal(stage.sections[section].timestamps[field], stage.sections[section].counts[count] > 0, + `${label} ${section}.${field} present exactly when ${count} is not zero`); + } + } + const raw = stage.sections; + assert.deepStrictEqual(Object.fromEntries(Object.keys(THRESHOLDS).map(key => [key, raw.history.counts[key]])), THRESHOLDS, + `${label} history budget thresholds`); + // A starting run's ready task counts once it is due. Past the redispatch + // window the waiting task is due whatever the clock's second. + assert.ok(raw.starts.counts.ready_tasks <= raw.tasks.counts.ready_due, `${label} ready start tasks are due`); + if (name === 'waiting_after_redispatch') { + assert.deepStrictEqual([raw.tasks.counts.ready_due, raw.starts.counts.ready_tasks], [1, 1], `${label} due task`); + } + const counts = normalized(raw); + for (const [section, values] of Object.entries(counts)) { + const expected = EXPECTED[name][section] ?? {}; + const nonzero = Object.fromEntries(Object.entries(values).filter(([, value]) => value !== 0)); + assert.deepStrictEqual(nonzero, expected, `${label} ${section} counts`); + } + assert.equal(stage.active_workers, ['empty', 'waiting', 'waiting_after_redispatch'].includes(name) ? 0 : 1, + `${label} active workers`); + assert.deepStrictEqual(stage.checks.map(check => [check.name, check.category]), CHECKS, `${label} check names`); + const statuses = stage.checks.filter(check => check.name !== UNCOMPARED_CHECK); + assert.ok(statuses.every(check => ['ok', 'warning'].includes(check.status)), `${label} no check errors`); + assert.deepStrictEqual(statuses.filter(check => check.status !== 'ok').map(check => check.name), WARNINGS[name], + `${label} warnings`); + const warning = WARNINGS[name].length > 0 || stage.checks.some(check => check.status === 'warning'); + assert.deepStrictEqual([stage.health_status, stage.health.status, stage.health.healthy], [200, warning ? 'warning' : 'ok', true], + `${label} health`); + assert.deepStrictEqual(stage.retention, ['bounded', 30], `${label} namespace retention`); + return counts; +} + +export function checkOperatorMetrics(fixture, observation, workflowId) { + const state = observation.operator_metrics; + assert.ok(state && typeof state === 'object', 'actual operator metrics observation'); + if (observation.mode === 'embedded') { + assert.deepStrictEqual(state, {applicable: false, reason: 'embedded_has_no_http_operator_metrics'}, + 'embedded operator metrics explicitly inapplicable'); + return {scope: 'operator_metrics', stages: STAGES}; + } + const namespace = `${workflowId}-metrics`.toLowerCase(); + assert.deepStrictEqual([state.applicable, state.namespace, Object.keys(state.stages)], [true, namespace, STAGES], + 'operator metrics stages'); + for (const name of STAGES) checkStage(name, state.stages[name], namespace); + return {scope: 'operator_metrics', stages: STAGES}; +} diff --git a/scripts/conformance/server-parity/operator-metrics-probe.php b/scripts/conformance/server-parity/operator-metrics-probe.php new file mode 100644 index 000000000..1aa0ba7ee --- /dev/null +++ b/scripts/conformance/server-parity/operator-metrics-probe.php @@ -0,0 +1,96 @@ +createNamespace($namespace, 'operator metrics', 30); + $token = getenv('DW_PARITY_TOKEN') ?: null; + $client = parityClient($url, namespace: $namespace, token: $token); + $http = new OperatorMetricsHttp(['base_uri' => rtrim($url, '/').'/api/', 'http_errors' => false, + 'connect_timeout' => 1, 'timeout' => 30]); + $get = static function (string $path) use ($http, $token, $namespace): array { + $response = $http->get($path, ['headers' => array_filter([ + 'Authorization' => $token === null ? null : 'Bearer '.$token, + 'Accept' => 'application/json', 'X-Namespace' => $namespace, + 'X-Durable-Workflow-Control-Plane-Version' => '2', + ])]); + + return ['status' => $response->getStatusCode(), + 'body' => json_decode((string) $response->getBody(), true, flags: JSON_THROW_ON_ERROR)]; + }; + $numbers = static fn (array $section): array => array_filter($section, + static fn ($value, string $key): bool => is_int($value) && ! str_contains($key, '_ms'), ARRAY_FILTER_USE_BOTH); + $present = static fn (array $section): array => array_map(static fn ($value): bool => $value !== null, + array_filter($section, static fn (string $key): bool => str_starts_with($key, 'oldest_'), ARRAY_FILTER_USE_KEY)); + $stage = static function () use ($get, $numbers, $present, $namespace): array { + $metrics = $get('system/operator-metrics'); + $health = $get('system/health'); + $document = $metrics['body']['operator_metrics'] ?? []; + $sections = []; + foreach (['runs', 'tasks', 'activities', 'backlog', 'starts', 'history', 'schedules', 'command_contracts'] as $name) { + $sections[$name] = ['counts' => $numbers($document[$name] ?? []), 'timestamps' => $present($document[$name] ?? [])]; + } + $checks = []; + foreach ($health['body']['health']['checks'] ?? [] as $check) { + $checks[] = ['name' => $check['name'], 'status' => $check['status'], 'category' => $check['category']]; + } + + return [ + 'metrics_status' => $metrics['status'], + 'metrics_namespace' => $metrics['body']['namespace'] ?? null, + 'metrics_keys' => array_keys($metrics['body'] ?? []), + 'sections' => $sections, + 'active_workers' => $document['workers']['active_workers'] ?? null, + 'worker_fleet' => array_map(static fn (array $entry): array => ['worker_id' => $entry['worker_id'] ?? null, + 'namespace' => $entry['namespace'] ?? null, 'queue' => $entry['queue'] ?? null], + $document['workers']['fleet'] ?? []), + 'health_status' => $health['status'], + 'health_namespace' => $health['body']['namespace'] ?? null, + 'retention' => [$health['body']['retention_mode'] ?? null, $health['body']['retention_days'] ?? null], + 'health' => ['status' => $health['body']['health']['status'] ?? null, 'healthy' => $health['body']['health']['healthy'] ?? null, + 'categories' => $health['body']['health']['categories'] ?? null], + 'checks' => $checks, + 'expected_namespace' => $namespace, + ]; + }; + $queue = $workflowId.'-metrics'; + $worker = $workflowId.'-metrics-worker'; + $input = $fixture['input']; + $stages = ['empty' => $stage()]; + $client->startWorkflow($definition['workflow_type'], $workflowId.'-metrics-run', $queue, [$input]); + $stages['waiting'] = $stage(); + // Past PHP's three-second redispatch window, the unclaimed task is overdue. + usleep((int) ($definition['redispatch_wait_seconds'] * 1_000_000)); + $stages['waiting_after_redispatch'] = $stage(); + $client->registerWorker($worker, $queue, [$definition['workflow_type']], [$definition['activity_type']]); + $task = $client->pollWorkflowTask($worker, $queue, 5); + $client->completeWorkflowTask($task['task_id'], $worker, $task['workflow_task_attempt'], [[ + 'type' => 'schedule_activity', 'activity_type' => $definition['activity_type'], + 'arguments' => $client->payloadCodec()->envelope([$input]), + 'retry_policy' => ['max_attempts' => 2, 'backoff_seconds' => [$definition['retry_backoff_seconds']]], + ]]); + $stages['activity_scheduled'] = $stage(); + $activity = $client->pollActivityTask($worker, $queue, 5); + $stages['activity_running'] = $stage(); + $client->failActivityTask($activity['task_id'], $activity['activity_attempt_id'], $worker, 'first attempt fails'); + $stages['activity_retrying'] = $stage(); + usleep((int) (($definition['retry_backoff_seconds'] + 1) * 1_000_000)); + $activity = $client->pollActivityTask($worker, $queue, 5); + $client->completeActivityTask($activity['task_id'], $activity['activity_attempt_id'], $worker, $input); + $task = $client->pollWorkflowTask($worker, $queue, 5); + $client->completeWorkflowTask($task['task_id'], $worker, $task['workflow_task_attempt'], + [['type' => 'complete_workflow', 'result' => $client->payloadCodec()->envelope($input)]]); + $stages['completed'] = $stage(); + + return ['applicable' => true, 'namespace' => $namespace, 'stages' => $stages]; +} diff --git a/scripts/conformance/server-parity/probe.php b/scripts/conformance/server-parity/probe.php index a929634f4..6814fcc46 100644 --- a/scripts/conformance/server-parity/probe.php +++ b/scripts/conformance/server-parity/probe.php @@ -429,6 +429,7 @@ function httpObservation(array $fixture, string $workflowId, string $namespace, $workerDeregistration = isset($fixture['worker_deregistration']) ? finishHttpWorkerDeregistration($client, $fixture, $workflowId, $queue, $url) : null; $workerListing = isset($fixture['worker_listing']) ? finishHttpWorkerListing($client, $fixture, $workflowId) : null; $workerReplacement = isset($fixture['worker_replacement']) ? finishHttpWorkerReplacement($client, $fixture, $workflowId) : null; + $operatorMetrics = isset($fixture['operator_metrics']) ? finishHttpOperatorMetrics($client, $fixture, $workflowId, $url) : null; $waitingHistory = isset($fixture['waiting_for_history']) ? finishHttpWaitingHistory($client, $fixture, $workflowId, $queue, $url) : null; $taskRetry = isset($fixture['workflow_task_retry']) ? finishHttpTaskRetry($client, $fixture, $workflowId, $queue, $url) : null; if ($scheduleState !== null) { @@ -517,6 +518,7 @@ function httpObservation(array $fixture, string $workflowId, string $namespace, 'worker_deregistration' => $workerDeregistration, 'worker_listing' => $workerListing, 'worker_replacement' => $workerReplacement, + 'operator_metrics' => $operatorMetrics, 'waiting_for_history' => $waitingHistory, 'workflow_task_retry' => $taskRetry, 'run_id' => $execution->runId, @@ -815,6 +817,7 @@ function embeddedObservation(array $fixture, string $workflowId, string $namespa require __DIR__.'/worker-deregistration-probe.php'; require __DIR__.'/worker-listing-probe.php'; require __DIR__.'/worker-replacement-probe.php'; + require __DIR__.'/operator-metrics-probe.php'; require __DIR__.'/waiting-history-probe.php'; require __DIR__.'/task-retry-probe.php'; require __DIR__.'/child-cancellation-probe.php'; @@ -948,6 +951,9 @@ function embeddedObservation(array $fixture, string $workflowId, string $namespa if ($mode === 'embedded' && isset($fixture['worker_replacement'])) { $observation['worker_replacement'] = ['applicable' => false, 'reason' => 'embedded_has_no_http_worker_registration_lifecycle']; } + if ($mode === 'embedded' && isset($fixture['operator_metrics'])) { + $observation['operator_metrics'] = ['applicable' => false, 'reason' => 'embedded_has_no_http_operator_metrics']; + } if ($mode === 'embedded' && isset($fixture['worker_listing'])) { $observation['worker_listing'] = ['applicable' => false, 'reason' => 'embedded_has_no_http_worker_registration_lifecycle']; } diff --git a/tests/Fixtures/ServerParity/operator-metrics.json b/tests/Fixtures/ServerParity/operator-metrics.json new file mode 100644 index 000000000..03a623bb0 --- /dev/null +++ b/tests/Fixtures/ServerParity/operator-metrics.json @@ -0,0 +1,21 @@ +{ + "schema": "durable-workflow.server-parity-fixture/v1", + "id": "operator-metrics", + "review_state": "reviewed", + "workflow_type": "parity.v1.echo", + "input": {"message": "operator metrics λ", "count": 9007199254740993, "optional": null}, + "typed_value": {"type": "map", "value": { + "message": {"type": "string", "value": "operator metrics λ"}, + "count": {"type": "int64", "value": "9007199254740993"}, + "optional": {"type": "null", "value": null} + }}, + "activity": false, + "expected_events": ["StartAccepted", "WorkflowStarted", "WorkflowCompleted"], + "operator_metrics": { + "workflow_type": "parity.v1.metrics", + "activity_type": "parity.v1.metrics_activity", + "redispatch_wait_seconds": 4, + "retry_backoff_seconds": 2 + }, + "contract": "GET /api/system/operator-metrics and GET /api/system/health report a fresh namespace through one run's stages: empty, a started run waiting for a worker, the same task past PHP's three-second redispatch window, an activity scheduled, running and retrying after a failure, and the completed run. At each stage the run, task, activity, backlog, start, history, schedule and command-contract counts match published PHP's, every oldest timestamp is present exactly when its count is not zero, and the health document answers 200 with PHP's twelve checks, categories and warnings: command-contract backfill while the run (started before its worker registered) is open, and task transport and routing once the unclaimed task is overdue. Ready-due and delayed tasks compare as one total, because Laravel compares the current time in whole seconds. Elapsed ages, history byte sizes and tasks dispatched in the last minute are not compared, nor the selected-run projection check, which PHP fills asynchronously. Embedded applications have no HTTP operator surface." +} diff --git a/tests/Unit/ServerParityOperatorMetricsContractTest.mjs b/tests/Unit/ServerParityOperatorMetricsContractTest.mjs new file mode 100644 index 000000000..a9bd6c001 --- /dev/null +++ b/tests/Unit/ServerParityOperatorMetricsContractTest.mjs @@ -0,0 +1,147 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import {readFileSync} from 'node:fs'; +import {checkOperatorMetrics} from '../../scripts/conformance/server-parity/operator-metrics-contract.mjs'; + +const fixture = JSON.parse(readFileSync(new URL('../Fixtures/ServerParity/operator-metrics.json', import.meta.url))); +const namespace = 'test-metrics'; +const SECTIONS = { + runs: ['total', 'current', 'running', 'completed', 'failed', 'cancelled', 'terminated', 'archived', 'repair_needed', + 'claim_failed', 'compatibility_blocked', 'waiting'], + tasks: ['open', 'ready', 'ready_due', 'delayed', 'leased', 'dispatch_failed', 'claim_failed', 'dispatch_overdue', + 'lease_expired', 'max_attempt_count', 'max_repair_count', 'unhealthy'], + activities: ['open', 'pending', 'running', 'local', 'local_open', 'local_pending', 'local_running', 'local_attempts', + 'local_failed_attempts', 'queued_open', 'queued_pending', 'queued_running', 'queued_failed_attempts', 'retrying', + 'timeout_overdue', 'failed_attempts', 'max_attempt_count'], + backlog: ['runnable_tasks', 'delayed_tasks', 'leased_tasks', 'tasks_added_last_minute', 'tasks_dispatched_last_minute', + 'retrying_activities', 'unhealthy_tasks', 'repair_needed_runs', 'claim_failed_runs', 'compatibility_blocked_runs'], + starts: ['pending_runs', 'pending_commands', 'ready_tasks'], + history: ['continue_as_new_recommended_runs', 'approaching_budget_runs', 'events', 'history_orphan_total', + 'max_event_count', 'max_size_bytes', 'max_fan_out', 'event_threshold', 'size_bytes_threshold', 'fan_out_threshold', + 'event_warning_threshold', 'size_bytes_warning_threshold', 'fan_out_warning_threshold'], + schedules: ['active', 'paused', 'missed', 'fires_total', 'failures_total'], + command_contracts: ['backfill_needed_runs', 'backfill_available_runs', 'backfill_unavailable_runs', + 'actionable_backfill_needed_runs', 'actionable_backfill_available_runs', 'actionable_backfill_unavailable_runs', + 'closed_backfill_needed_runs'], +}; +const STAMPS = { + runs: {oldest_repair_needed_at: 'repair_needed', oldest_wait_started_at: 'waiting'}, + tasks: {oldest_lease_expired_at: 'lease_expired', oldest_ready_due_at: 'ready_due', + oldest_dispatch_overdue_since: 'dispatch_overdue', oldest_claim_failed_at: 'claim_failed', + oldest_dispatch_failed_at: 'dispatch_failed', oldest_unhealthy_at: 'unhealthy'}, + activities: {oldest_retrying_started_at: 'retrying', oldest_timeout_overdue_at: 'timeout_overdue'}, + backlog: {oldest_compatibility_blocked_started_at: 'compatibility_blocked_runs'}, + starts: {oldest_pending_start_at: 'pending_runs'}, + schedules: {oldest_overdue_at: 'missed'}, + history: {}, command_contracts: {}, +}; +const NAMES = ['backend_capabilities', 'run_summary_projection', 'selected_run_projections', 'history_retention_invariant', + 'command_contract_snapshots', 'task_transport', 'activity_path', 'routing_health', 'durable_resume_paths', + 'worker_compatibility', 'scheduler_role', 'long_poll_wake_acceleration']; + +// PHP 2.5.14's recorded stages, with PHP's thresholds and sizes. +const WAITING = {runs: {total: 1, current: 1, running: 1, waiting: 1}, tasks: {open: 1, ready: 1, delayed: 1}, + backlog: {delayed_tasks: 1, tasks_added_last_minute: 1, tasks_dispatched_last_minute: 1}, + starts: {pending_runs: 1, pending_commands: 1}, history: {events: 2, max_event_count: 2}, + command_contracts: {backfill_needed_runs: 1, backfill_unavailable_runs: 1, actionable_backfill_needed_runs: 1, + actionable_backfill_unavailable_runs: 1}}; +const RECORDED = { + empty: [{}, []], + waiting: [WAITING, ['selected_run_projections', 'command_contract_snapshots']], + waiting_after_redispatch: [{...WAITING, + tasks: {open: 1, ready: 1, ready_due: 1, dispatch_overdue: 1, unhealthy: 1}, + backlog: {runnable_tasks: 1, tasks_added_last_minute: 1, tasks_dispatched_last_minute: 1, unhealthy_tasks: 1}, + starts: {pending_runs: 1, pending_commands: 1, ready_tasks: 1}}, + ['selected_run_projections', 'command_contract_snapshots', 'task_transport', 'routing_health']], + activity_scheduled: [{...WAITING, tasks: {open: 1, ready: 1, delayed: 1}, starts: {}, + activities: {open: 1, pending: 1, queued_open: 1, queued_pending: 1}, + backlog: {delayed_tasks: 1, tasks_added_last_minute: 2, tasks_dispatched_last_minute: 1}, + history: {events: 3, max_event_count: 3, max_size_bytes: 3000}}, ['command_contract_snapshots']], + activity_running: [{...WAITING, tasks: {open: 1, leased: 1, max_attempt_count: 1}, starts: {}, + activities: {open: 1, running: 1, queued_open: 1, queued_running: 1, max_attempt_count: 1}, + backlog: {leased_tasks: 1, tasks_added_last_minute: 2, tasks_dispatched_last_minute: 1}, + history: {events: 4, max_event_count: 4}}, ['command_contract_snapshots']], + activity_retrying: [{...WAITING, tasks: {open: 1, ready: 1, delayed: 1, max_attempt_count: 1}, starts: {}, + activities: {open: 1, pending: 1, queued_open: 1, queued_pending: 1, queued_failed_attempts: 1, retrying: 1, + failed_attempts: 1, max_attempt_count: 1}, + backlog: {delayed_tasks: 1, tasks_added_last_minute: 3, tasks_dispatched_last_minute: 2, retrying_activities: 1}, + history: {events: 5, max_event_count: 5}}, ['command_contract_snapshots']], + completed: [{runs: {total: 1, current: 1, completed: 1}, + activities: {queued_failed_attempts: 1, failed_attempts: 1, max_attempt_count: 2}, + backlog: {tasks_added_last_minute: 4, tasks_dispatched_last_minute: 3}, history: {events: 8, max_event_count: 8}, + command_contracts: {backfill_needed_runs: 1, backfill_unavailable_runs: 1, closed_backfill_needed_runs: 1}}, []], +}; + +const THRESHOLDS = {event_threshold: 10000, size_bytes_threshold: 5242880, fan_out_threshold: 200, + event_warning_threshold: 8000, size_bytes_warning_threshold: 4194304, fan_out_warning_threshold: 160}; +function stage(name) { + const [values, warnings] = RECORDED[name]; + const sections = {}; + for (const [section, fields] of Object.entries(SECTIONS)) { + const counts = Object.fromEntries(fields.map(field => [field, values[section]?.[field] + ?? (section === 'history' ? THRESHOLDS[field] : undefined) ?? 0])); + const timestamps = Object.fromEntries(Object.entries(STAMPS[section]).map(([field, count]) => [field, counts[count] > 0])); + sections[section] = {counts, timestamps}; + } + const warned = new Set(warnings); + return {metrics_status: 200, metrics_namespace: namespace, metrics_keys: ['namespace', 'operator_metrics'], sections, + active_workers: ['empty', 'waiting', 'waiting_after_redispatch'].includes(name) ? 0 : 1, worker_fleet: [], + health_status: 200, health_namespace: namespace, retention: ['bounded', 30], + health: {status: warned.size ? 'warning' : 'ok', healthy: true, categories: {}}, + checks: NAMES.map(check => ({name: check, status: warned.has(check) ? 'warning' : 'ok', + category: check === 'long_poll_wake_acceleration' ? 'acceleration' : 'correctness'})), + expected_namespace: namespace}; +} +const model = () => structuredClone({mode: 'http', operator_metrics: {applicable: true, namespace, + stages: Object.fromEntries(Object.keys(RECORDED).map(name => [name, stage(name)]))}}); +const projection = {scope: 'operator_metrics', stages: Object.keys(RECORDED)}; + +test('published PHP stages pass', () => assert.deepStrictEqual(checkOperatorMetrics(fixture, model(), 'TEST'), projection)); +test('a task made due within the same second passes', () => { + const observation = model(); + const tasks = observation.operator_metrics.stages.waiting.sections.tasks; + Object.assign(tasks.counts, {ready_due: 1, delayed: 0}); + tasks.timestamps.oldest_ready_due_at = true; + const backlog = observation.operator_metrics.stages.waiting.sections.backlog.counts; + Object.assign(backlog, {runnable_tasks: 1, delayed_tasks: 0}); + observation.operator_metrics.stages.waiting.sections.starts.counts.ready_tasks = 1; + assert.deepStrictEqual(checkOperatorMetrics(fixture, observation, 'TEST'), projection); +}); +test('an uncompared projection check may differ', () => { + const observation = model(); + observation.operator_metrics.stages.waiting.checks[2].status = 'ok'; + checkOperatorMetrics(fixture, observation, 'TEST'); +}); +test('embedded explicitly declares the HTTP metrics inapplicable', () => assert.deepStrictEqual(checkOperatorMetrics(fixture, + {mode: 'embedded', operator_metrics: {applicable: false, reason: 'embedded_has_no_http_operator_metrics'}}, 'test'), projection)); +for (const [label, mutate] of [ + ['a missing stage', s => delete s.stages.activity_running], + ['another namespace', s => s.stages.empty.metrics_namespace = 'default'], + ['a run never counted', s => s.stages.waiting.sections.runs.counts.total = 0], + ['a lost waiting run', s => s.stages.activity_scheduled.sections.runs.counts.waiting = 0], + ['an overdue task not reported', s => s.stages.waiting_after_redispatch.sections.tasks.counts.dispatch_overdue = 0], + ['an overdue task still delayed', s => Object.assign(s.stages.waiting_after_redispatch.sections.tasks.counts, {ready_due: 0, delayed: 1})], + ['a retry not counted', s => s.stages.activity_retrying.sections.activities.counts.retrying = 0], + ['a failed attempt forgotten', s => s.stages.completed.sections.activities.counts.failed_attempts = 0], + ['a timestamp without its count', s => s.stages.empty.sections.runs.timestamps.oldest_wait_started_at = true], + ['a count without its timestamp', s => s.stages.activity_retrying.sections.activities.timestamps.oldest_retrying_started_at = false], + ['a command contract backfill missed', s => s.stages.waiting.sections.command_contracts.counts.actionable_backfill_needed_runs = 0], + ['a closed run still actionable', s => s.stages.completed.sections.command_contracts.counts.actionable_backfill_needed_runs = 1], + ['history events miscounted', s => s.stages.completed.sections.history.counts.events = 7], + ['a worker never counted', s => s.stages.activity_running.active_workers = 0], + ['a transport warning missed', s => s.stages.waiting_after_redispatch.checks[5].status = 'ok'], + ['an unexpected warning', s => s.stages.completed.checks[9].status = 'warning'], + ['a check error', s => s.stages.empty.checks[0].status = 'error'], + ['a missing check', s => s.stages.empty.checks.pop()], + ['an unhealthy document', s => s.stages.empty.health.healthy = false], + ['health answered with 503', s => s.stages.empty.health_status = 503], + ['other retention', s => s.stages.empty.retention = ['forever', null]], + ['metrics refused', s => s.stages.empty.metrics_status = 403], + ['another history budget', s => s.stages.empty.sections.history.counts.fan_out_threshold = 100], +]) { + test(`rejects ${label}`, () => { + const observation = model(); + mutate(observation.operator_metrics); + assert.throws(() => checkOperatorMetrics(fixture, observation, 'TEST')); + }); +} From dedd08aef04cb5ca58dc861c29e6f7334c4ec6a8 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Wed, 21 Oct 2026 01:59:55 +0000 Subject: [PATCH 2/7] Deregister workers in named namespaces and accept PHP's task repair DELETE /api/workers/{id} now works in a named namespace and removes only that namespace's registration, as published PHP allows. The operator metrics probe deregisters its worker again. PHP's maintenance loop may redispatch the overdue task before the probe reads it, so the contract accepts PHP's repaired state for that stage; native runs no repair loop. Co-Authored-By: Claude Opus 5.5 --- docs/rust-server-port.md | 7 ++- rust/src/runtime/http.rs | 14 ++++-- rust/tests/execution/operator_metrics.rs | 50 +++++++++++++++++++ .../operator-metrics-contract.mjs | 16 ++++-- .../server-parity/operator-metrics-probe.php | 1 + .../ServerParity/operator-metrics.json | 2 +- ...erverParityOperatorMetricsContractTest.mjs | 14 ++++++ 7 files changed, 95 insertions(+), 9 deletions(-) diff --git a/docs/rust-server-port.md b/docs/rust-server-port.md index 0ffd35b0c..9670f7a06 100644 --- a/docs/rust-server-port.md +++ b/docs/rust-server-port.md @@ -244,6 +244,8 @@ October 21, operator metrics and health: sections describe native: no repair loop, sticky routing or worker sessions, in-process long-poll wake, and native's structural limits and workflow task lease. `capacity_evidence` reports itself unavailable. +- **Worker deregistration.** `DELETE /api/workers/{id}` now works in a named + namespace and removes only that namespace's registration, as in PHP. - **Decision: selected-run projections.** Native stores run waits in the transaction that records history, so they cannot drift. It reads timelines, timers and lineage from history, so those tables are reported as @@ -271,7 +273,10 @@ October 21, operator metrics and health: - PHP stamps a dispatch on some new tasks and not on an activity scheduled from a workflow task completion, so `tasks_dispatched_last_minute` differs. - - Native refuses worker deregistration in a named namespace. + - PHP's maintenance loop redispatches an overdue ready task, after which + PHP reports it repaired, not overdue. Native runs no repair loop, so the + task stays overdue until a worker claims it. The fixture accepts either + PHP state. - The operator dashboards, capacity evidence, and the repair, retention, activity-timeout and payload-cleanup pass endpoints remain. diff --git a/rust/src/runtime/http.rs b/rust/src/runtime/http.rs index 53ccb3632..26596f9b3 100644 --- a/rust/src/runtime/http.rs +++ b/rust/src/runtime/http.rs @@ -441,10 +441,14 @@ fn named_namespace_operation(method: &str, path: &str) -> bool { { return method == "DELETE"; } + // Deregistration deletes only the namespace's own registration. + if path + .strip_prefix("/api/workers/") + .is_some_and(|id| !id.is_empty() && !id.contains('/')) + { + return matches!(method, "GET" | "DELETE"); + } if path == "/api/workers" - || path - .strip_prefix("/api/workers/") - .is_some_and(|id| !id.is_empty() && !id.contains('/')) || path == "/api/system/operator-metrics" || path == "/api/system/health" { @@ -1004,7 +1008,9 @@ fn metrics_config(runtime: &Runtime) -> super::operator_metrics::MetricsConfig { } async fn operator_metrics(Extension(runtime): Extension) -> Result> { - Ok(Json(runtime.operator_metrics(&metrics_config(&runtime)).await?)) + Ok(Json( + runtime.operator_metrics(&metrics_config(&runtime)).await?, + )) } /// PHP answers 503 while a check reports an error. diff --git a/rust/tests/execution/operator_metrics.rs b/rust/tests/execution/operator_metrics.rs index 9afc170a7..c7d9fe82e 100644 --- a/rust/tests/execution/operator_metrics.rs +++ b/rust/tests/execution/operator_metrics.rs @@ -178,3 +178,53 @@ async fn operator_routes_are_admin_only_and_namespace_scoped() { runtime.close().await; database.remove().await; } + +#[tokio::test] +async fn a_named_namespace_deregisters_only_its_own_worker() { + let database = TestDatabase::new().await; + let runtime = database.open().await.unwrap(); + let app = router(runtime.clone()); + namespaces::create(&app, "metrics-workers").await; + for namespace in ["default", "metrics-workers"] { + let registered = namespaces::scoped( + &app, + namespace, + "POST", + "/api/worker/register", + json!({ + "worker_id":"shared-worker","task_queue":"test","runtime":"php", + "supported_workflow_types":["echo"],"supported_activity_types":[]}), + ) + .await; + assert_eq!(registered.0, StatusCode::CREATED, "{}", registered.1); + } + let removed = namespaces::scoped( + &app, + "metrics-workers", + "DELETE", + "/api/workers/shared-worker", + Value::Null, + ) + .await; + assert_eq!(removed.0, StatusCode::OK, "{}", removed.1); + let gone = namespaces::scoped( + &app, + "metrics-workers", + "GET", + "/api/workers/shared-worker", + Value::Null, + ) + .await; + assert_eq!(gone.0, StatusCode::NOT_FOUND, "{}", gone.1); + let kept = namespaces::scoped( + &app, + "default", + "GET", + "/api/workers/shared-worker", + Value::Null, + ) + .await; + assert_eq!(kept.0, StatusCode::OK, "{}", kept.1); + runtime.close().await; + database.remove().await; +} diff --git a/scripts/conformance/server-parity/operator-metrics-contract.mjs b/scripts/conformance/server-parity/operator-metrics-contract.mjs index 168bf8e83..918dbb2f0 100644 --- a/scripts/conformance/server-parity/operator-metrics-contract.mjs +++ b/scripts/conformance/server-parity/operator-metrics-contract.mjs @@ -69,6 +69,12 @@ const EXPECTED = { backlog: {tasks_added_last_minute: 4}, history: {events: 8, max_event_count: 8}, command_contracts: {backfill_needed_runs: 1, backfill_unavailable_runs: 1, closed_backfill_needed_runs: 1}}, }; +// PHP's maintenance loop redispatches an overdue ready task (TaskRepair), so +// past the redispatch window published PHP may already show the task repaired +// instead of overdue. Native runs no repair loop and keeps reporting it overdue. +const REPAIRED = {...EXPECTED.waiting_after_redispatch, tasks: {open: 1, ready: 1, ready_due_or_delayed: 1}, + backlog: {runnable_or_delayed_tasks: 1, tasks_added_last_minute: 1}}; +const REPAIRED_WARNINGS = ['command_contract_snapshots']; // Checks other than ok at each stage (the uncompared check aside). const WARNINGS = { empty: [], @@ -118,8 +124,12 @@ function checkStage(name, stage, namespace) { assert.deepStrictEqual([raw.tasks.counts.ready_due, raw.starts.counts.ready_tasks], [1, 1], `${label} due task`); } const counts = normalized(raw); + const repaired = name === 'waiting_after_redispatch' && counts.tasks.max_repair_count > 0; + if (repaired) delete counts.tasks.max_repair_count; + const expectedCounts = repaired ? REPAIRED : EXPECTED[name]; + const expectedWarnings = repaired ? REPAIRED_WARNINGS : WARNINGS[name]; for (const [section, values] of Object.entries(counts)) { - const expected = EXPECTED[name][section] ?? {}; + const expected = expectedCounts[section] ?? {}; const nonzero = Object.fromEntries(Object.entries(values).filter(([, value]) => value !== 0)); assert.deepStrictEqual(nonzero, expected, `${label} ${section} counts`); } @@ -128,9 +138,9 @@ function checkStage(name, stage, namespace) { assert.deepStrictEqual(stage.checks.map(check => [check.name, check.category]), CHECKS, `${label} check names`); const statuses = stage.checks.filter(check => check.name !== UNCOMPARED_CHECK); assert.ok(statuses.every(check => ['ok', 'warning'].includes(check.status)), `${label} no check errors`); - assert.deepStrictEqual(statuses.filter(check => check.status !== 'ok').map(check => check.name), WARNINGS[name], + assert.deepStrictEqual(statuses.filter(check => check.status !== 'ok').map(check => check.name), expectedWarnings, `${label} warnings`); - const warning = WARNINGS[name].length > 0 || stage.checks.some(check => check.status === 'warning'); + const warning = expectedWarnings.length > 0 || stage.checks.some(check => check.status === 'warning'); assert.deepStrictEqual([stage.health_status, stage.health.status, stage.health.healthy], [200, warning ? 'warning' : 'ok', true], `${label} health`); assert.deepStrictEqual(stage.retention, ['bounded', 30], `${label} namespace retention`); diff --git a/scripts/conformance/server-parity/operator-metrics-probe.php b/scripts/conformance/server-parity/operator-metrics-probe.php index 1aa0ba7ee..7617cfe5f 100644 --- a/scripts/conformance/server-parity/operator-metrics-probe.php +++ b/scripts/conformance/server-parity/operator-metrics-probe.php @@ -91,6 +91,7 @@ function finishHttpOperatorMetrics(Client $admin, array $fixture, string $workfl $client->completeWorkflowTask($task['task_id'], $worker, $task['workflow_task_attempt'], [['type' => 'complete_workflow', 'result' => $client->payloadCodec()->envelope($input)]]); $stages['completed'] = $stage(); + $client->deregisterWorker($worker); return ['applicable' => true, 'namespace' => $namespace, 'stages' => $stages]; } diff --git a/tests/Fixtures/ServerParity/operator-metrics.json b/tests/Fixtures/ServerParity/operator-metrics.json index 03a623bb0..00b03532c 100644 --- a/tests/Fixtures/ServerParity/operator-metrics.json +++ b/tests/Fixtures/ServerParity/operator-metrics.json @@ -17,5 +17,5 @@ "redispatch_wait_seconds": 4, "retry_backoff_seconds": 2 }, - "contract": "GET /api/system/operator-metrics and GET /api/system/health report a fresh namespace through one run's stages: empty, a started run waiting for a worker, the same task past PHP's three-second redispatch window, an activity scheduled, running and retrying after a failure, and the completed run. At each stage the run, task, activity, backlog, start, history, schedule and command-contract counts match published PHP's, every oldest timestamp is present exactly when its count is not zero, and the health document answers 200 with PHP's twelve checks, categories and warnings: command-contract backfill while the run (started before its worker registered) is open, and task transport and routing once the unclaimed task is overdue. Ready-due and delayed tasks compare as one total, because Laravel compares the current time in whole seconds. Elapsed ages, history byte sizes and tasks dispatched in the last minute are not compared, nor the selected-run projection check, which PHP fills asynchronously. Embedded applications have no HTTP operator surface." + "contract": "GET /api/system/operator-metrics and GET /api/system/health report a fresh namespace through one run's stages: empty, a started run waiting for a worker, the same task past PHP's three-second redispatch window, an activity scheduled, running and retrying after a failure, and the completed run. At each stage the run, task, activity, backlog, start, history, schedule and command-contract counts match published PHP's, every oldest timestamp is present exactly when its count is not zero, and the health document answers 200 with PHP's twelve checks, categories and warnings: command-contract backfill while the run (started before its worker registered) is open, and task transport and routing once the unclaimed task is overdue, unless PHP's maintenance loop has already redispatched it, which native never does. Ready-due and delayed tasks compare as one total, because Laravel compares the current time in whole seconds. Elapsed ages, history byte sizes and tasks dispatched in the last minute are not compared, nor the selected-run projection check, which PHP fills asynchronously. Embedded applications have no HTTP operator surface." } diff --git a/tests/Unit/ServerParityOperatorMetricsContractTest.mjs b/tests/Unit/ServerParityOperatorMetricsContractTest.mjs index a9bd6c001..06375c94a 100644 --- a/tests/Unit/ServerParityOperatorMetricsContractTest.mjs +++ b/tests/Unit/ServerParityOperatorMetricsContractTest.mjs @@ -112,6 +112,20 @@ test('an uncompared projection check may differ', () => { observation.operator_metrics.stages.waiting.checks[2].status = 'ok'; checkOperatorMetrics(fixture, observation, 'TEST'); }); +test('a task PHP\'s maintenance loop already redispatched passes', () => { + const observation = model(); + const stage = observation.operator_metrics.stages.waiting_after_redispatch; + Object.assign(stage.sections.tasks.counts, {dispatch_overdue: 0, unhealthy: 0, max_repair_count: 1}); + Object.assign(stage.sections.tasks.timestamps, {oldest_dispatch_overdue_since: false, oldest_unhealthy_at: false}); + stage.sections.backlog.counts.unhealthy_tasks = 0; + for (const check of stage.checks) if (['task_transport', 'routing_health'].includes(check.name)) check.status = 'ok'; + assert.deepStrictEqual(checkOperatorMetrics(fixture, observation, 'TEST'), projection); +}); +test('rejects a repaired task still reported overdue', () => { + const observation = model(); + observation.operator_metrics.stages.waiting_after_redispatch.sections.tasks.counts.max_repair_count = 1; + assert.throws(() => checkOperatorMetrics(fixture, observation, 'TEST')); +}); test('embedded explicitly declares the HTTP metrics inapplicable', () => assert.deepStrictEqual(checkOperatorMetrics(fixture, {mode: 'embedded', operator_metrics: {applicable: false, reason: 'embedded_has_no_http_operator_metrics'}}, 'test'), projection)); for (const [label, mutate] of [ From b0f61d8893f5821c5571a010560026e61822971e Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Wed, 21 Oct 2026 05:00:34 +0000 Subject: [PATCH 3/7] Accept published PHP's MariaDB health failure in the operator metrics fixture Published PHP's health check runs a task-queue build-id query that MariaDB rejects under ONLY_FULL_GROUP_BY once an activity exists (#569), so it answers 500 from the activity_scheduled stage on. The probe now records the runtime each stage reports and the MySQL-family job's database image; the contract accepts that 500 only for published PHP on MariaDB at those stages. Native must still answer 200 everywhere. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/server-parity.yml | 4 +-- .../operator-metrics-contract.mjs | 18 ++++++++++--- .../server-parity/operator-metrics-probe.php | 7 ++++- ...erverParityOperatorMetricsContractTest.mjs | 27 +++++++++++++++++++ 4 files changed, 50 insertions(+), 6 deletions(-) diff --git a/.github/workflows/server-parity.yml b/.github/workflows/server-parity.yml index 11a588252..b415cd73c 100644 --- a/.github/workflows/server-parity.yml +++ b/.github/workflows/server-parity.yml @@ -522,7 +522,7 @@ jobs: export DW_PARITY_CLI_PHAR=/source/parity-consumers/dw.phar docker run --rm --user=1000:1000 --entrypoint node --network "$RESOURCE_SCOPE" \ -v "$PWD:/source:ro" -v "$PWD/parity-evidence:/evidence" -w /source \ - -e DW_PARITY_TOKEN=parity-test-token -e DW_PARITY_RUNNER_REVISION -e DW_PARITY_CLI_PHAR "$image" scripts/conformance/server-parity.mjs record \ + -e DW_PARITY_TOKEN=parity-test-token -e DW_PARITY_RUNNER_REVISION -e DW_PARITY_CLI_PHAR -e MYSQL_IMAGE "$image" scripts/conformance/server-parity.mjs record \ --mode http --url http://parity-php:8080 --target php --output /evidence/php.json \ --include-fixture tests/Fixtures/ServerParityPending/patch-insertion-activity-pending.json \ --include-fixture tests/Fixtures/ServerParityPending/patch-insertion-activity-completed.json \ @@ -546,7 +546,7 @@ jobs: --include-fixture tests/Fixtures/ServerParityPending/workflow-task-transient-retry.json docker run --rm --user=1000:1000 --entrypoint node --network "$RESOURCE_SCOPE" \ -v "$PWD:/source:ro" -v "$PWD/parity-evidence:/evidence" -w /source \ - -e DW_PARITY_TOKEN=parity-test-token -e DW_PARITY_RUNNER_REVISION -e DW_PARITY_CLI_PHAR "$image" scripts/conformance/server-parity.mjs record \ + -e DW_PARITY_TOKEN=parity-test-token -e DW_PARITY_RUNNER_REVISION -e DW_PARITY_CLI_PHAR -e MYSQL_IMAGE "$image" scripts/conformance/server-parity.mjs record \ --mode http --url http://parity-native-a:8080 --target rust --output /evidence/rust.json \ --include-fixture tests/Fixtures/ServerParityPending/patch-insertion-activity-pending.json \ --include-fixture tests/Fixtures/ServerParityPending/patch-insertion-activity-completed.json \ diff --git a/scripts/conformance/server-parity/operator-metrics-contract.mjs b/scripts/conformance/server-parity/operator-metrics-contract.mjs index 918dbb2f0..df4481841 100644 --- a/scripts/conformance/server-parity/operator-metrics-contract.mjs +++ b/scripts/conformance/server-parity/operator-metrics-contract.mjs @@ -103,9 +103,19 @@ function normalized(sections) { return counts; } -function checkStage(name, stage, namespace) { +// Published PHP's health query is invalid SQL on MariaDB once an activity +// exists (server #569). Only that target there may answer 500 from then on. +const MARIADB_HEALTH_DEFECT_STAGES = ['activity_scheduled', 'activity_running', 'activity_retrying', 'completed']; + +export function publishedMariaDbHealthDefect(name, stage, family) { + return family === 'mariadb' && stage.runtime === 'php' && stage.health_status === 500 + && MARIADB_HEALTH_DEFECT_STAGES.includes(name); +} + +function checkStage(name, stage, namespace, family) { const label = `operator metrics stage ${name}`; - assert.deepStrictEqual([stage.metrics_status, stage.metrics_namespace, stage.health_namespace, stage.expected_namespace], + const defect = publishedMariaDbHealthDefect(name, stage, family); + assert.deepStrictEqual([stage.metrics_status, stage.metrics_namespace, defect ? namespace : stage.health_namespace, stage.expected_namespace], [200, namespace, namespace, namespace], `${label} namespace`); assert.deepStrictEqual(stage.metrics_keys, ['namespace', 'operator_metrics'], `${label} document`); for (const [section, pairs] of Object.entries(TIMESTAMPS)) { @@ -135,6 +145,7 @@ function checkStage(name, stage, namespace) { } assert.equal(stage.active_workers, ['empty', 'waiting', 'waiting_after_redispatch'].includes(name) ? 0 : 1, `${label} active workers`); + if (defect) return counts; assert.deepStrictEqual(stage.checks.map(check => [check.name, check.category]), CHECKS, `${label} check names`); const statuses = stage.checks.filter(check => check.name !== UNCOMPARED_CHECK); assert.ok(statuses.every(check => ['ok', 'warning'].includes(check.status)), `${label} no check errors`); @@ -158,6 +169,7 @@ export function checkOperatorMetrics(fixture, observation, workflowId) { const namespace = `${workflowId}-metrics`.toLowerCase(); assert.deepStrictEqual([state.applicable, state.namespace, Object.keys(state.stages)], [true, namespace, STAGES], 'operator metrics stages'); - for (const name of STAGES) checkStage(name, state.stages[name], namespace); + assert.ok([null, 'mariadb'].includes(state.database_family ?? null), 'operator metrics database family'); + for (const name of STAGES) checkStage(name, state.stages[name], namespace, state.database_family ?? null); return {scope: 'operator_metrics', stages: STAGES}; } diff --git a/scripts/conformance/server-parity/operator-metrics-probe.php b/scripts/conformance/server-parity/operator-metrics-probe.php index 7617cfe5f..b0fd674e4 100644 --- a/scripts/conformance/server-parity/operator-metrics-probe.php +++ b/scripts/conformance/server-parity/operator-metrics-probe.php @@ -54,6 +54,8 @@ function finishHttpOperatorMetrics(Client $admin, array $fixture, string $workfl 'worker_fleet' => array_map(static fn (array $entry): array => ['worker_id' => $entry['worker_id'] ?? null, 'namespace' => $entry['namespace'] ?? null, 'queue' => $entry['queue'] ?? null], $document['workers']['fleet'] ?? []), + // Native reports its runtime; published PHP reports none. + 'runtime' => $metrics['body']['operator_metrics']['backend']['runtime'] ?? 'php', 'health_status' => $health['status'], 'health_namespace' => $health['body']['namespace'] ?? null, 'retention' => [$health['body']['retention_mode'] ?? null, $health['body']['retention_days'] ?? null], @@ -93,5 +95,8 @@ function finishHttpOperatorMetrics(Client $admin, array $fixture, string $workfl $stages['completed'] = $stage(); $client->deregisterWorker($worker); - return ['applicable' => true, 'namespace' => $namespace, 'stages' => $stages]; + // The MySQL-family job names its server image (MariaDB or MySQL). + $family = str_contains((string) getenv('MYSQL_IMAGE'), '/mariadb@') ? 'mariadb' : null; + + return ['applicable' => true, 'namespace' => $namespace, 'database_family' => $family, 'stages' => $stages]; } diff --git a/tests/Unit/ServerParityOperatorMetricsContractTest.mjs b/tests/Unit/ServerParityOperatorMetricsContractTest.mjs index 06375c94a..e49a20545 100644 --- a/tests/Unit/ServerParityOperatorMetricsContractTest.mjs +++ b/tests/Unit/ServerParityOperatorMetricsContractTest.mjs @@ -86,6 +86,7 @@ function stage(name) { const warned = new Set(warnings); return {metrics_status: 200, metrics_namespace: namespace, metrics_keys: ['namespace', 'operator_metrics'], sections, active_workers: ['empty', 'waiting', 'waiting_after_redispatch'].includes(name) ? 0 : 1, worker_fleet: [], + runtime: 'php', health_status: 200, health_namespace: namespace, retention: ['bounded', 30], health: {status: warned.size ? 'warning' : 'ok', healthy: true, categories: {}}, checks: NAMES.map(check => ({name: check, status: warned.has(check) ? 'warning' : 'ok', @@ -126,6 +127,32 @@ test('rejects a repaired task still reported overdue', () => { observation.operator_metrics.stages.waiting_after_redispatch.sections.tasks.counts.max_repair_count = 1; assert.throws(() => checkOperatorMetrics(fixture, observation, 'TEST')); }); +// Published PHP on MariaDB answers health with 500 once an activity exists. +const mariaDbHealthDefect = (observation, runtime, stages) => { + observation.operator_metrics.database_family = 'mariadb'; + for (const name of stages) { + Object.assign(observation.operator_metrics.stages[name], {runtime, health_status: 500, health_namespace: null, + retention: [null, null], health: {status: null, healthy: null, categories: null}, checks: []}); + } + return observation; +}; +const ACTIVITY = ['activity_scheduled', 'activity_running', 'activity_retrying', 'completed']; +test('published PHP health failing on MariaDB after an activity passes', () => assert.deepStrictEqual( + checkOperatorMetrics(fixture, mariaDbHealthDefect(model(), 'php', ACTIVITY), 'TEST'), projection)); +test('rejects native health failing on MariaDB', () => assert.throws(() => + checkOperatorMetrics(fixture, mariaDbHealthDefect(model(), 'rust', ACTIVITY), 'TEST'))); +test('rejects PHP health failing on MariaDB before any activity', () => assert.throws(() => + checkOperatorMetrics(fixture, mariaDbHealthDefect(model(), 'php', ['waiting']), 'TEST'))); +test('rejects PHP health failing on another database', () => { + const observation = mariaDbHealthDefect(model(), 'php', ACTIVITY); + observation.operator_metrics.database_family = null; + assert.throws(() => checkOperatorMetrics(fixture, observation, 'TEST')); +}); +test('rejects an unknown database family', () => { + const observation = model(); + observation.operator_metrics.database_family = 'oracle'; + assert.throws(() => checkOperatorMetrics(fixture, observation, 'TEST')); +}); test('embedded explicitly declares the HTTP metrics inapplicable', () => assert.deepStrictEqual(checkOperatorMetrics(fixture, {mode: 'embedded', operator_metrics: {applicable: false, reason: 'embedded_has_no_http_operator_metrics'}}, 'test'), projection)); for (const [label, mutate] of [ From c69dfafb036f74566a8812841aae188f9cabb438 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Wed, 21 Oct 2026 09:44:12 +0000 Subject: [PATCH 4/7] WIP: query responses, admission order and route refusals like PHP Co-Authored-By: Claude Opus 5.5 --- docs/rust-server-port.md | 11 + rust/src/runtime/control_plane.rs | 45 +++- rust/src/runtime/execution.rs | 29 ++- rust/src/runtime/http.rs | 12 +- rust/src/runtime/queries.rs | 268 +++++++++++++++++++----- rust/tests/execution.rs | 2 + rust/tests/execution/query_arguments.rs | 38 +--- rust/tests/execution/query_responses.rs | 103 +++++++++ 8 files changed, 405 insertions(+), 103 deletions(-) create mode 100644 rust/tests/execution/query_responses.rs diff --git a/docs/rust-server-port.md b/docs/rust-server-port.md index fff342416..961497325 100644 --- a/docs/rust-server-port.md +++ b/docs/rust-server-port.md @@ -154,6 +154,17 @@ contract fixture before its correction. `upgrade_source_catalog_unqualified` until its recipe lands. The recipe must also refuse rows that carry trace context until native implements the contract. The boundary is agreed on #325. +- **Queries while a workflow task is open.** Native answers a query from a + history snapshot taken at admission, so it refuses with 409 + `query_snapshot_busy` while the run has a ready or leased workflow task. + Published PHP admits the query. A workflow task leased at admission fixes + the query's history cutoff at that point. The query's claim then waits + while a workflow task is leased, or while an earlier resume task (signal, + update, activity, child or timer) is ready and not yet in history. A + run's first workflow task does not hold the claim. A query sent right after + a signal therefore waits on PHP and is refused on native. The port takes the + snapshot at claim time and must keep query lease replay stable, so it needs + its own fixture. - **Terminating a child run.** Published Workflow resolves the parent with `ChildRunTerminated` and the `terminated` child call status. Native refuses this with `422 termination_propagation_not_available` until the parent-side diff --git a/rust/src/runtime/control_plane.rs b/rust/src/runtime/control_plane.rs index 51a11ea59..06549029c 100644 --- a/rust/src/runtime/control_plane.rs +++ b/rust/src/runtime/control_plane.rs @@ -17,6 +17,7 @@ pub(super) enum Operation<'a> { Debug, Update, Signal(&'a str), + Query(&'a str), } impl Operation<'_> { @@ -39,7 +40,7 @@ impl Operation<'_> { ) } super::RuntimeError::Refused { status, reason } - if matches!(self, Self::Signal(_)) + if matches!(self, Self::Signal(_) | Self::Query(_)) && reason == "historical_run_command_rejected" => { ( @@ -49,7 +50,7 @@ impl Operation<'_> { ) } super::RuntimeError::Protocol { status, response } - if matches!(self, Self::Signal(_)) => + if matches!(self, Self::Signal(_) | Self::Query(_)) => { (status, response) } @@ -101,6 +102,11 @@ impl Operation<'_> { &["workflow_id", "operation_name", "operation_name_field"], &["outcome"], ), + Self::Query(_) => ( + "query", + &["workflow_id", "operation_name", "operation_name_field"], + &["result", "result_envelope"], + ), Self::Signal(_) => ( "signal", &["workflow_id", "operation_name", "operation_name_field"], @@ -192,6 +198,41 @@ impl Operation<'_> { metadata[field] = body[field].clone(); } } + if let Self::Query(name) = self { + metadata["operation_name"] = json!(name); + metadata["operation_name_field"] = json!("query_name"); + metadata["contract"]["rejection_fields"] = json!([ + "workflow_id", + "run_id", + "query_name", + "target_scope", + "result", + "reason", + "message", + "run_status", + "is_terminal", + "blocked_reason", + "validation_errors" + ]); + metadata["contract"]["rejection_reasons"] = json!([ + "instance_not_found", + "historical_run_command_rejected", + "run_not_active", + "query_not_found", + "rejected_unknown_query", + "invalid_query_arguments", + "workflow_definition_unavailable", + "query_rejected", + "query_worker_unavailable", + "query_worker_incompatible", + "query_task_queue_full", + "query_task_queue_unavailable", + "query_task_not_claimed", + "query_worker_execution_timeout", + "query_worker_timeout", + "v1_projection_read_only" + ]); + } if let Self::Signal(name) = self { metadata["operation_name"] = json!(name); metadata["operation_name_field"] = json!("signal_name"); diff --git a/rust/src/runtime/execution.rs b/rust/src/runtime/execution.rs index 6f2fbae47..3146357f1 100644 --- a/rust/src/runtime/execution.rs +++ b/rust/src/runtime/execution.rs @@ -577,6 +577,7 @@ impl Runtime { } } } + delegate!(query_target(workflow_id: &str, selected_run: Option<&str>) -> Result<()>); pub(crate) async fn query_workflow( &self, workflow_id: &str, @@ -584,16 +585,26 @@ impl Runtime { name: &str, body: Value, ) -> Result<(StatusCode, Value)> { - if body.as_object().is_none_or(|m| m.len() > 1) { - return Err(refuse( - StatusCode::UNPROCESSABLE_ENTITY, - "unsupported_request_field", - )); + // `validateOperationName`, `workflowBound`, then `input` => + // nullable|array; other request fields are ignored, as in PHP. + let invalid = |field: &'static str, message: &'static str| { + let mut error = super::laravel_validation::failure(vec![(field, vec![message])]).expect("one message"); + if let super::RuntimeError::Protocol { response, .. } = &mut error { + response["workflow_id"] = json!(workflow_id); + } + error + }; + if name.len() > 256 { + return Err(invalid("query_name", "The query name exceeds the maximum length of 256 bytes.")); } - let arguments = if body.as_object().is_some_and(|m| m.is_empty()) { - super::queries::empty_arguments(self.signal_codec.clone()).await? - } else { - super::envelope(&body, "input")? + self.query_target(workflow_id, run_id).await?; + let input = super::laravel_validation::normalized(body.get("input")).filter(|value| !value.is_null()); + if input.as_ref().is_some_and(|value| !value.is_object() && !value.is_array()) { + return Err(invalid("input", "The input field must be an array.")); + } + let arguments = match input { + None => super::queries::empty_arguments(self.signal_codec.clone()).await?, + Some(_) => super::envelope(&body, "input")?, }; if arguments.len() > super::queries::ARGUMENT_BYTES { return Err(refuse( diff --git a/rust/src/runtime/http.rs b/rust/src/runtime/http.rs index 9f7377e6e..62d3458a2 100644 --- a/rust/src/runtime/http.rs +++ b/rust/src/runtime/http.rs @@ -1317,20 +1317,24 @@ async fn query_current( Path((workflow_id, name)): Path<(String, String)>, Json(body): Json, ) -> Result<(StatusCode, Json)> { - runtime + let (status, body) = runtime .query_workflow(&workflow_id, None, &name, body) .await - .map(|(status, body)| (status, Json(body))) + .map_err(|error| super::control_plane::Operation::Query(&name).refusal(error, &workflow_id, None))?; + Ok((status, Json(super::control_plane::Operation::Query(&name).response(body)))) } async fn query_run( Extension(runtime): Extension, Path((workflow_id, run_id, name)): Path<(String, String, String)>, Json(body): Json, ) -> Result<(StatusCode, Json)> { - runtime + let (status, body) = runtime .query_workflow(&workflow_id, Some(&run_id), &name, body) .await - .map(|(status, body)| (status, Json(body))) + .map_err(|error| { + super::control_plane::Operation::Query(&name).refusal(error, &workflow_id, Some(&run_id)) + })?; + Ok((status, Json(super::control_plane::Operation::Query(&name).response(body)))) } async fn poll_query( Extension(runtime): Extension, diff --git a/rust/src/runtime/queries.rs b/rust/src/runtime/queries.rs index 6c45ecb92..507df7d0a 100644 --- a/rust/src/runtime/queries.rs +++ b/rust/src/runtime/queries.rs @@ -96,6 +96,51 @@ pub(super) async fn decode( .map_err(|_| refuse(StatusCode::SERVICE_UNAVAILABLE, "query_codec_unavailable"))? } +/// One admission attempt: an answer, or no worker to route the query to. +pub(super) enum QueryAttempt { + Answered((StatusCode, Value)), + Unroutable { run_id: String, route: super::query_route::QueryRoute }, +} + +/// `queryFailed`: PHP's refusal body for one query. +fn query_refusal(status: StatusCode, workflow_id: &str, run_id: &str, name: &str, reason: &str, message: &str) -> super::RuntimeError { + super::RuntimeError::Protocol { + status, + response: json!({"workflow_id":workflow_id,"run_id":run_id,"target_scope":"instance","query_name":name, + "result":null,"reason":reason,"message":message}), + } +} + +/// A query refused because no worker can take it. +fn route_refusal(workflow_id: &str, run_id: &str, name: &str, route: &super::query_route::QueryRoute) -> super::RuntimeError { + query_refusal( + StatusCode::CONFLICT, + workflow_id, + run_id, + name, + route.reason.unwrap_or("query_worker_unavailable"), + route.message.as_deref().unwrap_or("No compatible query-capable worker is available for this workflow query."), + ) +} + +/// `terminalRunQueryFailure`: failed, cancelled and terminated runs refuse +/// queries. +fn run_not_active(workflow_id: &str, run_id: &str, name: &str, status: &str) -> super::RuntimeError { + let mut error = query_refusal( + StatusCode::CONFLICT, + workflow_id, + run_id, + name, + "run_not_active", + &format!("Workflow query [{name}] cannot execute because run [{run_id}] is terminal with status [{status}]."), + ); + if let super::RuntimeError::Protocol { response, .. } = &mut error { + response["run_status"] = json!(status); + response["is_terminal"] = json!(true); + } + error +} + impl Store where for<'c> &'c mut DB::Connection: Executor<'c, Database = DB>, @@ -156,6 +201,26 @@ where Ok(()) } + /// `workflowBound` and `withCurrentRunGuard` for a query: the workflow + /// is in this namespace, and a selected run is its current run. + pub(super) async fn query_target(&self, workflow_id: &str, selected_run: Option<&str>) -> Result<()> { + let instance = Self::query("SELECT current_run_id FROM workflow_instances WHERE id=$1 AND namespace=$2") + .bind(workflow_id) + .bind(self.namespace.as_ref()) + .fetch_optional(&self.pool) + .await? + .ok_or_else(|| refuse(StatusCode::NOT_FOUND, "instance_not_found"))?; + if let Some(selected) = selected_run + && DB::optional_string(&instance, "current_run_id")?.as_deref() != Some(selected) + { + return Err(refuse(StatusCode::CONFLICT, "historical_run_command_rejected")); + } + Ok(()) + } + + /// `WorkflowQueryTaskBroker::query`. While no worker can take the query, + /// it waits up to the query timeout for one to register + /// (`awaitRegisteredQueryRoute`), then refuses with the route's reason. pub(super) async fn query_workflow( &self, workflow_id: &str, @@ -165,15 +230,41 @@ where values: Vec, config: &QueryTaskConfig, ) -> Result<(StatusCode, Value)> { - if name.is_empty() || name.len() > 255 { - return Err(refuse( - StatusCode::UNPROCESSABLE_ENTITY, - "invalid_query_name", - )); + let wait_until = + tokio::time::Instant::now() + Duration::from_secs(config.timeout_seconds.max(0) as u64); + loop { + match self + .query_attempt(workflow_id, selected_run, name, &arguments, &values, config) + .await? + { + QueryAttempt::Answered(answer) => return Ok(answer), + QueryAttempt::Unroutable { run_id, route } => { + let now = tokio::time::Instant::now(); + if route.reason == Some("query_worker_unavailable") && now < wait_until { + tokio::time::sleep((wait_until - now).min(Duration::from_millis(250))).await; + continue; + } + return Err(route_refusal(workflow_id, &run_id, name, &route)); + } + } } - let mut tx = self.begin().await?; - let run = Self::query("SELECT r.* FROM workflow_runs r JOIN workflow_instances i ON i.id=r.workflow_instance_id WHERE i.id=$1 AND r.id=COALESCE($2,i.current_run_id) AND r.namespace=$3") - .bind(workflow_id).bind(selected_run).bind(self.namespace.as_ref()).fetch_optional(&mut *tx).await? + } + + async fn query_attempt( + &self, + workflow_id: &str, + selected_run: Option<&str>, + name: &str, + arguments: &str, + values: &[crate::codec::Value], + config: &QueryTaskConfig, + ) -> Result { + // PHP's refusal order: terminal run, query contract, then route. These + // reads run before the transaction so that no request holds one + // connection while waiting for another. + let lookup = "SELECT r.* FROM workflow_runs r JOIN workflow_instances i ON i.id=r.workflow_instance_id WHERE i.id=$1 AND r.id=COALESCE($2,i.current_run_id) AND r.namespace=$3"; + let run = Self::query(lookup) + .bind(workflow_id).bind(selected_run).bind(self.namespace.as_ref()).fetch_optional(&self.pool).await? .ok_or_else(|| refuse(StatusCode::NOT_FOUND, "instance_not_found"))?; if DB::string(&run, "namespace")? != self.namespace.as_ref() || DB::string(&run, "workflow_instance_id")? != workflow_id @@ -187,33 +278,64 @@ where status.as_str(), "pending" | "running" | "waiting" | "completed" ) { - return Err(refuse( - StatusCode::CONFLICT, - "query_unavailable_for_run_status", - )); + return Err(run_not_active(workflow_id, &run_id, name, &status)); + } + let contract = Self::query("SELECT payload FROM workflow_history_events WHERE workflow_run_id=$1 AND event_type='WorkflowStarted' ORDER BY sequence LIMIT 1") + .bind(&run_id).fetch_one(&self.pool).await?; + let contract = DB::document_row(&contract, "payload")?; + // `validateContractedQueryArguments`: a run that declares no queries + // leaves the name and arguments to the worker. + let declared = |field: &str| contract[field].as_array().is_some_and(|items| !items.is_empty()); + if declared("declared_queries") || declared("declared_query_contracts") { + if !contract["declared_queries"] + .as_array() + .is_some_and(|names| names.iter().any(|v| v == name)) + { + return Err(query_refusal(StatusCode::NOT_FOUND, workflow_id, &run_id, name, "rejected_unknown_query", + &format!("Workflow query [{name}] is not declared on workflow [{workflow_id}]."))); + } + // A declared query without a parameter contract takes its + // arguments as given. + if contract["declared_query_contracts"] + .as_array() + .is_some_and(|contracts| contracts.iter().any(|entry| entry["name"] == name)) + { + validate_arguments( + &contract, + "declared_query_contracts", + name, + values, + "unsupported_query_contract", + "invalid_query_arguments", + )?; + } + } + let route = self.query_route(&run, config.worker_stale_after_seconds).await?; + if !route.servable { + return Ok(QueryAttempt::Unroutable { run_id, route }); + } + let mut tx = self.begin().await?; + // The run read again under the transaction is the one admitted. + let run = Self::query(lookup) + .bind(workflow_id).bind(selected_run).bind(self.namespace.as_ref()).fetch_optional(&mut *tx).await? + .filter(|row| DB::string(row, "id").ok().as_deref() == Some(run_id.as_str())) + .ok_or_else(|| refuse(StatusCode::NOT_FOUND, "instance_not_found"))?; + let status = DB::string(&run, "status")?; + if !matches!( + status.as_str(), + "pending" | "running" | "waiting" | "completed" + ) { + return Err(run_not_active(workflow_id, &run_id, name, &status)); } let busy = Self::scalar("SELECT COUNT(*) FROM workflow_tasks WHERE workflow_run_id=$1 AND task_type='workflow' AND status IN ('ready','leased')") .bind(&run_id).fetch_one(&mut *tx).await?; + // Native answers from a snapshot taken here, so it refuses while a + // workflow task is open. PHP admits the query and holds its claim + // instead; see the port log's known gaps. if busy != 0 { - return Err(refuse(StatusCode::CONFLICT, "query_snapshot_busy")); + return Err(query_refusal(StatusCode::CONFLICT, workflow_id, &run_id, name, "query_snapshot_busy", + &format!("Workflow query [{name}] cannot run while run [{run_id}] has an open workflow task. Retry after the task completes."))); } - let contract = Self::query("SELECT payload FROM workflow_history_events WHERE workflow_run_id=$1 AND event_type='WorkflowStarted' ORDER BY sequence LIMIT 1") - .bind(&run_id).fetch_one(&mut *tx).await?; - let contract = DB::document_row(&contract, "payload")?; - if !contract["declared_queries"] - .as_array() - .is_some_and(|names| names.iter().any(|v| v == name)) - { - return Err(refuse(StatusCode::NOT_FOUND, "rejected_unknown_query")); - } - validate_arguments( - &contract, - "declared_query_contracts", - name, - &values, - "unsupported_query_contract", - "invalid_query_arguments", - )?; let entries = self.query_entries(&mut tx).await?; if entries.len() >= MAX_ENTRIES || entries @@ -264,32 +386,30 @@ where self.wake.notify_waiters(); // Never hold the transition lock or a pooled connection across the wait. // Periodic probes observe results committed by another server process. - let timed_out = |claimed| super::RuntimeError::Protocol { - status: StatusCode::GATEWAY_TIMEOUT, - response: json!({"workflow_id":workflow_id,"run_id":run_id,"query_name":name, - "reason":if claimed { "query_worker_execution_timeout" } else { "query_task_not_claimed" }, - "message":"Timed out waiting for the original query result."}), - }; - let mut claimed = false; + let mut last = json!({"status":"pending"}); loop { let wake = self.wake.notified(); let remaining = (deadline - authority::now()).to_std().unwrap_or_default(); let read = Self::query(DB::QUERY_CACHE_GET) .bind(&key) .fetch_optional(&self.pool); - let row = tokio::time::timeout(remaining, read) - .await - .map_err(|_| timed_out(claimed))?? - .ok_or_else(|| refuse(StatusCode::CONFLICT, "query_task_timed_out"))?; + let row = match tokio::time::timeout(remaining, read).await { + Err(_) => { + return Err(self.query_timeout(&run, workflow_id, &run_id, name, &last, config).await?); + } + Ok(row) => row?, + }; + let Some(row) = row else { + return Err(refuse(StatusCode::CONFLICT, "query_task_timed_out")); + }; let task: Value = serde_json::from_str(&DB::string(&row, "value")?)?; - claimed = task["query_task_attempt"].as_i64().unwrap_or(0) > 0; match task["status"].as_str() { Some("completed") => { - return Ok(( + return Ok(QueryAttempt::Answered(( StatusCode::OK, - json!({"workflow_id":workflow_id,"run_id":run_id, - "query_name":name,"result":null,"result_envelope":task["result_envelope"],"reason":null}), - )); + json!({"workflow_id":workflow_id,"run_id":run_id,"target_scope":"instance", + "query_name":name,"result":null,"result_envelope":task["result_envelope"],"reason":null}), + ))); } Some("failed") => { let status = match task["failure"]["reason"].as_str() { @@ -300,18 +420,23 @@ where let mut body = task["failure"].clone(); body["workflow_id"] = json!(workflow_id); body["run_id"] = json!(run_id); + body["target_scope"] = json!("instance"); body["query_name"] = json!(name); - return Ok((status, body)); + body["result"] = Value::Null; + if body["message"].as_str().is_none_or(str::is_empty) { + body["message"] = json!("Query failed on the worker."); + } + return Ok(QueryAttempt::Answered((status, body))); } Some("timed_out") => { - return Err(timed_out(claimed)); + return Err(self.query_timeout(&run, workflow_id, &run_id, name, &last, config).await?); } _ if authority::deadline(&task) .is_some_and(|deadline| authority::now() >= deadline) => { // The stored deadline already fences every new outcome. // Cleanup must not delay the caller behind a busy writer. - return Err(timed_out(claimed)); + return Err(self.query_timeout(&run, workflow_id, &run_id, name, &task, config).await?); } _ if authority::deadline(&task).is_none() => { let mut tx = self.begin().await?; @@ -333,10 +458,44 @@ where } _ => {} } + last = task; tokio::select! { _ = wake => {}, _ = tokio::time::sleep(Duration::from_millis(50)) => {} } } } + /// `timeoutFailure`: why a query got no answer before its deadline. + async fn query_timeout( + &self, + run: &DB::Row, + workflow_id: &str, + run_id: &str, + name: &str, + task: &Value, + config: &QueryTaskConfig, + ) -> Result { + let status = task["status"].as_str().unwrap_or("unknown"); + let (reason, message) = match status { + "pending" => { + let route = self.query_route(run, config.worker_stale_after_seconds).await?; + if !route.servable { + return Ok(route_refusal(workflow_id, run_id, name, &route)); + } + ("query_task_not_claimed", format!( + "Timed out waiting for a compatible worker to claim workflow query [{name}] on task queue [{}].", + route.task_queue + )) + } + "leased" => ("query_worker_execution_timeout", match task["lease_owner"].as_str().filter(|owner| !owner.is_empty()) { + Some(owner) => format!("Timed out waiting for worker [{owner}] to complete workflow query [{name}]."), + None => format!("Timed out waiting for the worker that leased workflow query [{name}] to complete it."), + }), + other => ("query_worker_timeout", format!( + "Timed out waiting for workflow query [{name}] to complete; last query task status was [{other}]." + )), + }; + Ok(query_refusal(StatusCode::GATEWAY_TIMEOUT, workflow_id, run_id, name, reason, &message)) + } + pub(super) async fn poll_query(&self, body: Value, config: &QueryTaskConfig) -> Result { let owner = text(&body, "worker_id")?; text(&body, "task_queue")?; @@ -546,7 +705,6 @@ where return Err(refuse(StatusCode::CONFLICT, "task_queue_mismatch")); } let types = DB::document_row(&worker, "supported_workflow_types")?; - let contracts = DB::document_row(&worker, "workflow_command_contracts")?; let capabilities = DB::document_row(&worker, "capabilities")?; if !capabilities .as_array() @@ -565,13 +723,9 @@ where { continue; } - let Some(workflow_type) = task["workflow_type"].as_str() else { - continue; - }; - if !contracts[workflow_type]["queries"] - .as_array() - .is_some_and(|names| names.contains(&task["query_name"])) - { + // `queryTaskMatchesPoller` matches the workflow type, not the + // query name: a worker answers a query it does not declare. + if task["workflow_type"].as_str().is_none() { continue; } if task["status"] == "leased" diff --git a/rust/tests/execution.rs b/rust/tests/execution.rs index 221e3cc8b..22d26be8a 100644 --- a/rust/tests/execution.rs +++ b/rust/tests/execution.rs @@ -79,6 +79,8 @@ mod activity_deadlines; mod lock_free_waiting; #[path = "execution/read_snapshots.rs"] mod read_snapshots; +#[path = "execution/query_responses.rs"] +mod query_responses; #[path = "execution/describe_actions.rs"] mod describe_actions; #[path = "execution/waterline_reads.rs"] diff --git a/rust/tests/execution/query_arguments.rs b/rust/tests/execution/query_arguments.rs index 88779c68b..a768a65fd 100644 --- a/rust/tests/execution/query_arguments.rs +++ b/rust/tests/execution/query_arguments.rs @@ -77,36 +77,6 @@ async fn omitted_query_input_uses_official_empty_arguments_without_mutating_the_ assert_eq!(query_entries(&database).await, 0); for (path, input, status, reason) in [ - ( - instance_query.clone(), - Value::Null, - StatusCode::UNPROCESSABLE_ENTITY, - "unsupported_request_field", - ), - ( - instance_query.clone(), - json!([]), - StatusCode::UNPROCESSABLE_ENTITY, - "unsupported_request_field", - ), - ( - instance_query.clone(), - json!({"foreign":true}), - StatusCode::UNPROCESSABLE_ENTITY, - "unsupported_payload_codec", - ), - ( - instance_query.clone(), - json!({"input":empty.clone(),"foreign":true}), - StatusCode::UNPROCESSABLE_ENTITY, - "unsupported_request_field", - ), - ( - instance_query.clone(), - json!({"input":null}), - StatusCode::UNPROCESSABLE_ENTITY, - "unsupported_payload_codec", - ), ( instance_query.clone(), json!({"input":{"codec":"json","blob":"[]"}}), @@ -192,9 +162,15 @@ async fn omitted_query_input_uses_official_empty_arguments_without_mutating_the_ .collect(), )); let mut task_ids = vec![]; + // As in PHP, `input` is nullable and other request fields are ignored. for (index, (path, input)) in [ - (instance_query, json!({})), + (instance_query.clone(), json!({})), (run_query, json!({"input":empty.clone()})), + (instance_query.clone(), Value::Null), + (instance_query.clone(), json!([])), + (instance_query.clone(), json!({"foreign":true})), + (instance_query.clone(), json!({"input":empty.clone(),"foreign":true})), + (instance_query, json!({"input":null})), ] .into_iter() .enumerate() diff --git a/rust/tests/execution/query_responses.rs b/rust/tests/execution/query_responses.rs new file mode 100644 index 000000000..2e29f0bb7 --- /dev/null +++ b/rust/tests/execution/query_responses.rs @@ -0,0 +1,103 @@ +use super::*; + +fn quick_queries(runtime: Runtime) -> Runtime { + runtime.with_query_task_config(durable_workflow_server::runtime::QueryTaskConfig { + timeout_seconds: 1, + ..Default::default() + }) +} + +async fn query(app: &Router, path: &str, body: Value) -> (StatusCode, Value) { + request(app, "POST", path, body).await +} + +fn assert_refusal(response: &(StatusCode, Value), status: StatusCode, reason: &str, message: &str) { + assert_eq!(response.0, status, "{}", response.1); + assert_eq!(response.1["reason"], reason, "{}", response.1); + assert_eq!(response.1["message"], message, "{}", response.1); + assert_eq!(response.1["control_plane"]["operation"], "query"); + assert_eq!(response.1["control_plane"]["operation_name_field"], "query_name"); + assert_eq!(response.1["control_plane"]["reason"], reason); +} + +// Published PHP refuses a query no worker can take with the first failing +// route check, after waiting up to the query timeout for a worker to +// register when none is available. +#[tokio::test] +async fn unroutable_queries_wait_then_refuse_like_php() { + let database = TestDatabase::new().await; + let app = router(quick_queries(database.open().await.unwrap())); + start(&app, "unrouted").await; + let started = std::time::Instant::now(); + let refused = query(&app, "/api/workflows/unrouted/query/state", json!({})).await; + assert!(started.elapsed() >= std::time::Duration::from_millis(900), "{:?}", started.elapsed()); + assert_refusal(&refused, StatusCode::CONFLICT, "query_worker_unavailable", + "No active worker is registered on task queue [test]."); + assert_eq!(refused.1["target_scope"], "instance"); + assert_eq!(refused.1["query_name"], "state"); + assert!(refused.1["result"].is_null() && refused.1.get("result").is_some()); + register(&app, "plain", json!(["echo"]), json!([])).await; + assert_refusal(&query(&app, "/api/workflows/unrouted/query/state", json!({})).await, StatusCode::CONFLICT, + "query_worker_unavailable", + "Active workers are registered on task queue [test], but none are accepting workflow query tasks."); + let (status, body) = request(&app, "POST", "/api/worker/register", json!({"worker_id":"other","task_queue":"test", + "runtime":"php","supported_workflow_types":["other"],"supported_activity_types":[],"capabilities":["query_tasks"]})).await; + assert_eq!(status, StatusCode::CREATED, "{body}"); + // An incompatible worker refuses at once: only an unavailable route waits. + let started = std::time::Instant::now(); + assert_refusal(&query(&app, "/api/workflows/unrouted/query/state", json!({})).await, StatusCode::CONFLICT, + "query_worker_incompatible", + "Query-capable workers on task queue [test] do not advertise workflow type [echo]."); + assert!(started.elapsed() < std::time::Duration::from_millis(900), "{:?}", started.elapsed()); +} + +// Admission checks run in PHP's order with PHP's bodies, and a run that +// declares no queries leaves the query name to the worker. +#[tokio::test] +async fn query_admission_follows_php() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + let missing = query(&app, "/api/workflows/missing/query/state", json!({})).await; + assert_refusal(&missing, StatusCode::NOT_FOUND, "instance_not_found", "Workflow not found."); + // Started before any worker registers, so the run records no contract. + let started = start(&app, "contractless").await; + query_worker(&app, "answering").await; + let task = poll(&app, "answering", "workflow").await; + let (status, body) = finish_task(&app, &task, json!([{"type":"complete_workflow","result":envelope(Payload::Long(1))}])).await; + assert_eq!(status, StatusCode::OK, "{body}"); + let historical = query(&app, "/api/workflows/contractless/runs/not-current/query/state", json!({})).await; + assert_eq!(historical.0, StatusCode::CONFLICT, "{}", historical.1); + assert_eq!(historical.1["reason"], "historical_run_command_rejected"); + assert_eq!(historical.1["control_plane"]["operation"], "query"); + let invalid = query(&app, "/api/workflows/contractless/query/state", json!({"input":"x"})).await; + assert_refusal(&invalid, StatusCode::UNPROCESSABLE_ENTITY, "validation_failed", "The input field must be an array."); + assert_eq!(invalid.1["validation_errors"], json!({"input":["The input field must be an array."]})); + let waiting = { + let app = app.clone(); + tokio::spawn(async move { query(&app, "/api/workflows/contractless/query/undeclared", json!({})).await }) + }; + let claim = await_query(&app, "answering").await; + assert_eq!(claim["run_id"], started["run_id"]); + assert_eq!(claim["query_name"], "undeclared"); + let result = envelope(Payload::Long(7)); + let receipt = request(&app, "POST", + &format!("/api/worker/query-tasks/{}/complete", claim["query_task_id"].as_str().unwrap()), + json!({"lease_owner":claim["lease_owner"],"query_task_attempt":claim["query_task_attempt"],"result_envelope":result})).await; + assert_eq!(receipt.0, StatusCode::OK, "{}", receipt.1); + let answer = waiting.await.unwrap(); + assert_eq!(answer.0, StatusCode::OK, "{}", answer.1); + assert_eq!(answer.1["target_scope"], "instance"); + assert_eq!(answer.1["result_envelope"], result); + assert_eq!(answer.1["control_plane"]["operation"], "query"); + assert_eq!(answer.1["control_plane"]["result_envelope"], result); + // Terminal runs other than completed ones refuse queries. + start(&app, "ended").await; + let (status, body) = request(&app, "POST", "/api/workflows/ended/terminate", json!({})).await; + assert_eq!(status, StatusCode::OK, "{body}"); + let ended = query(&app, "/api/workflows/ended/query/state", json!({})).await; + let run = ended.1["run_id"].as_str().unwrap().to_owned(); + assert_refusal(&ended, StatusCode::CONFLICT, "run_not_active", + &format!("Workflow query [state] cannot execute because run [{run}] is terminal with status [terminated].")); + assert_eq!(ended.1["run_status"], "terminated"); + assert_eq!(ended.1["is_terminal"], true); +} From 5d31b0bdb7f0732a34961033e588a4f11d3d7224 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Wed, 21 Oct 2026 09:59:29 +0000 Subject: [PATCH 5/7] Record the query responses checkpoint Co-Authored-By: Claude Opus 5.5 --- docs/rust-server-port.md | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/docs/rust-server-port.md b/docs/rust-server-port.md index 961497325..49b89e49c 100644 --- a/docs/rust-server-port.md +++ b/docs/rust-server-port.md @@ -238,6 +238,44 @@ support is not evidence of multi-node safety. Newest checkpoints first. Each entry is exact to the source, tree and runs it names, including unsuccessful observations. +October 21, query responses and routing: + +- **Found by.** The Waterline comparison and direct probes of published PHP + 2.5.14. Native answered every query on a run with an open workflow task + with `query_snapshot_busy`. It refused queries a run did not declare even + when the run declared none, and it refused unknown request fields and a + null `input`. Its responses lacked PHP's `control_plane` envelope and + messages. +- **Admission order.** As in PHP: a workflow outside the namespace is 404 + `Workflow not found.`. A selected run that is not current is 409 + `historical_run_command_rejected`. A non-array `input` fails Laravel's + rule with 422. A failed, cancelled or terminated run is 409 + `run_not_active` with `run_status` and `is_terminal`. Other request fields + are ignored, and a null `input` means no arguments. +- **Contracts.** A run that declares no queries leaves the name and arguments + to the worker. A declared query without a parameter contract takes its + arguments as given. Worker claims match the workflow type, not the query + name, so a worker answers queries it does not declare, as in PHP. +- **Routing.** Before admitting a query, native checks PHP's query route. A + missing route is refused with PHP's reason and message. While no worker is + available at all, the request waits up to the query timeout for one to + register, as `awaitRegisteredQueryRoute` does. +- **Timeouts.** A query nobody claimed re-checks the route. It is refused if + the route is gone, otherwise it is 504 `query_task_not_claimed`. A claimed + query that times out is 504 `query_worker_execution_timeout`, naming the + worker. Both carry PHP's messages. +- **Responses.** Answers and refusals carry `target_scope`, `query_name`, + `result` and PHP's `query` envelope, with its rejection fields and reasons. +- **Still different.** Native still refuses a query while a workflow task is + open (see the known gaps), now with PHP's body shape and an explanatory + message. +- **Evidence.** Probes of the route refusals, the terminal run, the missing + workflow, the historical run, invalid input and an unclaimed query match + PHP's status, reason, message and envelope. The open-task case is the only + difference. `state-queries`, `state-updates`, `signals-before-worker` and + `echo` pass on published PHP, embedded and native. New `query_responses` + tests and the existing query tests pass on SQLite, PostgreSQL and MySQL. + October 21, describe actions and run list: - **Found by.** The Waterline comparison. Waterline 2.12.0 disables cancel, From 4115e5d180e5410e57926a68bc7c58410b2738a7 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Wed, 21 Oct 2026 11:07:40 +0000 Subject: [PATCH 6/7] List control-plane rejection contracts in one table; check query contracts outside the store Sonar read the per-operation rejection lists as duplicates and the query attempt as too complex. The lists are unchanged. Co-Authored-By: Claude Opus 5.5 --- rust/src/runtime/control_plane.rs | 255 +++++++++++------------------- rust/src/runtime/queries.rs | 78 ++++----- 2 files changed, 136 insertions(+), 197 deletions(-) diff --git a/rust/src/runtime/control_plane.rs b/rust/src/runtime/control_plane.rs index 06549029c..e1b43c1fc 100644 --- a/rust/src/runtime/control_plane.rs +++ b/rust/src/runtime/control_plane.rs @@ -118,171 +118,51 @@ impl Operation<'_> { "contract":{"schema":"durable-workflow.v2.control-plane-response.contract", "version":1, "legacy_field_policy":"reject_non_canonical", "legacy_fields":{"query":"query_name","signal":"signal_name","update":"update_name","wait_policy":"wait_for"}, "required_fields":required, "success_fields":success}}); - if matches!(self, Self::Start) { - metadata["contract"]["rejection_fields"] = json!([ - "workflow_id", - "command_status", - "command_source", - "outcome", - "reason", - "rejection_reason", - "message" - ]); - metadata["contract"]["rejection_reasons"] = json!([ - "workflow_id_reserved_in_namespace", - "task_queue_draining", - "compatibility_blocked" - ]); + if let Some((fields, reasons)) = self.rejection_contract() { + metadata["contract"]["rejection_fields"] = words(fields); + metadata["contract"]["rejection_reasons"] = words(reasons); } - if matches!(self, Self::DescribeRun | Self::History) { - metadata["contract"]["rejection_fields"] = json!([ - "workflow_id", - "run_id", - "reason", - "message", - "retryable", - "error_id", - "exception" - ]); - metadata["contract"]["rejection_reasons"] = json!(["control_plane_internal_error"]); - } - if matches!(self, Self::Cancel | Self::Terminate | Self::Repair) { - metadata["contract"]["rejection_fields"] = - json!(["workflow_id", "run_id", "reason", "message", "remediation"]); - metadata["contract"]["rejection_reasons"] = json!(["v1_projection_read_only"]); - } - if matches!(self, Self::Update) { - metadata["operation_name"] = body["update_name"].clone(); - metadata["operation_name_field"] = json!("update_name"); - metadata["contract"]["rejection_fields"] = json!([ - "workflow_id", - "run_id", - "update_id", - "update_name", - "update_status", - "outcome", - "reason", - "rejection_reason", - "validation_errors", - "retryable", - "message", - "remediation" - ]); - metadata["contract"]["rejection_reasons"] = json!([ - "update_validator_rejected", - "update_validator_contract_missing", - "update_validation_request_id_required", - "update_validation_idempotency_conflict", - "update_validation_capability_unsupported", - "update_validator_worker_unavailable", - "update_validator_worker_incompatible", - "update_validator_worker_lost", - "update_validation_task_not_claimed", - "update_validation_execution_timeout", - "duplicate_update_validation_completion", - "stale_update_validation_completion", - "invalid_update_arguments", - "v1_projection_read_only" - ]); - for field in [ - "command_id", - "target_scope", - "result", - "result_envelope", - "update_id", - "update_status", - "wait_for", - "wait_timed_out", - "wait_timeout_seconds", - ] { - metadata[field] = body[field].clone(); + match self { + Self::Update => { + metadata["operation_name"] = body["update_name"].clone(); + metadata["operation_name_field"] = json!("update_name"); + for field in [ + "command_id", + "target_scope", + "result", + "result_envelope", + "update_id", + "update_status", + "wait_for", + "wait_timed_out", + "wait_timeout_seconds", + ] { + metadata[field] = body[field].clone(); + } } - } - if let Self::Query(name) = self { - metadata["operation_name"] = json!(name); - metadata["operation_name_field"] = json!("query_name"); - metadata["contract"]["rejection_fields"] = json!([ - "workflow_id", - "run_id", - "query_name", - "target_scope", - "result", - "reason", - "message", - "run_status", - "is_terminal", - "blocked_reason", - "validation_errors" - ]); - metadata["contract"]["rejection_reasons"] = json!([ - "instance_not_found", - "historical_run_command_rejected", - "run_not_active", - "query_not_found", - "rejected_unknown_query", - "invalid_query_arguments", - "workflow_definition_unavailable", - "query_rejected", - "query_worker_unavailable", - "query_worker_incompatible", - "query_task_queue_full", - "query_task_queue_unavailable", - "query_task_not_claimed", - "query_worker_execution_timeout", - "query_worker_timeout", - "v1_projection_read_only" - ]); - } - if let Self::Signal(name) = self { - metadata["operation_name"] = json!(name); - metadata["operation_name_field"] = json!("signal_name"); - metadata["contract"]["rejection_fields"] = json!([ - "workflow_id", - "run_id", - "signal_name", - "command_status", - "command_source", - "target_scope", - "outcome", - "reason", - "rejection_reason", - "rejection_category", - "message", - "command_contract_source", - "command_contract_backfill_needed", - "command_contract_backfill_available", - "declared_signals", - "signal_admission", - "retryable", - "error_id" - ]); - metadata["contract"]["rejection_reasons"] = json!([ - "instance_not_found", - "historical_run_command_rejected", - "unknown_signal", - "invalid_signal_arguments", - "run_not_active", - "instance_not_started", - "selected_run_not_current", - "configured_workflow_type_invalid", - "v1_projection_read_only", - "backend_lock_pressure", - "control_plane_internal_error" - ]); - // PHP's ControlPlaneResponseContract::PROJECTED_FIELDS for a signal. - for field in [ - "command_id", - "target_scope", - "command_contract_source", - "command_contract_backfill_needed", - "command_contract_backfill_available", - "declared_signals", - "signal_admission", - ] { - if let Some(value) = body.get(field) { - metadata[field] = value.clone(); + Self::Query(name) => { + metadata["operation_name"] = json!(name); + metadata["operation_name_field"] = json!("query_name"); + } + Self::Signal(name) => { + metadata["operation_name"] = json!(name); + metadata["operation_name_field"] = json!("signal_name"); + // PHP's ControlPlaneResponseContract::PROJECTED_FIELDS for a signal. + for field in [ + "command_id", + "target_scope", + "command_contract_source", + "command_contract_backfill_needed", + "command_contract_backfill_available", + "declared_signals", + "signal_admission", + ] { + if let Some(value) = body.get(field) { + metadata[field] = value.clone(); + } } } + _ => {} } // ControlPlaneResponseContract::PROJECTED_FIELDS: each present field. for field in [ @@ -346,6 +226,59 @@ impl Operation<'_> { body["control_plane"] = metadata; body } + /// `ControlPlaneResponseContract`'s rejection fields and reasons, each a + /// space-separated list. + fn rejection_contract(&self) -> Option<(&'static str, &'static str)> { + Some(match self { + Self::Start => ( + "workflow_id command_status command_source outcome reason rejection_reason message", + "workflow_id_reserved_in_namespace task_queue_draining compatibility_blocked", + ), + Self::DescribeRun | Self::History => ( + "workflow_id run_id reason message retryable error_id exception", + "control_plane_internal_error", + ), + Self::Cancel | Self::Terminate | Self::Repair => { + ("workflow_id run_id reason message remediation", "v1_projection_read_only") + } + Self::Update => ( + "workflow_id run_id update_id update_name update_status outcome reason \ + rejection_reason validation_errors retryable message remediation", + "update_validator_rejected update_validator_contract_missing \ + update_validation_request_id_required update_validation_idempotency_conflict \ + update_validation_capability_unsupported update_validator_worker_unavailable \ + update_validator_worker_incompatible update_validator_worker_lost \ + update_validation_task_not_claimed update_validation_execution_timeout \ + duplicate_update_validation_completion stale_update_validation_completion \ + invalid_update_arguments v1_projection_read_only", + ), + Self::Query(_) => ( + "workflow_id run_id query_name target_scope result reason message run_status \ + is_terminal blocked_reason validation_errors", + "instance_not_found historical_run_command_rejected run_not_active query_not_found \ + rejected_unknown_query invalid_query_arguments workflow_definition_unavailable \ + query_rejected query_worker_unavailable query_worker_incompatible \ + query_task_queue_full query_task_queue_unavailable query_task_not_claimed \ + query_worker_execution_timeout query_worker_timeout v1_projection_read_only", + ), + Self::Signal(_) => ( + "workflow_id run_id signal_name command_status command_source target_scope outcome \ + reason rejection_reason rejection_category message command_contract_source \ + command_contract_backfill_needed command_contract_backfill_available \ + declared_signals signal_admission retryable error_id", + "instance_not_found historical_run_command_rejected unknown_signal \ + invalid_signal_arguments run_not_active instance_not_started \ + selected_run_not_current configured_workflow_type_invalid v1_projection_read_only \ + backend_lock_pressure control_plane_internal_error", + ), + Self::List | Self::ListRuns | Self::Describe | Self::Observe | Self::Debug => return None, + }) + } +} + +/// A space-separated list as a JSON array of its words. +fn words(list: &str) -> Value { + list.split_whitespace().collect::>().into() } pub(super) async fn admission_response(request: Request, mut body: Value) -> Value { diff --git a/rust/src/runtime/queries.rs b/rust/src/runtime/queries.rs index 507df7d0a..efd293a26 100644 --- a/rust/src/runtime/queries.rs +++ b/rust/src/runtime/queries.rs @@ -141,6 +141,45 @@ fn run_not_active(workflow_id: &str, run_id: &str, name: &str, status: &str) -> error } +/// The run statuses PHP still answers queries for. +fn admits_queries(status: &str) -> bool { + matches!(status, "pending" | "running" | "waiting" | "completed") +} + +/// `validateContractedQueryArguments`: a run that declares no queries leaves +/// the name and arguments to the worker. A declared query without a +/// parameter contract takes its arguments as given. +fn check_query_contract( + contract: &Value, + workflow_id: &str, + run_id: &str, + name: &str, + values: &[crate::codec::Value], +) -> Result<()> { + let declared = |field: &str| contract[field].as_array().is_some_and(|items| !items.is_empty()); + if !declared("declared_queries") && !declared("declared_query_contracts") { + return Ok(()); + } + if !contract["declared_queries"].as_array().is_some_and(|names| names.iter().any(|v| v == name)) { + return Err(query_refusal(StatusCode::NOT_FOUND, workflow_id, run_id, name, "rejected_unknown_query", + &format!("Workflow query [{name}] is not declared on workflow [{workflow_id}]."))); + } + if contract["declared_query_contracts"] + .as_array() + .is_some_and(|contracts| contracts.iter().any(|entry| entry["name"] == name)) + { + validate_arguments( + contract, + "declared_query_contracts", + name, + values, + "unsupported_query_contract", + "invalid_query_arguments", + )?; + } + Ok(()) +} + impl Store where for<'c> &'c mut DB::Connection: Executor<'c, Database = DB>, @@ -274,42 +313,12 @@ where } let run_id = DB::string(&run, "id")?; let status = DB::string(&run, "status")?; - if !matches!( - status.as_str(), - "pending" | "running" | "waiting" | "completed" - ) { + if !admits_queries(&status) { return Err(run_not_active(workflow_id, &run_id, name, &status)); } let contract = Self::query("SELECT payload FROM workflow_history_events WHERE workflow_run_id=$1 AND event_type='WorkflowStarted' ORDER BY sequence LIMIT 1") .bind(&run_id).fetch_one(&self.pool).await?; - let contract = DB::document_row(&contract, "payload")?; - // `validateContractedQueryArguments`: a run that declares no queries - // leaves the name and arguments to the worker. - let declared = |field: &str| contract[field].as_array().is_some_and(|items| !items.is_empty()); - if declared("declared_queries") || declared("declared_query_contracts") { - if !contract["declared_queries"] - .as_array() - .is_some_and(|names| names.iter().any(|v| v == name)) - { - return Err(query_refusal(StatusCode::NOT_FOUND, workflow_id, &run_id, name, "rejected_unknown_query", - &format!("Workflow query [{name}] is not declared on workflow [{workflow_id}]."))); - } - // A declared query without a parameter contract takes its - // arguments as given. - if contract["declared_query_contracts"] - .as_array() - .is_some_and(|contracts| contracts.iter().any(|entry| entry["name"] == name)) - { - validate_arguments( - &contract, - "declared_query_contracts", - name, - values, - "unsupported_query_contract", - "invalid_query_arguments", - )?; - } - } + check_query_contract(&DB::document_row(&contract, "payload")?, workflow_id, &run_id, name, values)?; let route = self.query_route(&run, config.worker_stale_after_seconds).await?; if !route.servable { return Ok(QueryAttempt::Unroutable { run_id, route }); @@ -321,10 +330,7 @@ where .filter(|row| DB::string(row, "id").ok().as_deref() == Some(run_id.as_str())) .ok_or_else(|| refuse(StatusCode::NOT_FOUND, "instance_not_found"))?; let status = DB::string(&run, "status")?; - if !matches!( - status.as_str(), - "pending" | "running" | "waiting" | "completed" - ) { + if !admits_queries(&status) { return Err(run_not_active(workflow_id, &run_id, name, &status)); } let busy = Self::scalar("SELECT COUNT(*) FROM workflow_tasks WHERE workflow_run_id=$1 AND task_type='workflow' AND status IN ('ready','leased')") From 7562f43e6f0bdcae5d179415cb37c33d7d968b67 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Wed, 21 Oct 2026 11:08:04 +0000 Subject: [PATCH 7/7] Serve the operator dashboard like PHP so Waterline's landing page loads Native answers /api/system/operator-dashboard and /bounded from the run summaries by OperatorDashboardSummary's rules. The workflow-type variant validates its list, then answers PHP's 501 for an observer that cannot filter by type. Co-Authored-By: Claude Opus 5.5 --- docs/rust-server-port.md | 34 ++- rust/src/runtime/execution.rs | 1 + rust/src/runtime/http.rs | 31 +++ rust/src/runtime/mod.rs | 1 + rust/src/runtime/operator_dashboard.rs | 57 +++++ rust/src/runtime/operator_metrics.rs | 280 +++++++++++++++++++++ rust/src/runtime/run_diagnostics.rs | 2 +- rust/tests/execution.rs | 2 + rust/tests/execution/operator_dashboard.rs | 75 ++++++ 9 files changed, 480 insertions(+), 3 deletions(-) create mode 100644 rust/src/runtime/operator_dashboard.rs create mode 100644 rust/tests/execution/operator_dashboard.rs diff --git a/docs/rust-server-port.md b/docs/rust-server-port.md index 70e6bc664..9be5ec0c6 100644 --- a/docs/rust-server-port.md +++ b/docs/rust-server-port.md @@ -59,8 +59,8 @@ CLI and Waterline use the first group. - **Used by the CLI or Waterline.** - Task queue build IDs. - Namespace update, delete and external storage. - - `system` repair, retention and activity-timeout passes, metrics and - operator dashboards. + - `system` repair, retention and activity-timeout passes, metrics, and + the operator dashboard filtered by workflow type. - Activities (standalone). - **Worker protocol features.** - Worker sessions. @@ -227,6 +227,36 @@ support is not evidence of multi-node safety. Newest checkpoints first. Each entry is exact to the source, tree and runs it names, including unsuccessful observations. +October 21, operator dashboard: + +- **Found by.** Waterline 2.12.0's landing page in front of native: its + `/waterline/api/stats` request reached native's + `/api/system/operator-dashboard/bounded`, which answered 501. +- **Source.** Native serves admin-only `GET /api/system/operator-dashboard` + and `/bounded`, in the default or a named namespace, by the rules of + Workflow 2.5.5's `OperatorDashboardSummary` over the run summaries: + - the flow counts and rate, current running and failed runs, the hour's + completed and failed runs, the longest open wait and the longest + duration as summary rows, with the model's casts and appends; + - the week's ten busiest workflow types with pass rate, error count, + status breakdown and the upper-middle duration; + - PHP's attention alerts over workers and run summaries; + - hourly completed and failed counts for the last 169 hours; + - the operator metrics snapshot. The bounded variant marks the history + audits as not requested and reports capacity evidence unavailable. +- **By workflow type.** `/bounded/workflow-types` applies PHP's + `workflow_types` validation, then answers PHP's 501 + `backend_capability_unavailable` for an observer that cannot filter by + type. Native does not advertise `workflow_type_dashboard_summary`, so + Waterline does not ask for it. Published PHP 2.5.14 answers with + filtered totals; porting that filter is open. +- **Verification.** `operator_dashboard` integration tests pass on SQLite, + PostgreSQL and MySQL, and the full native suite passes on SQLite. With + identical seeds, Waterline's stats from native and from published PHP + 2.5.14 differ only in identifiers, timings and durations, summary + `history_size_bytes` (above) and sort-key precision. Before the scheduled + run summary correction, native also counted one run fewer. + October 21, operator metrics and health: - **Source.** Native serves admin-only `GET /api/system/operator-metrics` and diff --git a/rust/src/runtime/execution.rs b/rust/src/runtime/execution.rs index 19ca1cdb3..9e918da9b 100644 --- a/rust/src/runtime/execution.rs +++ b/rust/src/runtime/execution.rs @@ -298,6 +298,7 @@ impl Runtime { delegate!(describe_namespace(name: &str) -> Result); delegate!(operator_metrics(config: &super::operator_metrics::MetricsConfig) -> Result); delegate!(system_health(config: &super::operator_metrics::MetricsConfig) -> Result<(bool, Value)>); + delegate!(operator_dashboard(config: &super::operator_metrics::MetricsConfig, bounded: bool) -> Result); delegate!(create_namespace(body: Value) -> Result); delegate!(list_search_attributes() -> Result); delegate!( diff --git a/rust/src/runtime/http.rs b/rust/src/runtime/http.rs index 68248bd1f..db548b623 100644 --- a/rust/src/runtime/http.rs +++ b/rust/src/runtime/http.rs @@ -187,6 +187,12 @@ pub fn router(runtime: Runtime) -> Router { ) .route("/api/system/operator-metrics", get(operator_metrics)) .route("/api/system/health", get(system_health)) + .route("/api/system/operator-dashboard", get(operator_dashboard)) + .route("/api/system/operator-dashboard/bounded", get(bounded_operator_dashboard)) + .route( + "/api/system/operator-dashboard/bounded/workflow-types", + get(workflow_type_operator_dashboard), + ) .route("/api/workers", get(list_workers)) .route("/api/task-queues", get(list_task_queues)) .route("/api/task-queues/{task_queue}", get(describe_task_queue)) @@ -491,6 +497,9 @@ fn named_namespace_operation(method: &str, path: &str) -> bool { if path == "/api/workers" || path == "/api/system/operator-metrics" || path == "/api/system/health" + || path == "/api/system/operator-dashboard" + || path == "/api/system/operator-dashboard/bounded" + || path == "/api/system/operator-dashboard/bounded/workflow-types" { return method == "GET"; } @@ -1075,6 +1084,28 @@ async fn operator_metrics(Extension(runtime): Extension) -> Result) -> Result> { + Ok(Json(runtime.operator_dashboard(&metrics_config(&runtime), false).await?)) +} + +async fn bounded_operator_dashboard(Extension(runtime): Extension) -> Result> { + Ok(Json(runtime.operator_dashboard(&metrics_config(&runtime), true).await?)) +} + +/// PHP validates `workflow_types` and then answers 501 when its observer +/// cannot filter dashboard totals by workflow type; native cannot either. +async fn workflow_type_operator_dashboard( + Query(query): Query>, +) -> Result { + super::operator_dashboard::workflow_types(query.get("workflow_types").map(String::as_str))?; + Ok(( + StatusCode::NOT_IMPLEMENTED, + Json(json!({"message":"The installed workflow observer cannot filter dashboard totals by workflow type.", + "reason":"backend_capability_unavailable","capability":"workflow_type_dashboard"})), + ) + .into_response()) +} + /// PHP answers 503 while a check reports an error. async fn system_health(Extension(runtime): Extension) -> Result { let (unavailable, body) = runtime.system_health(&metrics_config(&runtime)).await?; diff --git a/rust/src/runtime/mod.rs b/rust/src/runtime/mod.rs index dc49fec32..96dc56ffc 100644 --- a/rust/src/runtime/mod.rs +++ b/rust/src/runtime/mod.rs @@ -36,6 +36,7 @@ mod memo_limits; mod memos; pub mod mysql; mod namespaces; +mod operator_dashboard; mod operator_metrics; mod poll_readiness; pub mod postgres; diff --git a/rust/src/runtime/operator_dashboard.rs b/rust/src/runtime/operator_dashboard.rs new file mode 100644 index 000000000..8fd6936d3 --- /dev/null +++ b/rust/src/runtime/operator_dashboard.rs @@ -0,0 +1,57 @@ +//! Request rules for published PHP's operator dashboard by workflow type. +//! The dashboard itself is built with the operator metrics. +use super::{RuntimeError, laravel_validation::failure}; + +/// `workflowTypeOperatorDashboard`'s validation: a JSON list of at most 4096 +/// encoded bytes whose items are nonempty strings of at most 255 characters. +pub(super) fn workflow_types(value: Option<&str>) -> Result, RuntimeError> { + let refuse = |message: &'static str| failure(vec![("workflow_types", vec![message])]).expect("one message"); + let text = value.map(str::trim).filter(|text| !text.is_empty()) + .ok_or_else(|| refuse("The workflow types field is required."))?; + if text.chars().count() > 4096 { + return Err(refuse("The workflow types field must not be greater than 4096 characters.")); + } + let decoded: serde_json::Value = + serde_json::from_str(text).map_err(|_| refuse("The workflow types field must be a valid JSON string."))?; + let encoded_len = url_encoded_len(text); + let Some(items) = decoded.as_array().filter(|_| encoded_len <= 4096) else { + return Err(refuse("Provide a JSON list of workflow types with at most 4096 encoded bytes.")); + }; + items + .iter() + .map(|item| { + item.as_str() + .filter(|kind| !kind.is_empty() && kind.chars().count() <= 255) + .map(str::to_owned) + .ok_or_else(|| refuse("Workflow types must be nonempty strings of at most 255 characters.")) + }) + .collect() +} + +/// `strlen(rawurlencode($value))`. +fn url_encoded_len(text: &str) -> usize { + text.bytes() + .map(|byte| if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.' | b'~') { 1 } else { 3 }) + .sum() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn message(value: Option<&str>) -> String { + match workflow_types(value) { + Err(RuntimeError::Protocol { response, .. }) => response["message"].as_str().unwrap().to_owned(), + other => panic!("{other:?}"), + } + } + + #[test] + fn workflow_types_follow_php_rules() { + assert_eq!(workflow_types(Some(r#"["orders","billing"]"#)).unwrap(), vec!["orders", "billing"]); + assert_eq!(message(None), "The workflow types field is required."); + assert_eq!(message(Some("{")), "The workflow types field must be a valid JSON string."); + assert_eq!(message(Some(r#"{"a":1}"#)), "Provide a JSON list of workflow types with at most 4096 encoded bytes."); + assert_eq!(message(Some(r#"["",1]"#)), "Workflow types must be nonempty strings of at most 255 characters."); + } +} diff --git a/rust/src/runtime/operator_metrics.rs b/rust/src/runtime/operator_metrics.rs index 067f7ea4e..182e17dd5 100644 --- a/rust/src/runtime/operator_metrics.rs +++ b/rust/src/runtime/operator_metrics.rs @@ -91,6 +91,68 @@ macro_rules! dispatch_overdue { }; } +/// The summary columns `WorkflowRunSummary::toArray()` reports, with flags +/// read as numbers on every backend. +macro_rules! summary_columns { + () => { + "SELECT id,workflow_instance_id,run_number,CASE WHEN is_current_run THEN 1 ELSE 0 END AS is_current_run,engine_source,projection_schema_version,class,workflow_type,namespace,compatibility,declared_entry_mode,declared_contract_source,business_key,visibility_labels,status,status_bucket,closed_reason,connection,queue,started_at,sort_timestamp,sort_key,closed_at,archived_at,archive_command_id,archive_reason,duration_ms,wait_kind,wait_reason,wait_started_at,wait_deadline_at,open_wait_id,resume_source_kind,resume_source_id,next_task_at,liveness_state,liveness_reason,repair_blocked_reason,CASE WHEN repair_attention THEN 1 ELSE 0 END AS repair_attention,CASE WHEN task_problem THEN 1 ELSE 0 END AS task_problem,next_task_id,next_task_type,next_task_status,next_task_lease_expires_at,exception_count,history_event_count,history_size_bytes,CASE WHEN continue_as_new_recommended THEN 1 ELSE 0 END AS continue_as_new_recommended,created_at,updated_at,history_fan_out,history_budget_pressure FROM workflow_run_summaries" + }; +} + +const SUMMARY_TEXT: &[&str] = &["id", "workflow_instance_id", "engine_source", "class", "workflow_type", "namespace", + "compatibility", "declared_entry_mode", "declared_contract_source", "business_key", "status", "status_bucket", + "closed_reason", "connection", "queue", "sort_key", "archive_command_id", "archive_reason", "wait_kind", "wait_reason", + "open_wait_id", "resume_source_kind", "resume_source_id", "liveness_state", "liveness_reason", "repair_blocked_reason", + "next_task_id", "next_task_type", "next_task_status", "history_budget_pressure"]; +const SUMMARY_NUMBERS: &[&str] = &["run_number", "projection_schema_version", "duration_ms", "exception_count", + "history_event_count", "history_size_bytes", "history_fan_out"]; +const SUMMARY_FLAGS: &[&str] = &["is_current_run", "repair_attention", "task_problem", "continue_as_new_recommended"]; +const SUMMARY_TIMES: &[&str] = &["started_at", "sort_timestamp", "closed_at", "archived_at", "wait_started_at", + "wait_deadline_at", "next_task_at", "next_task_lease_expires_at", "created_at", "updated_at"]; + +/// `RepairBlockedReason::metadata`. +fn repair_blocked(code: Option<&str>) -> Value { + let (label, description, tone, visible) = match code { + Some("unsupported_history") => ("Replay Blocked", "Repair is blocked because only unsupported diagnostic history remains.", "dark", true), + Some("waiting_for_compatible_worker") => ("Compat Blocked", "Repair is blocked because the task is waiting for a compatible worker.", "warning", true), + Some("selected_run_not_current") => ("Historical Run", "Repair is blocked because the selected run is no longer current.", "secondary", false), + Some("run_closed") => ("Run Closed", "Repair is blocked because the selected run is already closed.", "secondary", false), + Some("repair_not_needed") => ("Repair Not Needed", "Repair is blocked because the run already has a healthy durable resume path.", "secondary", false), + _ => return Value::Null, + }; + json!({"code": code, "label": label, "description": description, "tone": tone, "badge_visible": visible}) +} + +/// PHP's `$flowsPastHour / 60`: an integer when it divides evenly. +fn per_minute(flows: i64) -> Value { + if flows % 60 == 0 { json!(flows / 60) } else { json!(flows as f64 / 60.0) } +} + +/// Carbon's `toIso8601String()`. +fn iso8601(at: DateTime) -> String { + at.format("%Y-%m-%dT%H:%M:%S+00:00").to_string() +} + +fn start_of_hour(at: DateTime) -> DateTime { + at.with_minute(0).and_then(|at| at.with_second(0)).and_then(|at| at.with_nanosecond(0)).unwrap_or(at) +} + +/// `OperatorMetrics::snapshot(..., includeHistoryAudits: false)`: the history +/// audits are marked as not requested and their counts are null. +fn without_history_audits(metrics: &mut Value) { + metrics["history_audit_evaluation"] = json!("not_requested"); + if let Some(contracts) = metrics["command_contracts"].as_object_mut() { + for value in contracts.values_mut() { + *value = Value::Null; + } + } + for field in ["runs_with_waits", "projected_runs_with_waits", "missing_runs_with_waits", "stale_projected_runs", "needs_rebuild"] { + if metrics["projections"]["run_waits"].get(field).is_some() { + metrics["projections"]["run_waits"][field] = Value::Null; + } + } +} + impl Store where for<'c> &'c mut DB::Connection: Executor<'c, Database = DB>, @@ -189,6 +251,224 @@ where Ok(json!({"namespace": self.namespace.as_ref(), "operator_metrics": metrics})) } + /// `SystemController::dashboardResponse` for the whole namespace. The + /// bounded dashboard skips PHP's history audits. + pub(super) async fn operator_dashboard(&self, config: &MetricsConfig, bounded: bool) -> Result { + let now = now(); + let mut metrics = self.operator_metrics_snapshot(config).await?; + if bounded { + without_history_audits(&mut metrics); + metrics["capacity_evidence"] = + json!({"available": false, "reason": "native_capacity_evidence_unavailable"}); + } + metrics["worker_sessions"] = + json!({"total": 0, "active": 0, "closed": 0, "expired": 0, "failed": 0, "orphaned": 0}); + metrics["runtime_external_payload_cleanup"] = self.external_payload_cleanup_metrics(config).await?; + let hour = now - Duration::hours(1); + let day = now - Duration::days(1); + let week = now - Duration::weeks(1); + let flows_past_hour = self.tally(summaries!("(sort_timestamp>=$2 OR (sort_timestamp IS NULL AND created_at>=$3))"), &[hour, hour]).await?; + let mut dashboard = json!({ + "flows": self.tally("SELECT COUNT(*) FROM workflow_run_summaries WHERE namespace=$1", &[]).await?, + "flows_per_minute": per_minute(flows_past_hour), + "flows_past_hour": flows_past_hour, + "exceptions_past_hour": self.tally("SELECT COUNT(*) FROM workflow_failures f WHERE EXISTS (SELECT 1 FROM workflow_runs r WHERE r.id=f.workflow_run_id AND r.namespace=$1) AND f.created_at>=$2", &[hour]).await?, + "failed_flows_past_week": self.tally(summaries!("status='failed' AND updated_at>=$2"), &[now - Duration::days(7)]).await?, + "max_wait_time_workflow": self.summary_row(concat!(summary_columns!(), " WHERE namespace=$1 AND status_bucket='running' AND wait_started_at IS NOT NULL ORDER BY wait_started_at,sort_timestamp,created_at,id LIMIT 1")).await?, + "max_duration_workflow": self.summary_row(concat!(summary_columns!(), " WHERE namespace=$1 AND duration_ms IS NOT NULL ORDER BY duration_ms DESC,sort_timestamp DESC,created_at DESC,id DESC LIMIT 1")).await?, + "max_exceptions_workflow": self.summary_row(concat!(summary_columns!(), " WHERE namespace=$1 AND exception_count>0 ORDER BY exception_count DESC,sort_timestamp DESC,created_at DESC,id DESC LIMIT 1")).await?, + "fleet_overview": { + "current": self.bucket_counts("SELECT status_bucket AS bucket,COUNT(*) AS n FROM workflow_run_summaries WHERE namespace=$1 AND status_bucket<>'completed' GROUP BY status_bucket", None, &["running", "failed"]).await?, + "trends": { + "hour": self.closed_counts(hour).await?, + "day": self.closed_counts(day).await?, + "week": self.closed_counts(week).await?, + }, + }, + "workflow_type_health": self.workflow_type_health(week).await?, + "needs_attention": self.needs_attention(now).await?, + "fleet_trends_series": self.fleet_trends(now).await?, + "workflow_scope": {"namespace": self.namespace.as_ref(), "workflow_types": null, "all_workflow_types": true}, + "time_windows": {"generated_at": iso8601(now), "total_runs": "all_retained_runs", "current_status": "current", + "recent_hour_from": iso8601(hour), "recent_day_from": iso8601(day), "recent_week_from": iso8601(week), + "trends_resolution": "hour"}, + "operator_metrics_scope": {"namespace": self.namespace.as_ref(), "workflow_types": null}, + }); + dashboard["operator_metrics"] = metrics; + Ok(json!({"namespace": self.namespace.as_ref(), "dashboard": dashboard})) + } + + /// One summary as Eloquent's `WorkflowRunSummary::toArray()`, or null. + async fn summary_row(&self, sql: &'static str) -> Result { + let row = Self::query(sql).bind(self.namespace.as_ref()).fetch_optional(&self.pool).await?; + row.map(|row| Self::summary_array(&row)).transpose().map(|row| row.unwrap_or(Value::Null)) + } + + /// The summary's columns with the model's casts, then its appends. + fn summary_array(row: &DB::Row) -> Result { + let mut fields = serde_json::Map::new(); + for column in SUMMARY_TEXT { + fields.insert((*column).to_owned(), json!(DB::optional_string(row, column)?)); + } + for column in SUMMARY_NUMBERS { + fields.insert((*column).to_owned(), json!(DB::optional_number(row, column)?)); + } + for column in SUMMARY_FLAGS { + fields.insert((*column).to_owned(), json!(DB::optional_number(row, column)?.map(|flag| flag != 0))); + } + for column in SUMMARY_TIMES { + fields.insert((*column).to_owned(), time(DB::optional_instant(row, column)?)); + } + fields.insert("visibility_labels".into(), DB::optional_document_row(row, "visibility_labels")?.unwrap_or(Value::Null)); + let status = DB::optional_string(row, "status")?.unwrap_or_default(); + let text = |field: &str| fields.get(field).cloned().unwrap_or(Value::Null); + let appended = json!({"instance_id": text("workflow_instance_id"), "selected_run_id": text("id"), "run_id": text("id"), + "exceptions_count": DB::optional_number(row, "exception_count")?.unwrap_or_default(), + "is_terminal": matches!(status.as_str(), "completed" | "failed" | "cancelled" | "terminated"), + "repair_blocked": repair_blocked(DB::optional_string(row, "repair_blocked_reason")?.as_deref()), + "task_problem_badge": super::run_diagnostics::task_problem_badge( + fields["task_problem"] == true, + fields["liveness_state"].as_str(), + fields["wait_kind"].as_str(), + )}); + if let Value::Object(appended) = appended { + fields.extend(appended); + } + Ok(Value::Object(fields)) + } + + /// `pluck('count', 'status_bucket')` over `buckets`, zero when absent. + async fn bucket_counts(&self, sql: &'static str, since: Option>, buckets: &[&str]) -> Result { + let mut query = Self::query(sql).bind(self.namespace.as_ref()); + if let Some(since) = since { + query = query.bind(DB::bind_time(since)); + } + let mut counts = serde_json::Map::new(); + for bucket in buckets { + counts.insert((*bucket).to_owned(), json!(0)); + } + for row in query.fetch_all(&self.pool).await? { + let bucket = DB::string(&row, "bucket")?; + if counts.contains_key(&bucket) { + counts.insert(bucket, json!(DB::number(&row, "n")?)); + } + } + Ok(Value::Object(counts)) + } + + async fn closed_counts(&self, since: DateTime) -> Result { + self.bucket_counts("SELECT status_bucket AS bucket,COUNT(*) AS n FROM workflow_run_summaries WHERE namespace=$1 AND closed_at>=$2 AND status_bucket IN ('completed','failed') GROUP BY status_bucket", + Some(since), &["completed", "failed"]).await + } + + /// `workflowTypeHealth`: the week's ten busiest workflow types. + async fn workflow_type_health(&self, week: DateTime) -> Result { + let types = Self::query("SELECT workflow_type,COUNT(*) AS total_runs FROM workflow_run_summaries WHERE namespace=$1 AND created_at>=$2 GROUP BY workflow_type ORDER BY total_runs DESC LIMIT 10") + .bind(self.namespace.as_ref()).bind(DB::bind_time(week)).fetch_all(&self.pool).await?; + let mut health = Vec::with_capacity(types.len()); + for row in types { + let workflow_type = DB::string(&row, "workflow_type")?; + let total = DB::number(&row, "total_runs")?; + let mut statuses = std::collections::HashMap::new(); + for row in Self::query("SELECT status,COUNT(*) AS n FROM workflow_run_summaries WHERE namespace=$1 AND workflow_type=$2 AND created_at>=$3 GROUP BY status") + .bind(self.namespace.as_ref()).bind(workflow_type.as_str()).bind(DB::bind_time(week)).fetch_all(&self.pool).await? + { + statuses.insert(DB::string(&row, "status")?, DB::number(&row, "n")?); + } + let count = |status: &str| statuses.get(status).copied().unwrap_or(0); + let (completed, failed, cancelled, terminated) = (count("completed"), count("failed"), count("cancelled"), count("terminated")); + let terminal = completed + failed + cancelled + terminated; + let pass_rate = if terminal > 0 { completed as f64 / terminal as f64 * 100.0 } else { 0.0 }; + let durations = Self::scalar("SELECT COUNT(*) FROM workflow_run_summaries WHERE namespace=$1 AND workflow_type=$2 AND status='completed' AND created_at>=$3 AND duration_ms IS NOT NULL") + .bind(self.namespace.as_ref()).bind(workflow_type.as_str()).bind(DB::bind_time(week)).fetch_one(&self.pool).await?; + // The upper-middle duration, as PHP's offset query picks it. + let median = if durations > 0 { + Self::query("SELECT duration_ms FROM workflow_run_summaries WHERE namespace=$1 AND workflow_type=$2 AND status='completed' AND created_at>=$3 AND duration_ms IS NOT NULL ORDER BY duration_ms LIMIT 1 OFFSET $4") + .bind(self.namespace.as_ref()).bind(workflow_type.as_str()).bind(DB::bind_time(week)).bind(durations / 2) + .fetch_optional(&self.pool).await? + .map(|row| DB::optional_number(&row, "duration_ms")).transpose()?.flatten() + } else { + None + }; + health.push(json!({"workflow_type": workflow_type, "workflow_class": workflow_type, "total_runs": total, + "pass_rate": (pass_rate * 10.0).round() / 10.0, "median_duration_ms": median, + "error_count": failed + cancelled + terminated, + "status_breakdown": {"completed": completed, "failed": failed, "cancelled": cancelled, "terminated": terminated}})); + } + Ok(json!(health)) + } + + /// `needsAttention`: PHP's alert rules over workers and run summaries. + async fn needs_attention(&self, now: DateTime) -> Result { + let hour = now - Duration::hours(1); + let mut alerts = Vec::new(); + let stuck = self.tally("SELECT COUNT(*) FROM workflow_worker_compatibility_heartbeats WHERE namespace=$1 AND recorded_at<$2 AND recorded_at>$3", + &[now - Duration::minutes(5), hour]).await?; + if stuck > 0 { + alerts.push(json!({"type": "stuck_workers", "scope": "operator_workers", "severity": "warning", + "message": format!("{stuck} worker(s) have not sent heartbeat in 5+ minutes"), "count": stuck, + "action": "Check worker health and restart if needed"})); + } + let long_running = self.tally(summaries!("status_bucket='running' AND started_at<$2 AND wait_started_at IS NULL"), &[hour]).await?; + if long_running > 0 { + alerts.push(json!({"type": "long_running", "severity": "info", + "message": format!("{long_running} workflow(s) running longer than 1 hour"), "count": long_running, + "action": "Review workflow duration expectations"})); + } + let storms = self.tally(summaries!("status_bucket='running' AND updated_at>=$2 AND exception_count>=10"), &[hour]).await?; + if storms > 0 { + alerts.push(json!({"type": "retry_storm", "severity": "error", + "message": format!("{storms} workflow(s) with 10+ exceptions in last hour"), "count": storms, + "action": "Investigate activity failures and fix root cause"})); + } + let failed = self.tally(summaries!("status='failed' AND closed_at>=$2"), &[hour]).await?; + let completed = self.tally(summaries!("status='completed' AND closed_at>=$2"), &[hour]).await?; + let total = failed + completed; + if total >= 10 && failed as f64 / total as f64 * 100.0 > 20.0 { + alerts.push(json!({"type": "high_failure_rate", "severity": "error", + "message": format!("{:.0}% failure rate in last hour ({failed}/{total})", failed as f64 / total as f64 * 100.0), + "count": failed, "action": "Review failed workflows for common patterns"})); + } + let overdue = now - Duration::minutes(30); + let long_waits = self.tally(summaries!("status_bucket='running' AND wait_started_at IS NOT NULL AND (task_problem=TRUE OR repair_attention=TRUE OR (wait_kind IN ('activity','child','signal','condition','update') AND wait_deadline_at<=$2) OR (next_task_status='ready' AND next_task_at<$3) OR (wait_kind='timer' AND wait_deadline_at<$4 AND (next_task_status IS NULL OR next_task_status<>'leased')))"), + &[now, overdue, overdue]).await?; + if long_waits > 0 { + alerts.push(json!({"type": "long_waits", "severity": "warning", + "message": format!("{long_waits} workflow(s) with blocked or overdue waits"), "count": long_waits, + "action": "Inspect wait deadlines and task dispatch before recovering work"})); + } + let critical = alerts.iter().any(|alert| alert["severity"] == "error"); + Ok(json!({"total_alerts": alerts.len(), "has_critical": critical, "alerts": alerts})) + } + + /// `fleetTrendsSeries`: completed and failed runs per hour for a week. + async fn fleet_trends(&self, now: DateTime) -> Result { + let week = now - Duration::weeks(1); + let start = start_of_hour(week); + let end = start_of_hour(now); + let sql = match DB::NAME { + "PostgreSQL" => "SELECT TO_CHAR(closed_at, 'YYYY-MM-DD HH24:00:00') AS hour_bucket,status_bucket,COUNT(*) AS run_count FROM workflow_run_summaries WHERE namespace=$1 AND closed_at IS NOT NULL AND closed_at>=$2 AND closed_at<$3 AND status_bucket IN ('completed','failed') GROUP BY TO_CHAR(closed_at, 'YYYY-MM-DD HH24:00:00'),status_bucket", + "MySQL" => "SELECT DATE_FORMAT(closed_at, '%Y-%m-%d %H:00:00') AS hour_bucket,status_bucket,COUNT(*) AS run_count FROM workflow_run_summaries WHERE namespace=$1 AND closed_at IS NOT NULL AND closed_at>=$2 AND closed_at<$3 AND status_bucket IN ('completed','failed') GROUP BY DATE_FORMAT(closed_at, '%Y-%m-%d %H:00:00'),status_bucket", + _ => "SELECT STRFTIME('%Y-%m-%d %H:00:00', closed_at) AS hour_bucket,status_bucket,COUNT(*) AS run_count FROM workflow_run_summaries WHERE namespace=$1 AND closed_at IS NOT NULL AND closed_at>=$2 AND closed_at<$3 AND status_bucket IN ('completed','failed') GROUP BY STRFTIME('%Y-%m-%d %H:00:00', closed_at),status_bucket", + }; + let rows = Self::query(sql).bind(self.namespace.as_ref()).bind(DB::bind_time(week)).bind(DB::bind_time(end + Duration::hours(1))) + .fetch_all(&self.pool).await?; + let mut counts = std::collections::HashMap::new(); + for row in rows { + counts.insert((DB::string(&row, "hour_bucket")?, DB::string(&row, "status_bucket")?), DB::number(&row, "run_count")?); + } + let (mut timestamps, mut completed, mut failed) = (Vec::new(), Vec::new(), Vec::new()); + let mut hour = start; + while hour <= end { + let key = hour.format("%Y-%m-%d %H:00:00").to_string(); + timestamps.push(hour.timestamp() * 1000); + completed.push(counts.get(&(key.clone(), "completed".to_owned())).copied().unwrap_or(0)); + failed.push(counts.get(&(key, "failed".to_owned())).copied().unwrap_or(0)); + hour += Duration::hours(1); + } + Ok(json!({"timestamps": timestamps, "completed": completed, "failed": failed})) + } + /// `GET /api/system/health`, with PHP's HTTP status. pub(super) async fn system_health(&self, config: &MetricsConfig) -> Result<(bool, Value)> { let health = super::system_health::snapshot(&self.operator_metrics_snapshot(config).await?); diff --git a/rust/src/runtime/run_diagnostics.rs b/rust/src/runtime/run_diagnostics.rs index 1df12d31d..96abcfe06 100644 --- a/rust/src/runtime/run_diagnostics.rs +++ b/rust/src/runtime/run_diagnostics.rs @@ -51,7 +51,7 @@ fn trimmed(value: Option) -> Option { } /// `WorkflowTaskProblem::metadata`. -fn task_problem_badge(problem: bool, liveness: Option<&str>, wait: Option<&str>) -> Value { +pub(super) fn task_problem_badge(problem: bool, liveness: Option<&str>, wait: Option<&str>) -> Value { if !problem { return Value::Null; } diff --git a/rust/tests/execution.rs b/rust/tests/execution.rs index 606ccd696..850f0dbd1 100644 --- a/rust/tests/execution.rs +++ b/rust/tests/execution.rs @@ -81,6 +81,8 @@ mod activity_deadlines; mod lock_free_waiting; #[path = "execution/read_snapshots.rs"] mod read_snapshots; +#[path = "execution/operator_dashboard.rs"] +mod operator_dashboard; #[path = "execution/waterline_reads.rs"] mod waterline_reads; #[path = "execution/start_timeouts.rs"] diff --git a/rust/tests/execution/operator_dashboard.rs b/rust/tests/execution/operator_dashboard.rs new file mode 100644 index 000000000..f029faa92 --- /dev/null +++ b/rust/tests/execution/operator_dashboard.rs @@ -0,0 +1,75 @@ +use super::*; + +async fn dashboard(app: &Router, path: &str) -> Value { + let (status, body) = request(app, "GET", path, Value::Null).await; + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!(body["namespace"], "default"); + body["dashboard"].clone() +} + +// Published PHP's operator dashboard summarizes the namespace's runs from +// their summaries; the bounded variant skips the history audits. +#[tokio::test] +async fn operator_dashboard_summarizes_runs_like_php() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + register(&app, "worker", json!(["echo"]), json!([])).await; + for workflow in ["done", "broken", "open"] { + start(&app, workflow).await; + } + let done = poll(&app, "worker", "workflow").await; + let (status, body) = finish_task(&app, &done, json!([{"type":"complete_workflow","result":envelope(Payload::Long(1))}])).await; + assert_eq!(status, StatusCode::OK, "{body}"); + let broken = poll(&app, "worker", "workflow").await; + let (status, body) = finish_task(&app, &broken, json!([{"type":"fail_workflow","message":"boom","exception_class":"RuntimeException"}])).await; + assert_eq!(status, StatusCode::OK, "{body}"); + let bounded = dashboard(&app, "/api/system/operator-dashboard/bounded").await; + assert_eq!(bounded["flows"], 3); + assert_eq!(bounded["flows_past_hour"], 3); + assert_eq!(bounded["flows_per_minute"], json!(0.05)); + assert_eq!(bounded["failed_flows_past_week"], 1); + assert_eq!(bounded["fleet_overview"]["current"], json!({"running":1,"failed":1})); + assert_eq!(bounded["fleet_overview"]["trends"]["hour"], json!({"completed":1,"failed":1})); + let health = &bounded["workflow_type_health"][0]; + assert_eq!(health["workflow_type"], "echo"); + assert_eq!(health["total_runs"], 3); + assert_eq!(health["pass_rate"], json!(50.0)); + assert_eq!(health["error_count"], 1); + assert_eq!(health["status_breakdown"], json!({"completed":1,"failed":1,"cancelled":0,"terminated":0})); + assert_eq!(bounded["needs_attention"], json!({"alerts":[],"total_alerts":0,"has_critical":false})); + let series = &bounded["fleet_trends_series"]; + assert_eq!(series["timestamps"].as_array().unwrap().len(), 169); + assert_eq!(series["completed"].as_array().unwrap().iter().filter_map(Value::as_i64).sum::(), 1); + assert_eq!(series["failed"].as_array().unwrap().iter().filter_map(Value::as_i64).sum::(), 1); + assert_eq!(bounded["workflow_scope"], json!({"namespace":"default","workflow_types":null,"all_workflow_types":true})); + assert!(bounded["time_windows"]["recent_hour_from"].as_str().unwrap().ends_with("+00:00")); + // The longest-running open wait is the run still waiting on its task. + let waiting = &bounded["max_wait_time_workflow"]; + assert_eq!(waiting["instance_id"], "open"); + assert_eq!(waiting["is_terminal"], false); + assert_eq!(waiting["exceptions_count"], 0); + assert!(waiting["is_current_run"].is_boolean()); + let metrics = &bounded["operator_metrics"]; + assert_eq!(metrics["history_audit_evaluation"], "not_requested"); + assert!(metrics["command_contracts"].as_object().unwrap().values().all(Value::is_null)); + assert_eq!(metrics["capacity_evidence"]["available"], false); + let full = dashboard(&app, "/api/system/operator-dashboard").await; + assert_eq!(full["flows"], 3); + assert!(full["operator_metrics"].get("history_audit_evaluation").is_none()); +} + +// Totals filtered by workflow type need an observer native does not have, so +// native answers as PHP does when its observer cannot filter. +#[tokio::test] +async fn workflow_type_dashboard_validates_then_reports_the_missing_capability() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + let path = "/api/system/operator-dashboard/bounded/workflow-types"; + let (status, body) = request(&app, "GET", path, Value::Null).await; + assert_eq!(status, StatusCode::UNPROCESSABLE_ENTITY, "{body}"); + assert_eq!(body["errors"]["workflow_types"], json!(["The workflow types field is required."])); + let (status, body) = request(&app, "GET", &format!("{path}?workflow_types=%5B%22echo%22%5D"), Value::Null).await; + assert_eq!(status, StatusCode::NOT_IMPLEMENTED, "{body}"); + assert_eq!(body["reason"], "backend_capability_unavailable"); + assert_eq!(body["capability"], "workflow_type_dashboard"); +}