diff --git a/.github/workflows/server-parity.yml b/.github/workflows/server-parity.yml index 5457019d6..73711cf55 100644 --- a/.github/workflows/server-parity.yml +++ b/.github/workflows/server-parity.yml @@ -66,6 +66,7 @@ on: - tests/Unit/ServerParityKilledAuthorPollTest.mjs - tests/Unit/ServerParityImagePullTest.mjs - tests/Unit/ServerParityMessageStreamsContractTest.mjs + - tests/Unit/ServerParityWorkflowStreamWritesContractTest.mjs - tests/Unit/ServerParityWaitingHistoryContractTest.mjs - tests/Unit/ServerParityTaskRetryContractTest.mjs - tests/Unit/ServerParityPublishedPatchContractTest.mjs @@ -1806,7 +1807,7 @@ jobs: bash scripts/ci/run-composer.sh --rm -v "$PWD:/source" -w /source -- \ install --working-dir=scripts/conformance/server-parity --no-interaction --no-progress docker run --rm --user=1000:1000 --entrypoint node -v "$PWD:/source:ro" -w /source \ - "$image" --test tests/Unit/ServerParityRunnerTest.mjs tests/Unit/ServerParitySdkDirectionLanesTest.mjs tests/Unit/ServerParityCodecDeadlineCellsTest.mjs tests/Unit/ServerParityCooperativeContractTest.mjs tests/Unit/ServerParityChildCancellationContractTest.mjs tests/Unit/ServerParityScheduleContractTest.mjs tests/Unit/ServerParityVisibilityContractTest.mjs tests/Unit/ServerParityAdmissionContractTest.mjs tests/Unit/ServerParityRoleTokenContractTest.mjs tests/Unit/ServerParityWorkerDeregistrationContractTest.mjs tests/Unit/ServerParitySearchAttributeContractTest.mjs tests/Unit/ServerParityWorkerListingContractTest.mjs tests/Unit/ServerParityWorkerReplacementContractTest.mjs tests/Unit/ServerParityPatchDeploymentContractTest.mjs tests/Unit/ServerParityWaitingHistoryContractTest.mjs tests/Unit/ServerParityTaskRetryContractTest.mjs tests/Unit/ServerParityPublishedPatchContractTest.mjs tests/Unit/ServerParityPublishedPatchTransportTest.mjs tests/Unit/ServerParityRecordFileTest.mjs tests/Unit/ServerParityFailureLocationTest.mjs tests/Unit/ServerParityFailureExcerptTest.mjs tests/Unit/ServerParityLateWorkflowCompletionContractTest.mjs tests/Unit/ServerParityStorageTestContractTest.mjs tests/Unit/ServerParitySchedulePoliciesContractTest.mjs tests/Unit/ServerParitySignalsBeforeWorkerContractTest.mjs tests/Unit/ServerParityScheduleCatalogContractTest.mjs tests/Unit/ServerParityOperatorMetricsContractTest.mjs tests/Unit/ServerParityActivityLeaseRenewalContractTest.mjs tests/Unit/ServerParityRecordLocalActivityContractTest.mjs tests/Unit/ServerParityActivityCancellationAckContractTest.mjs tests/Unit/ServerParityServiceCatalogContractTest.mjs tests/Unit/ServerParityWorkflowImportContractTest.mjs tests/Unit/ServerParityWaterlineImportContractTest.mjs tests/Unit/ServerParitySystemMetricsContractTest.mjs tests/Unit/ServerParityOperatorDashboardTypesContractTest.mjs tests/Unit/ServerParityPrometheusMetricsContractTest.mjs tests/Unit/ServerParityPayloadCleanupContractTest.mjs tests/Unit/ServerParityRepairPassContractTest.mjs tests/Unit/ServerParityClusterInfoContractTest.mjs tests/Unit/ServerParityNamespaceDeleteContractTest.mjs tests/Unit/ServerParityMessageStreamsContractTest.mjs tests/Unit/ServerParityCancellationScopesContractTest.mjs tests/Unit/ServerParityWorkflowPollRequestContractTest.mjs tests/Unit/ServerParityUpdateValidationContractTest.mjs tests/Unit/ServerParityFixtureSelectionTest.mjs tests/Unit/ServerParityUpdateValidationRetryTest.mjs tests/Unit/ServerParityKilledAuthorPollTest.mjs tests/Unit/ServerParityImagePullTest.mjs tests/Unit/ServerParityWorkerSessionsContractTest.mjs tests/Unit/ServerParityWorkerSessionActivitiesContractTest.mjs tests/Unit/ServerParityNexusOperationsContractTest.mjs tests/Unit/ServerParityBridgeAdaptersContractTest.mjs > parity-evidence/comparator-tests.txt + "$image" --test tests/Unit/ServerParityRunnerTest.mjs tests/Unit/ServerParitySdkDirectionLanesTest.mjs tests/Unit/ServerParityCodecDeadlineCellsTest.mjs tests/Unit/ServerParityCooperativeContractTest.mjs tests/Unit/ServerParityChildCancellationContractTest.mjs tests/Unit/ServerParityScheduleContractTest.mjs tests/Unit/ServerParityVisibilityContractTest.mjs tests/Unit/ServerParityAdmissionContractTest.mjs tests/Unit/ServerParityRoleTokenContractTest.mjs tests/Unit/ServerParityWorkerDeregistrationContractTest.mjs tests/Unit/ServerParitySearchAttributeContractTest.mjs tests/Unit/ServerParityWorkerListingContractTest.mjs tests/Unit/ServerParityWorkerReplacementContractTest.mjs tests/Unit/ServerParityPatchDeploymentContractTest.mjs tests/Unit/ServerParityWaitingHistoryContractTest.mjs tests/Unit/ServerParityTaskRetryContractTest.mjs tests/Unit/ServerParityPublishedPatchContractTest.mjs tests/Unit/ServerParityPublishedPatchTransportTest.mjs tests/Unit/ServerParityRecordFileTest.mjs tests/Unit/ServerParityFailureLocationTest.mjs tests/Unit/ServerParityFailureExcerptTest.mjs tests/Unit/ServerParityLateWorkflowCompletionContractTest.mjs tests/Unit/ServerParityStorageTestContractTest.mjs tests/Unit/ServerParitySchedulePoliciesContractTest.mjs tests/Unit/ServerParitySignalsBeforeWorkerContractTest.mjs tests/Unit/ServerParityScheduleCatalogContractTest.mjs tests/Unit/ServerParityOperatorMetricsContractTest.mjs tests/Unit/ServerParityActivityLeaseRenewalContractTest.mjs tests/Unit/ServerParityRecordLocalActivityContractTest.mjs tests/Unit/ServerParityActivityCancellationAckContractTest.mjs tests/Unit/ServerParityServiceCatalogContractTest.mjs tests/Unit/ServerParityWorkflowImportContractTest.mjs tests/Unit/ServerParityWaterlineImportContractTest.mjs tests/Unit/ServerParitySystemMetricsContractTest.mjs tests/Unit/ServerParityOperatorDashboardTypesContractTest.mjs tests/Unit/ServerParityPrometheusMetricsContractTest.mjs tests/Unit/ServerParityPayloadCleanupContractTest.mjs tests/Unit/ServerParityRepairPassContractTest.mjs tests/Unit/ServerParityClusterInfoContractTest.mjs tests/Unit/ServerParityNamespaceDeleteContractTest.mjs tests/Unit/ServerParityMessageStreamsContractTest.mjs tests/Unit/ServerParityCancellationScopesContractTest.mjs tests/Unit/ServerParityWorkflowPollRequestContractTest.mjs tests/Unit/ServerParityUpdateValidationContractTest.mjs tests/Unit/ServerParityFixtureSelectionTest.mjs tests/Unit/ServerParityUpdateValidationRetryTest.mjs tests/Unit/ServerParityKilledAuthorPollTest.mjs tests/Unit/ServerParityImagePullTest.mjs tests/Unit/ServerParityWorkerSessionsContractTest.mjs tests/Unit/ServerParityWorkerSessionActivitiesContractTest.mjs tests/Unit/ServerParityNexusOperationsContractTest.mjs tests/Unit/ServerParityBridgeAdaptersContractTest.mjs tests/Unit/ServerParityWorkflowStreamWritesContractTest.mjs > parity-evidence/comparator-tests.txt docker run --rm --user=1000:1000 --cpus=1 --memory=256m --memory-swap=256m \ --network=none --entrypoint node -v "$PWD:/source:ro" -w /source \ "$image" --test tests/Unit/ServerParityCodecDeadlineLoadTest.mjs > parity-evidence/codec-load-tests.txt diff --git a/docs/rust-server-port.md b/docs/rust-server-port.md index 9fc053771..a82e4a1dc 100644 --- a/docs/rust-server-port.md +++ b/docs/rust-server-port.md @@ -82,13 +82,14 @@ Updated October 24, 2026. ([#737](https://github.com/durable-workflow/server/issues/737)). No expectation is edited to make native pass. -- **Parity surface.** Native serves 146 of the 175 API route methods in - published 2.11.1's route table, with message streams, update validation, - activity cancellation acknowledgement, worker sessions, workflow-task - cancellation scopes, caller-indexed Nexus operation reads, bridge adapter - webhooks, and embedded v2 and Waterline v1 workflow import. An execution - test sends each one through the native router, and the ones still - answering `501 rust_capability_not_implemented` must match +- **Parity surface.** Native serves 148 of the 175 API route methods in + published 2.11.1's route table, with message streams, durable workflow + stream writes, update validation, activity cancellation acknowledgement, + worker sessions, workflow-task cancellation scopes, caller-indexed Nexus + operation reads, bridge adapter webhooks, and embedded v2 and Waterline v1 + workflow import. An execution test sends each one through the native + router, and the ones still answering `501 rust_capability_not_implemented` + must match [`unserved-php-routes.v2.11.1.txt`](../rust/tests/fixtures/unserved-php-routes.v2.11.1.txt). Unlike published PHP, native's embedded v2 import refuses a bundle whose rows reference rows outside it, links only runs of one namespace, and reads @@ -160,7 +161,6 @@ any other unknown path or method answers PHP's own 404 or 405 ([checkpoint](rust-server-port-checkpoints/2026-10-23-message-streams.md)), except external payload references as input, stream retention cleanup and takeover of PHP data with streams. - - Durable workflow stream items and close. - Service endpoints. (The service catalog reads from Server 2.11 are served: native never holds catalog rows, so every list is PHP's empty page and every identity its not-found answer; the `service-catalog` fixture checks 37 cases. @@ -931,12 +931,38 @@ October 21, Waterline reads: - **Task queues.** `GET /api/task-queues` and `GET /api/task-queues/{name}` follow PHP's `TaskQueueController`: pollers, stats, current leases, repair candidates, recent task flow, activity attempt states and admission. -- **Durable streams.** The run's stream list, stream description and item - reads answer as PHP does, including the name pattern, the `from`, - `max_items` and `wait_seconds` rules and the refusal bodies. Native runs - never produce durable streams, and takeover refuses a database that holds - any, so every list is empty. Producer appends and closes are not - implemented. +- **Durable streams.** The run's stream list, stream description, item + reads, appends and closes answer as PHP does, with the + `workflow-stream-writes` fixture recorded from published 2.11.1: offsets + from 0, idempotent retries, bounded waits, final closes, whole-second times + and every refusal body. The shared contract fix of Server #774 is native + too: + - `max_pending_items` is at most 10000 and `retention_seconds` at most + 2592000, both refused with 422; + - a batch whose fresh items would pass the cap is refused whole, so + nothing of it is written (published 2.11.1 committed the items before the + cap; the fixture bounds that answer as a published defect); + - a closed stream past its retention reads as not found, append + included, and keeps its row until its run is pruned, so the name is + never opened again; + - each deadline is a row in the native-owned `dw_stream_reclaims` + (schema 6), indexed `(namespace, reclaim_at, stream_id)`. A bounded pass + reads due streams from that index, from the worker-heartbeat retention + pass and from a general `POST /api/system/retention/pass`. It deletes + items in batches under the transition lock and stops blocked when + items hold external references, which native does not delete yet; + - run retention deletes a pruned run's streams unless they hold external + references, and an append to a run whose details are pruned answers + 404 `stream_not_found`. + A continued run keeps its streams, as in PHP. Native differs in three + places, documented in `rust/src/runtime/workflow_streams.rs`: + - a batch whose items hold no item fields gets 400, not PHP's 500; + - a read at the head never clears the count of a later append; + - a runtime external payload reference object in `payload_reference` + gets 501. + Still missing: the workflow-side `workflow_stream` command, so cluster + info does not advertise `workflow_streams_contract`, and takeover still + refuses a PHP database that holds streams. - **Last progress.** Runs record `last_progress_at` when they start, as PHP does at start and child start. - **Deliberate differences.** diff --git a/rust/migrations/mariadb/0005_catalog.json b/rust/migrations/mariadb/0006_catalog.json similarity index 99% rename from rust/migrations/mariadb/0005_catalog.json rename to rust/migrations/mariadb/0006_catalog.json index 57d64e99a..737a3c180 100644 --- a/rust/migrations/mariadb/0005_catalog.json +++ b/rust/migrations/mariadb/0006_catalog.json @@ -19,6 +19,10 @@ "dw_server_schema", "CREATE TABLE `dw_server_schema` (\n `engine` varchar(64) NOT NULL,\n `version` bigint(20) NOT NULL,\n `bootstrap_checksum` varbinary(48) NOT NULL,\n PRIMARY KEY (`engine`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin" ], + [ + "dw_stream_reclaims", + "CREATE TABLE `dw_stream_reclaims` (\n `stream_id` bigint(20) unsigned NOT NULL,\n `namespace` varchar(128) NOT NULL,\n `workflow_run_id` varchar(26) NOT NULL,\n `reclaim_at` datetime(6) NOT NULL,\n PRIMARY KEY (`stream_id`),\n KEY `dw_stream_reclaims_due` (`namespace`,`reclaim_at`,`stream_id`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci" + ], [ "dw_task_completions", "CREATE TABLE `dw_task_completions` (\n `task_id` varchar(26) NOT NULL,\n `receipt` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL CHECK (json_valid(`receipt`)),\n PRIMARY KEY (`task_id`),\n CONSTRAINT `dw_task_completions_task` FOREIGN KEY (`task_id`) REFERENCES `workflow_tasks` (`id`) ON DELETE CASCADE\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci" diff --git a/rust/migrations/mariadb/0006_stream_reclaims.sql b/rust/migrations/mariadb/0006_stream_reclaims.sql new file mode 100644 index 000000000..2adaf3ba0 --- /dev/null +++ b/rust/migrations/mariadb/0006_stream_reclaims.sql @@ -0,0 +1,13 @@ +-- Native development bootstrap version 6: the reclaim deadline of each closed +-- or errored durable workflow stream (Server #774), closed_at plus its +-- retention. The stream retention pass reads a namespace's due streams in +-- deadline order from (namespace, reclaim_at, stream_id). A stream leaves the +-- table once its items are reclaimed; its row stays until its run is pruned. +CREATE TABLE dw_stream_reclaims ( + stream_id BIGINT UNSIGNED NOT NULL PRIMARY KEY, + namespace VARCHAR(128) NOT NULL, + workflow_run_id VARCHAR(26) NOT NULL, + reclaim_at DATETIME(6) NOT NULL, + KEY dw_stream_reclaims_due (namespace, reclaim_at, stream_id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +UPDATE dw_server_schema SET version=6 WHERE engine='rust-development' AND version=5; diff --git a/rust/migrations/mysql/0005_catalog.json b/rust/migrations/mysql/0006_catalog.json similarity index 99% rename from rust/migrations/mysql/0005_catalog.json rename to rust/migrations/mysql/0006_catalog.json index 4f67db5a6..b57c5f919 100644 --- a/rust/migrations/mysql/0005_catalog.json +++ b/rust/migrations/mysql/0006_catalog.json @@ -23,6 +23,10 @@ "dw_server_schema", "CREATE TABLE `dw_server_schema` (\n `engine` varchar(64) COLLATE utf8mb4_bin NOT NULL,\n `version` bigint NOT NULL,\n `bootstrap_checksum` varbinary(48) NOT NULL,\n PRIMARY KEY (`engine`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin" ], + [ + "dw_stream_reclaims", + "CREATE TABLE `dw_stream_reclaims` (\n `stream_id` bigint unsigned NOT NULL,\n `namespace` varchar(128) COLLATE utf8mb4_unicode_ci NOT NULL,\n `workflow_run_id` varchar(26) COLLATE utf8mb4_unicode_ci NOT NULL,\n `reclaim_at` datetime(6) NOT NULL,\n PRIMARY KEY (`stream_id`),\n KEY `dw_stream_reclaims_due` (`namespace`,`reclaim_at`,`stream_id`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci" + ], [ "dw_task_completions", "CREATE TABLE `dw_task_completions` (\n `task_id` varchar(26) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL,\n `receipt` json NOT NULL,\n PRIMARY KEY (`task_id`),\n CONSTRAINT `dw_task_completions_task` FOREIGN KEY (`task_id`) REFERENCES `workflow_tasks` (`id`) ON DELETE CASCADE\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci" diff --git a/rust/migrations/mysql/0006_stream_reclaims.sql b/rust/migrations/mysql/0006_stream_reclaims.sql new file mode 100644 index 000000000..2adaf3ba0 --- /dev/null +++ b/rust/migrations/mysql/0006_stream_reclaims.sql @@ -0,0 +1,13 @@ +-- Native development bootstrap version 6: the reclaim deadline of each closed +-- or errored durable workflow stream (Server #774), closed_at plus its +-- retention. The stream retention pass reads a namespace's due streams in +-- deadline order from (namespace, reclaim_at, stream_id). A stream leaves the +-- table once its items are reclaimed; its row stays until its run is pruned. +CREATE TABLE dw_stream_reclaims ( + stream_id BIGINT UNSIGNED NOT NULL PRIMARY KEY, + namespace VARCHAR(128) NOT NULL, + workflow_run_id VARCHAR(26) NOT NULL, + reclaim_at DATETIME(6) NOT NULL, + KEY dw_stream_reclaims_due (namespace, reclaim_at, stream_id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +UPDATE dw_server_schema SET version=6 WHERE engine='rust-development' AND version=5; diff --git a/rust/migrations/postgres/0005_catalog.json b/rust/migrations/postgres/0006_catalog.json similarity index 99% rename from rust/migrations/postgres/0005_catalog.json rename to rust/migrations/postgres/0006_catalog.json index 4dd16a521..38675b901 100644 --- a/rust/migrations/postgres/0005_catalog.json +++ b/rust/migrations/postgres/0006_catalog.json @@ -766,6 +766,58 @@ "generated": "" } ], + [ + "column", + "dw_stream_reclaims.1", + { + "collation": null, + "default": null, + "generated": "", + "identity": "", + "name": "stream_id", + "not_null": true, + "type": "bigint" + } + ], + [ + "column", + "dw_stream_reclaims.2", + { + "collation": "default", + "default": null, + "generated": "", + "identity": "", + "name": "namespace", + "not_null": true, + "type": "text" + } + ], + [ + "column", + "dw_stream_reclaims.3", + { + "collation": "default", + "default": null, + "generated": "", + "identity": "", + "name": "workflow_run_id", + "not_null": true, + "type": "text" + } + ], + [ + "column", + "dw_stream_reclaims.4", + { + "collation": null, + "default": null, + "generated": "", + "identity": "", + "name": "reclaim_at", + "not_null": true, + "type": "timestamp(6) without time zone" + } + ], [ "column", "dw_task_completions.1", @@ -11987,6 +12039,16 @@ "definition": "PRIMARY KEY (key)" } ], + [ + "constraint", + "dw_stream_reclaims.dw_stream_reclaims_pkey", + { + "deferrable": false, + "deferred": false, + "definition": "PRIMARY KEY (stream_id)", + "validated": true + } + ], [ "constraint", "dw_task_completions.dw_task_completions_pkey", @@ -13082,6 +13144,28 @@ "definition": "CREATE UNIQUE INDEX dw_query_cache_pkey ON public.dw_query_cache USING btree (key)" } ], + [ + "index", + "dw_stream_reclaims_due", + { + "definition": "CREATE INDEX dw_stream_reclaims_due ON public.dw_stream_reclaims USING btree (namespace, reclaim_at, stream_id)", + "primary": false, + "ready": true, + "unique": false, + "valid": true + } + ], + [ + "index", + "dw_stream_reclaims_pkey", + { + "definition": "CREATE UNIQUE INDEX dw_stream_reclaims_pkey ON public.dw_stream_reclaims USING btree (stream_id)", + "primary": true, + "ready": true, + "unique": true, + "valid": true + } + ], [ "index", "dw_task_completions_pkey", @@ -17308,6 +17392,21 @@ "force_row_security": false } ], + [ + "relation", + "dw_stream_reclaims", + { + "access_method": "heap", + "force_row_security": false, + "kind": "r", + "options": null, + "partition": false, + "partition_key": null, + "persistence": "p", + "row_security": false, + "view": null + } + ], [ "relation", "dw_task_completions", diff --git a/rust/migrations/postgres/0006_stream_reclaims.sql b/rust/migrations/postgres/0006_stream_reclaims.sql new file mode 100644 index 000000000..bf7114bdc --- /dev/null +++ b/rust/migrations/postgres/0006_stream_reclaims.sql @@ -0,0 +1,13 @@ +-- Native development bootstrap version 6: the reclaim deadline of each closed +-- or errored durable workflow stream (Server #774), closed_at plus its +-- retention. The stream retention pass reads a namespace's due streams in +-- deadline order from (namespace, reclaim_at, stream_id). A stream leaves the +-- table once its items are reclaimed; its row stays until its run is pruned. +CREATE TABLE dw_stream_reclaims ( + stream_id bigint NOT NULL PRIMARY KEY, + namespace text NOT NULL, + workflow_run_id text NOT NULL, + reclaim_at timestamp(6) without time zone NOT NULL +); +CREATE INDEX dw_stream_reclaims_due ON dw_stream_reclaims (namespace, reclaim_at, stream_id); +UPDATE dw_server_schema SET version=6 WHERE engine='rust-development' AND version=5; diff --git a/rust/migrations/sqlite/0006_stream_reclaims.sql b/rust/migrations/sqlite/0006_stream_reclaims.sql new file mode 100644 index 000000000..481e73c9f --- /dev/null +++ b/rust/migrations/sqlite/0006_stream_reclaims.sql @@ -0,0 +1,13 @@ +-- Native development bootstrap version 6: the reclaim deadline of each closed +-- or errored durable workflow stream (Server #774), closed_at plus its +-- retention. The stream retention pass reads a namespace's due streams in +-- deadline order from (namespace, reclaim_at, stream_id). A stream leaves the +-- table once its items are reclaimed; its row stays until its run is pruned. +CREATE TABLE dw_stream_reclaims ( + stream_id INTEGER NOT NULL PRIMARY KEY, + namespace TEXT NOT NULL, + workflow_run_id TEXT NOT NULL, + reclaim_at TEXT NOT NULL +); +CREATE INDEX dw_stream_reclaims_due ON dw_stream_reclaims (namespace, reclaim_at, stream_id); +UPDATE dw_server_schema SET version=6 WHERE engine='rust-development' AND version=5; diff --git a/rust/src/runtime/activity_failures.rs b/rust/src/runtime/activity_failures.rs index 61b89964c..0e89d3084 100644 --- a/rust/src/runtime/activity_failures.rs +++ b/rust/src/runtime/activity_failures.rs @@ -219,7 +219,7 @@ fn resolve_details(details: &Value) -> Result<(Value, Option<&'static str>)> { /// A PHP array round trip: an object keyed 0..n, including an empty one, /// comes back as a list. -fn php_array(value: Value) -> Value { +pub(super) fn php_array(value: Value) -> Value { match value { Value::Array(items) => Value::Array(items.into_iter().map(php_array).collect()), Value::Object(fields) => { diff --git a/rust/src/runtime/backend.rs b/rust/src/runtime/backend.rs index 1229d146c..6cd8306a9 100644 --- a/rust/src/runtime/backend.rs +++ b/rust/src/runtime/backend.rs @@ -82,6 +82,20 @@ pub(super) trait Backend: Database { const SEARCH_ATTRIBUTE_DELETE: &str = "DELETE FROM workflow_search_attributes WHERE workflow_run_id=$1 AND \"key\"=$2"; const SEARCH_ATTRIBUTE_KEY_ROWS: &str = "SELECT a.workflow_run_id,a.\"key\" AS attribute_key,a.type,a.value_string,a.value_keyword,a.value_int,a.value_float,CASE WHEN a.value_bool IS NULL THEN NULL WHEN a.value_bool THEN 1 ELSE 0 END AS value_bool,a.value_datetime,a.value_keyword_list FROM workflow_search_attributes a JOIN workflow_runs r ON r.id=a.workflow_run_id WHERE r.namespace=$1 AND a.\"key\"=$2"; const SEARCH_ATTRIBUTE_INHERIT: &str = "INSERT INTO workflow_search_attributes(workflow_run_id,workflow_instance_id,\"key\",type,value_string,value_keyword,value_keyword_list,value_int,value_float,value_bool,value_datetime,upserted_at_sequence,inherited_from_parent,created_at,updated_at) SELECT $1,$2,\"key\",type,value_string,value_keyword,value_keyword_list,value_int,value_float,value_bool,value_datetime,1,true,$3,$4 FROM workflow_search_attributes WHERE workflow_run_id=$5"; + // Durable workflow stream items; `offset` is a reserved word. + const STREAM_ITEM_BY_KEY: &str = "SELECT \"offset\" FROM workflow_durable_stream_items WHERE stream_id=$1 AND idempotency_key=$2 LIMIT 1"; + const STREAM_ITEM_INSERT: &str = "INSERT INTO workflow_durable_stream_items(stream_id,namespace,workflow_run_id,stream_name,\"offset\",idempotency_key,origin,origin_reference,payload,payload_reference,payload_codec,item_type,content_type,emitted_at,created_at,updated_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16)"; + const STREAM_ITEMS_WINDOW: &str = "SELECT * FROM workflow_durable_stream_items WHERE stream_id=$1 AND \"offset\">=$2 ORDER BY \"offset\" LIMIT $3"; + const STREAM_SWEEP_BOUNDARY: &str = "SELECT \"offset\" FROM workflow_durable_stream_items WHERE stream_id=$1 ORDER BY \"offset\" LIMIT 1 OFFSET $2"; + const STREAM_SWEEP_THROUGH: &str = "DELETE FROM workflow_durable_stream_items WHERE stream_id=$1 AND \"offset\"<=$2"; + // Durable stream items, and streams, holding an external payload + // reference, which native does not settle (Server #774). PostgreSQL + // stores these payloads as `json`, matched through its text. + const STREAM_ITEM_REFERENCES: &str = concat!("SELECT COUNT(*) FROM workflow_durable_stream_items WHERE stream_id=$1 AND (payload_reference IS NOT NULL OR ", super::retention::reference!("payload::text"), ")"); + const RUN_STREAM_REFERENCES: &str = concat!( + "SELECT (SELECT COUNT(*) FROM workflow_durable_stream_items WHERE workflow_run_id=$1 AND (payload_reference IS NOT NULL OR ", super::retention::reference!("payload::text"), "))", + "+(SELECT COUNT(*) FROM workflow_durable_streams WHERE workflow_run_id=$2 AND ", super::retention::reference!("metadata::text"), ")" + ); // `duplicateBridgeCommand`: the latest accepted command PHP recorded for // this adapter, action, key and target name ($6 names the target field), // filtered in the database as PHP's JSON-path query is. PHP's bridge @@ -197,6 +211,11 @@ WHERE r.status IN ('pending','running','waiting') ORDER BY t.available_order,t.id LIMIT 100 "#; const SELECTION_RESOLUTION_SQL: &str = "SELECT id,payload FROM workflow_history_events WHERE workflow_run_id=$1 AND event_type='SelectionResolved' AND json_extract(payload,'$.selection_group_id')=$2 ORDER BY sequence LIMIT 1"; + const STREAM_ITEM_REFERENCES: &str = concat!("SELECT COUNT(*) FROM workflow_durable_stream_items WHERE stream_id=$1 AND (payload_reference IS NOT NULL OR ", super::retention::reference!("payload"), ")"); + const RUN_STREAM_REFERENCES: &str = concat!( + "SELECT (SELECT COUNT(*) FROM workflow_durable_stream_items WHERE workflow_run_id=$1 AND (payload_reference IS NOT NULL OR ", super::retention::reference!("payload"), "))", + "+(SELECT COUNT(*) FROM workflow_durable_streams WHERE workflow_run_id=$2 AND ", super::retention::reference!("metadata"), ")" + ); const BRIDGE_COMMAND: &str = "SELECT workflow_run_id,resolved_workflow_run_id FROM workflow_commands WHERE workflow_instance_id=$1 AND command_type=$2 AND status='accepted' AND json_extract(context,'$.server.metadata.adapter')=$3 AND json_extract(context,'$.server.metadata.action')=$4 AND json_extract(context,'$.server.metadata.idempotency_key')=$5 AND json_extract(context,'$.server.metadata.'||$6)=$7 AND (request_id IS NULL OR request_id=$8) ORDER BY accepted_at DESC,command_sequence DESC LIMIT 1"; const REPAIR_COUNTER_MAX: i64 = i64::MAX; const RANGE_REPAIR_TASK_SQL: &str = "SELECT * FROM workflow_tasks WHERE workflow_run_id=$1 AND namespace=$2 AND task_type='workflow' AND status='failed' AND json_extract(payload,'$.replay_blocked')=1 ORDER BY CASE WHEN available_at IS NULL THEN 1 ELSE 0 END,available_at,created_at,id LIMIT 1"; @@ -447,6 +466,16 @@ impl Backend for MySql { const SEARCH_ATTRIBUTE_DELETE: &str = "DELETE FROM workflow_search_attributes WHERE workflow_run_id=$1 AND `key`=$2"; const SEARCH_ATTRIBUTE_KEY_ROWS: &str = "SELECT a.workflow_run_id,a.`key` AS attribute_key,a.type,a.value_string,a.value_keyword,a.value_int,a.value_float,CASE WHEN a.value_bool IS NULL THEN NULL WHEN a.value_bool THEN 1 ELSE 0 END AS value_bool,a.value_datetime,a.value_keyword_list FROM workflow_search_attributes a JOIN workflow_runs r ON r.id=a.workflow_run_id WHERE r.namespace=$1 AND a.`key`=$2"; const SEARCH_ATTRIBUTE_INHERIT: &str = "INSERT INTO workflow_search_attributes(workflow_run_id,workflow_instance_id,`key`,type,value_string,value_keyword,value_keyword_list,value_int,value_float,value_bool,value_datetime,upserted_at_sequence,inherited_from_parent,created_at,updated_at) SELECT $1,$2,`key`,type,value_string,value_keyword,value_keyword_list,value_int,value_float,value_bool,value_datetime,1,true,$3,$4 FROM workflow_search_attributes WHERE workflow_run_id=$5"; + const STREAM_ITEM_BY_KEY: &str = "SELECT `offset` FROM workflow_durable_stream_items WHERE stream_id=$1 AND idempotency_key=$2 LIMIT 1"; + const STREAM_ITEM_INSERT: &str = "INSERT INTO workflow_durable_stream_items(stream_id,namespace,workflow_run_id,stream_name,`offset`,idempotency_key,origin,origin_reference,payload,payload_reference,payload_codec,item_type,content_type,emitted_at,created_at,updated_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16)"; + const STREAM_ITEMS_WINDOW: &str = "SELECT * FROM workflow_durable_stream_items WHERE stream_id=$1 AND `offset`>=$2 ORDER BY `offset` LIMIT $3"; + const STREAM_SWEEP_BOUNDARY: &str = "SELECT `offset` FROM workflow_durable_stream_items WHERE stream_id=$1 ORDER BY `offset` LIMIT 1 OFFSET $2"; + const STREAM_SWEEP_THROUGH: &str = "DELETE FROM workflow_durable_stream_items WHERE stream_id=$1 AND `offset`<=$2"; + const STREAM_ITEM_REFERENCES: &str = concat!("SELECT COUNT(*) FROM workflow_durable_stream_items WHERE stream_id=$1 AND (payload_reference IS NOT NULL OR ", super::retention::reference!("payload"), ")"); + const RUN_STREAM_REFERENCES: &str = concat!( + "SELECT (SELECT COUNT(*) FROM workflow_durable_stream_items WHERE workflow_run_id=$1 AND (payload_reference IS NOT NULL OR ", super::retention::reference!("payload"), "))", + "+(SELECT COUNT(*) FROM workflow_durable_streams WHERE workflow_run_id=$2 AND ", super::retention::reference!("metadata"), ")" + ); const READ_SNAPSHOT_BEGIN: &str = "START TRANSACTION WITH CONSISTENT SNAPSHOT, READ ONLY"; const ACTIVITY_METADATA_HISTORY: &str = "SELECT h.sequence,h.event_type,CASE WHEN page.projection_bytes<=8388608 THEN h.payload ELSE NULL END AS payload,h.workflow_task_id,h.recorded_at,page.projection_count FROM (SELECT id,sequence,COUNT(*) OVER () AS projection_count,SUM(payload_bytes) OVER (ORDER BY sequence) AS projection_bytes FROM (SELECT id,sequence,octet_length(CAST(payload AS CHAR CHARACTER SET utf8mb4)) AS payload_bytes FROM workflow_history_events FORCE INDEX (workflow_history_events_workflow_run_id_sequence_unique) WHERE workflow_run_id=$1 AND event_type IN ('ActivityScheduled','ActivityStarted','ActivityHeartbeatRecorded','ActivityRetryScheduled','ActivityCompleted','ActivityFailed','ActivityTimedOut','ActivityCancelled')) AS sizes ORDER BY sequence LIMIT 10001) AS page JOIN workflow_history_events h ON h.id=page.id ORDER BY page.sequence"; const PHP_CONNECTION: &str = "mysql"; diff --git a/rust/src/runtime/continuations.rs b/rust/src/runtime/continuations.rs index 84097ff43..b0867117b 100644 --- a/rust/src/runtime/continuations.rs +++ b/rust/src/runtime/continuations.rs @@ -112,13 +112,10 @@ where // timers, children and waits stay with the closed predecessor, and no // parent-close policy applies. Message streams pass to the next run // when the completion records its stream acknowledgements. Durable - // output streams, service calls and live queries are not transferred - // by native yet. + // output streams are rows of their run and stay with the predecessor; + // PHP's continue-as-new does not touch them either. + // Service calls and live queries are not transferred by native yet. for (sql, selected) in [ - ( - "SELECT COUNT(*) FROM workflow_durable_streams WHERE workflow_instance_id=$1", - workflow.as_str(), - ), ( "SELECT COUNT(*) FROM workflow_service_calls WHERE caller_workflow_run_id=$1", old.as_str(), diff --git a/rust/src/runtime/execution.rs b/rust/src/runtime/execution.rs index 9b9887e05..1aa083fe4 100644 --- a/rust/src/runtime/execution.rs +++ b/rust/src/runtime/execution.rs @@ -911,6 +911,24 @@ impl Runtime { } delegate!(run_in_namespace(workflow_id: &str, run_id: &str) -> Result); + delegate!(stream_run(workflow_id: &str, run_id: &str) -> Result>); + delegate!(list_workflow_streams(run_id: &str) -> Result>); + delegate!(show_workflow_stream(run_id: &str, name: &str) -> Result>); + delegate!(append_workflow_stream( + workflow_id: &str, + run_id: &str, + name: &str, + items: Vec, + max_pending: i64, + ) -> Result); + delegate!(read_workflow_stream(run_id: &str, name: &str, from: i64, max_items: i64) + -> Result>); + delegate!(close_workflow_stream( + run_id: &str, + name: &str, + error_reason: Option, + retention_seconds: Option, + ) -> Result>); pub(crate) async fn run_diagnostics( &self, workflow_id: &str, diff --git a/rust/src/runtime/http.rs b/rust/src/runtime/http.rs index 667cc8622..ad36fc574 100644 --- a/rust/src/runtime/http.rs +++ b/rust/src/runtime/http.rs @@ -145,7 +145,11 @@ pub fn router(runtime: Runtime) -> Router { ) .route( "/api/workflows/{workflow_id}/runs/{run_id}/streams/{stream_name}/items", - get(super::workflow_streams::items), + get(super::workflow_streams::items).post(super::workflow_streams::append), + ) + .route( + "/api/workflows/{workflow_id}/runs/{run_id}/streams/{stream_name}/close", + post(super::workflow_streams::close), ) .route( "/api/workflows/{workflow_id}/request-cancellation", @@ -780,9 +784,11 @@ fn workflow_route(rest: &str) -> Option { | [_, "runs", _, "history", "export"] | [_, "runs", _, "streams"] | [_, "runs", _, "streams", _] - | [_, "runs", _, "streams", _, "items"] | [_, "nexus-operations"] | [_, "runs", _, "nexus-operations"] => Some(READ), + // A run's durable streams are rows in the request's namespace. + [_, "runs", _, "streams", _, "items"] => Some(READ_WRITE), + [_, "runs", _, "streams", _, "close"] => Some(WRITE), [_, "signal" | "query" | "update", _] | [_, "runs", _, "signal" | "query" | "update", _] | [_, "repair" | "request-cancellation"] diff --git a/rust/src/runtime/mysql.rs b/rust/src/runtime/mysql.rs index 53d7ae4d8..81bc63261 100644 --- a/rust/src/runtime/mysql.rs +++ b/rust/src/runtime/mysql.rs @@ -9,7 +9,7 @@ use sqlx::{ }; use std::{collections::BTreeMap, sync::OnceLock, time::Duration}; -pub const VERSION: i64 = 5; +pub const VERSION: i64 = 6; type Catalog = Vec<(String, String)>; pub const MARKER_SQL: &str = "CREATE TABLE dw_server_schema (engine VARCHAR(64) NOT NULL PRIMARY KEY,version BIGINT NOT NULL,bootstrap_checksum VARBINARY(48) NOT NULL) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin"; @@ -46,7 +46,9 @@ impl Flavor { "\n", include_str!("../../migrations/mysql/0004_worker_incarnations.sql"), "\n", - include_str!("../../migrations/mysql/0005_worker_session_holders.sql") + include_str!("../../migrations/mysql/0005_worker_session_holders.sql"), + "\n", + include_str!("../../migrations/mysql/0006_stream_reclaims.sql") ), ), Self::MariaDb => ( @@ -56,14 +58,16 @@ impl Flavor { "\n", include_str!("../../migrations/mariadb/0004_worker_incarnations.sql"), "\n", - include_str!("../../migrations/mariadb/0005_worker_session_holders.sql") + include_str!("../../migrations/mariadb/0005_worker_session_holders.sql"), + "\n", + include_str!("../../migrations/mariadb/0006_stream_reclaims.sql") ), ), }; slot.get_or_init(|| { let mut migrations = Migrator::with_migrations(vec![Migration::new( VERSION, - "full frozen PHP schema, native worker incarnations and session holders".into(), + "full frozen PHP schema, native worker incarnations, session holders and stream reclaim deadlines".into(), MigrationType::Simple, sql.into_sql_str(), false, @@ -76,8 +80,8 @@ impl Flavor { } fn expected(self) -> Result { Ok(serde_json::from_str(&canonical(match self { - Self::MySql => include_str!("../../migrations/mysql/0005_catalog.json"), - Self::MariaDb => include_str!("../../migrations/mariadb/0005_catalog.json"), + Self::MySql => include_str!("../../migrations/mysql/0006_catalog.json"), + Self::MariaDb => include_str!("../../migrations/mariadb/0006_catalog.json"), }))?) } /// The single native migration, which a takeover records as applied. @@ -94,7 +98,9 @@ impl Flavor { concat!( include_str!("../../migrations/mysql/0004_worker_incarnations.sql"), "\n", - include_str!("../../migrations/mysql/0005_worker_session_holders.sql") + include_str!("../../migrations/mysql/0005_worker_session_holders.sql"), + "\n", + include_str!("../../migrations/mysql/0006_stream_reclaims.sql") ), ), Self::MariaDb => ( @@ -102,7 +108,9 @@ impl Flavor { concat!( include_str!("../../migrations/mariadb/0004_worker_incarnations.sql"), "\n", - include_str!("../../migrations/mariadb/0005_worker_session_holders.sql") + include_str!("../../migrations/mariadb/0005_worker_session_holders.sql"), + "\n", + include_str!("../../migrations/mariadb/0006_stream_reclaims.sql") ), ), }; @@ -118,6 +126,10 @@ impl Flavor { .replace( "CREATE TABLE dw_worker_session_holders", "CREATE TABLE IF NOT EXISTS dw_worker_session_holders", + ) + .replace( + "CREATE TABLE dw_stream_reclaims", + "CREATE TABLE IF NOT EXISTS dw_stream_reclaims", )) } pub(super) fn checksum(self) -> &'static [u8] { diff --git a/rust/src/runtime/namespace_cleanup.rs b/rust/src/runtime/namespace_cleanup.rs index 780382dc0..40aeb114b 100644 --- a/rust/src/runtime/namespace_cleanup.rs +++ b/rust/src/runtime/namespace_cleanup.rs @@ -149,6 +149,7 @@ const NATIVE_CLEANUP: &[&str] = &[ "DELETE FROM dw_poll_receipts WHERE namespace=$1", "DELETE FROM dw_worker_registration_incarnations WHERE namespace=$1", "DELETE FROM dw_worker_session_holders WHERE namespace=$1", + "DELETE FROM dw_stream_reclaims WHERE namespace=$1", ]; /// Registered external payloads and payload columns holding a reference. diff --git a/rust/src/runtime/operator_metrics.rs b/rust/src/runtime/operator_metrics.rs index 8691494cd..815835b3c 100644 --- a/rust/src/runtime/operator_metrics.rs +++ b/rust/src/runtime/operator_metrics.rs @@ -1234,6 +1234,15 @@ where /// `POST /api/system/retention/pass`: the report, and whether any run failed. pub(super) async fn retention_pass(&self, limit: i64, run_ids: Vec) -> Result<(Value, bool)> { + // A general pass also reclaims closed workflow streams past their own + // retention (Server #774), at most `limit` streams and `limit` * 100 + // items. The report is the run pass's; a stream failure leaves its + // items for the next pass and does not stop the run pass. + if run_ids.is_empty() + && let Err(error) = self.prune_expired_streams(limit, limit * 100).await + { + eprintln!("workflow_stream_retention_failed: {error}"); + } let policy = self.retention_policy().await?; let report = self.prune_runs(&policy, limit, run_ids).await?; let failed = report["failed"].as_i64().is_some_and(|failed| failed > 0); @@ -1280,6 +1289,15 @@ where /// `runInlinePass`, reported on the worker heartbeat. pub(super) async fn retention_inline_pass(&self) -> Result { + // Closed workflow streams carry their own retention, whatever the + // history retention mode (Server #774); the stream pass has its own + // throttle and is not part of this report. + if retention::inline_stream_pass_due(self.namespace.as_ref()) { + let (streams, items) = super::workflow_streams::INLINE_SWEEP; + if let Err(error) = self.prune_expired_streams(streams, items).await { + eprintln!("workflow_stream_retention_failed: {error}"); + } + } let policy = self.retention_policy().await?; if policy.forever { return Ok(retention::inline_report(false, None)); @@ -1359,6 +1377,12 @@ where } deleted.insert((*key).into(), json!(DB::affected(query.execute(&mut *tx).await?))); } + for (key, sql) in retention::DURABLE_STREAMS { + let affected = DB::affected(Self::query(sql).bind(run_id).execute(&mut *tx).await?); + if let Some(key) = key { + deleted.insert((*key).into(), json!(affected)); + } + } // The run's history is gone; no cached read of it may survive. tx.scope_history = None; Self::query("UPDATE workflow_runs SET details_pruned_at=$1 WHERE id=$2 AND details_pruned_at IS NULL") @@ -1386,6 +1410,9 @@ where return Ok(Some(reason)); } } + if Self::scalar(DB::RUN_STREAM_REFERENCES).bind(run_id).bind(run_id).fetch_one(&self.pool).await? > 0 { + return Ok(Some("external_payload_cleanup_unavailable")); + } Ok(None) } diff --git a/rust/src/runtime/postgres.rs b/rust/src/runtime/postgres.rs index fc3077fd6..66553bb59 100644 --- a/rust/src/runtime/postgres.rs +++ b/rust/src/runtime/postgres.rs @@ -13,7 +13,7 @@ use sqlx::{ use super::{Result, refuse}; -pub const VERSION: i64 = 5; +pub const VERSION: i64 = 6; pub const CATALOG_QUERY: &str = include_str!("postgres-catalog.sql"); type Catalog = Vec<(String, String, Json)>; @@ -22,14 +22,16 @@ fn migrator() -> &'static Migrator { MIGRATOR.get_or_init(|| { let mut migrations = Migrator::with_migrations(vec![Migration::new( VERSION, - "full frozen PHP PostgreSQL schema, native worker incarnations and session holders".into(), + "full frozen PHP PostgreSQL schema, native worker incarnations, session holders and stream reclaim deadlines".into(), MigrationType::Simple, concat!( include_str!("../../migrations/postgres/0003_full_schema.sql"), "\n", include_str!("../../migrations/postgres/0004_worker_incarnations.sql"), "\n", - include_str!("../../migrations/postgres/0005_worker_session_holders.sql") + include_str!("../../migrations/postgres/0005_worker_session_holders.sql"), + "\n", + include_str!("../../migrations/postgres/0006_stream_reclaims.sql") ) .into_sql_str(), false, @@ -57,7 +59,9 @@ pub(super) fn native_relations_sql() -> Result { concat!( include_str!("../../migrations/postgres/0004_worker_incarnations.sql"), "\n", - include_str!("../../migrations/postgres/0005_worker_session_holders.sql") + include_str!("../../migrations/postgres/0005_worker_session_holders.sql"), + "\n", + include_str!("../../migrations/postgres/0006_stream_reclaims.sql") ), ) } @@ -91,7 +95,7 @@ pub(super) async fn php_source_catalog(connection: &mut PgConnection) -> Result< )); } let native: Catalog = - serde_json::from_str(include_str!("../../migrations/postgres/0005_catalog.json"))?; + serde_json::from_str(include_str!("../../migrations/postgres/0006_catalog.json"))?; let expected: Catalog = native .into_iter() .filter(|(kind, name, _)| { @@ -186,7 +190,7 @@ pub(super) async fn verify(connection: &mut PgConnection) -> Result<()> { // from SQL text. It retains types, precision, defaults, indexes, foreign // keys and sequence ownership while excluding live rows/sequence values. let expected: Catalog = - serde_json::from_str(include_str!("../../migrations/postgres/0005_catalog.json"))?; + serde_json::from_str(include_str!("../../migrations/postgres/0006_catalog.json"))?; let actual = catalog(connection).await?; if actual != expected { return Err(refuse( @@ -398,7 +402,7 @@ mod tests { #[test] fn compiled_catalog_preserves_long_object_identities_and_int64_limits() { let rows: Catalog = - serde_json::from_str(include_str!("../../migrations/postgres/0005_catalog.json")) + serde_json::from_str(include_str!("../../migrations/postgres/0006_catalog.json")) .unwrap(); let identities: std::collections::BTreeSet<_> = rows.iter().map(|(kind, name, _)| (kind, name)).collect(); diff --git a/rust/src/runtime/retention.rs b/rust/src/runtime/retention.rs index b1a8e1047..3053e0c8b 100644 --- a/rust/src/runtime/retention.rs +++ b/rust/src/runtime/retention.rs @@ -73,17 +73,28 @@ const REFERENCES_SQL: &str = concat!( "+(SELECT COUNT(*) FROM activity_executions WHERE workflow_run_id=$5 AND (", reference!("arguments"), " OR ", reference!("result"), "))", ); +/// The run's durable workflow streams, deleted with it as PHP's +/// `HistoryRetentionEnforcer::pruneRun` deletes them (Server #774): report +/// key, if any, and statement binding the run id once. +pub(super) const DURABLE_STREAMS: &[(Option<&str>, &str)] = &[ + (Some("durable_stream_items_deleted"), "DELETE FROM workflow_durable_stream_items WHERE workflow_run_id=$1"), + (None, "DELETE FROM dw_stream_reclaims WHERE workflow_run_id=$1"), + (Some("durable_streams_deleted"), "DELETE FROM workflow_durable_streams WHERE workflow_run_id=$1"), +]; + /// `MessageStreamRetentionCleanup::apply`'s counts in the prune report. pub(super) const STREAM_COUNTS: [&str; 3] = ["message_streams_deleted", "message_stream_items_deleted", "message_stream_items_compacted"]; /// Counts that hold a run's details, the run-id binds of each, and the /// skip reason. An abandoned activity's hold is read from history. A run -/// whose instance has inbound message streams is held too: native does not -/// compact or remove stream state as PHP's MessageStreamRetentionCleanup -/// does yet, so it keeps the run rather than orphan the streams. +/// with messages or whose instance has inbound message streams is held too: +/// native does not compact or remove that state as PHP's +/// MessageStreamRetentionCleanup does yet, so it keeps the run rather than +/// orphan it. Durable workflow streams go with the run (`DURABLE_STREAMS`) +/// unless they hold an external reference (`Backend::RUN_STREAM_REFERENCES`). pub(super) const HOLDS: &[(&str, usize, &str)] = &[ - ("SELECT (SELECT COUNT(*) FROM workflow_durable_streams WHERE workflow_run_id=$1)+(SELECT COUNT(*) FROM workflow_messages WHERE workflow_run_id=$2)+(SELECT COUNT(*) FROM workflow_inbound_streams WHERE workflow_instance_id IN (SELECT workflow_instance_id FROM workflow_runs WHERE id=$3))", 3, "message_stream_cleanup_unavailable"), + ("SELECT (SELECT COUNT(*) FROM workflow_messages WHERE workflow_run_id=$1)+(SELECT COUNT(*) FROM workflow_inbound_streams WHERE workflow_instance_id IN (SELECT workflow_instance_id FROM workflow_runs WHERE id=$2))", 2, "message_stream_cleanup_unavailable"), (REFERENCES_SQL, 5, "external_payload_cleanup_unavailable"), ]; @@ -229,6 +240,21 @@ pub(super) fn inline_pass_due(namespace: &str) -> bool { } } +/// `WorkflowStreamRetention::runInlinePass`'s throttle: at most one stream +/// pass a minute per namespace in this process. +pub(super) fn inline_stream_pass_due(namespace: &str) -> bool { + static LAST: OnceLock>> = OnceLock::new(); + let mut last = LAST.get_or_init(Default::default).lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let now = Instant::now(); + match last.get(namespace) { + Some(at) if now.duration_since(*at).as_secs() < INLINE_THROTTLE_SECONDS => false, + _ => { + last.insert(namespace.to_owned(), now); + true + } + } +} + /// `runInlinePass`'s heartbeat report. pub(super) fn inline_report(throttled: bool, report: Option<&Map>) -> Value { let count = |key: &str| report.and_then(|report| report.get(key)).cloned().unwrap_or(json!(0)); diff --git a/rust/src/runtime/schema.rs b/rust/src/runtime/schema.rs index 55db13331..62fd112b4 100644 --- a/rust/src/runtime/schema.rs +++ b/rust/src/runtime/schema.rs @@ -11,21 +11,23 @@ use tokio::sync::OnceCell; use super::{Result, refuse}; -pub(super) const VERSION: i64 = 5; +pub(super) const VERSION: i64 = 6; fn migrator() -> &'static Migrator { static MIGRATOR: OnceLock = OnceLock::new(); MIGRATOR.get_or_init(|| { Migrator::with_migrations(vec![Migration::new( VERSION, - "full frozen PHP schema, native worker incarnations and session holders".into(), + "full frozen PHP schema, native worker incarnations, session holders and stream reclaim deadlines".into(), MigrationType::Simple, concat!( include_str!("../../migrations/sqlite/0003_full_schema.sql"), "\n", include_str!("../../migrations/sqlite/0004_worker_incarnations.sql"), "\n", - include_str!("../../migrations/sqlite/0005_worker_session_holders.sql") + include_str!("../../migrations/sqlite/0005_worker_session_holders.sql"), + "\n", + include_str!("../../migrations/sqlite/0006_stream_reclaims.sql") ) .into_sql_str(), false, @@ -43,7 +45,9 @@ pub(super) fn native_relations_sql() -> Result { concat!( include_str!("../../migrations/sqlite/0004_worker_incarnations.sql"), "\n", - include_str!("../../migrations/sqlite/0005_worker_session_holders.sql") + include_str!("../../migrations/sqlite/0005_worker_session_holders.sql"), + "\n", + include_str!("../../migrations/sqlite/0006_stream_reclaims.sql") ), ) } diff --git a/rust/src/runtime/upgrade.rs b/rust/src/runtime/upgrade.rs index 612e0977c..eabf0db5a 100644 --- a/rust/src/runtime/upgrade.rs +++ b/rust/src/runtime/upgrade.rs @@ -106,6 +106,7 @@ pub(super) fn native_relation(name: &str) -> bool { | "dw_query_cache" | "dw_worker_registration_incarnations" | "dw_worker_session_holders" + | "dw_stream_reclaims" ) } @@ -122,6 +123,8 @@ pub(super) fn native_object(name: &str) -> bool { | "dw_worker_incarnation_authority" | "dw_worker_incarnation_retention" | "dw_worker_session_holders_pkey" + | "dw_stream_reclaims_pkey" + | "dw_stream_reclaims_due" ) } diff --git a/rust/src/runtime/upgrade_tests.rs b/rust/src/runtime/upgrade_tests.rs index a76db6497..c2953c883 100644 --- a/rust/src/runtime/upgrade_tests.rs +++ b/rust/src/runtime/upgrade_tests.rs @@ -27,12 +27,15 @@ async fn php_layout() -> (tempfile::TempDir, String) { "\n", include_str!("../../migrations/sqlite/0004_worker_incarnations.sql"), "\n", - include_str!("../../migrations/sqlite/0005_worker_session_holders.sql") + include_str!("../../migrations/sqlite/0005_worker_session_holders.sql"), + "\n", + include_str!("../../migrations/sqlite/0006_stream_reclaims.sql") )) .execute(&mut connection) .await .unwrap(); for table in [ + "dw_stream_reclaims", "dw_worker_session_holders", "dw_worker_registration_incarnations", "dw_task_completions", diff --git a/rust/src/runtime/workflow_streams.rs b/rust/src/runtime/workflow_streams.rs index efe6bc0d4..a9c69e59f 100644 --- a/rust/src/runtime/workflow_streams.rs +++ b/rust/src/runtime/workflow_streams.rs @@ -1,18 +1,161 @@ -//! Durable workflow stream reads as published PHP's `WorkflowStreamController` -//! answers them. Native runs never produce durable streams, and takeover -//! refuses a PHP database that holds any, so every run has none: the list is -//! empty, a valid name is not found and an invalid one is refused. Producer -//! appends and closes are not implemented. +//! Durable workflow streams as published PHP's `WorkflowStreamController` +//! and `WorkflowStreamService` serve them. A producer appends ordered items +//! to a named, run-scoped stream, which the first append opens. Offsets +//! start at 0, a repeated idempotency key returns the item it already +//! names, and a batch whose fresh items would pass the pending-items cap is +//! refused whole with 429 (Server #774). A close records a clean or errored +//! end once. Readers page by offset and may wait, and a read that reaches +//! the head resets `pending_items`. A closed stream past its retention +//! reads as not found, and a bounded pass reclaims its rows. +//! +//! PHP's models write these times in Eloquent's `Y-m-d H:i:s` format, so +//! every stored time is a whole second. Native differs in three places: +//! - An append whose items each hold none of the item fields fails in PHP, +//! whose validated data then has no `items` (an undefined index, 500). +//! Native refuses it with the service's own 400. A batch with some such +//! items drops them, as PHP does. +//! - A read that reaches the head resets `pending_items` only while the +//! stream is still at that head. PHP's unlocked save can also clear the +//! count of an append committed after its read. +//! - A runtime external payload reference object in `payload_reference` is +//! refused with 501; PHP resolves and claims it. Other values are refused +//! as PHP's transport refuses them. +use super::laravel_rules::{self, Rule}; use super::laravel_validation::{failure, integer, normalized}; -use super::{Result, Runtime, RuntimeError, backend::Backend, store::Store}; +use super::{ + Result, Runtime, RuntimeError, + backend::Backend, + refuse, + store::{Store, now}, +}; use axum::{ Extension, Json, extract::{Path, Query}, http::StatusCode, }; +use chrono::{DateTime, SubsecRound, Utc}; use serde_json::{Value, json}; use sqlx::{ColumnIndex, Decode, Encode, Executor, IntoArguments, Type}; -use std::collections::HashMap; +use std::{collections::HashMap, time::Duration}; +use tokio::time::Instant; + +/// `WorkflowStreamService::DEFAULT_MAX_PENDING_ITEMS`. +const DEFAULT_MAX_PENDING_ITEMS: i64 = 10_000; +/// `WorkflowStreamService::DEFAULT_MAX_ITEMS_PER_SUBSCRIBE`. +const DEFAULT_MAX_ITEMS_PER_SUBSCRIBE: i64 = 100; +/// `WorkflowStreamService::SUBSCRIBE_MAX_ITEMS`. +const SUBSCRIBE_MAX_ITEMS: i64 = 500; +/// `WorkflowStreamService::DEFAULT_CLOSED_RETENTION_SECONDS`. +const DEFAULT_CLOSED_RETENTION_SECONDS: i64 = 3600; +/// `WorkflowStreamService::MAX_PENDING_ITEMS_MAX`: a producer may lower its +/// cap, not raise it. +const MAX_PENDING_ITEMS_MAX: i64 = 10_000; +/// `WorkflowStreamService::RETENTION_SECONDS_MAX` (30 days). +const RETENTION_SECONDS_MAX: i64 = 2_592_000; +/// `WorkflowStreamRetention::ITEM_BATCH`: items deleted per transaction. +const SWEEP_ITEM_BATCH: i64 = 100; +/// `WorkflowStreamRetention::INLINE_STREAM_LIMIT` and `INLINE_ITEM_BUDGET`. +pub(super) const INLINE_SWEEP: (i64, i64) = (10, 500); +/// A namespace's due streams in deadline order, read from the +/// `(namespace, reclaim_at, stream_id)` index of `dw_stream_reclaims`. +pub(super) const DUE_STREAMS: &str = + "SELECT stream_id FROM dw_stream_reclaims WHERE namespace=$1 AND reclaim_at<=$2 ORDER BY reclaim_at,stream_id LIMIT $3"; +/// `LegacyV1ProjectionContract::IMPORT_SOURCE`. +const PROJECTION_SOURCE: &str = "waterline_v1_projection"; +/// `readWithWait` checks again every 250 ms. +const WAIT_INTERVAL: Duration = Duration::from_millis(250); + +const STREAM_BY_NAME: &str = + "SELECT * FROM workflow_durable_streams WHERE namespace=$1 AND workflow_run_id=$2 AND stream_name=$3"; +const STREAM_BY_ID: &str = "SELECT * FROM workflow_durable_streams WHERE id=$1"; + +/// `append()`'s Laravel rules. +const APPEND_RULES: &laravel_rules::Rules = &[ + ("items", &[Rule::Required, Rule::Array, Rule::Min(1), Rule::Max(500)]), + ("items.*", &[Rule::Array]), + ("items.*.payload", &[Rule::Nullable]), + ("items.*.payload_reference", &[Rule::Nullable, Rule::String, Rule::Max(191)]), + ("items.*.payload_codec", &[Rule::Nullable, Rule::String, Rule::Max(64)]), + ("items.*.idempotency_key", &[Rule::Nullable, Rule::String, Rule::Max(191)]), + ("items.*.origin", &[Rule::Nullable, Rule::String, Rule::Max(64)]), + ("items.*.origin_reference", &[Rule::Nullable, Rule::String, Rule::Max(191)]), + ("items.*.item_type", &[Rule::Nullable, Rule::String, Rule::Max(64)]), + ("items.*.content_type", &[Rule::Nullable, Rule::String, Rule::Max(191)]), + ("max_pending_items", &[Rule::Nullable, Rule::Integer, Rule::Min(1), Rule::Max(MAX_PENDING_ITEMS_MAX)]), +]; + +/// `close()`'s Laravel rules. +const CLOSE_RULES: &laravel_rules::Rules = &[ + ("error_reason", &[Rule::Nullable, Rule::String, Rule::Max(191)]), + ("retention_seconds", &[Rule::Nullable, Rule::Integer, Rule::Min(1), Rule::Max(RETENTION_SECONDS_MAX)]), +]; + +/// The item fields `append()` validates, which are all its validated data +/// keeps of an item. +const ITEM_FIELDS: [&str; 8] = [ + "payload", + "payload_reference", + "payload_codec", + "idempotency_key", + "origin", + "origin_reference", + "item_type", + "content_type", +]; + +/// One validated item as `WorkflowStreamService::append` reads it. +#[derive(Debug, Default, PartialEq)] +pub(crate) struct StreamItem { + payload: Value, + payload_reference: Option, + payload_codec: Option, + idempotency_key: Option, + origin: Option, + origin_reference: Option, + item_type: Option, + content_type: Option, +} + +pub(crate) enum Appended { + Accepted { + stream: Value, + offsets: Vec, + accepted: usize, + deduped: usize, + }, + /// The batch would pass the cap, so none of it was written. + Full { pending_items: i64 }, + Closed, + Errored, + /// Past its retention the stream reads as not found, append included. + Expired, + /// The run is gone, or retention has pruned its details: it has no + /// streams to write (Server #774). + RunPruned, +} + +/// One retention pass over a namespace's due streams. +#[derive(Default)] +pub(crate) struct StreamReclaim { + pub(crate) streams_reclaimed: i64, + pub(crate) items_removed: i64, + pub(crate) budget_exhausted: bool, + /// Why the pass stopped short of a due stream. + pub(crate) blocked_reason: Option<&'static str>, +} + +/// One `read()` window. +pub(crate) struct Window { + stream: Value, + items: Vec, + next_offset: i64, + terminal: bool, +} + +/// Carbon's `toJSON`. +fn carbon_json(at: DateTime) -> String { + at.format("%Y-%m-%dT%H:%M:%S%.6fZ").to_string() +} impl Store where @@ -39,6 +182,419 @@ where .await?; Ok(found > 0) } + + /// `locateRun`, then `LegacyV1Projection::isProjectedRun`: nothing when + /// the run is not in this namespace, else whether it is a read-only + /// Waterline v1 projection. + pub(super) async fn stream_run(&self, workflow_id: &str, run_id: &str) -> Result> { + let row = Self::query("SELECT import_source FROM workflow_runs WHERE namespace=$1 AND workflow_instance_id=$2 AND id=$3 AND EXISTS (SELECT 1 FROM workflow_instances WHERE id=$4 AND namespace=$5)") + .bind(self.namespace.as_ref()) + .bind(workflow_id) + .bind(run_id) + .bind(workflow_id) + .bind(self.namespace.as_ref()) + .fetch_optional(&self.pool) + .await?; + match row { + Some(row) => Ok(Some(DB::optional_string(&row, "import_source")?.as_deref() == Some(PROJECTION_SOURCE))), + None => Ok(None), + } + } + + /// `formatStream`. + /// `WorkflowStreamService::isExpired`: a closed or errored stream past + /// `closed_at` plus its retention (3600 seconds when none was recorded). + fn stream_expired(row: &DB::Row, at: DateTime) -> Result { + if DB::string(row, "status")? == "open" { + return Ok(false); + } + let Some(closed) = DB::optional_instant(row, "closed_at")? else { + return Ok(false); + }; + let retention = DB::optional_number(row, "retention_seconds")?.unwrap_or(DEFAULT_CLOSED_RETENTION_SECONDS); + Ok(closed + chrono::Duration::seconds(retention) <= at) + } + + fn stream_json(row: &DB::Row) -> Result { + let time = |field: &str| Ok::<_, RuntimeError>(DB::optional_instant(row, field)?.map(carbon_json)); + Ok(json!({ + "stream_name": DB::string(row, "stream_name")?, + "status": DB::string(row, "status")?, + "last_offset": DB::number(row, "last_offset")?, + "total_items": DB::number(row, "total_items")?, + "pending_items": DB::number(row, "pending_items")?, + "opened_at": time("opened_at")?, + "last_appended_at": time("last_appended_at")?, + "closed_at": time("closed_at")?, + "error_reason": DB::optional_string(row, "error_reason")?, + "retention_seconds": DB::optional_number(row, "retention_seconds")?, + })) + } + + /// One item as `items()` renders it. + fn stream_item_json(row: &DB::Row) -> Result { + Ok(json!({ + "offset": DB::number(row, "offset")?, + "idempotency_key": DB::optional_string(row, "idempotency_key")?, + "origin": DB::string(row, "origin")?, + "origin_reference": DB::optional_string(row, "origin_reference")?, + "item_type": DB::optional_string(row, "item_type")?, + "content_type": DB::optional_string(row, "content_type")?, + "payload": DB::optional_document_row(row, "payload")?.unwrap_or(Value::Null), + "payload_reference": DB::optional_string(row, "payload_reference")?, + "payload_codec": DB::optional_string(row, "payload_codec")?, + "emitted_at": carbon_json(DB::instant(row, "emitted_at")?), + })) + } + + /// `listForRun`: the run's streams by name, in the database's order. + pub(super) async fn list_workflow_streams(&self, run_id: &str) -> Result> { + let at = now(); + let mut streams = Vec::new(); + for row in Self::query("SELECT * FROM workflow_durable_streams WHERE namespace=$1 AND workflow_run_id=$2 ORDER BY stream_name") + .bind(self.namespace.as_ref()) + .bind(run_id) + .fetch_all(&self.pool) + .await? + { + if !Self::stream_expired(&row, at)? { + streams.push(Self::stream_json(&row)?); + } + } + Ok(streams) + } + + /// `describe` for a valid name. + pub(super) async fn show_workflow_stream(&self, run_id: &str, name: &str) -> Result> { + let row = Self::query(STREAM_BY_NAME) + .bind(self.namespace.as_ref()) + .bind(run_id) + .bind(name) + .fetch_optional(&self.pool) + .await?; + match row { + Some(row) if !Self::stream_expired(&row, now())? => Ok(Some(Self::stream_json(&row)?)), + _ => Ok(None), + } + } + + /// `WorkflowStreamService::append` for a valid name and a non-empty + /// batch, in one transaction. + pub(super) async fn append_workflow_stream( + &self, + workflow_id: &str, + run_id: &str, + name: &str, + items: Vec, + max_pending: i64, + ) -> Result { + let at = now().trunc_subsecs(0); + let mut tx = self.begin().await?; + // Run retention deletes a run's streams and marks its details pruned + // under the same transition lock, so an append after it must not open + // a stream on the pruned run, as PHP's `lockStream` refuses it. + let pruned = Self::query("SELECT details_pruned_at FROM workflow_runs WHERE id=$1 AND namespace=$2") + .bind(run_id) + .bind(self.namespace.as_ref()) + .fetch_optional(&mut *tx) + .await?; + match pruned { + Some(row) if DB::optional_instant(&row, "details_pruned_at")?.is_none() => {} + _ => return Ok(Appended::RunPruned), + } + let found = Self::query(STREAM_BY_NAME) + .bind(self.namespace.as_ref()) + .bind(run_id) + .bind(name) + .fetch_optional(&mut *tx) + .await?; + let row = match found { + Some(row) => row, + None => { + Self::query("INSERT INTO workflow_durable_streams(namespace,workflow_instance_id,workflow_run_id,stream_name,status,last_offset,total_items,pending_items,opened_at,created_at,updated_at) VALUES ($1,$2,$3,$4,'open',-1,0,0,$5,$6,$7)") + .bind(self.namespace.as_ref()) + .bind(workflow_id) + .bind(run_id) + .bind(name) + .bind(DB::bind_time(at)) + .bind(DB::bind_time(at)) + .bind(DB::bind_time(at)) + .execute(&mut *tx) + .await?; + Self::query(STREAM_BY_NAME) + .bind(self.namespace.as_ref()) + .bind(run_id) + .bind(name) + .fetch_one(&mut *tx) + .await? + } + }; + // From the expiry instant a stream reads as not found for append + // too. Its row stays until its run is pruned, so the name is never + // opened again at offset 0 (Server #774). + if Self::stream_expired(&row, at)? { + return Ok(Appended::Expired); + } + match DB::string(&row, "status")?.as_str() { + "closed" => return Ok(Appended::Closed), + "errored" => return Ok(Appended::Errored), + _ => {} + } + let stream_id = DB::number(&row, "id")?; + let mut last_offset = DB::number(&row, "last_offset")?; + let mut total = DB::number(&row, "total_items")?; + let mut pending = DB::number(&row, "pending_items")?; + // The whole batch is admitted or refused before any item is written + // (Server #774). An item whose key is already stored, or earlier in + // this batch, takes no capacity. + let mut stored: HashMap<&str, Option> = HashMap::new(); + let mut fresh = 0; + for item in &items { + match item.idempotency_key.as_deref() { + Some(key) if stored.contains_key(key) => {} + Some(key) => { + let existing = Self::scalar(DB::STREAM_ITEM_BY_KEY).bind(stream_id).bind(key).fetch_optional(&mut *tx).await?; + if existing.is_none() { + fresh += 1; + } + stored.insert(key, existing); + } + None => fresh += 1, + } + } + if fresh > 0 && pending + fresh > max_pending { + // Dropping the transaction rolls back a stream this append opened. + return Ok(Appended::Full { pending_items: pending }); + } + let mut offsets = Vec::with_capacity(items.len()); + let (mut accepted, mut deduped) = (0, 0); + for item in &items { + if let Some(key) = item.idempotency_key.as_deref() + && let Some(Some(existing)) = stored.get(key) + { + offsets.push(*existing); + deduped += 1; + continue; + } + last_offset += 1; + Self::query(DB::STREAM_ITEM_INSERT) + .bind(stream_id) + .bind(self.namespace.as_ref()) + .bind(run_id) + .bind(name) + .bind(last_offset) + .bind(item.idempotency_key.as_deref()) + .bind(item.origin.as_deref().unwrap_or("workflow_command")) + .bind(item.origin_reference.as_deref()) + .bind(DB::optional_document(&item.payload)) + .bind(item.payload_reference.as_deref()) + .bind(item.payload_codec.as_deref()) + .bind(item.item_type.as_deref()) + .bind(item.content_type.as_deref()) + .bind(DB::bind_time(at)) + .bind(DB::bind_time(at)) + .bind(DB::bind_time(at)) + .execute(&mut *tx) + .await?; + if let Some(key) = item.idempotency_key.as_deref() { + stored.insert(key, Some(last_offset)); + } + total += 1; + pending += 1; + accepted += 1; + offsets.push(last_offset); + } + // Eloquent saves only changed attributes, so a batch that adds + // nothing leaves the row as it was. + if accepted > 0 { + Self::query("UPDATE workflow_durable_streams SET last_offset=$1,total_items=$2,pending_items=$3,last_appended_at=$4,updated_at=$5 WHERE id=$6") + .bind(last_offset) + .bind(total) + .bind(pending) + .bind(DB::bind_time(at)) + .bind(DB::bind_time(at)) + .bind(stream_id) + .execute(&mut *tx) + .await?; + } + let row = Self::query(STREAM_BY_ID).bind(stream_id).fetch_one(&mut *tx).await?; + tx.commit().await?; + Ok(Appended::Accepted { stream: Self::stream_json(&row)?, offsets, accepted, deduped }) + } + + /// `WorkflowStreamRetention::pruneExpired` for this namespace (Server + /// #774). It reads at most `stream_limit` due streams in deadline order + /// from `dw_stream_reclaims` and deletes at most `item_budget` of their + /// items, in transactions of at most SWEEP_ITEM_BATCH under the + /// transition lock. A stream with no items left leaves + /// `dw_stream_reclaims`; its row stays until its run is pruned. Native + /// settles no external payloads, so a stream whose items hold an + /// external reference stops the pass, blocked, with its items kept. + pub(super) async fn prune_expired_streams(&self, stream_limit: i64, item_budget: i64) -> Result { + let at = now(); + let due = Self::query(DUE_STREAMS) + .bind(self.namespace.as_ref()) + .bind(DB::bind_time(at)) + .bind(stream_limit.max(1)) + .fetch_all(&self.pool) + .await? + .iter() + .map(|row| DB::number(row, "stream_id")) + .collect::>>()?; + let budget = item_budget.max(1); + let mut report = StreamReclaim::default(); + for stream_id in due { + loop { + if report.items_removed >= budget { + report.budget_exhausted = true; + return Ok(report); + } + let take = SWEEP_ITEM_BATCH.min(budget - report.items_removed); + let mut tx = self.begin().await?; + let Some(row) = Self::query(STREAM_BY_ID).bind(stream_id).fetch_optional(&mut *tx).await? else { + // The stream went with its run; its deadline goes too. + Self::query("DELETE FROM dw_stream_reclaims WHERE stream_id=$1").bind(stream_id).execute(&mut *tx).await?; + tx.commit().await?; + break; + }; + if !Self::stream_expired(&row, at)? { + break; + } + if Self::scalar(DB::STREAM_ITEM_REFERENCES).bind(stream_id).fetch_one(&mut *tx).await? > 0 { + report.blocked_reason = Some("external_payload_cleanup_unavailable"); + return Ok(report); + } + // The offset of the `take`th item, when that many remain. + let through = Self::scalar(DB::STREAM_SWEEP_BOUNDARY) + .bind(stream_id) + .bind(take - 1) + .fetch_optional(&mut *tx) + .await?; + if let Some(offset) = through { + let deleted = Self::query(DB::STREAM_SWEEP_THROUGH).bind(stream_id).bind(offset).execute(&mut *tx).await?; + tx.commit().await?; + report.items_removed += DB::affected(deleted) as i64; + continue; + } + let deleted = Self::query("DELETE FROM workflow_durable_stream_items WHERE stream_id=$1") + .bind(stream_id) + .execute(&mut *tx) + .await?; + // The terminal row keeps the name; without a deadline the + // pass no longer reads it. + Self::query("DELETE FROM dw_stream_reclaims WHERE stream_id=$1").bind(stream_id).execute(&mut *tx).await?; + tx.commit().await?; + report.items_removed += DB::affected(deleted) as i64; + report.streams_reclaimed += 1; + break; + } + } + Ok(report) + } + + /// `WorkflowStreamService::read` for a valid name and offset. + pub(super) async fn read_workflow_stream( + &self, + run_id: &str, + name: &str, + from: i64, + max_items: i64, + ) -> Result> { + let Some(mut row) = Self::query(STREAM_BY_NAME) + .bind(self.namespace.as_ref()) + .bind(run_id) + .bind(name) + .fetch_optional(&self.pool) + .await? + else { + return Ok(None); + }; + if Self::stream_expired(&row, now())? { + return Ok(None); + } + let stream_id = DB::number(&row, "id")?; + let rows = Self::query(DB::STREAM_ITEMS_WINDOW) + .bind(stream_id) + .bind(from) + .bind(max_items.clamp(1, SUBSCRIBE_MAX_ITEMS)) + .fetch_all(&self.pool) + .await?; + let last = rows.last().map(|item| DB::number(item, "offset")).transpose()?; + let next_offset = last.map_or(from, |offset| offset + 1); + let head = DB::number(&row, "last_offset")?; + // The window reached the head: nothing is pending past it. + if last == Some(head) && DB::number(&row, "pending_items")? > 0 { + let mut tx = self.begin().await?; + Self::query("UPDATE workflow_durable_streams SET pending_items=0,updated_at=$1 WHERE id=$2 AND last_offset=$3") + .bind(DB::bind_time(now().trunc_subsecs(0))) + .bind(stream_id) + .bind(head) + .execute(&mut *tx) + .await?; + row = Self::query(STREAM_BY_ID).bind(stream_id).fetch_one(&mut *tx).await?; + tx.commit().await?; + } + let terminal = DB::string(&row, "status")? != "open" && next_offset > DB::number(&row, "last_offset")?; + Ok(Some(Window { + stream: Self::stream_json(&row)?, + items: rows.iter().map(Self::stream_item_json).collect::>()?, + next_offset, + terminal, + })) + } + + /// `WorkflowStreamService::close` for a valid name: a terminal stream is + /// returned unchanged. + pub(super) async fn close_workflow_stream( + &self, + run_id: &str, + name: &str, + error_reason: Option, + retention_seconds: Option, + ) -> Result> { + let mut tx = self.begin().await?; + let Some(row) = Self::query(STREAM_BY_NAME) + .bind(self.namespace.as_ref()) + .bind(run_id) + .bind(name) + .fetch_optional(&mut *tx) + .await? + else { + return Ok(None); + }; + if Self::stream_expired(&row, now())? { + return Ok(None); + } + if DB::string(&row, "status")? != "open" { + return Ok(Some(Self::stream_json(&row)?)); + } + let at = now().trunc_subsecs(0); + let retention = match retention_seconds { + Some(seconds) if seconds > 0 => seconds, + _ => DB::optional_number(&row, "retention_seconds")?.unwrap_or(DEFAULT_CLOSED_RETENTION_SECONDS), + }; + let stream_id = DB::number(&row, "id")?; + Self::query("UPDATE workflow_durable_streams SET status=$1,error_reason=$2,closed_at=$3,retention_seconds=$4,updated_at=$5 WHERE id=$6") + .bind(if error_reason.is_some() { "errored" } else { "closed" }) + .bind(error_reason.as_deref()) + .bind(DB::bind_time(at)) + .bind(retention) + .bind(DB::bind_time(at)) + .bind(stream_id) + .execute(&mut *tx) + .await?; + // The retention pass's deadline: the instant the stream expires. + Self::query("INSERT INTO dw_stream_reclaims(stream_id,namespace,workflow_run_id,reclaim_at) VALUES ($1,$2,$3,$4)") + .bind(stream_id) + .bind(self.namespace.as_ref()) + .bind(run_id) + .bind(DB::bind_time(at + chrono::Duration::seconds(retention))) + .execute(&mut *tx) + .await?; + let row = Self::query(STREAM_BY_ID).bind(stream_id).fetch_one(&mut *tx).await?; + tx.commit().await?; + Ok(Some(Self::stream_json(&row)?)) + } } fn protocol(status: StatusCode, response: Value) -> RuntimeError { @@ -49,6 +605,10 @@ fn run_not_found() -> RuntimeError { protocol(StatusCode::NOT_FOUND, json!({"message":"Workflow run not found.","reason":"instance_not_found"})) } +fn stream_not_found() -> RuntimeError { + protocol(StatusCode::NOT_FOUND, json!({"message":"Stream not found.","reason":"stream_not_found"})) +} + /// `WorkflowStreamService::STREAM_NAME_PATTERN`. fn valid_name(name: &str) -> bool { let bytes = name.as_bytes(); @@ -58,31 +618,123 @@ fn valid_name(name: &str) -> bool { && bytes[1..].iter().all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b':' | b'-')) } -fn stream_refusal(name: &str) -> RuntimeError { - if valid_name(name) { - protocol(StatusCode::NOT_FOUND, json!({"message":"Stream not found.","reason":"stream_not_found"})) - } else { - protocol(StatusCode::BAD_REQUEST, json!({"message":"invalid stream name","reason":"invalid_stream_name"})) +/// The service's invalid-name refusal, which `append()` reports with its +/// general reason and the other routes with their own. +fn invalid_name(reason: &str) -> RuntimeError { + protocol(StatusCode::BAD_REQUEST, json!({"message":"invalid stream name","reason":reason})) +} + +/// The run lookup every route makes first. +async fn located(runtime: &Runtime, workflow_id: &str, run_id: &str) -> Result { + runtime.stream_run(workflow_id, run_id).await?.ok_or_else(run_not_found) +} + +/// `projectedRunReadOnly`. +fn projected_read_only(workflow_id: &str, run_id: &str) -> RuntimeError { + protocol(StatusCode::CONFLICT, json!({ + "message": "Durable streams cannot mutate a read-only v1 projection.", + "reason": "v1_projection_read_only", + "remediation": "Continue execution and stream production on the source v1 application.", + "workflow_id": workflow_id, + "run_id": run_id, + "execution_owner": "v1", + })) +} + +/// The items of a normalized append body, in data order, whether `items` +/// is a list or keyed. +fn body_items(body: &Value) -> Vec<&Value> { + match &body["items"] { + Value::Array(items) => items.iter().collect(), + Value::Object(items) => items.values().collect(), + _ => Vec::new(), } } +/// `RuntimeExternalPayloadTransport::incoming` on `items.*.payload_reference`, +/// which runs before the controller: a value other than null or a reference +/// object is refused. Native does not resolve reference objects yet. +fn references(body: &Value) -> Result<()> { + for item in body_items(body) { + match item.get("payload_reference") { + None | Some(Value::Null) => {} + Some(Value::Array(_) | Value::Object(_)) => { + return Err(refuse(StatusCode::NOT_IMPLEMENTED, "rust_capability_not_implemented")); + } + Some(_) => { + return Err(super::external_transport::typed_error( + StatusCode::UNPROCESSABLE_ENTITY, + "external_payload_unsupported", + "Payload reference fields require an opaque runtime external payload reference object.", + )); + } + } + } + Ok(()) +} + +/// The validated items `append()` passes on: Laravel's validated data keeps +/// only the item fields, so an item with none of them is left out. A +/// present codec is canonicalized, and the first one refused fails the +/// request as `items.{index}.payload_codec`. +fn validated_items(body: &Value) -> Result> { + let keyed: Vec<(String, &Value)> = match &body["items"] { + Value::Array(items) => items.iter().enumerate().map(|(index, item)| (index.to_string(), item)).collect(), + Value::Object(items) => items.iter().map(|(key, item)| (key.clone(), item)).collect(), + _ => Vec::new(), + }; + let mut items = Vec::with_capacity(keyed.len()); + for (key, item) in keyed { + let Some(fields) = item.as_object() else { + continue; + }; + if !ITEM_FIELDS.iter().any(|field| fields.contains_key(*field)) { + continue; + } + let text = |field: &str| fields.get(field).and_then(Value::as_str).map(str::to_owned); + let payload_codec = match fields.get("payload_codec") { + None => None, + Some(codec) => match super::input_envelope::codec_refusal(codec) { + None => Some("avro".to_owned()), + Some(message) => { + return Err(super::laravel_validation::failure_messages(vec![( + format!("items.{key}.payload_codec"), + vec![message], + )]) + .expect("one message")); + } + }, + }; + items.push(StreamItem { + // The array cast stores a PHP array, so an empty object reads back + // as a list, as PHP's other array round trips do. + payload: super::activity_failures::php_array(fields.get("payload").cloned().unwrap_or(Value::Null)), + payload_reference: text("payload_reference"), + payload_codec, + // The service treats an empty key as none; normalization already + // turned empty strings into null. + idempotency_key: text("idempotency_key"), + origin: text("origin"), + origin_reference: text("origin_reference"), + item_type: text("item_type"), + content_type: text("content_type"), + }); + } + Ok(items) +} + /// `items()`'s Laravel rules for `from`, `max_items` and `wait_seconds`. fn items_rules(query: &HashMap) -> Result<()> { - // Query values are strings; empty ones become null and pass `nullable`. - let integer = |field: &str| -> Option> { - let value = normalized(query.get(field).map(|text| json!(text)).as_ref())?; - (!value.is_null()).then(|| integer(&value).ok_or(())) - }; let mut errors: Vec<(&'static str, Vec<&'static str>)> = Vec::new(); let mut from = vec![]; - match integer("from") { + match query_integer(query, "from") { Some(Err(())) => from.push("The from field must be an integer."), Some(Ok(value)) if value < 0 => from.push("The from field must be at least 0."), _ => {} } errors.push(("from", from)); let mut max_items = vec![]; - match integer("max_items") { + match query_integer(query, "max_items") { Some(Err(())) => max_items.push("The max items field must be an integer."), Some(Ok(value)) if value < 1 => max_items.push("The max items field must be at least 1."), Some(Ok(value)) if value > 500 => max_items.push("The max items field must not be greater than 500."), @@ -90,7 +742,7 @@ fn items_rules(query: &HashMap) -> Result<()> { } errors.push(("max_items", max_items)); let mut wait = vec![]; - match integer("wait_seconds") { + match query_integer(query, "wait_seconds") { Some(Err(())) => wait.push("The wait seconds field must be an integer."), Some(Ok(value)) if value < 0 => wait.push("The wait seconds field must be at least 0."), Some(Ok(value)) if value > 60 => wait.push("The wait seconds field must not be greater than 60."), @@ -103,15 +755,21 @@ fn items_rules(query: &HashMap) -> Result<()> { } } +/// A query value as the `integer` rule reads it. Query values are strings; +/// empty ones become null and pass `nullable`. +fn query_integer(query: &HashMap, field: &str) -> Option> { + let value = normalized(query.get(field).map(|text| json!(text)).as_ref())?; + (!value.is_null()).then(|| integer(&value).ok_or(())) +} + /// `GET /api/workflows/{id}/runs/{run}/streams`. pub(super) async fn list( Extension(runtime): Extension, Path((workflow_id, run_id)): Path<(String, String)>, ) -> Result> { - if !runtime.run_in_namespace(&workflow_id, &run_id).await? { - return Err(run_not_found()); - } - Ok(Json(json!({"workflow_id":workflow_id,"workflow_run_id":run_id,"count":0,"streams":[]}))) + located(&runtime, &workflow_id, &run_id).await?; + let streams = runtime.list_workflow_streams(&run_id).await?; + Ok(Json(json!({"workflow_id":workflow_id,"workflow_run_id":run_id,"count":streams.len(),"streams":streams}))) } /// `GET /api/workflows/{id}/runs/{run}/streams/{name}`. @@ -119,29 +777,130 @@ pub(super) async fn show( Extension(runtime): Extension, Path((workflow_id, run_id, name)): Path<(String, String, String)>, ) -> Result> { - if !runtime.run_in_namespace(&workflow_id, &run_id).await? { - return Err(run_not_found()); + located(&runtime, &workflow_id, &run_id).await?; + if !valid_name(&name) { + return Err(invalid_name("invalid_stream_name")); } - Err(stream_refusal(&name)) + let stream = runtime.show_workflow_stream(&run_id, &name).await?.ok_or_else(stream_not_found)?; + Ok(Json(json!({"workflow_id":workflow_id,"workflow_run_id":run_id,"stream":stream}))) } -/// `GET /api/workflows/{id}/runs/{run}/streams/{name}/items`. +/// `GET /api/workflows/{id}/runs/{run}/streams/{name}/items`, with +/// `readWithWait`'s bounded wait for an open stream with nothing new. pub(super) async fn items( Extension(runtime): Extension, Path((workflow_id, run_id, name)): Path<(String, String, String)>, Query(query): Query>, ) -> Result> { - if !runtime.run_in_namespace(&workflow_id, &run_id).await? { - return Err(run_not_found()); - } + located(&runtime, &workflow_id, &run_id).await?; items_rules(&query)?; - Err(stream_refusal(&name)) + let value = |field: &str, default: i64| query_integer(&query, field).and_then(|value| value.ok()).unwrap_or(default); + let from = value("from", 0); + let max_items = value("max_items", DEFAULT_MAX_ITEMS_PER_SUBSCRIBE); + let wait_seconds = value("wait_seconds", 0); + if !valid_name(&name) { + return Err(invalid_name("invalid_stream_name")); + } + let (reader, run, stream) = (&runtime, run_id.as_str(), name.as_str()); + let read = move || async move { + reader.read_workflow_stream(run, stream, from, max_items).await?.ok_or_else(stream_not_found) + }; + let mut window = read().await?; + if wait_seconds > 0 && window.items.is_empty() && !window.terminal { + let deadline = Instant::now() + Duration::from_secs(wait_seconds as u64); + while Instant::now() < deadline { + tokio::time::sleep(WAIT_INTERVAL).await; + window = read().await?; + if !window.items.is_empty() || window.terminal { + break; + } + } + } + Ok(Json(json!({"workflow_id":workflow_id,"workflow_run_id":run_id,"stream":window.stream, + "items":window.items,"next_offset":window.next_offset,"terminal":window.terminal}))) +} + +/// `POST /api/workflows/{id}/runs/{run}/streams/{name}/items`. +pub(super) async fn append( + Extension(runtime): Extension, + Path((workflow_id, run_id, name)): Path<(String, String, String)>, + Json(body): Json, +) -> Result> { + // Laravel trims every string, payloads included, before anything reads it. + let body = super::start_request::normalized(body); + references(&body)?; + if located(&runtime, &workflow_id, &run_id).await? { + return Err(projected_read_only(&workflow_id, &run_id)); + } + laravel_rules::validate(&body, APPEND_RULES)?; + let items = validated_items(&body)?; + let max_pending = body.get("max_pending_items").and_then(integer).unwrap_or(DEFAULT_MAX_PENDING_ITEMS); + if !valid_name(&name) { + return Err(invalid_name("invalid_request")); + } + if items.is_empty() { + return Err(protocol(StatusCode::BAD_REQUEST, + json!({"message":"append requires at least one item","reason":"invalid_request"}))); + } + match runtime.append_workflow_stream(&workflow_id, &run_id, &name, items, max_pending).await? { + Appended::Accepted { stream, offsets, accepted, deduped } => Ok(Json(json!({ + "workflow_id": workflow_id, + "workflow_run_id": run_id, + "stream": stream, + "accepted_offsets": offsets, + "accepted": accepted, + "deduped": deduped, + }))), + Appended::Full { pending_items } => Err(protocol(StatusCode::TOO_MANY_REQUESTS, json!({ + "message": "Stream pending-items cap reached; consumer is draining too slowly.", + "reason": "stream_full", + "pending_items": pending_items, + "max_pending_items": max_pending, + }))), + Appended::Expired | Appended::RunPruned => Err(stream_not_found()), + Appended::Closed => Err(protocol(StatusCode::CONFLICT, json!({"message":"Stream is closed.","reason":"stream_closed"}))), + Appended::Errored => Err(protocol(StatusCode::CONFLICT, json!({"message":"Stream is errored.","reason":"stream_errored"}))), + } +} + +/// `POST /api/workflows/{id}/runs/{run}/streams/{name}/close`. +pub(super) async fn close( + Extension(runtime): Extension, + Path((workflow_id, run_id, name)): Path<(String, String, String)>, + Json(body): Json, +) -> Result> { + let body = super::start_request::normalized(body); + if located(&runtime, &workflow_id, &run_id).await? { + return Err(projected_read_only(&workflow_id, &run_id)); + } + laravel_rules::validate(&body, CLOSE_RULES)?; + let error_reason = body.get("error_reason").and_then(Value::as_str).map(str::to_owned); + let retention_seconds = body.get("retention_seconds").and_then(integer); + if !valid_name(&name) { + return Err(invalid_name("invalid_stream_name")); + } + let stream = runtime + .close_workflow_stream(&run_id, &name, error_reason, retention_seconds) + .await? + .ok_or_else(stream_not_found)?; + Ok(Json(json!({"workflow_id":workflow_id,"workflow_run_id":run_id,"stream":stream}))) } #[cfg(test)] mod tests { use super::*; + fn query(pairs: &[(&str, &str)]) -> HashMap { + pairs.iter().map(|(key, value)| ((*key).to_owned(), (*value).to_owned())).collect() + } + + fn refusal(result: Result) -> (StatusCode, Value) { + match result { + Err(RuntimeError::Protocol { status, response }) => (status, response), + other => panic!("{other:?}"), + } + } + #[test] fn stream_names_follow_php_pattern() { assert!(valid_name("orders.v1:main-2")); @@ -153,26 +912,61 @@ mod tests { #[test] fn item_rules_report_laravel_messages() { - let query: HashMap = [("from", "x"), ("max_items", "501"), ("wait_seconds", "61")] - .into_iter() - .map(|(key, value)| (key.to_owned(), value.to_owned())) - .collect(); - match items_rules(&query) { - Err(RuntimeError::Protocol { response, .. }) => { - assert_eq!(response["message"], "The from field must be an integer. (and 2 more errors)"); - } - other => panic!("{other:?}"), - } + let (_, response) = refusal(items_rules(&query(&[("from", "x"), ("max_items", "501"), ("wait_seconds", "61")]))); + assert_eq!(response["message"], "The from field must be an integer. (and 2 more errors)"); assert!(items_rules(&HashMap::new()).is_ok()); - let query: HashMap = [("from", "05"), ("max_items", "+5"), ("wait_seconds", " ")] - .into_iter() - .map(|(key, value)| (key.to_owned(), value.to_owned())) - .collect(); - match items_rules(&query) { - Err(RuntimeError::Protocol { response, .. }) => { - assert_eq!(response["errors"], json!({"from":["The from field must be an integer."]})); - } - other => panic!("{other:?}"), - } + let (_, response) = refusal(items_rules(&query(&[("from", "05"), ("max_items", "+5"), ("wait_seconds", " ")]))); + assert_eq!(response["errors"], json!({"from":["The from field must be an integer."]})); + } + + #[test] + fn append_rules_report_laravel_messages() { + let check = |body: Value| refusal(laravel_rules::validate(&super::super::start_request::normalized(body), APPEND_RULES)).1; + assert_eq!(check(json!({"items":[]}))["message"], "The items field is required."); + assert_eq!(check(json!({"items":["text"]}))["errors"], json!({"items.0":["The items.0 field must be an array."]})); + assert_eq!(check(json!({"items":[{"payload":1}],"max_pending_items":0}))["message"], + "The max pending items field must be at least 1."); + assert_eq!(check(json!({"items":[{"idempotency_key":"k".repeat(192)}]}))["message"], + "The items.0.idempotency_key field must not be greater than 191 characters."); + // Plain attributes report before the expanded item attributes. + assert_eq!(check(json!({"items":[{"origin":5}],"max_pending_items":"x"}))["message"], + "The max pending items field must be an integer. (and 1 more error)"); + } + + #[test] + fn validated_items_keep_only_item_fields_and_canonicalize_codecs() { + let body = super::super::start_request::normalized(json!({"items":[ + {"payload":{"token":" Hello ","empty":{}},"payload_codec":null,"idempotency_key":" "}, + {}, + {"unknown":1}, + {"payload":null}, + ]})); + let items = validated_items(&body).unwrap(); + assert_eq!(items.len(), 2); + assert_eq!(items[0].payload, json!({"token":"Hello","empty":[]})); + assert_eq!(items[0].payload_codec.as_deref(), Some("avro")); + assert_eq!(items[0].idempotency_key, None); + assert_eq!(items[1], StreamItem::default()); + let (status, response) = refusal(validated_items(&json!({"items":[{"payload":1},{"payload_codec":"json"}]}))); + assert_eq!(status, StatusCode::UNPROCESSABLE_ENTITY); + assert!(response["errors"]["items.1.payload_codec"][0].as_str().unwrap() + .starts_with("unsupported_payload_codec: workflow payload codec \"json\"")); + } + + #[test] + fn references_refuse_strings_and_defer_reference_objects() { + assert!(references(&json!({"items":[{"payload_reference":null},{"payload":1}]})).is_ok()); + assert!(references(&json!({"items":"text"})).is_ok()); + let (status, response) = refusal(references(&json!({"items":[{"payload_reference":"dw-ref"}]}))); + assert_eq!(status, StatusCode::UNPROCESSABLE_ENTITY); + assert_eq!(response["reason"], "external_payload_unsupported"); + assert!(matches!(references(&json!({"items":{"a":{"payload_reference":{"reference_id":"x"}}}})), + Err(RuntimeError::Refused { status: StatusCode::NOT_IMPLEMENTED, reason: "rust_capability_not_implemented" }))); + } + + #[test] + fn times_render_as_carbon_json() { + let at = DateTime::parse_from_rfc3339("2026-10-25T06:42:31.987Z").unwrap().with_timezone(&Utc); + assert_eq!(carbon_json(at.trunc_subsecs(0)), "2026-10-25T06:42:31.000000Z"); } } diff --git a/rust/tests/execution.rs b/rust/tests/execution.rs index cf9f88e60..de8866abf 100644 --- a/rust/tests/execution.rs +++ b/rust/tests/execution.rs @@ -212,6 +212,8 @@ mod describe_actions; mod run_deadlines; #[path = "execution/waterline_projection_guards.rs"] mod waterline_projection_guards; +#[path = "execution/workflow_stream_bounds.rs"] +mod workflow_stream_bounds; #[path = "execution/operator_dashboard.rs"] mod operator_dashboard; #[path = "execution/waterline_reads.rs"] diff --git a/rust/tests/execution/waterline_projection_guards.rs b/rust/tests/execution/waterline_projection_guards.rs index 5edb30911..9a16513df 100644 --- a/rust/tests/execution/waterline_projection_guards.rs +++ b/rust/tests/execution/waterline_projection_guards.rs @@ -60,6 +60,33 @@ async fn appends_to_a_projection_are_refused_before_their_body_is_read() { assert_eq!(after, described); } +// Durable output streams on a projection are read-only too: an append and a +// close are refused as PHP's `projectedRunReadOnly` refuses them, and no +// stream or item is written. +#[tokio::test] +async fn durable_stream_writes_to_a_projection_are_refused() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + let (status, report) = import(&app, "stream-guard-src", running(1)).await; + assert_eq!(status, StatusCode::CREATED, "{report}"); + let workflow_id = report["identity"]["standalone"]["workflow_id"].as_str().unwrap().replace(':', "%3A"); + let run_id = report["identity"]["standalone"]["run_id"].as_str().unwrap().to_owned(); + let stream = format!("/api/workflows/{workflow_id}/runs/{run_id}/streams/tokens"); + let before = rows(&database).await; + for (path, body) in [ + (format!("{stream}/items"), json!({"items": [{"payload": "x", "idempotency_key": "k1"}]})), + (format!("{stream}/close"), json!({})), + (format!("{stream}/close"), json!({"error_reason": "failed"})), + ] { + let (status, refusal) = request(&app, "POST", &path, body.clone()).await; + assert_eq!((status, &refusal["reason"]), (StatusCode::CONFLICT, &json!("v1_projection_read_only")), "{path} {body}: {refusal}"); + assert_eq!((&refusal["run_id"], &refusal["execution_owner"]), (&json!(run_id), &json!("v1")), "{refusal}"); + } + let (status, listed) = request(&app, "GET", &format!("/api/workflows/{workflow_id}/runs/{run_id}/streams"), Value::Null).await; + assert_eq!((status, &listed["count"]), (StatusCode::OK, &json!(0)), "{listed}"); + assert_eq!(rows(&database).await, before); +} + #[tokio::test] async fn a_projection_past_the_event_limit_is_refused_before_any_row() { let database = TestDatabase::new().await; diff --git a/rust/tests/execution/waterline_reads.rs b/rust/tests/execution/waterline_reads.rs index 489655cd9..be9146b37 100644 --- a/rust/tests/execution/waterline_reads.rs +++ b/rust/tests/execution/waterline_reads.rs @@ -29,7 +29,7 @@ async fn task_queue_views_follow_registrations_and_tasks() { assert_eq!(other["task_queues"], json!([]), "{other}"); } -// Native runs have no durable streams: lists are empty, a valid name is not +// Durable stream reads: an empty run lists nothing, a valid name is not // found and an invalid name, bad item paging or a foreign run is refused. #[tokio::test] async fn durable_stream_reads_answer_like_php() { @@ -53,6 +53,84 @@ async fn durable_stream_reads_answer_like_php() { assert_eq!((status, body["reason"].clone()), (StatusCode::NOT_FOUND, json!("instance_not_found")), "{body}"); } +// Producer writes: appends open the stream and keep their offsets across a +// retry, reads at the head clear the pending count, a close is final, and +// another namespace can neither read nor write the run's streams. +#[tokio::test] +async fn durable_stream_writes_answer_like_php() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + namespaces::create(&app, "streams").await; + let started = start(&app, "producing").await; + let run = format!("/api/workflows/producing/runs/{}", started["run_id"].as_str().unwrap()); + let items = format!("{run}/streams/tokens/items"); + let (status, body) = request(&app, "POST", &items, json!({"items":[ + {"payload":{"token":" Hello ","empty":{}},"idempotency_key":"k1"},{"payload":"plain","payload_codec":null}]})).await; + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!((body["accepted_offsets"].clone(), body["accepted"].clone(), body["deduped"].clone()), (json!([0, 1]), json!(2), json!(0))); + assert_eq!(body["stream"]["pending_items"], 2); + let opened = body["stream"]["opened_at"].as_str().unwrap().to_owned(); + assert!(opened.ends_with(".000000Z"), "{opened}"); + let (status, body) = request(&app, "POST", &items, json!({"items":[{"payload":1,"idempotency_key":"k1"},{"payload":2}], + "max_pending_items":3})).await; + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!((body["accepted_offsets"].clone(), body["deduped"].clone()), (json!([0, 2]), json!(1))); + // At the cap, the batch keeps nothing past it and answers 429. + let (status, body) = request(&app, "POST", &items, json!({"items":[{"payload":3}],"max_pending_items":3})).await; + assert_eq!(status, StatusCode::TOO_MANY_REQUESTS, "{body}"); + assert_eq!((body["reason"].clone(), body["pending_items"].clone(), body["max_pending_items"].clone()), + (json!("stream_full"), json!(3), json!(3))); + let (status, body) = get(&app, "default", &format!("{items}?from=0")).await; + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!(body["items"][0]["payload"], json!({"token":"Hello","empty":[]})); + assert_eq!(body["items"][0]["origin"], "workflow_command"); + assert_eq!(body["items"][1]["payload_codec"], "avro"); + assert_eq!((body["next_offset"].clone(), body["terminal"].clone(), body["stream"]["pending_items"].clone()), + (json!(3), json!(false), json!(0))); + // Another namespace sees no such run. + let (status, body) = namespaces::scoped(&app, "streams", "POST", &items, json!({"items":[{"payload":1}]})).await; + assert_eq!((status, body["reason"].clone()), (StatusCode::NOT_FOUND, json!("instance_not_found")), "{body}"); + let close = format!("{run}/streams/tokens/close"); + let (status, body) = request(&app, "POST", &close, json!({})).await; + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!((body["stream"]["status"].clone(), body["stream"]["retention_seconds"].clone()), (json!("closed"), json!(3600))); + let (status, again) = request(&app, "POST", &close, json!({"error_reason":"late","retention_seconds":5})).await; + assert_eq!(status, StatusCode::OK, "{again}"); + assert_eq!(again["stream"], body["stream"]); + let (status, body) = request(&app, "POST", &items, json!({"items":[{"payload":1}]})).await; + assert_eq!((status, body["reason"].clone()), (StatusCode::CONFLICT, json!("stream_closed")), "{body}"); + let (status, body) = get(&app, "default", &format!("{items}?from=3&wait_seconds=5")).await; + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!((body["items"].clone(), body["terminal"].clone()), (json!([]), json!(true))); + let (status, body) = request(&app, "POST", &format!("{run}/streams/bad%20name/items"), json!({"items":[{"payload":1}]})).await; + assert_eq!((status, body["reason"].clone()), (StatusCode::BAD_REQUEST, json!("invalid_request")), "{body}"); + let (status, body) = request(&app, "POST", &items, json!({"items":[{"payload_reference":"dw-unstored"}]})).await; + assert_eq!((status, body["reason"].clone()), (StatusCode::UNPROCESSABLE_ENTITY, json!("external_payload_unsupported")), "{body}"); + let (status, body) = get(&app, "default", &format!("{run}/streams")).await; + assert_eq!((status, body["count"].clone()), (StatusCode::OK, json!(1)), "{body}"); +} + +// A run's durable streams stay with it when the workflow continues as new, +// as PHP keeps them; the continuation is not refused. +#[tokio::test] +async fn continue_as_new_keeps_durable_streams_with_the_predecessor() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + register(&app, "stream-worker", json!(["echo"]), json!([])).await; + let started = start(&app, "stream-continue").await; + let first = started["run_id"].as_str().unwrap().to_owned(); + let (status, body) = request(&app, "POST", &format!("/api/workflows/stream-continue/runs/{first}/streams/out/items"), + json!({"items":[{"payload":"before"}]})).await; + assert_eq!(status, StatusCode::OK, "{body}"); + let task = poll(&app, "stream-worker", "workflow").await; + let (status, body) = finish_task(&app, &task, json!([{"type":"continue_as_new"}])).await; + assert_eq!(status, StatusCode::OK, "{body}"); + let (_, runs) = request(&app, "GET", "/api/workflows/stream-continue/runs", Value::Null).await; + assert_eq!(runs["runs"].as_array().unwrap().len(), 2, "{runs}"); + let (status, body) = get(&app, "default", &format!("/api/workflows/stream-continue/runs/{first}/streams")).await; + assert_eq!((status, body["count"].clone()), (StatusCode::OK, json!(1)), "{body}"); +} + // The workflow list filters cancelled and terminated runs by their exact // status, as PHP does, instead of refusing those filters. #[tokio::test] diff --git a/rust/tests/execution/workflow_stream_bounds.rs b/rust/tests/execution/workflow_stream_bounds.rs new file mode 100644 index 000000000..8beafc128 --- /dev/null +++ b/rust/tests/execution/workflow_stream_bounds.rs @@ -0,0 +1,358 @@ +use super::*; + +// The durable stream producer contract of Server #774, natively: bounded +// producer controls, whole-batch capacity under contention, expiry for every +// operation that keeps the stream's identity, and a bounded, resumable +// retention pass that reads due streams from an index. + +fn route(workflow: &Value, run: &Value, stream: &str, tail: &str) -> String { + format!("/api/workflows/{}/runs/{}/streams/{stream}{tail}", workflow.as_str().unwrap(), run.as_str().unwrap()) +} + +fn items(count: usize) -> Value { + json!((0..count).map(|n| json!({"payload": n})).collect::>()) +} + +async fn usage(app: &Router, resource: &str) -> i64 { + let (status, body) = request(app, "GET", "/api/system/metrics", Value::Null).await; + assert_eq!(status, StatusCode::OK, "{body}"); + body["metrics"]["dw_namespace_durable_state_usage"]["usage"][resource].as_i64().unwrap() +} + +#[tokio::test] +async fn a_capacity_refusal_writes_nothing_of_the_batch() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + let run = start(&app, "stream-batch").await; + let (workflow, run) = (&run["workflow_id"], &run["run_id"]); + let append = |body: Value| { + let app = app.clone(); + let path = route(workflow, run, "tokens", "/items"); + async move { request(&app, "POST", &path, body).await } + }; + // Three fresh items past a cap of two: nothing is written and the stream + // the append would have opened does not exist. + let (status, refused) = append(json!({"max_pending_items":2,"items":items(3)})).await; + assert_eq!((status, &refused["reason"]), (StatusCode::TOO_MANY_REQUESTS, &json!("stream_full")), "{refused}"); + let (_, listed) = request(&app, "GET", &route(workflow, run, "", "").trim_end_matches('/').to_owned(), Value::Null).await; + assert_eq!(listed["streams"], json!([]), "{listed}"); + assert_eq!(usage(&app, "workflow_stream_items").await, 0); + + let (status, body) = append(json!({"max_pending_items":3,"items":[{"payload":0,"idempotency_key":"k0"},{"payload":1},{"payload":2}]})).await; + assert_eq!(status, StatusCode::OK, "{body}"); + // At the cap one fresh item is refused, and a stored key with it is not + // answered either; duplicates alone take no capacity. + let (status, _) = append(json!({"max_pending_items":3,"items":[{"payload":3}]})).await; + assert_eq!(status, StatusCode::TOO_MANY_REQUESTS); + let (status, _) = append(json!({"max_pending_items":3,"items":[{"payload":0,"idempotency_key":"k0"},{"payload":4}]})).await; + assert_eq!(status, StatusCode::TOO_MANY_REQUESTS); + let (status, body) = append(json!({"max_pending_items":3,"items":[{"payload":0,"idempotency_key":"k0"},{"payload":0,"idempotency_key":"k0"}]})).await; + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!((&body["accepted"], &body["deduped"], &body["accepted_offsets"]), (&json!(0), &json!(2), &json!([0, 0])), "{body}"); + assert_eq!(usage(&app, "workflow_stream_items").await, 3); +} + +#[tokio::test] +async fn producer_controls_are_bounded() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + let run = start(&app, "stream-bounds").await; + let (workflow, run) = (&run["workflow_id"], &run["run_id"]); + let (status, refused) = request(&app, "POST", &route(workflow, run, "tokens", "/items"), + json!({"max_pending_items":10_001,"items":items(1)})).await; + assert_eq!(status, StatusCode::UNPROCESSABLE_ENTITY, "{refused}"); + assert_eq!(refused["errors"]["max_pending_items"], json!(["The max pending items field must not be greater than 10000."])); + assert_eq!(usage(&app, "workflow_streams").await, 0); + let (status, _) = request(&app, "POST", &route(workflow, run, "tokens", "/items"), + json!({"max_pending_items":10_000,"items":items(1)})).await; + assert_eq!(status, StatusCode::OK); + let (status, refused) = request(&app, "POST", &route(workflow, run, "tokens", "/close"), json!({"retention_seconds":2_592_001})).await; + assert_eq!(status, StatusCode::UNPROCESSABLE_ENTITY, "{refused}"); + assert_eq!(refused["errors"]["retention_seconds"], json!(["The retention seconds field must not be greater than 2592000."])); + let (status, closed) = request(&app, "POST", &route(workflow, run, "tokens", "/close"), json!({"retention_seconds":2_592_000})).await; + assert_eq!(status, StatusCode::OK, "{closed}"); + assert_eq!(closed["stream"]["retention_seconds"], 2_592_000); +} + +/// Eight appends of three items at once through two pools, against a cap of +/// five: `stored` items exist first. Returns the accepted and refused counts. +async fn contend(stored: usize) -> (usize, usize, i64) { + let database = TestDatabase::new().await; + let apps = [router(database.open().await.unwrap()), router(database.open().await.unwrap())]; + let run = start(&apps[0], "stream-race").await; + let path = route(&run["workflow_id"], &run["run_id"], "tokens", "/items"); + if stored > 0 { + let (status, _) = request(&apps[0], "POST", &path, json!({"items":items(stored)})).await; + assert_eq!(status, StatusCode::OK); + } + let mut tasks = tokio::task::JoinSet::new(); + for index in 0..8 { + let app = apps[index % 2].clone(); + let path = path.clone(); + tasks.spawn(async move { request(&app, "POST", &path, json!({"max_pending_items":5,"items":items(3)})).await }); + } + let (mut accepted, mut refused) = (0, 0); + while let Some(answer) = tasks.join_next().await { + let (status, body) = answer.unwrap(); + match status { + StatusCode::OK => accepted += 1, + StatusCode::TOO_MANY_REQUESTS => refused += 1, + other => panic!("{other}: {body}"), + } + } + (accepted, refused, usage(&apps[0], "workflow_stream_items").await) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn concurrent_appends_never_pass_the_cap() { + // Concurrent first appends open one stream; the first batch fits, every + // other one is refused whole. + assert_eq!(contend(0).await, (1, 7, 3)); + // With one item stored, one batch reaches four and the rest are refused. + assert_eq!(contend(1).await, (1, 7, 4)); +} + +#[tokio::test] +async fn an_expired_stream_reads_as_not_found() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + let run = start(&app, "stream-expiry").await; + let (workflow, run) = (&run["workflow_id"], &run["run_id"]); + for name in ["tokens", "kept"] { + let (status, _) = request(&app, "POST", &route(workflow, run, name, "/items"), json!({"items":items(1)})).await; + assert_eq!(status, StatusCode::OK); + } + let (status, _) = request(&app, "POST", &route(workflow, run, "tokens", "/close"), json!({"retention_seconds":1})).await; + assert_eq!(status, StatusCode::OK); + tokio::time::sleep(std::time::Duration::from_millis(2100)).await; + + let (status, body) = request(&app, "GET", &route(workflow, run, "tokens", ""), Value::Null).await; + assert_eq!((status, &body["reason"]), (StatusCode::NOT_FOUND, &json!("stream_not_found")), "{body}"); + let (status, body) = request(&app, "GET", &route(workflow, run, "tokens", "/items"), Value::Null).await; + assert_eq!((status, &body["reason"]), (StatusCode::NOT_FOUND, &json!("stream_not_found")), "{body}"); + let (status, body) = request(&app, "POST", &route(workflow, run, "tokens", "/close"), json!({})).await; + assert_eq!((status, &body["reason"]), (StatusCode::NOT_FOUND, &json!("stream_not_found")), "{body}"); + let list = format!("/api/workflows/{}/runs/{}/streams", workflow.as_str().unwrap(), run.as_str().unwrap()); + let (_, listed) = request(&app, "GET", &list, Value::Null).await; + let names: Vec<_> = listed["streams"].as_array().unwrap().iter().map(|stream| stream["stream_name"].clone()).collect(); + assert_eq!(names, vec![json!("kept")], "{listed}"); + // Append is not found too, and writes nothing. + let (status, body) = request(&app, "POST", &route(workflow, run, "tokens", "/items"), json!({"items":items(1)})).await; + assert_eq!((status, &body["reason"]), (StatusCode::NOT_FOUND, &json!("stream_not_found")), "{body}"); + assert_eq!((usage(&app, "workflow_streams").await, usage(&app, "workflow_stream_items").await), (2, 2)); +} + +#[tokio::test] +async fn the_retention_pass_reclaims_expired_streams_in_bounded_resumable_steps() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + let run = start(&app, "stream-sweep").await; + let (workflow, run) = (&run["workflow_id"], &run["run_id"]); + let fill = |name: &'static str, count: usize| { + let app = app.clone(); + let path = route(workflow, run, name, "/items"); + async move { + let mut left = count; + while left > 0 { + let batch = left.min(500); + let (status, body) = request(&app, "POST", &path, json!({"items":items(batch)})).await; + assert_eq!(status, StatusCode::OK, "{body}"); + left -= batch; + } + } + }; + fill("large", 150).await; + fill("small", 1).await; + fill("long", 1).await; + fill("open", 1).await; + for (name, retention) in [("large", 1), ("small", 1), ("long", 3600)] { + let (status, _) = request(&app, "POST", &route(workflow, run, name, "/close"), json!({"retention_seconds":retention})).await; + assert_eq!(status, StatusCode::OK); + } + tokio::time::sleep(std::time::Duration::from_millis(2100)).await; + let pass = |limit: i64| { + let app = app.clone(); + async move { + let (status, body) = request(&app, "POST", "/api/system/retention/pass", json!({"limit":limit})).await; + assert!(status == StatusCode::OK, "{status}: {body}"); + } + }; + let deadlines = || task_retry::rows(&database, "SELECT COUNT(*) FROM dw_stream_reclaims"); + assert_eq!(deadlines().await, 3); + // limit 1: one stream and at most 100 items per pass; the next pass + // resumes where it stopped. Reclaimed streams keep their rows. + pass(1).await; + assert_eq!((usage(&app, "workflow_streams").await, usage(&app, "workflow_stream_items").await), (4, 53)); + pass(1).await; + assert_eq!((usage(&app, "workflow_streams").await, usage(&app, "workflow_stream_items").await), (4, 3)); + pass(1).await; + assert_eq!((usage(&app, "workflow_streams").await, usage(&app, "workflow_stream_items").await), (4, 2)); + // The long-retention and open streams stay; only the long one still has + // a deadline. + pass(100).await; + assert_eq!((usage(&app, "workflow_streams").await, usage(&app, "workflow_stream_items").await), (4, 2)); + assert_eq!(deadlines().await, 1); + // A reclaimed stream's name is never opened again. + let (status, body) = request(&app, "POST", &route(workflow, run, "large", "/items"), json!({"items":items(1)})).await; + assert_eq!((status, &body["reason"]), (StatusCode::NOT_FOUND, &json!("stream_not_found")), "{body}"); + assert_eq!(usage(&app, "workflow_stream_items").await, 2); +} + +#[tokio::test] +async fn a_stream_whose_items_hold_external_references_is_left_with_its_items() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + let run = start(&app, "stream-references").await; + let (workflow, run) = (&run["workflow_id"], &run["run_id"]); + let (status, _) = request(&app, "POST", &route(workflow, run, "tokens", "/items"), json!({"items":items(2)})).await; + assert_eq!(status, StatusCode::OK); + let (status, _) = request(&app, "POST", &route(workflow, run, "tokens", "/close"), json!({"retention_seconds":1})).await; + assert_eq!(status, StatusCode::OK); + // Native admits no external reference and settles none, so a row + // holding one, written here directly, keeps its stream unreclaimed. + database.execute("UPDATE workflow_durable_stream_items SET payload_reference='file:///php/stream-item.bin'").await; + tokio::time::sleep(std::time::Duration::from_millis(2100)).await; + let (status, body) = request(&app, "POST", "/api/system/retention/pass", json!({})).await; + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!(usage(&app, "workflow_stream_items").await, 2); + assert_eq!(task_retry::rows(&database, "SELECT COUNT(*) FROM dw_stream_reclaims").await, 1); +} + +async fn completed_run(app: &Router, workflow: &str, stream_items: usize) -> String { + let started = start(app, workflow).await; + let (status, _) = request(app, "POST", &route(&started["workflow_id"], &started["run_id"], "tokens", "/items"), json!({"items":items(stream_items)})).await; + assert_eq!(status, StatusCode::OK); + let (status, _) = request(app, "POST", &route(&started["workflow_id"], &started["run_id"], "tokens", "/close"), json!({})).await; + assert_eq!(status, StatusCode::OK); + let task = poll(app, "worker", "workflow").await; + let complete = json!({"type":"complete_workflow","result":envelope(Payload::Long(1))}); + assert_eq!(finish_task(app, &task, json!([complete])).await.0, StatusCode::OK); + started["run_id"].as_str().unwrap().to_owned() +} + +#[tokio::test] +async fn run_retention_removes_a_pruned_runs_streams_unless_they_hold_references() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + register(&app, "worker", json!(["echo"]), json!([])).await; + let pruned = completed_run(&app, "stream-run-pruned", 2).await; + let held = completed_run(&app, "stream-run-held", 1).await; + for run in [&pruned, &held] { + for table in ["workflow_runs", "workflow_run_summaries"] { + database + .execute(sqlx::AssertSqlSafe(format!("UPDATE {table} SET closed_at='2020-01-01 00:00:00.000000' WHERE id='{run}'"))) + .await; + } + } + database + .execute(sqlx::AssertSqlSafe(format!("UPDATE workflow_durable_stream_items SET payload_reference='file:///php/held.bin' WHERE workflow_run_id='{held}'"))) + .await; + + let (status, body) = request(&app, "POST", "/api/system/retention/pass", json!({"run_ids":[pruned, held]})).await; + assert_eq!(status, StatusCode::OK, "{body}"); + let result = &body["results"][0]; + assert_eq!(result["outcome"], "pruned", "{body}"); + assert_eq!(result["deleted"]["durable_stream_items_deleted"], 2, "{body}"); + assert_eq!(result["deleted"]["durable_streams_deleted"], 1, "{body}"); + assert_eq!(body["results"][1]["reason"], "external_payload_cleanup_unavailable", "{body}"); + // A pruned run has no streams to write: a later append opens none. + let (status, body) = request(&app, "POST", &route(&json!("stream-run-pruned"), &json!(pruned), "late", "/items"), json!({"items":items(1)})).await; + assert_eq!((status, body["reason"].clone()), (StatusCode::NOT_FOUND, json!("stream_not_found")), "{body}"); + for (sql, expected) in [ + (format!("SELECT COUNT(*) FROM workflow_durable_streams WHERE workflow_run_id='{pruned}'"), 0), + (format!("SELECT COUNT(*) FROM dw_stream_reclaims WHERE workflow_run_id='{pruned}'"), 0), + (format!("SELECT COUNT(*) FROM workflow_durable_stream_items WHERE workflow_run_id='{held}'"), 1), + (format!("SELECT COUNT(*) FROM dw_stream_reclaims WHERE workflow_run_id='{held}'"), 1), + ] { + let sql: &'static str = Box::leak(sql.into_boxed_str()); + assert_eq!(task_retry::rows(&database, sql).await, expected, "{sql}"); + } +} + +/// The retention pass reads a namespace's due streams from +/// `dw_stream_reclaims_due`: an index range in deadline order that stops at +/// its limit, whatever else the namespaces hold. +#[tokio::test] +async fn due_stream_discovery_reads_the_deadline_index() { + use sqlx::{Connection, Row}; + let database = TestDatabase::new().await; + let _app = router(database.open().await.unwrap()); + // 2,000 deadlines in two namespaces; 30 of `default`'s are due. + let mut values = Vec::new(); + for (n, namespace) in ["default", "elsewhere"].into_iter().enumerate() { + for i in 0..1000 { + let id = n * 1000 + i + 1; + let at = if namespace == "default" && i < 30 { "2020-01-01 00:00:00" } else { "2099-01-01 00:00:00" }; + values.push(format!("({id},'{namespace}','run-{id}','{at}')")); + } + } + for chunk in values.chunks(500) { + database + .execute(sqlx::AssertSqlSafe(format!( + "INSERT INTO dw_stream_reclaims(stream_id,namespace,workflow_run_id,reclaim_at) VALUES {}", + chunk.join(",") + ))) + .await; + } + let sql = "SELECT stream_id FROM dw_stream_reclaims WHERE namespace='default' AND reclaim_at<='2026-01-01 00:00:00' ORDER BY reclaim_at,stream_id LIMIT 10"; + let plan = match &database { + TestDatabase::Sqlite(dir) => { + let pool = sqlx::SqlitePool::connect(&format!("sqlite://{}", dir.path().join("runtime.sqlite").display())).await.unwrap(); + let rows: Vec<(i64, i64, i64, String)> = + sqlx::query_as(sqlx::AssertSqlSafe(format!("EXPLAIN QUERY PLAN {sql}"))).fetch_all(&pool).await.unwrap(); + pool.close().await; + let plan = rows.into_iter().map(|row| row.3).collect::>().join("\n"); + assert!(plan.contains("USING COVERING INDEX dw_stream_reclaims_due (namespace=? AND reclaim_at { + let mut connection = sqlx::PgConnection::connect_with(options).await.unwrap(); + sqlx::query("ANALYZE dw_stream_reclaims").execute(&mut connection).await.unwrap(); + // A small table plans as a sequential scan; without one, the plan + // shows the index serves both the predicate and the order. + let mut tx = connection.begin().await.unwrap(); + sqlx::query("SET LOCAL enable_seqscan=off").execute(&mut *tx).await.unwrap(); + let rows: Vec<(String,)> = sqlx::query_as(sqlx::AssertSqlSafe(format!( + "EXPLAIN (ANALYZE, COSTS OFF, TIMING OFF, SUMMARY OFF) {sql}" + ))) + .fetch_all(&mut *tx) + .await + .unwrap(); + tx.rollback().await.unwrap(); + let plan = rows.into_iter().map(|row| row.0).collect::>().join("\n"); + assert!(plan.contains("Index Only Scan using dw_stream_reclaims_due"), "{plan}"); + assert!(plan.contains("actual rows=10 loops=1"), "{plan}"); + assert!(!plan.contains("Sort"), "{plan}"); + plan + } + TestDatabase::MySql { options, .. } => { + let mut connection = sqlx::MySqlConnection::connect_with(options).await.unwrap(); + sqlx::raw_sql("ANALYZE TABLE dw_stream_reclaims").execute(&mut connection).await.unwrap(); + let explained = sqlx::raw_sql(sqlx::AssertSqlSafe(format!("EXPLAIN {sql}"))).fetch_all(&mut connection).await.unwrap(); + assert_eq!(explained.len(), 1); + let field = |name: &str| explained[0].try_get::, _>(name).unwrap().unwrap_or_default(); + let plan = format!("type={} key={} extra={}", field("type"), field("key"), field("Extra")); + assert_eq!((field("type"), field("key")), ("range".into(), "dw_stream_reclaims_due".into()), "{plan}"); + assert!(!plan.contains("filesort"), "{plan}"); + // The first use of a table or status view reads the data + // dictionary, which MySQL counts too; measure a second read. + sqlx::raw_sql(sql).fetch_all(&mut connection).await.unwrap(); + sqlx::raw_sql("SHOW SESSION STATUS LIKE 'Handler_read%'").fetch_all(&mut connection).await.unwrap(); + sqlx::raw_sql("FLUSH STATUS").execute(&mut connection).await.unwrap(); + assert_eq!(sqlx::raw_sql(sql).fetch_all(&mut connection).await.unwrap().len(), 10); + let reads: std::collections::BTreeMap = sqlx::raw_sql("SHOW SESSION STATUS LIKE 'Handler_read%'") + .fetch_all(&mut connection) + .await + .unwrap() + .iter() + .map(|row| (row.try_get::(0).unwrap(), row.try_get::(1).unwrap().parse().unwrap())) + .collect(); + // One seek and nine index steps: the limit, not the namespace. + assert!(reads["Handler_read_key"] + reads["Handler_read_next"] <= 11, "{reads:?}"); + assert_eq!(reads["Handler_read_rnd_next"], 0, "{reads:?}"); + format!("{plan} reads={reads:?}") + } + }; + eprintln!("[due-stream plan]\n{plan}"); +} diff --git a/rust/tests/fixtures/unserved-php-routes.v2.11.1.txt b/rust/tests/fixtures/unserved-php-routes.v2.11.1.txt index a67da9d4a..eea755f16 100644 --- a/rust/tests/fixtures/unserved-php-routes.v2.11.1.txt +++ b/rust/tests/fixtures/unserved-php-routes.v2.11.1.txt @@ -28,5 +28,3 @@ POST /api/worker/workflow-tasks/{taskId}/local-activities/{attemptId}/acknowledg POST /api/worker/workflow-tasks/{taskId}/local-activities/{attemptId}/control POST /api/worker/workflow-tasks/{taskId}/local-activities/{attemptId}/heartbeat POST /api/worker/workflow-tasks/{taskId}/local-activities/{attemptId}/outcome -POST /api/workflows/{workflowId}/runs/{runId}/streams/{streamName}/close -POST /api/workflows/{workflowId}/runs/{runId}/streams/{streamName}/items diff --git a/scripts/conformance/server-parity/contract.mjs b/scripts/conformance/server-parity/contract.mjs index f5a237c5c..df9c973f5 100644 --- a/scripts/conformance/server-parity/contract.mjs +++ b/scripts/conformance/server-parity/contract.mjs @@ -34,6 +34,7 @@ import {checkRepairPass} from './repair-pass-contract.mjs'; import {checkNamespaceDelete} from './namespace-delete-contract.mjs'; import {checkUpdateValidation} from './update-validation-contract.mjs'; import {checkMessageStreams} from './message-streams-contract.mjs'; +import {checkWorkflowStreamWrites} from './workflow-stream-writes-contract.mjs'; import {checkSchedulePolicies} from './schedule-policies-contract.mjs'; import {checkScheduleCatalog} from './schedule-catalog-contract.mjs'; import {checkSignalsBeforeWorker} from './signals-before-worker-contract.mjs'; @@ -186,6 +187,7 @@ export function checkObservation(fixture, observation, workflowId, artifacts) { const namespaceDelete = fixture.namespace_delete ? checkNamespaceDelete(fixture, observation, workflowId) : null; const updateValidation = fixture.update_validation ? checkUpdateValidation(fixture, observation, workflowId) : null; const messageStreams = fixture.message_streams ? checkMessageStreams(fixture, observation, workflowId) : null; + const workflowStreamWrites = fixture.workflow_stream_writes ? checkWorkflowStreamWrites(fixture, observation, workflowId) : null; const schedulePolicies = fixture.schedule_policies ? checkSchedulePolicies(fixture, observation, workflowId) : null; const scheduleCatalog = fixture.schedule_catalog ? checkScheduleCatalog(fixture, observation, workflowId) : null; const signalsBeforeWorker = fixture.signals_before_worker ? checkSignalsBeforeWorker(fixture, observation, workflowId) : null; @@ -754,6 +756,7 @@ export function checkObservation(fixture, observation, workflowId, artifacts) { ...(namespaceDelete ? {namespace_delete: namespaceDelete} : {}), ...(updateValidation ? {update_validation: updateValidation} : {}), ...(messageStreams ? {message_streams: messageStreams} : {}), + ...(workflowStreamWrites ? {workflow_stream_writes: workflowStreamWrites} : {}), ...(schedulePolicies ? {schedule_policies: schedulePolicies} : {}), ...(scheduleCatalog ? {schedule_catalog: scheduleCatalog} : {}), ...(signalsBeforeWorker ? {signals_before_worker: signalsBeforeWorker} : {}), diff --git a/scripts/conformance/server-parity/mysql-schema.php b/scripts/conformance/server-parity/mysql-schema.php index 1a71793d7..0d828a2cf 100644 --- a/scripts/conformance/server-parity/mysql-schema.php +++ b/scripts/conformance/server-parity/mysql-schema.php @@ -106,7 +106,7 @@ function catalog(PDO $pdo): array } $pdo->exec('ROLLBACK'); $pdo = connection($argv[3]); -$native = array_values(array_filter(catalog($pdo), static fn (array $row): bool => ! in_array($row[0], ['dw_server_schema', '_sqlx_migrations', 'dw_task_completions', 'dw_poll_receipts', 'dw_query_cache', 'dw_worker_registration_incarnations', 'dw_worker_session_holders'], true))); +$native = array_values(array_filter(catalog($pdo), static fn (array $row): bool => ! in_array($row[0], ['dw_server_schema', '_sqlx_migrations', 'dw_task_completions', 'dw_poll_receipts', 'dw_query_cache', 'dw_worker_registration_incarnations', 'dw_worker_session_holders', 'dw_stream_reclaims'], true))); $pdo->exec('ROLLBACK'); // Native polling uses an additional index on an existing table. Retain all // original column, index and constraint definitions in the comparison. diff --git a/scripts/conformance/server-parity/postgres-schema.php b/scripts/conformance/server-parity/postgres-schema.php index 7ea05ff49..236486e51 100644 --- a/scripts/conformance/server-parity/postgres-schema.php +++ b/scripts/conformance/server-parity/postgres-schema.php @@ -70,8 +70,8 @@ function catalog(string $database): array throw new InvalidArgumentException('Compare requires two independent databases.'); } $native = catalog($argv[3]); -$tables = ['_sqlx_migrations', 'dw_server_schema', 'dw_task_completions', 'dw_poll_receipts', 'dw_query_cache', 'dw_worker_registration_incarnations', 'dw_worker_session_holders']; -$indexes = ['_sqlx_migrations_pkey', 'dw_task_completions_pkey', 'dw_poll_receipts_pkey', 'dw_workflow_tasks_poll', 'dw_query_cache_pkey', 'dw_worker_registration_incarnations_pkey', 'dw_worker_incarnation_authority', 'dw_worker_incarnation_retention', 'dw_worker_session_holders_pkey']; +$tables = ['_sqlx_migrations', 'dw_server_schema', 'dw_task_completions', 'dw_poll_receipts', 'dw_query_cache', 'dw_worker_registration_incarnations', 'dw_worker_session_holders', 'dw_stream_reclaims']; +$indexes = ['_sqlx_migrations_pkey', 'dw_task_completions_pkey', 'dw_poll_receipts_pkey', 'dw_workflow_tasks_poll', 'dw_query_cache_pkey', 'dw_worker_registration_incarnations_pkey', 'dw_worker_incarnation_authority', 'dw_worker_incarnation_retention', 'dw_worker_session_holders_pkey', 'dw_stream_reclaims_pkey', 'dw_stream_reclaims_due']; $native = array_values(array_filter($native, static function (array $row) use ($tables, $indexes): bool { if (in_array($row[0], ['column', 'constraint'], true)) { return ! in_array(explode('.', $row[1], 2)[0], $tables, true); diff --git a/scripts/conformance/server-parity/probe.php b/scripts/conformance/server-parity/probe.php index ec38d7b86..46a0e3bae 100644 --- a/scripts/conformance/server-parity/probe.php +++ b/scripts/conformance/server-parity/probe.php @@ -453,6 +453,7 @@ function httpObservation(array $fixture, string $workflowId, string $namespace, $namespaceDelete = isset($fixture['namespace_delete']) ? finishHttpNamespaceDelete($client, $fixture, $workflowId, $queue, $url) : null; $updateValidation = isset($fixture['update_validation']) ? finishHttpUpdateValidation($client, $fixture, $workflowId, $url) : null; $messageStreams = isset($fixture['message_streams']) ? finishHttpMessageStreams($client, $fixture, $workflowId, $url) : null; + $workflowStreamWrites = isset($fixture['workflow_stream_writes']) ? finishHttpWorkflowStreamWrites($client, $fixture, $workflowId, $url) : null; $schedulePolicies = isset($fixture['schedule_policies']) ? finishHttpSchedulePolicies($client, $fixture, $workflowId, $url) : null; $scheduleCatalog = isset($fixture['schedule_catalog']) ? finishHttpSchedulePolicies($client, $fixture, $workflowId, $url, 'schedule_catalog') @@ -574,6 +575,7 @@ function httpObservation(array $fixture, string $workflowId, string $namespace, 'namespace_delete' => $namespaceDelete, 'update_validation' => $updateValidation, 'message_streams' => $messageStreams, + 'workflow_stream_writes' => $workflowStreamWrites, 'schedule_policies' => $schedulePolicies, 'schedule_catalog' => $scheduleCatalog, 'signals_before_worker' => $signalsBeforeWorker, @@ -899,6 +901,7 @@ function embeddedObservation(array $fixture, string $workflowId, string $namespa require __DIR__.'/namespace-delete-probe.php'; require __DIR__.'/update-validation-probe.php'; require __DIR__.'/message-streams-probe.php'; + require __DIR__.'/workflow-stream-writes-probe.php'; require __DIR__.'/schedule-policies-probe.php'; require __DIR__.'/signals-before-worker-probe.php'; require __DIR__.'/worker-deregistration-probe.php'; @@ -1123,6 +1126,9 @@ function embeddedObservation(array $fixture, string $workflowId, string $namespa if ($mode === 'embedded' && isset($fixture['message_streams'])) { $observation['message_streams'] = ['applicable' => false, 'reason' => 'embedded_has_no_http_message_streams']; } + if ($mode === 'embedded' && isset($fixture['workflow_stream_writes'])) { + $observation['workflow_stream_writes'] = ['applicable' => false, 'reason' => 'embedded_has_no_http_workflow_streams']; + } if ($mode === 'embedded' && isset($fixture['schedule_policies'])) { $observation['schedule_policies'] = ['applicable' => false, 'reason' => 'embedded_has_no_http_schedules']; } diff --git a/scripts/conformance/server-parity/schema.php b/scripts/conformance/server-parity/schema.php index 5929ce8db..f52b18e84 100644 --- a/scripts/conformance/server-parity/schema.php +++ b/scripts/conformance/server-parity/schema.php @@ -38,7 +38,7 @@ function catalog(string $filename): array $php = catalog($argv[1]); $native = catalog($argv[2]); -$nativeOnly = ['_sqlx_migrations', 'dw_server_schema', 'dw_task_completions', 'dw_poll_receipts', 'dw_workflow_tasks_poll', 'dw_query_cache', 'dw_worker_registration_incarnations', 'dw_worker_incarnation_authority', 'dw_worker_incarnation_retention', 'dw_worker_session_holders']; +$nativeOnly = ['_sqlx_migrations', 'dw_server_schema', 'dw_task_completions', 'dw_poll_receipts', 'dw_workflow_tasks_poll', 'dw_query_cache', 'dw_worker_registration_incarnations', 'dw_worker_incarnation_authority', 'dw_worker_incarnation_retention', 'dw_worker_session_holders', 'dw_stream_reclaims', 'dw_stream_reclaims_due']; $native = array_values(array_filter($native, static fn (array $row): bool => ! in_array($row['name'], $nativeOnly, true))); if ($php !== $native) { $byName = static fn (array $rows): array => array_column($rows, null, 'name'); diff --git a/scripts/conformance/server-parity/workflow-stream-writes-contract.mjs b/scripts/conformance/server-parity/workflow-stream-writes-contract.mjs new file mode 100644 index 000000000..7b77b4e18 --- /dev/null +++ b/scripts/conformance/server-parity/workflow-stream-writes-contract.mjs @@ -0,0 +1,205 @@ +import assert from 'node:assert/strict'; + +// Published 2.11.1 stores stream times in whole seconds and answers them in +// UTC with six fractional digits. +const TIME = /^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d\.000000Z$/; +const STEPS = ['applicable', 'workflow_id', 'run_id', 'list_empty', 'show_missing', 'items_missing', 'close_missing', + 'append_first', 'append_retry', 'show_after_appends', 'items_window', 'show_after_window', 'items_all', + 'show_after_catch_up', 'items_wait_empty', 'items_wait_returned_after_wait', 'append_full', 'show_after_full', + 'items_after_full', 'refusals', 'list_after_refusals', 'close', 'close_again', 'append_closed', 'items_closed_tail', + 'items_closed_head', 'append_other', 'close_errored', 'append_errored', 'items_errored', 'list_final']; + +const notFound = {message: 'Stream not found.', reason: 'stream_not_found'}; +const runNotFound = {message: 'Workflow run not found.', reason: 'instance_not_found'}; + +function exact(read, status, response, label) { + assert.equal(read?.status, status, `${label} status`); + assert.deepStrictEqual(read.response, response, `${label} body`); +} + +// Laravel's 422 with one attribute and its one message. +function refused(read, field, message, label) { + exact(read, 422, {message, reason: 'validation_failed', errors: {[field]: [message]}, + validation_errors: {[field]: [message]}}, label); +} + +// One stream as formatStream answers it: exact counters and states, times in +// whole seconds that never run backwards. +function stream(actual, expected, label) { + const {opened_at: opened, last_appended_at: appended, closed_at: closed, ...rest} = actual; + const {closed: terminal = false, ...fields} = expected; + assert.deepStrictEqual(rest, {error_reason: null, retention_seconds: null, ...fields}, `${label} stream`); + assert.match(opened ?? '', TIME, `${label} opened_at`); + assert.match(appended ?? '', TIME, `${label} last_appended_at`); + assert.ok(appended >= opened, `${label} appended after opening`); + if (terminal) { + assert.match(closed ?? '', TIME, `${label} closed_at`); + assert.ok(closed >= appended, `${label} closed after the last append`); + } else { + assert.equal(closed, null, `${label} is not closed`); + } + return {opened, appended, closed}; +} + +function streamRead(read, ids, expected, label) { + assert.equal(read?.status, 200, `${label} status`); + const {stream: body, ...rest} = read.response; + assert.deepStrictEqual(rest, ids, `${label} identity`); + return stream(body, expected, label); +} + +function appended(read, ids, expected, offsets, accepted, deduped, label) { + assert.equal(read?.status, 200, `${label} status`); + const {stream: body, ...rest} = read.response; + assert.deepStrictEqual(rest, {...ids, accepted_offsets: offsets, accepted, deduped}, `${label} receipt`); + return stream(body, expected, label); +} + +function item(offset, fields) { + return {offset, idempotency_key: null, origin: 'workflow_command', origin_reference: null, item_type: null, content_type: null, + payload: null, payload_reference: null, payload_codec: null, ...fields}; +} + +function items(read, ids, expected, rows, nextOffset, terminal, label) { + assert.equal(read?.status, 200, `${label} status`); + const {stream: body, items: actual, next_offset: next, terminal: done, ...rest} = read.response; + assert.deepStrictEqual(rest, ids, `${label} identity`); + stream(body, expected, label); + assert.equal(next, nextOffset, `${label} next_offset`); + assert.equal(done, terminal, `${label} terminal`); + assert.equal(actual.length, rows.length, `${label} item count`); + actual.forEach((row, index) => { + const {emitted_at: emitted, ...fields} = row; + assert.deepStrictEqual(fields, rows[index], `${label} item ${index}`); + assert.match(emitted ?? '', TIME, `${label} item ${index} emitted_at`); + }); +} + +export function checkWorkflowStreamWrites(fixture, observation, workflowId) { + const spec = fixture.workflow_stream_writes; + const state = observation.workflow_stream_writes; + if (observation.mode === 'embedded') { + assert.deepStrictEqual(state, {applicable: false, reason: 'embedded_has_no_http_workflow_streams'}, + 'embedded workflow streams are inapplicable'); + return {scope: 'workflow_stream_writes'}; + } + assert.deepStrictEqual(Object.keys(state ?? {}), STEPS, 'workflow stream writes steps'); + assert.equal(state.applicable, true); + const id = `${workflowId}-wsw`; + assert.equal(state.workflow_id, id); + assert.equal(typeof state.run_id, 'string'); + const ids = {workflow_id: id, workflow_run_id: state.run_id}; + const tokens = spec.stream; + const audit = spec.other_stream; + const batch = spec.batches; + const label = name => `workflow stream writes ${name}`; + + exact(state.list_empty, 200, {...ids, count: 0, streams: []}, label('list_empty')); + for (const name of ['show_missing', 'items_missing', 'close_missing']) exact(state[name], 404, notFound, label(name)); + + // The first append opens the stream at offset 0; a repeated key returns the + // existing item's offset and only new items move the counters. + const open = {stream_name: tokens, status: 'open'}; + const first = appended(state.append_first, ids, {...open, last_offset: 1, total_items: 2, pending_items: 2}, + [0, 1], 2, 0, label('append_first')); + appended(state.append_retry, ids, {...open, last_offset: 2, total_items: 3, pending_items: 3}, + [0, 2], 1, 1, label('append_retry')); + const three = {...open, last_offset: 2, total_items: 3}; + streamRead(state.show_after_appends, ids, {...three, pending_items: 3}, label('show_after_appends')); + + const rows = [ + item(0, {...batch.first[0], idempotency_key: 'k1'}), + item(1, {payload: batch.first[1].payload, payload_codec: 'avro', idempotency_key: 'k2'}), + item(2, {payload: batch.retry[1].payload}), + ]; + // A window that stops short of the head leaves pending_items alone; one + // that reaches the head clears it, and a bounded wait past it returns empty. + items(state.items_window, ids, {...three, pending_items: 3}, [rows[1]], 2, false, label('items_window')); + streamRead(state.show_after_window, ids, {...three, pending_items: 3}, label('show_after_window')); + items(state.items_all, ids, {...three, pending_items: 0}, rows, 3, false, label('items_all')); + streamRead(state.show_after_catch_up, ids, {...three, pending_items: 0}, label('show_after_catch_up')); + items(state.items_wait_empty, ids, {...three, pending_items: 0}, [], 3, false, label('items_wait_empty')); + assert.equal(state.items_wait_returned_after_wait, true, label('a bounded wait on an open stream waits')); + + // Server #774 refuses the over-cap batch whole: nothing is written, so the + // stream keeps its three items. Published 2.11.1 committed the items before + // the cap and then refused the rest (a published defect, bounded here): its + // answers are accepted as a set, and every later step follows the branch + // the observation took. + const whole = state.show_after_full?.response?.stream?.total_items === 3; + exact(state.append_full, 429, {message: 'Stream pending-items cap reached; consumer is draining too slowly.', + reason: 'stream_full', pending_items: whole ? 0 : spec.max_pending_items, max_pending_items: spec.max_pending_items}, + label('append_full')); + const five = whole ? three : {...open, last_offset: 4, total_items: 5}; + const full = streamRead(state.show_after_full, ids, {...five, pending_items: whole ? 0 : 2}, label('show_after_full')); + assert.ok(full.appended >= first.appended, label('the capped append kept last_appended_at in order')); + items(state.items_after_full, ids, {...five, pending_items: 0}, + whole ? [] : [item(3, {...batch.over_cap[0]}), item(4, {...batch.over_cap[1]})], whole ? 3 : 5, false, + label('items_after_full')); + + const r = state.refusals; + const refusal = name => label(`refusal ${name}`); + refused(r.no_items, 'items', 'The items field is required.', refusal('no_items')); + refused(r.empty_items, 'items', 'The items field is required.', refusal('empty_items')); + refused(r.too_many_items, 'items', 'The items field must not have more than 500 items.', refusal('too_many_items')); + refused(r.item_not_object, 'items.0', 'The items.0 field must be an array.', refusal('item_not_object')); + const codecMessage = `unsupported_payload_codec: workflow payload codec "${spec.bad_codec}" is not supported by Durable Workflow 2.0; ` + + 'use codec="avro" with the fixed Avro Value schema and single-object framing. JSON remains the HTTP document ' + + 'transport, not a workflow payload codec.'; + refused(r.bad_codec, 'items.0.payload_codec', codecMessage, refusal('bad_codec')); + refused(r.long_idempotency_key, 'items.0.idempotency_key', + 'The items.0.idempotency_key field must not be greater than 191 characters.', refusal('long_idempotency_key')); + refused(r.zero_max_pending, 'max_pending_items', 'The max pending items field must be at least 1.', refusal('zero_max_pending')); + exact(r.bad_stream_append, 400, {message: 'invalid stream name', reason: 'invalid_request'}, refusal('bad_stream_append')); + exact(r.unstored_reference, 422, {schema: 'durable-workflow.v2.runtime-external-payload-error.v1', + reason: 'external_payload_unsupported', + message: 'Payload reference fields require an opaque runtime external payload reference object.', + retryable: false, status: 422}, refusal('unstored_reference')); + for (const name of ['unknown_run_append', 'unknown_workflow_append', 'unknown_run_close']) { + exact(r[name], 404, runNotFound, refusal(name)); + } + for (const name of ['bad_stream_show', 'bad_stream_items', 'bad_stream_close']) { + exact(r[name], 400, {message: 'invalid stream name', reason: 'invalid_stream_name'}, refusal(name)); + } + refused(r.negative_from, 'from', 'The from field must be at least 0.', refusal('negative_from')); + refused(r.too_many_read, 'max_items', 'The max items field must not be greater than 500.', refusal('too_many_read')); + refused(r.long_wait, 'wait_seconds', 'The wait seconds field must not be greater than 60.', refusal('long_wait')); + refused(r.long_error_reason, 'error_reason', 'The error reason field must not be greater than 191 characters.', + refusal('long_error_reason')); + refused(r.zero_retention, 'retention_seconds', 'The retention seconds field must be at least 1.', refusal('zero_retention')); + assert.deepStrictEqual(Object.keys(r).sort(), ['bad_codec', 'bad_stream_append', 'bad_stream_close', 'bad_stream_items', + 'bad_stream_show', 'empty_items', 'item_not_object', 'long_error_reason', 'long_idempotency_key', 'long_wait', + 'negative_from', 'no_items', 'too_many_items', 'too_many_read', 'unknown_run_append', 'unknown_run_close', + 'unknown_workflow_append', 'unstored_reference', 'zero_max_pending', 'zero_retention'], label('refusal set')); + + // No refusal wrote anything: the list still shows one stream as it was. + assert.equal(state.list_after_refusals?.status, 200, label('list_after_refusals status')); + assert.deepStrictEqual({...state.list_after_refusals.response, streams: undefined}, {...ids, count: 1, streams: undefined}, + label('list_after_refusals identity')); + stream(state.list_after_refusals.response.streams[0], {...five, pending_items: 0}, label('list_after_refusals stream')); + + // Close once; a terminal stream keeps its first close, error and retention. + const closed = {...five, pending_items: 0, status: 'closed', retention_seconds: 3600, closed: true}; + const firstClose = streamRead(state.close, ids, closed, label('close')); + const again = streamRead(state.close_again, ids, closed, label('close_again')); + assert.equal(again.closed, firstClose.closed, label('a repeated close keeps closed_at')); + exact(state.append_closed, 409, {message: 'Stream is closed.', reason: 'stream_closed'}, label('append_closed')); + items(state.items_closed_tail, ids, closed, [], 5, true, label('items_closed_tail')); + items(state.items_closed_head, ids, closed, whole ? [] : [item(4, {...batch.over_cap[1]})], whole ? 4 : 5, true, + label('items_closed_head')); + + const other = {stream_name: audit, status: 'open', last_offset: 0, total_items: 1, pending_items: 1}; + appended(state.append_other, ids, other, [0], 1, 0, label('append_other')); + const errored = {...other, status: 'errored', error_reason: spec.error_reason, retention_seconds: spec.retention_seconds, closed: true}; + streamRead(state.close_errored, ids, errored, label('close_errored')); + exact(state.append_errored, 409, {message: 'Stream is errored.', reason: 'stream_errored'}, label('append_errored')); + items(state.items_errored, ids, {...errored, pending_items: 0}, [item(0, {...batch.other[0]})], 1, true, label('items_errored')); + + assert.equal(state.list_final?.status, 200, label('list_final status')); + const final = state.list_final.response; + assert.equal(final.count, 2, label('list_final count')); + assert.deepStrictEqual(final.streams.map(entry => entry.stream_name), [audit, tokens], label('list_final is by name')); + stream(final.streams[0], {...errored, pending_items: 0}, label('list_final errored')); + stream(final.streams[1], closed, label('list_final closed')); + return {scope: 'workflow_stream_writes'}; +} diff --git a/scripts/conformance/server-parity/workflow-stream-writes-probe.php b/scripts/conformance/server-parity/workflow-stream-writes-probe.php new file mode 100644 index 000000000..fab2fe631 --- /dev/null +++ b/scripts/conformance/server-parity/workflow-stream-writes-probe.php @@ -0,0 +1,103 @@ +push(LockPressurePollRetries::middleware(), 'lock_pressure_poll_retry'); + $http = new WorkflowStreamHttp(['base_uri' => rtrim($url, '/').'/api/', 'http_errors' => false, + 'connect_timeout' => 1, 'timeout' => 30, 'handler' => $stack]); + $raw = static function (string $method, string $path, ?array $body = null) use ($http, $token): array { + $response = $http->request($method, $path, ['headers' => array_filter([ + 'Authorization' => $token === null ? null : 'Bearer '.$token, + 'Accept' => 'application/json', 'X-Namespace' => 'default', + 'X-Durable-Workflow-Control-Plane-Version' => '2', + ]), ...($body === null ? [] : ['json' => $body])]); + $decoded = json_decode((string) $response->getBody(), true, flags: JSON_THROW_ON_ERROR); + if (is_array($decoded)) { + unset($decoded['protocol_version'], $decoded['server_capabilities'], $decoded['control_plane']); + } + + return ['status' => $response->getStatusCode(), 'response' => $decoded]; + }; + $state = ['applicable' => true, 'workflow_id' => $id]; + $state['run_id'] = $client->startWorkflow($definition['workflow_type'], $id, $id.'-queue', [$fixture['input']])->selectedRunId; + $run = "workflows/{$id}/runs/{$state['run_id']}"; + $stream = $definition['stream']; + $other = $definition['other_stream']; + $bad = rawurlencode($definition['bad_stream']); + $items = static fn (string $name): string => "{$run}/streams/".rawurlencode($name).'/items'; + $append = static fn (string $name, array $body): array => $raw('POST', $items($name), $body); + $read = static fn (string $name, string $query = ''): array => $raw('GET', $items($name).($query === '' ? '' : '?'.$query)); + $close = static fn (string $name, array $body = []): array => $raw('POST', "{$run}/streams/".rawurlencode($name).'/close', $body); + $show = static fn (string $name): array => $raw('GET', "{$run}/streams/".rawurlencode($name)); + $batches = $definition['batches']; + + $state['list_empty'] = $raw('GET', "{$run}/streams"); + $state['show_missing'] = $show($stream); + $state['items_missing'] = $read($stream); + $state['close_missing'] = $close($stream); + $state['append_first'] = $append($stream, ['items' => $batches['first']]); + $state['append_retry'] = $append($stream, ['items' => $batches['retry']]); + $state['show_after_appends'] = $show($stream); + $state['items_window'] = $read($stream, 'from=1&max_items=1'); + $state['show_after_window'] = $show($stream); + $state['items_all'] = $read($stream, 'from=0'); + $state['show_after_catch_up'] = $show($stream); + $started = microtime(true); + $state['items_wait_empty'] = $read($stream, 'from=3&wait_seconds=1'); + $state['items_wait_returned_after_wait'] = microtime(true) - $started >= 0.9; + $state['append_full'] = $append($stream, ['items' => $batches['over_cap'], 'max_pending_items' => $definition['max_pending_items']]); + $state['show_after_full'] = $show($stream); + $state['items_after_full'] = $read($stream, 'from=3'); + $state['refusals'] = [ + 'no_items' => $append($stream, []), + 'empty_items' => $append($stream, ['items' => []]), + 'too_many_items' => $append($stream, ['items' => array_fill(0, $definition['too_many_items'], ['payload' => 1])]), + 'item_not_object' => $append($stream, ['items' => ['text']]), + 'bad_codec' => $append($stream, ['items' => [['payload' => 1, 'payload_codec' => $definition['bad_codec']]]]), + 'long_idempotency_key' => $append($stream, ['items' => [['payload' => 1, 'idempotency_key' => str_repeat('k', $definition['long_key_length'])]]]), + 'zero_max_pending' => $append($stream, ['items' => [['payload' => 1]], 'max_pending_items' => 0]), + 'bad_stream_append' => $raw('POST', "{$run}/streams/{$bad}/items", ['items' => [['payload' => 1]]]), + 'unstored_reference' => $append($other, ['items' => [['payload_reference' => $definition['unstored_reference']]]]), + 'unknown_run_append' => $raw('POST', "workflows/{$id}/runs/{$state['run_id']}x/streams/{$stream}/items", ['items' => [['payload' => 1]]]), + 'unknown_workflow_append' => $raw('POST', "workflows/{$id}-missing/runs/{$state['run_id']}/streams/{$stream}/items", ['items' => [['payload' => 1]]]), + 'bad_stream_show' => $raw('GET', "{$run}/streams/{$bad}"), + 'bad_stream_items' => $raw('GET', "{$run}/streams/{$bad}/items"), + 'negative_from' => $read($stream, 'from=-1'), + 'too_many_read' => $read($stream, 'max_items='.$definition['too_many_read']), + 'long_wait' => $read($stream, 'wait_seconds='.$definition['too_long_wait']), + 'bad_stream_close' => $raw('POST', "{$run}/streams/{$bad}/close", []), + 'long_error_reason' => $close($stream, ['error_reason' => str_repeat('e', $definition['long_key_length'])]), + 'zero_retention' => $close($stream, ['retention_seconds' => 0]), + 'unknown_run_close' => $raw('POST', "workflows/{$id}/runs/{$state['run_id']}x/streams/{$stream}/close", []), + ]; + $state['list_after_refusals'] = $raw('GET', "{$run}/streams"); + $state['close'] = $close($stream); + $state['close_again'] = $close($stream, ['error_reason' => 'late error', 'retention_seconds' => 5]); + $state['append_closed'] = $append($stream, ['items' => [['payload' => 1, 'idempotency_key' => 'k1']]]); + $state['items_closed_tail'] = $read($stream, 'from=5'); + $state['items_closed_head'] = $read($stream, 'from=4'); + $state['append_other'] = $append($other, ['items' => $batches['other']]); + $state['close_errored'] = $close($other, ['error_reason' => $definition['error_reason'], 'retention_seconds' => $definition['retention_seconds']]); + $state['append_errored'] = $append($other, ['items' => [['payload' => 1]]]); + $state['items_errored'] = $read($other, 'from=0&wait_seconds=1'); + $state['list_final'] = $raw('GET', "{$run}/streams"); + + return $state; +} diff --git a/tests/Fixtures/ServerParity/workflow-stream-writes.json b/tests/Fixtures/ServerParity/workflow-stream-writes.json new file mode 100644 index 000000000..4670776ff --- /dev/null +++ b/tests/Fixtures/ServerParity/workflow-stream-writes.json @@ -0,0 +1,113 @@ +{ + "schema": "durable-workflow.server-parity-fixture/v1", + "id": "workflow-stream-writes", + "review_state": "reviewed", + "contract": "Published Server 2.11.1 durable workflow stream producer routes: POST runs/{run}/streams/{name}/items opens a stream on first append, assigns offsets from 0, returns an existing item for a repeated idempotency key, refuses a batch whose fresh items would pass the pending-items cap whole with 429 stream_full (Server #774; published 2.11.1 committed the items before the cap first, a bounded published defect), and refuses closed and errored streams with 409. POST .../close closes or errors a stream once, keeps a terminal stream unchanged and defaults retention to 3600 seconds. Reads report the rows, reset pending_items when a read reaches the head, and a bounded wait on an open stream returns empty. Every validation, name and unknown-run refusal is recorded.", + "workflow_type": "parity.v1.echo", + "input": { + "message": "message streams λ", + "count": 9007199254740993, + "optional": null + }, + "typed_value": { + "type": "map", + "value": { + "message": { + "type": "string", + "value": "message streams λ" + }, + "count": { + "type": "int64", + "value": "9007199254740993" + }, + "optional": { + "type": "null", + "value": null + } + } + }, + "activity": false, + "expected_events": [ + "StartAccepted", + "WorkflowStarted", + "WorkflowCompleted" + ], + "workflow_stream_writes": { + "workflow_type": "parity.v1.workflow_streams", + "stream": "tokens", + "other_stream": "audit", + "bad_stream": "bad name!", + "max_pending_items": 2, + "too_many_items": 501, + "too_many_read": 501, + "too_long_wait": 61, + "long_key_length": 192, + "bad_codec": "not-a-codec", + "unstored_reference": "dw-external-payload-reference-that-was-never-stored", + "error_reason": "producer crashed", + "retention_seconds": 60, + "batches": { + "first": [ + { + "payload": { + "token": "Hello", + "count": 9007199254740993, + "unicode": "λ" + }, + "idempotency_key": "k1", + "origin": "worker", + "origin_reference": "attempt-1", + "item_type": "token", + "content_type": "application/json" + }, + { + "payload": "plain text", + "payload_codec": "avro", + "idempotency_key": "k2" + } + ], + "retry": [ + { + "payload": { + "token": "changed" + }, + "idempotency_key": "k1" + }, + { + "payload": [ + 1, + 2, + 3 + ] + } + ], + "over_cap": [ + { + "payload": { + "n": 3 + }, + "idempotency_key": "k3" + }, + { + "payload": { + "n": 4 + } + }, + { + "payload": { + "n": 5 + }, + "idempotency_key": "k5" + } + ], + "other": [ + { + "payload": { + "audit": true + }, + "item_type": "event" + } + ] + } + } +} diff --git a/tests/Fixtures/ServerParityExpected/workflow-stream-writes-observation.json b/tests/Fixtures/ServerParityExpected/workflow-stream-writes-observation.json new file mode 100644 index 000000000..0b07560d6 --- /dev/null +++ b/tests/Fixtures/ServerParityExpected/workflow-stream-writes-observation.json @@ -0,0 +1,858 @@ +{ + "applicable": true, + "workflow_id": "parity-v1-workflow-stream-writes-wsw", + "run_id": "01m5rwqt9swt5hthyvnqaac50h", + "list_empty": { + "status": 200, + "response": { + "workflow_id": "parity-v1-workflow-stream-writes-wsw", + "workflow_run_id": "01m5rwqt9swt5hthyvnqaac50h", + "count": 0, + "streams": [] + } + }, + "show_missing": { + "status": 404, + "response": { + "message": "Stream not found.", + "reason": "stream_not_found" + } + }, + "items_missing": { + "status": 404, + "response": { + "message": "Stream not found.", + "reason": "stream_not_found" + } + }, + "close_missing": { + "status": 404, + "response": { + "message": "Stream not found.", + "reason": "stream_not_found" + } + }, + "append_first": { + "status": 200, + "response": { + "workflow_id": "parity-v1-workflow-stream-writes-wsw", + "workflow_run_id": "01m5rwqt9swt5hthyvnqaac50h", + "stream": { + "stream_name": "tokens", + "status": "open", + "last_offset": 1, + "total_items": 2, + "pending_items": 2, + "opened_at": "2026-10-25T06:42:31.000000Z", + "last_appended_at": "2026-10-25T06:42:31.000000Z", + "closed_at": null, + "error_reason": null, + "retention_seconds": null + }, + "accepted_offsets": [ + 0, + 1 + ], + "accepted": 2, + "deduped": 0 + } + }, + "append_retry": { + "status": 200, + "response": { + "workflow_id": "parity-v1-workflow-stream-writes-wsw", + "workflow_run_id": "01m5rwqt9swt5hthyvnqaac50h", + "stream": { + "stream_name": "tokens", + "status": "open", + "last_offset": 2, + "total_items": 3, + "pending_items": 3, + "opened_at": "2026-10-25T06:42:31.000000Z", + "last_appended_at": "2026-10-25T06:42:31.000000Z", + "closed_at": null, + "error_reason": null, + "retention_seconds": null + }, + "accepted_offsets": [ + 0, + 2 + ], + "accepted": 1, + "deduped": 1 + } + }, + "show_after_appends": { + "status": 200, + "response": { + "workflow_id": "parity-v1-workflow-stream-writes-wsw", + "workflow_run_id": "01m5rwqt9swt5hthyvnqaac50h", + "stream": { + "stream_name": "tokens", + "status": "open", + "last_offset": 2, + "total_items": 3, + "pending_items": 3, + "opened_at": "2026-10-25T06:42:31.000000Z", + "last_appended_at": "2026-10-25T06:42:31.000000Z", + "closed_at": null, + "error_reason": null, + "retention_seconds": null + } + } + }, + "items_window": { + "status": 200, + "response": { + "workflow_id": "parity-v1-workflow-stream-writes-wsw", + "workflow_run_id": "01m5rwqt9swt5hthyvnqaac50h", + "stream": { + "stream_name": "tokens", + "status": "open", + "last_offset": 2, + "total_items": 3, + "pending_items": 3, + "opened_at": "2026-10-25T06:42:31.000000Z", + "last_appended_at": "2026-10-25T06:42:31.000000Z", + "closed_at": null, + "error_reason": null, + "retention_seconds": null + }, + "items": [ + { + "offset": 1, + "idempotency_key": "k2", + "origin": "workflow_command", + "origin_reference": null, + "item_type": null, + "content_type": null, + "payload": "plain text", + "payload_reference": null, + "payload_codec": "avro", + "emitted_at": "2026-10-25T06:42:31.000000Z" + } + ], + "next_offset": 2, + "terminal": false + } + }, + "show_after_window": { + "status": 200, + "response": { + "workflow_id": "parity-v1-workflow-stream-writes-wsw", + "workflow_run_id": "01m5rwqt9swt5hthyvnqaac50h", + "stream": { + "stream_name": "tokens", + "status": "open", + "last_offset": 2, + "total_items": 3, + "pending_items": 3, + "opened_at": "2026-10-25T06:42:31.000000Z", + "last_appended_at": "2026-10-25T06:42:31.000000Z", + "closed_at": null, + "error_reason": null, + "retention_seconds": null + } + } + }, + "items_all": { + "status": 200, + "response": { + "workflow_id": "parity-v1-workflow-stream-writes-wsw", + "workflow_run_id": "01m5rwqt9swt5hthyvnqaac50h", + "stream": { + "stream_name": "tokens", + "status": "open", + "last_offset": 2, + "total_items": 3, + "pending_items": 0, + "opened_at": "2026-10-25T06:42:31.000000Z", + "last_appended_at": "2026-10-25T06:42:31.000000Z", + "closed_at": null, + "error_reason": null, + "retention_seconds": null + }, + "items": [ + { + "offset": 0, + "idempotency_key": "k1", + "origin": "worker", + "origin_reference": "attempt-1", + "item_type": "token", + "content_type": "application/json", + "payload": { + "token": "Hello", + "count": 9007199254740993, + "unicode": "λ" + }, + "payload_reference": null, + "payload_codec": null, + "emitted_at": "2026-10-25T06:42:31.000000Z" + }, + { + "offset": 1, + "idempotency_key": "k2", + "origin": "workflow_command", + "origin_reference": null, + "item_type": null, + "content_type": null, + "payload": "plain text", + "payload_reference": null, + "payload_codec": "avro", + "emitted_at": "2026-10-25T06:42:31.000000Z" + }, + { + "offset": 2, + "idempotency_key": null, + "origin": "workflow_command", + "origin_reference": null, + "item_type": null, + "content_type": null, + "payload": [ + 1, + 2, + 3 + ], + "payload_reference": null, + "payload_codec": null, + "emitted_at": "2026-10-25T06:42:31.000000Z" + } + ], + "next_offset": 3, + "terminal": false + } + }, + "show_after_catch_up": { + "status": 200, + "response": { + "workflow_id": "parity-v1-workflow-stream-writes-wsw", + "workflow_run_id": "01m5rwqt9swt5hthyvnqaac50h", + "stream": { + "stream_name": "tokens", + "status": "open", + "last_offset": 2, + "total_items": 3, + "pending_items": 0, + "opened_at": "2026-10-25T06:42:31.000000Z", + "last_appended_at": "2026-10-25T06:42:31.000000Z", + "closed_at": null, + "error_reason": null, + "retention_seconds": null + } + } + }, + "items_wait_empty": { + "status": 200, + "response": { + "workflow_id": "parity-v1-workflow-stream-writes-wsw", + "workflow_run_id": "01m5rwqt9swt5hthyvnqaac50h", + "stream": { + "stream_name": "tokens", + "status": "open", + "last_offset": 2, + "total_items": 3, + "pending_items": 0, + "opened_at": "2026-10-25T06:42:31.000000Z", + "last_appended_at": "2026-10-25T06:42:31.000000Z", + "closed_at": null, + "error_reason": null, + "retention_seconds": null + }, + "items": [], + "next_offset": 3, + "terminal": false + } + }, + "items_wait_returned_after_wait": true, + "append_full": { + "status": 429, + "response": { + "message": "Stream pending-items cap reached; consumer is draining too slowly.", + "reason": "stream_full", + "pending_items": 2, + "max_pending_items": 2 + } + }, + "show_after_full": { + "status": 200, + "response": { + "workflow_id": "parity-v1-workflow-stream-writes-wsw", + "workflow_run_id": "01m5rwqt9swt5hthyvnqaac50h", + "stream": { + "stream_name": "tokens", + "status": "open", + "last_offset": 4, + "total_items": 5, + "pending_items": 2, + "opened_at": "2026-10-25T06:42:31.000000Z", + "last_appended_at": "2026-10-25T06:42:33.000000Z", + "closed_at": null, + "error_reason": null, + "retention_seconds": null + } + } + }, + "items_after_full": { + "status": 200, + "response": { + "workflow_id": "parity-v1-workflow-stream-writes-wsw", + "workflow_run_id": "01m5rwqt9swt5hthyvnqaac50h", + "stream": { + "stream_name": "tokens", + "status": "open", + "last_offset": 4, + "total_items": 5, + "pending_items": 0, + "opened_at": "2026-10-25T06:42:31.000000Z", + "last_appended_at": "2026-10-25T06:42:33.000000Z", + "closed_at": null, + "error_reason": null, + "retention_seconds": null + }, + "items": [ + { + "offset": 3, + "idempotency_key": "k3", + "origin": "workflow_command", + "origin_reference": null, + "item_type": null, + "content_type": null, + "payload": { + "n": 3 + }, + "payload_reference": null, + "payload_codec": null, + "emitted_at": "2026-10-25T06:42:33.000000Z" + }, + { + "offset": 4, + "idempotency_key": null, + "origin": "workflow_command", + "origin_reference": null, + "item_type": null, + "content_type": null, + "payload": { + "n": 4 + }, + "payload_reference": null, + "payload_codec": null, + "emitted_at": "2026-10-25T06:42:33.000000Z" + } + ], + "next_offset": 5, + "terminal": false + } + }, + "refusals": { + "no_items": { + "status": 422, + "response": { + "message": "The items field is required.", + "reason": "validation_failed", + "errors": { + "items": [ + "The items field is required." + ] + }, + "validation_errors": { + "items": [ + "The items field is required." + ] + } + } + }, + "empty_items": { + "status": 422, + "response": { + "message": "The items field is required.", + "reason": "validation_failed", + "errors": { + "items": [ + "The items field is required." + ] + }, + "validation_errors": { + "items": [ + "The items field is required." + ] + } + } + }, + "too_many_items": { + "status": 422, + "response": { + "message": "The items field must not have more than 500 items.", + "reason": "validation_failed", + "errors": { + "items": [ + "The items field must not have more than 500 items." + ] + }, + "validation_errors": { + "items": [ + "The items field must not have more than 500 items." + ] + } + } + }, + "item_not_object": { + "status": 422, + "response": { + "message": "The items.0 field must be an array.", + "reason": "validation_failed", + "errors": { + "items.0": [ + "The items.0 field must be an array." + ] + }, + "validation_errors": { + "items.0": [ + "The items.0 field must be an array." + ] + } + } + }, + "bad_codec": { + "status": 422, + "response": { + "message": "unsupported_payload_codec: workflow payload codec \"not-a-codec\" is not supported by Durable Workflow 2.0; use codec=\"avro\" with the fixed Avro Value schema and single-object framing. JSON remains the HTTP document transport, not a workflow payload codec.", + "reason": "validation_failed", + "errors": { + "items.0.payload_codec": [ + "unsupported_payload_codec: workflow payload codec \"not-a-codec\" is not supported by Durable Workflow 2.0; use codec=\"avro\" with the fixed Avro Value schema and single-object framing. JSON remains the HTTP document transport, not a workflow payload codec." + ] + }, + "validation_errors": { + "items.0.payload_codec": [ + "unsupported_payload_codec: workflow payload codec \"not-a-codec\" is not supported by Durable Workflow 2.0; use codec=\"avro\" with the fixed Avro Value schema and single-object framing. JSON remains the HTTP document transport, not a workflow payload codec." + ] + } + } + }, + "long_idempotency_key": { + "status": 422, + "response": { + "message": "The items.0.idempotency_key field must not be greater than 191 characters.", + "reason": "validation_failed", + "errors": { + "items.0.idempotency_key": [ + "The items.0.idempotency_key field must not be greater than 191 characters." + ] + }, + "validation_errors": { + "items.0.idempotency_key": [ + "The items.0.idempotency_key field must not be greater than 191 characters." + ] + } + } + }, + "zero_max_pending": { + "status": 422, + "response": { + "message": "The max pending items field must be at least 1.", + "reason": "validation_failed", + "errors": { + "max_pending_items": [ + "The max pending items field must be at least 1." + ] + }, + "validation_errors": { + "max_pending_items": [ + "The max pending items field must be at least 1." + ] + } + } + }, + "bad_stream_append": { + "status": 400, + "response": { + "message": "invalid stream name", + "reason": "invalid_request" + } + }, + "unstored_reference": { + "status": 422, + "response": { + "schema": "durable-workflow.v2.runtime-external-payload-error.v1", + "reason": "external_payload_unsupported", + "message": "Payload reference fields require an opaque runtime external payload reference object.", + "retryable": false, + "status": 422 + } + }, + "unknown_run_append": { + "status": 404, + "response": { + "message": "Workflow run not found.", + "reason": "instance_not_found" + } + }, + "unknown_workflow_append": { + "status": 404, + "response": { + "message": "Workflow run not found.", + "reason": "instance_not_found" + } + }, + "bad_stream_show": { + "status": 400, + "response": { + "message": "invalid stream name", + "reason": "invalid_stream_name" + } + }, + "bad_stream_items": { + "status": 400, + "response": { + "message": "invalid stream name", + "reason": "invalid_stream_name" + } + }, + "negative_from": { + "status": 422, + "response": { + "message": "The from field must be at least 0.", + "reason": "validation_failed", + "errors": { + "from": [ + "The from field must be at least 0." + ] + }, + "validation_errors": { + "from": [ + "The from field must be at least 0." + ] + } + } + }, + "too_many_read": { + "status": 422, + "response": { + "message": "The max items field must not be greater than 500.", + "reason": "validation_failed", + "errors": { + "max_items": [ + "The max items field must not be greater than 500." + ] + }, + "validation_errors": { + "max_items": [ + "The max items field must not be greater than 500." + ] + } + } + }, + "long_wait": { + "status": 422, + "response": { + "message": "The wait seconds field must not be greater than 60.", + "reason": "validation_failed", + "errors": { + "wait_seconds": [ + "The wait seconds field must not be greater than 60." + ] + }, + "validation_errors": { + "wait_seconds": [ + "The wait seconds field must not be greater than 60." + ] + } + } + }, + "bad_stream_close": { + "status": 400, + "response": { + "message": "invalid stream name", + "reason": "invalid_stream_name" + } + }, + "long_error_reason": { + "status": 422, + "response": { + "message": "The error reason field must not be greater than 191 characters.", + "reason": "validation_failed", + "errors": { + "error_reason": [ + "The error reason field must not be greater than 191 characters." + ] + }, + "validation_errors": { + "error_reason": [ + "The error reason field must not be greater than 191 characters." + ] + } + } + }, + "zero_retention": { + "status": 422, + "response": { + "message": "The retention seconds field must be at least 1.", + "reason": "validation_failed", + "errors": { + "retention_seconds": [ + "The retention seconds field must be at least 1." + ] + }, + "validation_errors": { + "retention_seconds": [ + "The retention seconds field must be at least 1." + ] + } + } + }, + "unknown_run_close": { + "status": 404, + "response": { + "message": "Workflow run not found.", + "reason": "instance_not_found" + } + } + }, + "list_after_refusals": { + "status": 200, + "response": { + "workflow_id": "parity-v1-workflow-stream-writes-wsw", + "workflow_run_id": "01m5rwqt9swt5hthyvnqaac50h", + "count": 1, + "streams": [ + { + "stream_name": "tokens", + "status": "open", + "last_offset": 4, + "total_items": 5, + "pending_items": 0, + "opened_at": "2026-10-25T06:42:31.000000Z", + "last_appended_at": "2026-10-25T06:42:33.000000Z", + "closed_at": null, + "error_reason": null, + "retention_seconds": null + } + ] + } + }, + "close": { + "status": 200, + "response": { + "workflow_id": "parity-v1-workflow-stream-writes-wsw", + "workflow_run_id": "01m5rwqt9swt5hthyvnqaac50h", + "stream": { + "stream_name": "tokens", + "status": "closed", + "last_offset": 4, + "total_items": 5, + "pending_items": 0, + "opened_at": "2026-10-25T06:42:31.000000Z", + "last_appended_at": "2026-10-25T06:42:33.000000Z", + "closed_at": "2026-10-25T06:42:35.000000Z", + "error_reason": null, + "retention_seconds": 3600 + } + } + }, + "close_again": { + "status": 200, + "response": { + "workflow_id": "parity-v1-workflow-stream-writes-wsw", + "workflow_run_id": "01m5rwqt9swt5hthyvnqaac50h", + "stream": { + "stream_name": "tokens", + "status": "closed", + "last_offset": 4, + "total_items": 5, + "pending_items": 0, + "opened_at": "2026-10-25T06:42:31.000000Z", + "last_appended_at": "2026-10-25T06:42:33.000000Z", + "closed_at": "2026-10-25T06:42:35.000000Z", + "error_reason": null, + "retention_seconds": 3600 + } + } + }, + "append_closed": { + "status": 409, + "response": { + "message": "Stream is closed.", + "reason": "stream_closed" + } + }, + "items_closed_tail": { + "status": 200, + "response": { + "workflow_id": "parity-v1-workflow-stream-writes-wsw", + "workflow_run_id": "01m5rwqt9swt5hthyvnqaac50h", + "stream": { + "stream_name": "tokens", + "status": "closed", + "last_offset": 4, + "total_items": 5, + "pending_items": 0, + "opened_at": "2026-10-25T06:42:31.000000Z", + "last_appended_at": "2026-10-25T06:42:33.000000Z", + "closed_at": "2026-10-25T06:42:35.000000Z", + "error_reason": null, + "retention_seconds": 3600 + }, + "items": [], + "next_offset": 5, + "terminal": true + } + }, + "items_closed_head": { + "status": 200, + "response": { + "workflow_id": "parity-v1-workflow-stream-writes-wsw", + "workflow_run_id": "01m5rwqt9swt5hthyvnqaac50h", + "stream": { + "stream_name": "tokens", + "status": "closed", + "last_offset": 4, + "total_items": 5, + "pending_items": 0, + "opened_at": "2026-10-25T06:42:31.000000Z", + "last_appended_at": "2026-10-25T06:42:33.000000Z", + "closed_at": "2026-10-25T06:42:35.000000Z", + "error_reason": null, + "retention_seconds": 3600 + }, + "items": [ + { + "offset": 4, + "idempotency_key": null, + "origin": "workflow_command", + "origin_reference": null, + "item_type": null, + "content_type": null, + "payload": { + "n": 4 + }, + "payload_reference": null, + "payload_codec": null, + "emitted_at": "2026-10-25T06:42:33.000000Z" + } + ], + "next_offset": 5, + "terminal": true + } + }, + "append_other": { + "status": 200, + "response": { + "workflow_id": "parity-v1-workflow-stream-writes-wsw", + "workflow_run_id": "01m5rwqt9swt5hthyvnqaac50h", + "stream": { + "stream_name": "audit", + "status": "open", + "last_offset": 0, + "total_items": 1, + "pending_items": 1, + "opened_at": "2026-10-25T06:42:35.000000Z", + "last_appended_at": "2026-10-25T06:42:35.000000Z", + "closed_at": null, + "error_reason": null, + "retention_seconds": null + }, + "accepted_offsets": [ + 0 + ], + "accepted": 1, + "deduped": 0 + } + }, + "close_errored": { + "status": 200, + "response": { + "workflow_id": "parity-v1-workflow-stream-writes-wsw", + "workflow_run_id": "01m5rwqt9swt5hthyvnqaac50h", + "stream": { + "stream_name": "audit", + "status": "errored", + "last_offset": 0, + "total_items": 1, + "pending_items": 1, + "opened_at": "2026-10-25T06:42:35.000000Z", + "last_appended_at": "2026-10-25T06:42:35.000000Z", + "closed_at": "2026-10-25T06:42:35.000000Z", + "error_reason": "producer crashed", + "retention_seconds": 60 + } + } + }, + "append_errored": { + "status": 409, + "response": { + "message": "Stream is errored.", + "reason": "stream_errored" + } + }, + "items_errored": { + "status": 200, + "response": { + "workflow_id": "parity-v1-workflow-stream-writes-wsw", + "workflow_run_id": "01m5rwqt9swt5hthyvnqaac50h", + "stream": { + "stream_name": "audit", + "status": "errored", + "last_offset": 0, + "total_items": 1, + "pending_items": 0, + "opened_at": "2026-10-25T06:42:35.000000Z", + "last_appended_at": "2026-10-25T06:42:35.000000Z", + "closed_at": "2026-10-25T06:42:35.000000Z", + "error_reason": "producer crashed", + "retention_seconds": 60 + }, + "items": [ + { + "offset": 0, + "idempotency_key": null, + "origin": "workflow_command", + "origin_reference": null, + "item_type": "event", + "content_type": null, + "payload": { + "audit": true + }, + "payload_reference": null, + "payload_codec": null, + "emitted_at": "2026-10-25T06:42:35.000000Z" + } + ], + "next_offset": 1, + "terminal": true + } + }, + "list_final": { + "status": 200, + "response": { + "workflow_id": "parity-v1-workflow-stream-writes-wsw", + "workflow_run_id": "01m5rwqt9swt5hthyvnqaac50h", + "count": 2, + "streams": [ + { + "stream_name": "audit", + "status": "errored", + "last_offset": 0, + "total_items": 1, + "pending_items": 0, + "opened_at": "2026-10-25T06:42:35.000000Z", + "last_appended_at": "2026-10-25T06:42:35.000000Z", + "closed_at": "2026-10-25T06:42:35.000000Z", + "error_reason": "producer crashed", + "retention_seconds": 60 + }, + { + "stream_name": "tokens", + "status": "closed", + "last_offset": 4, + "total_items": 5, + "pending_items": 0, + "opened_at": "2026-10-25T06:42:31.000000Z", + "last_appended_at": "2026-10-25T06:42:33.000000Z", + "closed_at": "2026-10-25T06:42:35.000000Z", + "error_reason": null, + "retention_seconds": 3600 + } + ] + } + } +} diff --git a/tests/Unit/ServerParityWorkflowStreamWritesContractTest.mjs b/tests/Unit/ServerParityWorkflowStreamWritesContractTest.mjs new file mode 100644 index 000000000..14a1035b6 --- /dev/null +++ b/tests/Unit/ServerParityWorkflowStreamWritesContractTest.mjs @@ -0,0 +1,103 @@ +import assert from 'node:assert/strict'; +import {readFileSync} from 'node:fs'; +import test from 'node:test'; +import {checkWorkflowStreamWrites} from '../../scripts/conformance/server-parity/workflow-stream-writes-contract.mjs'; + +const fixture = JSON.parse(readFileSync(new URL('../Fixtures/ServerParity/workflow-stream-writes.json', import.meta.url))); +// The observation published Server 2.11.1 returned for this fixture. +const recorded = JSON.parse(readFileSync( + new URL('../Fixtures/ServerParityExpected/workflow-stream-writes-observation.json', import.meta.url))); +const id = 'parity-v1-workflow-stream-writes'; +const observation = (change = () => {}) => { + const state = structuredClone(recorded); + change(state); + return {mode: 'http', workflow_stream_writes: state}; +}; +const check = state => checkWorkflowStreamWrites(fixture, state, id); + +test('the published 2.11.1 recording satisfies the contract', () => { + assert.deepStrictEqual(check(observation()), {scope: 'workflow_stream_writes'}); +}); + +// The same run under Server #774: the over-cap batch is refused whole, so the +// tokens stream keeps its three items through every later step. +const three = stream => Object.assign(stream, {last_offset: 2, total_items: 3}); +const wholeBatch = state => { + state.append_full.response.pending_items = 0; + three(state.show_after_full.response.stream).pending_items = 0; + Object.assign(state.items_after_full.response, {items: [], next_offset: 3}); + three(state.items_after_full.response.stream); + three(state.list_after_refusals.response.streams[0]); + three(state.close.response.stream); + three(state.close_again.response.stream); + three(state.items_closed_tail.response.stream); + Object.assign(state.items_closed_head.response, {items: [], next_offset: 4}); + three(state.items_closed_head.response.stream); + three(state.list_final.response.streams.find(entry => entry.stream_name === 'tokens')); +}; + +test('a whole-batch capacity refusal (Server #774) satisfies the contract', () => { + assert.deepStrictEqual(check(observation(wholeBatch)), {scope: 'workflow_stream_writes'}); +}); + +test('embedded applications are explicitly inapplicable', () => { + assert.deepStrictEqual(check({mode: 'embedded', workflow_stream_writes: + {applicable: false, reason: 'embedded_has_no_http_workflow_streams'}}), {scope: 'workflow_stream_writes'}); + assert.throws(() => check({mode: 'embedded', workflow_stream_writes: {applicable: true}}), /embedded workflow streams/); +}); + +for (const [label, change, message] of [ + // Native answered 501 for both producer routes before this fixture. + ['an unimplemented append route', state => { + state.append_first = {status: 501, response: {reason: 'rust_capability_not_implemented'}}; + }, /append_first status/], + ['offsets that start at 1', state => { state.append_first.response.accepted_offsets = [1, 2]; }, /append_first receipt/], + ['a repeated idempotency key appended again', state => { + state.append_retry.response.accepted_offsets = [2, 3]; + state.append_retry.response.accepted = 2; + state.append_retry.response.deduped = 0; + }, /append_retry receipt/], + ['a partial window that clears pending items', state => { + state.show_after_window.response.stream.pending_items = 0; + }, /show_after_window stream/], + ['a read at the head that leaves pending items', state => { + state.items_all.response.stream.pending_items = 3; + }, /items_all stream/], + ['a whole refusal that reports the published pending count', state => { + wholeBatch(state); + state.append_full.response.pending_items = 2; + }, /append_full body/], + ['a whole refusal whose later reads still show the capped items', state => { + wholeBatch(state); + state.items_closed_head.response.next_offset = 5; + }, /items_closed_head next_offset/], + ['a cap answered as accepted', state => { state.append_full.status = 200; }, /append_full status/], + ['a repeated close that replaces the first', state => { + state.close_again.response.stream.error_reason = 'late error'; + state.close_again.response.stream.status = 'errored'; + }, /close_again stream/], + ['a closed stream that accepts appends', state => { + state.append_closed = {status: 200, response: {}}; + }, /append_closed status/], + ['a closed tail that is not terminal', state => { state.items_closed_tail.response.terminal = false; }, /items_closed_tail terminal/], + ['an errored stream without its reason', state => { state.close_errored.response.stream.error_reason = null; }, /close_errored stream/], + ['the default retention missing', state => { state.close.response.stream.retention_seconds = null; }, /close stream/], + ['an invalid name refused with the read reason', state => { + state.refusals.bad_stream_append.response.reason = 'invalid_stream_name'; + }, /refusal bad_stream_append/], + ['a string reference accepted', state => { + state.refusals.unstored_reference = {status: 200, response: {}}; + }, /refusal unstored_reference/], + // The runner parses records losslessly; JSON.parse here cannot hold the + // int64 itself, so this checks any change to a stored payload value. + ['a changed payload value', state => { state.items_all.response.items[0].payload.token = 'Hellp'; }, /items_all item 0/], + ['microsecond times', state => { + state.append_first.response.stream.opened_at = '2026-10-25T06:42:31.123456Z'; + }, /append_first opened_at/], + ['a list out of name order', state => { state.list_final.response.streams.reverse(); }, /list_final is by name/], + ['a missing step', state => { delete state.items_window; }, /steps/], +]) { + test(`the contract refuses ${label}`, () => { + assert.throws(() => check(observation(change)), message); + }); +}