From 25d76fae3f134215677fadf377da9826aa23c165 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Sun, 25 Oct 2026 06:46:16 +0000 Subject: [PATCH 1/6] Server parity: workflow stream writes fixture, recorded from published 2.11.1 The producer routes for durable workflow streams, POST runs/{run}/streams/{name}/items and /close, are the last stream routes native does not serve, and native's reads only ever see an empty run. This fixture records what published 2.11.1 answers over raw HTTP: - an append opens a stream at offset 0, a repeated idempotency key returns the existing offset, and only new items move the counters; - the pending-items cap commits the items before it and then answers 429 stream_full; - a read window that stops short of the head keeps pending_items, one that reaches the head clears it, and a bounded wait on an open stream returns empty after waiting; - close closes once and keeps a terminal stream's first close, error and retention (3600 seconds by default); closed and errored streams refuse appends with 409 and their tails read as terminal; - every validation, stream-name, external-reference and unknown-run refusal, including the different 400 reasons append and close give. The contract checks exact bodies, whole-second UTC times that never run backwards, and that no refusal wrote anything. A unit test runs it on the recorded 2.11.1 observation and on 18 mutations it must refuse. A fresh published PHP recording passes it. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/server-parity.yml | 3 +- .../conformance/server-parity/contract.mjs | 3 + scripts/conformance/server-parity/probe.php | 6 + .../workflow-stream-writes-contract.mjs | 198 ++++ .../workflow-stream-writes-probe.php | 103 +++ .../ServerParity/workflow-stream-writes.json | 113 +++ .../workflow-stream-writes-observation.json | 858 ++++++++++++++++++ ...ParityWorkflowStreamWritesContractTest.mjs | 78 ++ 8 files changed, 1361 insertions(+), 1 deletion(-) create mode 100644 scripts/conformance/server-parity/workflow-stream-writes-contract.mjs create mode 100644 scripts/conformance/server-parity/workflow-stream-writes-probe.php create mode 100644 tests/Fixtures/ServerParity/workflow-stream-writes.json create mode 100644 tests/Fixtures/ServerParityExpected/workflow-stream-writes-observation.json create mode 100644 tests/Unit/ServerParityWorkflowStreamWritesContractTest.mjs diff --git a/.github/workflows/server-parity.yml b/.github/workflows/server-parity.yml index f2c8a8373..0b9303452 100644 --- a/.github/workflows/server-parity.yml +++ b/.github/workflows/server-parity.yml @@ -53,6 +53,7 @@ on: - tests/Unit/ServerParityUpdateValidationContractTest.mjs - tests/Unit/ServerParityFixtureSelectionTest.mjs - tests/Unit/ServerParityMessageStreamsContractTest.mjs + - tests/Unit/ServerParityWorkflowStreamWritesContractTest.mjs - tests/Unit/ServerParityWaitingHistoryContractTest.mjs - tests/Unit/ServerParityTaskRetryContractTest.mjs - tests/Unit/ServerParityPublishedPatchContractTest.mjs @@ -1669,7 +1670,7 @@ jobs: -e COMPOSER_HOME=/tmp/composer -v "$PWD:/source" -w /source "$image" \ install --working-dir=scripts/conformance/server-parity --no-interaction --no-progress docker run --rm --user=1000:1000 --entrypoint node -v "$PWD:/source:ro" -w /source \ - "$image" --test tests/Unit/ServerParityRunnerTest.mjs tests/Unit/ServerParitySdkDirectionLanesTest.mjs tests/Unit/ServerParityCooperativeContractTest.mjs tests/Unit/ServerParityChildCancellationContractTest.mjs tests/Unit/ServerParityScheduleContractTest.mjs tests/Unit/ServerParityVisibilityContractTest.mjs tests/Unit/ServerParityAdmissionContractTest.mjs tests/Unit/ServerParityRoleTokenContractTest.mjs tests/Unit/ServerParityWorkerDeregistrationContractTest.mjs tests/Unit/ServerParitySearchAttributeContractTest.mjs tests/Unit/ServerParityWorkerListingContractTest.mjs tests/Unit/ServerParityWorkerReplacementContractTest.mjs tests/Unit/ServerParityPatchDeploymentContractTest.mjs tests/Unit/ServerParityWaitingHistoryContractTest.mjs tests/Unit/ServerParityTaskRetryContractTest.mjs tests/Unit/ServerParityPublishedPatchContractTest.mjs tests/Unit/ServerParityPublishedPatchTransportTest.mjs tests/Unit/ServerParityRecordFileTest.mjs tests/Unit/ServerParityFailureLocationTest.mjs tests/Unit/ServerParityFailureExcerptTest.mjs tests/Unit/ServerParityLateWorkflowCompletionContractTest.mjs tests/Unit/ServerParityStorageTestContractTest.mjs tests/Unit/ServerParitySchedulePoliciesContractTest.mjs tests/Unit/ServerParitySignalsBeforeWorkerContractTest.mjs tests/Unit/ServerParityScheduleCatalogContractTest.mjs tests/Unit/ServerParityOperatorMetricsContractTest.mjs tests/Unit/ServerParityActivityLeaseRenewalContractTest.mjs tests/Unit/ServerParityActivityCancellationAckContractTest.mjs tests/Unit/ServerParityServiceCatalogContractTest.mjs tests/Unit/ServerParitySystemMetricsContractTest.mjs tests/Unit/ServerParityOperatorDashboardTypesContractTest.mjs tests/Unit/ServerParityPrometheusMetricsContractTest.mjs tests/Unit/ServerParityPayloadCleanupContractTest.mjs tests/Unit/ServerParityRepairPassContractTest.mjs tests/Unit/ServerParityClusterInfoContractTest.mjs tests/Unit/ServerParityNamespaceDeleteContractTest.mjs tests/Unit/ServerParityMessageStreamsContractTest.mjs tests/Unit/ServerParityWorkflowPollRequestContractTest.mjs tests/Unit/ServerParityUpdateValidationContractTest.mjs tests/Unit/ServerParityFixtureSelectionTest.mjs > parity-evidence/comparator-tests.txt + "$image" --test tests/Unit/ServerParityRunnerTest.mjs tests/Unit/ServerParitySdkDirectionLanesTest.mjs tests/Unit/ServerParityCooperativeContractTest.mjs tests/Unit/ServerParityChildCancellationContractTest.mjs tests/Unit/ServerParityScheduleContractTest.mjs tests/Unit/ServerParityVisibilityContractTest.mjs tests/Unit/ServerParityAdmissionContractTest.mjs tests/Unit/ServerParityRoleTokenContractTest.mjs tests/Unit/ServerParityWorkerDeregistrationContractTest.mjs tests/Unit/ServerParitySearchAttributeContractTest.mjs tests/Unit/ServerParityWorkerListingContractTest.mjs tests/Unit/ServerParityWorkerReplacementContractTest.mjs tests/Unit/ServerParityPatchDeploymentContractTest.mjs tests/Unit/ServerParityWaitingHistoryContractTest.mjs tests/Unit/ServerParityTaskRetryContractTest.mjs tests/Unit/ServerParityPublishedPatchContractTest.mjs tests/Unit/ServerParityPublishedPatchTransportTest.mjs tests/Unit/ServerParityRecordFileTest.mjs tests/Unit/ServerParityFailureLocationTest.mjs tests/Unit/ServerParityFailureExcerptTest.mjs tests/Unit/ServerParityLateWorkflowCompletionContractTest.mjs tests/Unit/ServerParityStorageTestContractTest.mjs tests/Unit/ServerParitySchedulePoliciesContractTest.mjs tests/Unit/ServerParitySignalsBeforeWorkerContractTest.mjs tests/Unit/ServerParityScheduleCatalogContractTest.mjs tests/Unit/ServerParityOperatorMetricsContractTest.mjs tests/Unit/ServerParityActivityLeaseRenewalContractTest.mjs tests/Unit/ServerParityActivityCancellationAckContractTest.mjs tests/Unit/ServerParityServiceCatalogContractTest.mjs tests/Unit/ServerParitySystemMetricsContractTest.mjs tests/Unit/ServerParityOperatorDashboardTypesContractTest.mjs tests/Unit/ServerParityPrometheusMetricsContractTest.mjs tests/Unit/ServerParityPayloadCleanupContractTest.mjs tests/Unit/ServerParityRepairPassContractTest.mjs tests/Unit/ServerParityClusterInfoContractTest.mjs tests/Unit/ServerParityNamespaceDeleteContractTest.mjs tests/Unit/ServerParityMessageStreamsContractTest.mjs tests/Unit/ServerParityWorkflowStreamWritesContractTest.mjs tests/Unit/ServerParityWorkflowPollRequestContractTest.mjs tests/Unit/ServerParityUpdateValidationContractTest.mjs tests/Unit/ServerParityFixtureSelectionTest.mjs > parity-evidence/comparator-tests.txt docker run --rm --user=1000:1000 --cpus=1 --memory=256m --memory-swap=256m \ --network=none --entrypoint node -v "$PWD:/source:ro" -w /source \ "$image" --test tests/Unit/ServerParityCodecDeadlineLoadTest.mjs > parity-evidence/codec-load-tests.txt diff --git a/scripts/conformance/server-parity/contract.mjs b/scripts/conformance/server-parity/contract.mjs index e57210c3c..ac18b6c7e 100644 --- a/scripts/conformance/server-parity/contract.mjs +++ b/scripts/conformance/server-parity/contract.mjs @@ -26,6 +26,7 @@ import {checkRepairPass} from './repair-pass-contract.mjs'; import {checkNamespaceDelete} from './namespace-delete-contract.mjs'; import {checkUpdateValidation} from './update-validation-contract.mjs'; import {checkMessageStreams} from './message-streams-contract.mjs'; +import {checkWorkflowStreamWrites} from './workflow-stream-writes-contract.mjs'; import {checkSchedulePolicies} from './schedule-policies-contract.mjs'; import {checkScheduleCatalog} from './schedule-catalog-contract.mjs'; import {checkSignalsBeforeWorker} from './signals-before-worker-contract.mjs'; @@ -170,6 +171,7 @@ export function checkObservation(fixture, observation, workflowId, artifacts) { const namespaceDelete = fixture.namespace_delete ? checkNamespaceDelete(fixture, observation, workflowId) : null; const updateValidation = fixture.update_validation ? checkUpdateValidation(fixture, observation, workflowId) : null; const messageStreams = fixture.message_streams ? checkMessageStreams(fixture, observation, workflowId) : null; + const workflowStreamWrites = fixture.workflow_stream_writes ? checkWorkflowStreamWrites(fixture, observation, workflowId) : null; const schedulePolicies = fixture.schedule_policies ? checkSchedulePolicies(fixture, observation, workflowId) : null; const scheduleCatalog = fixture.schedule_catalog ? checkScheduleCatalog(fixture, observation, workflowId) : null; const signalsBeforeWorker = fixture.signals_before_worker ? checkSignalsBeforeWorker(fixture, observation, workflowId) : null; @@ -730,6 +732,7 @@ export function checkObservation(fixture, observation, workflowId, artifacts) { ...(namespaceDelete ? {namespace_delete: namespaceDelete} : {}), ...(updateValidation ? {update_validation: updateValidation} : {}), ...(messageStreams ? {message_streams: messageStreams} : {}), + ...(workflowStreamWrites ? {workflow_stream_writes: workflowStreamWrites} : {}), ...(schedulePolicies ? {schedule_policies: schedulePolicies} : {}), ...(scheduleCatalog ? {schedule_catalog: scheduleCatalog} : {}), ...(signalsBeforeWorker ? {signals_before_worker: signalsBeforeWorker} : {}), diff --git a/scripts/conformance/server-parity/probe.php b/scripts/conformance/server-parity/probe.php index cd96f2d22..8d342b7e3 100644 --- a/scripts/conformance/server-parity/probe.php +++ b/scripts/conformance/server-parity/probe.php @@ -445,6 +445,7 @@ function httpObservation(array $fixture, string $workflowId, string $namespace, $namespaceDelete = isset($fixture['namespace_delete']) ? finishHttpNamespaceDelete($client, $fixture, $workflowId, $queue, $url) : null; $updateValidation = isset($fixture['update_validation']) ? finishHttpUpdateValidation($client, $fixture, $workflowId, $url) : null; $messageStreams = isset($fixture['message_streams']) ? finishHttpMessageStreams($client, $fixture, $workflowId, $url) : null; + $workflowStreamWrites = isset($fixture['workflow_stream_writes']) ? finishHttpWorkflowStreamWrites($client, $fixture, $workflowId, $url) : null; $schedulePolicies = isset($fixture['schedule_policies']) ? finishHttpSchedulePolicies($client, $fixture, $workflowId, $url) : null; $scheduleCatalog = isset($fixture['schedule_catalog']) ? finishHttpSchedulePolicies($client, $fixture, $workflowId, $url, 'schedule_catalog') @@ -558,6 +559,7 @@ function httpObservation(array $fixture, string $workflowId, string $namespace, 'namespace_delete' => $namespaceDelete, 'update_validation' => $updateValidation, 'message_streams' => $messageStreams, + 'workflow_stream_writes' => $workflowStreamWrites, 'schedule_policies' => $schedulePolicies, 'schedule_catalog' => $scheduleCatalog, 'signals_before_worker' => $signalsBeforeWorker, @@ -875,6 +877,7 @@ function embeddedObservation(array $fixture, string $workflowId, string $namespa require __DIR__.'/namespace-delete-probe.php'; require __DIR__.'/update-validation-probe.php'; require __DIR__.'/message-streams-probe.php'; + require __DIR__.'/workflow-stream-writes-probe.php'; require __DIR__.'/schedule-policies-probe.php'; require __DIR__.'/signals-before-worker-probe.php'; require __DIR__.'/worker-deregistration-probe.php'; @@ -1075,6 +1078,9 @@ function embeddedObservation(array $fixture, string $workflowId, string $namespa if ($mode === 'embedded' && isset($fixture['message_streams'])) { $observation['message_streams'] = ['applicable' => false, 'reason' => 'embedded_has_no_http_message_streams']; } + if ($mode === 'embedded' && isset($fixture['workflow_stream_writes'])) { + $observation['workflow_stream_writes'] = ['applicable' => false, 'reason' => 'embedded_has_no_http_workflow_streams']; + } if ($mode === 'embedded' && isset($fixture['schedule_policies'])) { $observation['schedule_policies'] = ['applicable' => false, 'reason' => 'embedded_has_no_http_schedules']; } diff --git a/scripts/conformance/server-parity/workflow-stream-writes-contract.mjs b/scripts/conformance/server-parity/workflow-stream-writes-contract.mjs new file mode 100644 index 000000000..e567e3330 --- /dev/null +++ b/scripts/conformance/server-parity/workflow-stream-writes-contract.mjs @@ -0,0 +1,198 @@ +import assert from 'node:assert/strict'; + +// Published 2.11.1 stores stream times in whole seconds and answers them in +// UTC with six fractional digits. +const TIME = /^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d\.000000Z$/; +const STEPS = ['applicable', 'workflow_id', 'run_id', 'list_empty', 'show_missing', 'items_missing', 'close_missing', + 'append_first', 'append_retry', 'show_after_appends', 'items_window', 'show_after_window', 'items_all', + 'show_after_catch_up', 'items_wait_empty', 'items_wait_returned_after_wait', 'append_full', 'show_after_full', + 'items_after_full', 'refusals', 'list_after_refusals', 'close', 'close_again', 'append_closed', 'items_closed_tail', + 'items_closed_head', 'append_other', 'close_errored', 'append_errored', 'items_errored', 'list_final']; + +const notFound = {message: 'Stream not found.', reason: 'stream_not_found'}; +const runNotFound = {message: 'Workflow run not found.', reason: 'instance_not_found'}; + +function exact(read, status, response, label) { + assert.equal(read?.status, status, `${label} status`); + assert.deepStrictEqual(read.response, response, `${label} body`); +} + +// Laravel's 422 with one attribute and its one message. +function refused(read, field, message, label) { + exact(read, 422, {message, reason: 'validation_failed', errors: {[field]: [message]}, + validation_errors: {[field]: [message]}}, label); +} + +// One stream as formatStream answers it: exact counters and states, times in +// whole seconds that never run backwards. +function stream(actual, expected, label) { + const {opened_at: opened, last_appended_at: appended, closed_at: closed, ...rest} = actual; + const {closed: terminal = false, ...fields} = expected; + assert.deepStrictEqual(rest, {error_reason: null, retention_seconds: null, ...fields}, `${label} stream`); + assert.match(opened ?? '', TIME, `${label} opened_at`); + assert.match(appended ?? '', TIME, `${label} last_appended_at`); + assert.ok(appended >= opened, `${label} appended after opening`); + if (terminal) { + assert.match(closed ?? '', TIME, `${label} closed_at`); + assert.ok(closed >= appended, `${label} closed after the last append`); + } else { + assert.equal(closed, null, `${label} is not closed`); + } + return {opened, appended, closed}; +} + +function streamRead(read, ids, expected, label) { + assert.equal(read?.status, 200, `${label} status`); + const {stream: body, ...rest} = read.response; + assert.deepStrictEqual(rest, ids, `${label} identity`); + return stream(body, expected, label); +} + +function appended(read, ids, expected, offsets, accepted, deduped, label) { + assert.equal(read?.status, 200, `${label} status`); + const {stream: body, ...rest} = read.response; + assert.deepStrictEqual(rest, {...ids, accepted_offsets: offsets, accepted, deduped}, `${label} receipt`); + return stream(body, expected, label); +} + +function item(offset, fields) { + return {offset, idempotency_key: null, origin: 'workflow_command', origin_reference: null, item_type: null, content_type: null, + payload: null, payload_reference: null, payload_codec: null, ...fields}; +} + +function items(read, ids, expected, rows, nextOffset, terminal, label) { + assert.equal(read?.status, 200, `${label} status`); + const {stream: body, items: actual, next_offset: next, terminal: done, ...rest} = read.response; + assert.deepStrictEqual(rest, ids, `${label} identity`); + stream(body, expected, label); + assert.equal(next, nextOffset, `${label} next_offset`); + assert.equal(done, terminal, `${label} terminal`); + assert.equal(actual.length, rows.length, `${label} item count`); + actual.forEach((row, index) => { + const {emitted_at: emitted, ...fields} = row; + assert.deepStrictEqual(fields, rows[index], `${label} item ${index}`); + assert.match(emitted ?? '', TIME, `${label} item ${index} emitted_at`); + }); +} + +export function checkWorkflowStreamWrites(fixture, observation, workflowId) { + const spec = fixture.workflow_stream_writes; + const state = observation.workflow_stream_writes; + if (observation.mode === 'embedded') { + assert.deepStrictEqual(state, {applicable: false, reason: 'embedded_has_no_http_workflow_streams'}, + 'embedded workflow streams are inapplicable'); + return {scope: 'workflow_stream_writes'}; + } + assert.deepStrictEqual(Object.keys(state ?? {}), STEPS, 'workflow stream writes steps'); + assert.equal(state.applicable, true); + const id = `${workflowId}-wsw`; + assert.equal(state.workflow_id, id); + assert.equal(typeof state.run_id, 'string'); + const ids = {workflow_id: id, workflow_run_id: state.run_id}; + const tokens = spec.stream; + const audit = spec.other_stream; + const batch = spec.batches; + const label = name => `workflow stream writes ${name}`; + + exact(state.list_empty, 200, {...ids, count: 0, streams: []}, label('list_empty')); + for (const name of ['show_missing', 'items_missing', 'close_missing']) exact(state[name], 404, notFound, label(name)); + + // The first append opens the stream at offset 0; a repeated key returns the + // existing item's offset and only new items move the counters. + const open = {stream_name: tokens, status: 'open'}; + const first = appended(state.append_first, ids, {...open, last_offset: 1, total_items: 2, pending_items: 2}, + [0, 1], 2, 0, label('append_first')); + appended(state.append_retry, ids, {...open, last_offset: 2, total_items: 3, pending_items: 3}, + [0, 2], 1, 1, label('append_retry')); + const three = {...open, last_offset: 2, total_items: 3}; + streamRead(state.show_after_appends, ids, {...three, pending_items: 3}, label('show_after_appends')); + + const rows = [ + item(0, {...batch.first[0], idempotency_key: 'k1'}), + item(1, {payload: batch.first[1].payload, payload_codec: 'avro', idempotency_key: 'k2'}), + item(2, {payload: batch.retry[1].payload}), + ]; + // A window that stops short of the head leaves pending_items alone; one + // that reaches the head clears it, and a bounded wait past it returns empty. + items(state.items_window, ids, {...three, pending_items: 3}, [rows[1]], 2, false, label('items_window')); + streamRead(state.show_after_window, ids, {...three, pending_items: 3}, label('show_after_window')); + items(state.items_all, ids, {...three, pending_items: 0}, rows, 3, false, label('items_all')); + streamRead(state.show_after_catch_up, ids, {...three, pending_items: 0}, label('show_after_catch_up')); + items(state.items_wait_empty, ids, {...three, pending_items: 0}, [], 3, false, label('items_wait_empty')); + assert.equal(state.items_wait_returned_after_wait, true, label('a bounded wait on an open stream waits')); + + // The cap commits the items before it and then refuses the rest. + exact(state.append_full, 429, {message: 'Stream pending-items cap reached; consumer is draining too slowly.', + reason: 'stream_full', pending_items: spec.max_pending_items, max_pending_items: spec.max_pending_items}, + label('append_full')); + const five = {...open, last_offset: 4, total_items: 5}; + const full = streamRead(state.show_after_full, ids, {...five, pending_items: 2}, label('show_after_full')); + assert.ok(full.appended >= first.appended, label('the capped append moved last_appended_at forward')); + items(state.items_after_full, ids, {...five, pending_items: 0}, + [item(3, {...batch.over_cap[0]}), item(4, {...batch.over_cap[1]})], 5, false, label('items_after_full')); + + const r = state.refusals; + const refusal = name => label(`refusal ${name}`); + refused(r.no_items, 'items', 'The items field is required.', refusal('no_items')); + refused(r.empty_items, 'items', 'The items field is required.', refusal('empty_items')); + refused(r.too_many_items, 'items', 'The items field must not have more than 500 items.', refusal('too_many_items')); + refused(r.item_not_object, 'items.0', 'The items.0 field must be an array.', refusal('item_not_object')); + const codecMessage = `unsupported_payload_codec: workflow payload codec "${spec.bad_codec}" is not supported by Durable Workflow 2.0; ` + + 'use codec="avro" with the fixed Avro Value schema and single-object framing. JSON remains the HTTP document ' + + 'transport, not a workflow payload codec.'; + refused(r.bad_codec, 'items.0.payload_codec', codecMessage, refusal('bad_codec')); + refused(r.long_idempotency_key, 'items.0.idempotency_key', + 'The items.0.idempotency_key field must not be greater than 191 characters.', refusal('long_idempotency_key')); + refused(r.zero_max_pending, 'max_pending_items', 'The max pending items field must be at least 1.', refusal('zero_max_pending')); + exact(r.bad_stream_append, 400, {message: 'invalid stream name', reason: 'invalid_request'}, refusal('bad_stream_append')); + exact(r.unstored_reference, 422, {schema: 'durable-workflow.v2.runtime-external-payload-error.v1', + reason: 'external_payload_unsupported', + message: 'Payload reference fields require an opaque runtime external payload reference object.', + retryable: false, status: 422}, refusal('unstored_reference')); + for (const name of ['unknown_run_append', 'unknown_workflow_append', 'unknown_run_close']) { + exact(r[name], 404, runNotFound, refusal(name)); + } + for (const name of ['bad_stream_show', 'bad_stream_items', 'bad_stream_close']) { + exact(r[name], 400, {message: 'invalid stream name', reason: 'invalid_stream_name'}, refusal(name)); + } + refused(r.negative_from, 'from', 'The from field must be at least 0.', refusal('negative_from')); + refused(r.too_many_read, 'max_items', 'The max items field must not be greater than 500.', refusal('too_many_read')); + refused(r.long_wait, 'wait_seconds', 'The wait seconds field must not be greater than 60.', refusal('long_wait')); + refused(r.long_error_reason, 'error_reason', 'The error reason field must not be greater than 191 characters.', + refusal('long_error_reason')); + refused(r.zero_retention, 'retention_seconds', 'The retention seconds field must be at least 1.', refusal('zero_retention')); + assert.deepStrictEqual(Object.keys(r).sort(), ['bad_codec', 'bad_stream_append', 'bad_stream_close', 'bad_stream_items', + 'bad_stream_show', 'empty_items', 'item_not_object', 'long_error_reason', 'long_idempotency_key', 'long_wait', + 'negative_from', 'no_items', 'too_many_items', 'too_many_read', 'unknown_run_append', 'unknown_run_close', + 'unknown_workflow_append', 'unstored_reference', 'zero_max_pending', 'zero_retention'], label('refusal set')); + + // No refusal wrote anything: the list still shows one stream as it was. + assert.equal(state.list_after_refusals?.status, 200, label('list_after_refusals status')); + assert.deepStrictEqual({...state.list_after_refusals.response, streams: undefined}, {...ids, count: 1, streams: undefined}, + label('list_after_refusals identity')); + stream(state.list_after_refusals.response.streams[0], {...five, pending_items: 0}, label('list_after_refusals stream')); + + // Close once; a terminal stream keeps its first close, error and retention. + const closed = {...five, pending_items: 0, status: 'closed', retention_seconds: 3600, closed: true}; + const firstClose = streamRead(state.close, ids, closed, label('close')); + const again = streamRead(state.close_again, ids, closed, label('close_again')); + assert.equal(again.closed, firstClose.closed, label('a repeated close keeps closed_at')); + exact(state.append_closed, 409, {message: 'Stream is closed.', reason: 'stream_closed'}, label('append_closed')); + items(state.items_closed_tail, ids, closed, [], 5, true, label('items_closed_tail')); + items(state.items_closed_head, ids, closed, [item(4, {...batch.over_cap[1]})], 5, true, label('items_closed_head')); + + const other = {stream_name: audit, status: 'open', last_offset: 0, total_items: 1, pending_items: 1}; + appended(state.append_other, ids, other, [0], 1, 0, label('append_other')); + const errored = {...other, status: 'errored', error_reason: spec.error_reason, retention_seconds: spec.retention_seconds, closed: true}; + streamRead(state.close_errored, ids, errored, label('close_errored')); + exact(state.append_errored, 409, {message: 'Stream is errored.', reason: 'stream_errored'}, label('append_errored')); + items(state.items_errored, ids, {...errored, pending_items: 0}, [item(0, {...batch.other[0]})], 1, true, label('items_errored')); + + assert.equal(state.list_final?.status, 200, label('list_final status')); + const final = state.list_final.response; + assert.equal(final.count, 2, label('list_final count')); + assert.deepStrictEqual(final.streams.map(entry => entry.stream_name), [audit, tokens], label('list_final is by name')); + stream(final.streams[0], {...errored, pending_items: 0}, label('list_final errored')); + stream(final.streams[1], closed, label('list_final closed')); + return {scope: 'workflow_stream_writes'}; +} diff --git a/scripts/conformance/server-parity/workflow-stream-writes-probe.php b/scripts/conformance/server-parity/workflow-stream-writes-probe.php new file mode 100644 index 000000000..fab2fe631 --- /dev/null +++ b/scripts/conformance/server-parity/workflow-stream-writes-probe.php @@ -0,0 +1,103 @@ +push(LockPressurePollRetries::middleware(), 'lock_pressure_poll_retry'); + $http = new WorkflowStreamHttp(['base_uri' => rtrim($url, '/').'/api/', 'http_errors' => false, + 'connect_timeout' => 1, 'timeout' => 30, 'handler' => $stack]); + $raw = static function (string $method, string $path, ?array $body = null) use ($http, $token): array { + $response = $http->request($method, $path, ['headers' => array_filter([ + 'Authorization' => $token === null ? null : 'Bearer '.$token, + 'Accept' => 'application/json', 'X-Namespace' => 'default', + 'X-Durable-Workflow-Control-Plane-Version' => '2', + ]), ...($body === null ? [] : ['json' => $body])]); + $decoded = json_decode((string) $response->getBody(), true, flags: JSON_THROW_ON_ERROR); + if (is_array($decoded)) { + unset($decoded['protocol_version'], $decoded['server_capabilities'], $decoded['control_plane']); + } + + return ['status' => $response->getStatusCode(), 'response' => $decoded]; + }; + $state = ['applicable' => true, 'workflow_id' => $id]; + $state['run_id'] = $client->startWorkflow($definition['workflow_type'], $id, $id.'-queue', [$fixture['input']])->selectedRunId; + $run = "workflows/{$id}/runs/{$state['run_id']}"; + $stream = $definition['stream']; + $other = $definition['other_stream']; + $bad = rawurlencode($definition['bad_stream']); + $items = static fn (string $name): string => "{$run}/streams/".rawurlencode($name).'/items'; + $append = static fn (string $name, array $body): array => $raw('POST', $items($name), $body); + $read = static fn (string $name, string $query = ''): array => $raw('GET', $items($name).($query === '' ? '' : '?'.$query)); + $close = static fn (string $name, array $body = []): array => $raw('POST', "{$run}/streams/".rawurlencode($name).'/close', $body); + $show = static fn (string $name): array => $raw('GET', "{$run}/streams/".rawurlencode($name)); + $batches = $definition['batches']; + + $state['list_empty'] = $raw('GET', "{$run}/streams"); + $state['show_missing'] = $show($stream); + $state['items_missing'] = $read($stream); + $state['close_missing'] = $close($stream); + $state['append_first'] = $append($stream, ['items' => $batches['first']]); + $state['append_retry'] = $append($stream, ['items' => $batches['retry']]); + $state['show_after_appends'] = $show($stream); + $state['items_window'] = $read($stream, 'from=1&max_items=1'); + $state['show_after_window'] = $show($stream); + $state['items_all'] = $read($stream, 'from=0'); + $state['show_after_catch_up'] = $show($stream); + $started = microtime(true); + $state['items_wait_empty'] = $read($stream, 'from=3&wait_seconds=1'); + $state['items_wait_returned_after_wait'] = microtime(true) - $started >= 0.9; + $state['append_full'] = $append($stream, ['items' => $batches['over_cap'], 'max_pending_items' => $definition['max_pending_items']]); + $state['show_after_full'] = $show($stream); + $state['items_after_full'] = $read($stream, 'from=3'); + $state['refusals'] = [ + 'no_items' => $append($stream, []), + 'empty_items' => $append($stream, ['items' => []]), + 'too_many_items' => $append($stream, ['items' => array_fill(0, $definition['too_many_items'], ['payload' => 1])]), + 'item_not_object' => $append($stream, ['items' => ['text']]), + 'bad_codec' => $append($stream, ['items' => [['payload' => 1, 'payload_codec' => $definition['bad_codec']]]]), + 'long_idempotency_key' => $append($stream, ['items' => [['payload' => 1, 'idempotency_key' => str_repeat('k', $definition['long_key_length'])]]]), + 'zero_max_pending' => $append($stream, ['items' => [['payload' => 1]], 'max_pending_items' => 0]), + 'bad_stream_append' => $raw('POST', "{$run}/streams/{$bad}/items", ['items' => [['payload' => 1]]]), + 'unstored_reference' => $append($other, ['items' => [['payload_reference' => $definition['unstored_reference']]]]), + 'unknown_run_append' => $raw('POST', "workflows/{$id}/runs/{$state['run_id']}x/streams/{$stream}/items", ['items' => [['payload' => 1]]]), + 'unknown_workflow_append' => $raw('POST', "workflows/{$id}-missing/runs/{$state['run_id']}/streams/{$stream}/items", ['items' => [['payload' => 1]]]), + 'bad_stream_show' => $raw('GET', "{$run}/streams/{$bad}"), + 'bad_stream_items' => $raw('GET', "{$run}/streams/{$bad}/items"), + 'negative_from' => $read($stream, 'from=-1'), + 'too_many_read' => $read($stream, 'max_items='.$definition['too_many_read']), + 'long_wait' => $read($stream, 'wait_seconds='.$definition['too_long_wait']), + 'bad_stream_close' => $raw('POST', "{$run}/streams/{$bad}/close", []), + 'long_error_reason' => $close($stream, ['error_reason' => str_repeat('e', $definition['long_key_length'])]), + 'zero_retention' => $close($stream, ['retention_seconds' => 0]), + 'unknown_run_close' => $raw('POST', "workflows/{$id}/runs/{$state['run_id']}x/streams/{$stream}/close", []), + ]; + $state['list_after_refusals'] = $raw('GET', "{$run}/streams"); + $state['close'] = $close($stream); + $state['close_again'] = $close($stream, ['error_reason' => 'late error', 'retention_seconds' => 5]); + $state['append_closed'] = $append($stream, ['items' => [['payload' => 1, 'idempotency_key' => 'k1']]]); + $state['items_closed_tail'] = $read($stream, 'from=5'); + $state['items_closed_head'] = $read($stream, 'from=4'); + $state['append_other'] = $append($other, ['items' => $batches['other']]); + $state['close_errored'] = $close($other, ['error_reason' => $definition['error_reason'], 'retention_seconds' => $definition['retention_seconds']]); + $state['append_errored'] = $append($other, ['items' => [['payload' => 1]]]); + $state['items_errored'] = $read($other, 'from=0&wait_seconds=1'); + $state['list_final'] = $raw('GET', "{$run}/streams"); + + return $state; +} diff --git a/tests/Fixtures/ServerParity/workflow-stream-writes.json b/tests/Fixtures/ServerParity/workflow-stream-writes.json new file mode 100644 index 000000000..5fbe11aaa --- /dev/null +++ b/tests/Fixtures/ServerParity/workflow-stream-writes.json @@ -0,0 +1,113 @@ +{ + "schema": "durable-workflow.server-parity-fixture/v1", + "id": "workflow-stream-writes", + "review_state": "reviewed", + "contract": "Published Server 2.11.1 durable workflow stream producer routes: POST runs/{run}/streams/{name}/items opens a stream on first append, assigns offsets from 0, returns an existing item for a repeated idempotency key, commits the items before the pending-items cap and then answers 429 stream_full, and refuses closed and errored streams with 409. POST .../close closes or errors a stream once, keeps a terminal stream unchanged and defaults retention to 3600 seconds. Reads report the rows, reset pending_items when a read reaches the head, and a bounded wait on an open stream returns empty. Every validation, name and unknown-run refusal is recorded.", + "workflow_type": "parity.v1.echo", + "input": { + "message": "message streams λ", + "count": 9007199254740993, + "optional": null + }, + "typed_value": { + "type": "map", + "value": { + "message": { + "type": "string", + "value": "message streams λ" + }, + "count": { + "type": "int64", + "value": "9007199254740993" + }, + "optional": { + "type": "null", + "value": null + } + } + }, + "activity": false, + "expected_events": [ + "StartAccepted", + "WorkflowStarted", + "WorkflowCompleted" + ], + "workflow_stream_writes": { + "workflow_type": "parity.v1.workflow_streams", + "stream": "tokens", + "other_stream": "audit", + "bad_stream": "bad name!", + "max_pending_items": 2, + "too_many_items": 501, + "too_many_read": 501, + "too_long_wait": 61, + "long_key_length": 192, + "bad_codec": "not-a-codec", + "unstored_reference": "dw-external-payload-reference-that-was-never-stored", + "error_reason": "producer crashed", + "retention_seconds": 60, + "batches": { + "first": [ + { + "payload": { + "token": "Hello", + "count": 9007199254740993, + "unicode": "λ" + }, + "idempotency_key": "k1", + "origin": "worker", + "origin_reference": "attempt-1", + "item_type": "token", + "content_type": "application/json" + }, + { + "payload": "plain text", + "payload_codec": "avro", + "idempotency_key": "k2" + } + ], + "retry": [ + { + "payload": { + "token": "changed" + }, + "idempotency_key": "k1" + }, + { + "payload": [ + 1, + 2, + 3 + ] + } + ], + "over_cap": [ + { + "payload": { + "n": 3 + }, + "idempotency_key": "k3" + }, + { + "payload": { + "n": 4 + } + }, + { + "payload": { + "n": 5 + }, + "idempotency_key": "k5" + } + ], + "other": [ + { + "payload": { + "audit": true + }, + "item_type": "event" + } + ] + } + } +} diff --git a/tests/Fixtures/ServerParityExpected/workflow-stream-writes-observation.json b/tests/Fixtures/ServerParityExpected/workflow-stream-writes-observation.json new file mode 100644 index 000000000..0b07560d6 --- /dev/null +++ b/tests/Fixtures/ServerParityExpected/workflow-stream-writes-observation.json @@ -0,0 +1,858 @@ +{ + "applicable": true, + "workflow_id": "parity-v1-workflow-stream-writes-wsw", + "run_id": "01m5rwqt9swt5hthyvnqaac50h", + "list_empty": { + "status": 200, + "response": { + "workflow_id": "parity-v1-workflow-stream-writes-wsw", + "workflow_run_id": "01m5rwqt9swt5hthyvnqaac50h", + "count": 0, + "streams": [] + } + }, + "show_missing": { + "status": 404, + "response": { + "message": "Stream not found.", + "reason": "stream_not_found" + } + }, + "items_missing": { + "status": 404, + "response": { + "message": "Stream not found.", + "reason": "stream_not_found" + } + }, + "close_missing": { + "status": 404, + "response": { + "message": "Stream not found.", + "reason": "stream_not_found" + } + }, + "append_first": { + "status": 200, + "response": { + "workflow_id": "parity-v1-workflow-stream-writes-wsw", + "workflow_run_id": "01m5rwqt9swt5hthyvnqaac50h", + "stream": { + "stream_name": "tokens", + "status": "open", + "last_offset": 1, + "total_items": 2, + "pending_items": 2, + "opened_at": "2026-10-25T06:42:31.000000Z", + "last_appended_at": "2026-10-25T06:42:31.000000Z", + "closed_at": null, + "error_reason": null, + "retention_seconds": null + }, + "accepted_offsets": [ + 0, + 1 + ], + "accepted": 2, + "deduped": 0 + } + }, + "append_retry": { + "status": 200, + "response": { + "workflow_id": "parity-v1-workflow-stream-writes-wsw", + "workflow_run_id": "01m5rwqt9swt5hthyvnqaac50h", + "stream": { + "stream_name": "tokens", + "status": "open", + "last_offset": 2, + "total_items": 3, + "pending_items": 3, + "opened_at": "2026-10-25T06:42:31.000000Z", + "last_appended_at": "2026-10-25T06:42:31.000000Z", + "closed_at": null, + "error_reason": null, + "retention_seconds": null + }, + "accepted_offsets": [ + 0, + 2 + ], + "accepted": 1, + "deduped": 1 + } + }, + "show_after_appends": { + "status": 200, + "response": { + "workflow_id": "parity-v1-workflow-stream-writes-wsw", + "workflow_run_id": "01m5rwqt9swt5hthyvnqaac50h", + "stream": { + "stream_name": "tokens", + "status": "open", + "last_offset": 2, + "total_items": 3, + "pending_items": 3, + "opened_at": "2026-10-25T06:42:31.000000Z", + "last_appended_at": "2026-10-25T06:42:31.000000Z", + "closed_at": null, + "error_reason": null, + "retention_seconds": null + } + } + }, + "items_window": { + "status": 200, + "response": { + "workflow_id": "parity-v1-workflow-stream-writes-wsw", + "workflow_run_id": "01m5rwqt9swt5hthyvnqaac50h", + "stream": { + "stream_name": "tokens", + "status": "open", + "last_offset": 2, + "total_items": 3, + "pending_items": 3, + "opened_at": "2026-10-25T06:42:31.000000Z", + "last_appended_at": "2026-10-25T06:42:31.000000Z", + "closed_at": null, + "error_reason": null, + "retention_seconds": null + }, + "items": [ + { + "offset": 1, + "idempotency_key": "k2", + "origin": "workflow_command", + "origin_reference": null, + "item_type": null, + "content_type": null, + "payload": "plain text", + "payload_reference": null, + "payload_codec": "avro", + "emitted_at": "2026-10-25T06:42:31.000000Z" + } + ], + "next_offset": 2, + "terminal": false + } + }, + "show_after_window": { + "status": 200, + "response": { + "workflow_id": "parity-v1-workflow-stream-writes-wsw", + "workflow_run_id": "01m5rwqt9swt5hthyvnqaac50h", + "stream": { + "stream_name": "tokens", + "status": "open", + "last_offset": 2, + "total_items": 3, + "pending_items": 3, + "opened_at": "2026-10-25T06:42:31.000000Z", + "last_appended_at": "2026-10-25T06:42:31.000000Z", + "closed_at": null, + "error_reason": null, + "retention_seconds": null + } + } + }, + "items_all": { + "status": 200, + "response": { + "workflow_id": "parity-v1-workflow-stream-writes-wsw", + "workflow_run_id": "01m5rwqt9swt5hthyvnqaac50h", + "stream": { + "stream_name": "tokens", + "status": "open", + "last_offset": 2, + "total_items": 3, + "pending_items": 0, + "opened_at": "2026-10-25T06:42:31.000000Z", + "last_appended_at": "2026-10-25T06:42:31.000000Z", + "closed_at": null, + "error_reason": null, + "retention_seconds": null + }, + "items": [ + { + "offset": 0, + "idempotency_key": "k1", + "origin": "worker", + "origin_reference": "attempt-1", + "item_type": "token", + "content_type": "application/json", + "payload": { + "token": "Hello", + "count": 9007199254740993, + "unicode": "λ" + }, + "payload_reference": null, + "payload_codec": null, + "emitted_at": "2026-10-25T06:42:31.000000Z" + }, + { + "offset": 1, + "idempotency_key": "k2", + "origin": "workflow_command", + "origin_reference": null, + "item_type": null, + "content_type": null, + "payload": "plain text", + "payload_reference": null, + "payload_codec": "avro", + "emitted_at": "2026-10-25T06:42:31.000000Z" + }, + { + "offset": 2, + "idempotency_key": null, + "origin": "workflow_command", + "origin_reference": null, + "item_type": null, + "content_type": null, + "payload": [ + 1, + 2, + 3 + ], + "payload_reference": null, + "payload_codec": null, + "emitted_at": "2026-10-25T06:42:31.000000Z" + } + ], + "next_offset": 3, + "terminal": false + } + }, + "show_after_catch_up": { + "status": 200, + "response": { + "workflow_id": "parity-v1-workflow-stream-writes-wsw", + "workflow_run_id": "01m5rwqt9swt5hthyvnqaac50h", + "stream": { + "stream_name": "tokens", + "status": "open", + "last_offset": 2, + "total_items": 3, + "pending_items": 0, + "opened_at": "2026-10-25T06:42:31.000000Z", + "last_appended_at": "2026-10-25T06:42:31.000000Z", + "closed_at": null, + "error_reason": null, + "retention_seconds": null + } + } + }, + "items_wait_empty": { + "status": 200, + "response": { + "workflow_id": "parity-v1-workflow-stream-writes-wsw", + "workflow_run_id": "01m5rwqt9swt5hthyvnqaac50h", + "stream": { + "stream_name": "tokens", + "status": "open", + "last_offset": 2, + "total_items": 3, + "pending_items": 0, + "opened_at": "2026-10-25T06:42:31.000000Z", + "last_appended_at": "2026-10-25T06:42:31.000000Z", + "closed_at": null, + "error_reason": null, + "retention_seconds": null + }, + "items": [], + "next_offset": 3, + "terminal": false + } + }, + "items_wait_returned_after_wait": true, + "append_full": { + "status": 429, + "response": { + "message": "Stream pending-items cap reached; consumer is draining too slowly.", + "reason": "stream_full", + "pending_items": 2, + "max_pending_items": 2 + } + }, + "show_after_full": { + "status": 200, + "response": { + "workflow_id": "parity-v1-workflow-stream-writes-wsw", + "workflow_run_id": "01m5rwqt9swt5hthyvnqaac50h", + "stream": { + "stream_name": "tokens", + "status": "open", + "last_offset": 4, + "total_items": 5, + "pending_items": 2, + "opened_at": "2026-10-25T06:42:31.000000Z", + "last_appended_at": "2026-10-25T06:42:33.000000Z", + "closed_at": null, + "error_reason": null, + "retention_seconds": null + } + } + }, + "items_after_full": { + "status": 200, + "response": { + "workflow_id": "parity-v1-workflow-stream-writes-wsw", + "workflow_run_id": "01m5rwqt9swt5hthyvnqaac50h", + "stream": { + "stream_name": "tokens", + "status": "open", + "last_offset": 4, + "total_items": 5, + "pending_items": 0, + "opened_at": "2026-10-25T06:42:31.000000Z", + "last_appended_at": "2026-10-25T06:42:33.000000Z", + "closed_at": null, + "error_reason": null, + "retention_seconds": null + }, + "items": [ + { + "offset": 3, + "idempotency_key": "k3", + "origin": "workflow_command", + "origin_reference": null, + "item_type": null, + "content_type": null, + "payload": { + "n": 3 + }, + "payload_reference": null, + "payload_codec": null, + "emitted_at": "2026-10-25T06:42:33.000000Z" + }, + { + "offset": 4, + "idempotency_key": null, + "origin": "workflow_command", + "origin_reference": null, + "item_type": null, + "content_type": null, + "payload": { + "n": 4 + }, + "payload_reference": null, + "payload_codec": null, + "emitted_at": "2026-10-25T06:42:33.000000Z" + } + ], + "next_offset": 5, + "terminal": false + } + }, + "refusals": { + "no_items": { + "status": 422, + "response": { + "message": "The items field is required.", + "reason": "validation_failed", + "errors": { + "items": [ + "The items field is required." + ] + }, + "validation_errors": { + "items": [ + "The items field is required." + ] + } + } + }, + "empty_items": { + "status": 422, + "response": { + "message": "The items field is required.", + "reason": "validation_failed", + "errors": { + "items": [ + "The items field is required." + ] + }, + "validation_errors": { + "items": [ + "The items field is required." + ] + } + } + }, + "too_many_items": { + "status": 422, + "response": { + "message": "The items field must not have more than 500 items.", + "reason": "validation_failed", + "errors": { + "items": [ + "The items field must not have more than 500 items." + ] + }, + "validation_errors": { + "items": [ + "The items field must not have more than 500 items." + ] + } + } + }, + "item_not_object": { + "status": 422, + "response": { + "message": "The items.0 field must be an array.", + "reason": "validation_failed", + "errors": { + "items.0": [ + "The items.0 field must be an array." + ] + }, + "validation_errors": { + "items.0": [ + "The items.0 field must be an array." + ] + } + } + }, + "bad_codec": { + "status": 422, + "response": { + "message": "unsupported_payload_codec: workflow payload codec \"not-a-codec\" is not supported by Durable Workflow 2.0; use codec=\"avro\" with the fixed Avro Value schema and single-object framing. JSON remains the HTTP document transport, not a workflow payload codec.", + "reason": "validation_failed", + "errors": { + "items.0.payload_codec": [ + "unsupported_payload_codec: workflow payload codec \"not-a-codec\" is not supported by Durable Workflow 2.0; use codec=\"avro\" with the fixed Avro Value schema and single-object framing. JSON remains the HTTP document transport, not a workflow payload codec." + ] + }, + "validation_errors": { + "items.0.payload_codec": [ + "unsupported_payload_codec: workflow payload codec \"not-a-codec\" is not supported by Durable Workflow 2.0; use codec=\"avro\" with the fixed Avro Value schema and single-object framing. JSON remains the HTTP document transport, not a workflow payload codec." + ] + } + } + }, + "long_idempotency_key": { + "status": 422, + "response": { + "message": "The items.0.idempotency_key field must not be greater than 191 characters.", + "reason": "validation_failed", + "errors": { + "items.0.idempotency_key": [ + "The items.0.idempotency_key field must not be greater than 191 characters." + ] + }, + "validation_errors": { + "items.0.idempotency_key": [ + "The items.0.idempotency_key field must not be greater than 191 characters." + ] + } + } + }, + "zero_max_pending": { + "status": 422, + "response": { + "message": "The max pending items field must be at least 1.", + "reason": "validation_failed", + "errors": { + "max_pending_items": [ + "The max pending items field must be at least 1." + ] + }, + "validation_errors": { + "max_pending_items": [ + "The max pending items field must be at least 1." + ] + } + } + }, + "bad_stream_append": { + "status": 400, + "response": { + "message": "invalid stream name", + "reason": "invalid_request" + } + }, + "unstored_reference": { + "status": 422, + "response": { + "schema": "durable-workflow.v2.runtime-external-payload-error.v1", + "reason": "external_payload_unsupported", + "message": "Payload reference fields require an opaque runtime external payload reference object.", + "retryable": false, + "status": 422 + } + }, + "unknown_run_append": { + "status": 404, + "response": { + "message": "Workflow run not found.", + "reason": "instance_not_found" + } + }, + "unknown_workflow_append": { + "status": 404, + "response": { + "message": "Workflow run not found.", + "reason": "instance_not_found" + } + }, + "bad_stream_show": { + "status": 400, + "response": { + "message": "invalid stream name", + "reason": "invalid_stream_name" + } + }, + "bad_stream_items": { + "status": 400, + "response": { + "message": "invalid stream name", + "reason": "invalid_stream_name" + } + }, + "negative_from": { + "status": 422, + "response": { + "message": "The from field must be at least 0.", + "reason": "validation_failed", + "errors": { + "from": [ + "The from field must be at least 0." + ] + }, + "validation_errors": { + "from": [ + "The from field must be at least 0." + ] + } + } + }, + "too_many_read": { + "status": 422, + "response": { + "message": "The max items field must not be greater than 500.", + "reason": "validation_failed", + "errors": { + "max_items": [ + "The max items field must not be greater than 500." + ] + }, + "validation_errors": { + "max_items": [ + "The max items field must not be greater than 500." + ] + } + } + }, + "long_wait": { + "status": 422, + "response": { + "message": "The wait seconds field must not be greater than 60.", + "reason": "validation_failed", + "errors": { + "wait_seconds": [ + "The wait seconds field must not be greater than 60." + ] + }, + "validation_errors": { + "wait_seconds": [ + "The wait seconds field must not be greater than 60." + ] + } + } + }, + "bad_stream_close": { + "status": 400, + "response": { + "message": "invalid stream name", + "reason": "invalid_stream_name" + } + }, + "long_error_reason": { + "status": 422, + "response": { + "message": "The error reason field must not be greater than 191 characters.", + "reason": "validation_failed", + "errors": { + "error_reason": [ + "The error reason field must not be greater than 191 characters." + ] + }, + "validation_errors": { + "error_reason": [ + "The error reason field must not be greater than 191 characters." + ] + } + } + }, + "zero_retention": { + "status": 422, + "response": { + "message": "The retention seconds field must be at least 1.", + "reason": "validation_failed", + "errors": { + "retention_seconds": [ + "The retention seconds field must be at least 1." + ] + }, + "validation_errors": { + "retention_seconds": [ + "The retention seconds field must be at least 1." + ] + } + } + }, + "unknown_run_close": { + "status": 404, + "response": { + "message": "Workflow run not found.", + "reason": "instance_not_found" + } + } + }, + "list_after_refusals": { + "status": 200, + "response": { + "workflow_id": "parity-v1-workflow-stream-writes-wsw", + "workflow_run_id": "01m5rwqt9swt5hthyvnqaac50h", + "count": 1, + "streams": [ + { + "stream_name": "tokens", + "status": "open", + "last_offset": 4, + "total_items": 5, + "pending_items": 0, + "opened_at": "2026-10-25T06:42:31.000000Z", + "last_appended_at": "2026-10-25T06:42:33.000000Z", + "closed_at": null, + "error_reason": null, + "retention_seconds": null + } + ] + } + }, + "close": { + "status": 200, + "response": { + "workflow_id": "parity-v1-workflow-stream-writes-wsw", + "workflow_run_id": "01m5rwqt9swt5hthyvnqaac50h", + "stream": { + "stream_name": "tokens", + "status": "closed", + "last_offset": 4, + "total_items": 5, + "pending_items": 0, + "opened_at": "2026-10-25T06:42:31.000000Z", + "last_appended_at": "2026-10-25T06:42:33.000000Z", + "closed_at": "2026-10-25T06:42:35.000000Z", + "error_reason": null, + "retention_seconds": 3600 + } + } + }, + "close_again": { + "status": 200, + "response": { + "workflow_id": "parity-v1-workflow-stream-writes-wsw", + "workflow_run_id": "01m5rwqt9swt5hthyvnqaac50h", + "stream": { + "stream_name": "tokens", + "status": "closed", + "last_offset": 4, + "total_items": 5, + "pending_items": 0, + "opened_at": "2026-10-25T06:42:31.000000Z", + "last_appended_at": "2026-10-25T06:42:33.000000Z", + "closed_at": "2026-10-25T06:42:35.000000Z", + "error_reason": null, + "retention_seconds": 3600 + } + } + }, + "append_closed": { + "status": 409, + "response": { + "message": "Stream is closed.", + "reason": "stream_closed" + } + }, + "items_closed_tail": { + "status": 200, + "response": { + "workflow_id": "parity-v1-workflow-stream-writes-wsw", + "workflow_run_id": "01m5rwqt9swt5hthyvnqaac50h", + "stream": { + "stream_name": "tokens", + "status": "closed", + "last_offset": 4, + "total_items": 5, + "pending_items": 0, + "opened_at": "2026-10-25T06:42:31.000000Z", + "last_appended_at": "2026-10-25T06:42:33.000000Z", + "closed_at": "2026-10-25T06:42:35.000000Z", + "error_reason": null, + "retention_seconds": 3600 + }, + "items": [], + "next_offset": 5, + "terminal": true + } + }, + "items_closed_head": { + "status": 200, + "response": { + "workflow_id": "parity-v1-workflow-stream-writes-wsw", + "workflow_run_id": "01m5rwqt9swt5hthyvnqaac50h", + "stream": { + "stream_name": "tokens", + "status": "closed", + "last_offset": 4, + "total_items": 5, + "pending_items": 0, + "opened_at": "2026-10-25T06:42:31.000000Z", + "last_appended_at": "2026-10-25T06:42:33.000000Z", + "closed_at": "2026-10-25T06:42:35.000000Z", + "error_reason": null, + "retention_seconds": 3600 + }, + "items": [ + { + "offset": 4, + "idempotency_key": null, + "origin": "workflow_command", + "origin_reference": null, + "item_type": null, + "content_type": null, + "payload": { + "n": 4 + }, + "payload_reference": null, + "payload_codec": null, + "emitted_at": "2026-10-25T06:42:33.000000Z" + } + ], + "next_offset": 5, + "terminal": true + } + }, + "append_other": { + "status": 200, + "response": { + "workflow_id": "parity-v1-workflow-stream-writes-wsw", + "workflow_run_id": "01m5rwqt9swt5hthyvnqaac50h", + "stream": { + "stream_name": "audit", + "status": "open", + "last_offset": 0, + "total_items": 1, + "pending_items": 1, + "opened_at": "2026-10-25T06:42:35.000000Z", + "last_appended_at": "2026-10-25T06:42:35.000000Z", + "closed_at": null, + "error_reason": null, + "retention_seconds": null + }, + "accepted_offsets": [ + 0 + ], + "accepted": 1, + "deduped": 0 + } + }, + "close_errored": { + "status": 200, + "response": { + "workflow_id": "parity-v1-workflow-stream-writes-wsw", + "workflow_run_id": "01m5rwqt9swt5hthyvnqaac50h", + "stream": { + "stream_name": "audit", + "status": "errored", + "last_offset": 0, + "total_items": 1, + "pending_items": 1, + "opened_at": "2026-10-25T06:42:35.000000Z", + "last_appended_at": "2026-10-25T06:42:35.000000Z", + "closed_at": "2026-10-25T06:42:35.000000Z", + "error_reason": "producer crashed", + "retention_seconds": 60 + } + } + }, + "append_errored": { + "status": 409, + "response": { + "message": "Stream is errored.", + "reason": "stream_errored" + } + }, + "items_errored": { + "status": 200, + "response": { + "workflow_id": "parity-v1-workflow-stream-writes-wsw", + "workflow_run_id": "01m5rwqt9swt5hthyvnqaac50h", + "stream": { + "stream_name": "audit", + "status": "errored", + "last_offset": 0, + "total_items": 1, + "pending_items": 0, + "opened_at": "2026-10-25T06:42:35.000000Z", + "last_appended_at": "2026-10-25T06:42:35.000000Z", + "closed_at": "2026-10-25T06:42:35.000000Z", + "error_reason": "producer crashed", + "retention_seconds": 60 + }, + "items": [ + { + "offset": 0, + "idempotency_key": null, + "origin": "workflow_command", + "origin_reference": null, + "item_type": "event", + "content_type": null, + "payload": { + "audit": true + }, + "payload_reference": null, + "payload_codec": null, + "emitted_at": "2026-10-25T06:42:35.000000Z" + } + ], + "next_offset": 1, + "terminal": true + } + }, + "list_final": { + "status": 200, + "response": { + "workflow_id": "parity-v1-workflow-stream-writes-wsw", + "workflow_run_id": "01m5rwqt9swt5hthyvnqaac50h", + "count": 2, + "streams": [ + { + "stream_name": "audit", + "status": "errored", + "last_offset": 0, + "total_items": 1, + "pending_items": 0, + "opened_at": "2026-10-25T06:42:35.000000Z", + "last_appended_at": "2026-10-25T06:42:35.000000Z", + "closed_at": "2026-10-25T06:42:35.000000Z", + "error_reason": "producer crashed", + "retention_seconds": 60 + }, + { + "stream_name": "tokens", + "status": "closed", + "last_offset": 4, + "total_items": 5, + "pending_items": 0, + "opened_at": "2026-10-25T06:42:31.000000Z", + "last_appended_at": "2026-10-25T06:42:33.000000Z", + "closed_at": "2026-10-25T06:42:35.000000Z", + "error_reason": null, + "retention_seconds": 3600 + } + ] + } + } +} diff --git a/tests/Unit/ServerParityWorkflowStreamWritesContractTest.mjs b/tests/Unit/ServerParityWorkflowStreamWritesContractTest.mjs new file mode 100644 index 000000000..5942455b5 --- /dev/null +++ b/tests/Unit/ServerParityWorkflowStreamWritesContractTest.mjs @@ -0,0 +1,78 @@ +import assert from 'node:assert/strict'; +import {readFileSync} from 'node:fs'; +import test from 'node:test'; +import {checkWorkflowStreamWrites} from '../../scripts/conformance/server-parity/workflow-stream-writes-contract.mjs'; + +const fixture = JSON.parse(readFileSync(new URL('../Fixtures/ServerParity/workflow-stream-writes.json', import.meta.url))); +// The observation published Server 2.11.1 returned for this fixture. +const recorded = JSON.parse(readFileSync( + new URL('../Fixtures/ServerParityExpected/workflow-stream-writes-observation.json', import.meta.url))); +const id = 'parity-v1-workflow-stream-writes'; +const observation = (change = () => {}) => { + const state = structuredClone(recorded); + change(state); + return {mode: 'http', workflow_stream_writes: state}; +}; +const check = state => checkWorkflowStreamWrites(fixture, state, id); + +test('the published 2.11.1 recording satisfies the contract', () => { + assert.deepStrictEqual(check(observation()), {scope: 'workflow_stream_writes'}); +}); + +test('embedded applications are explicitly inapplicable', () => { + assert.deepStrictEqual(check({mode: 'embedded', workflow_stream_writes: + {applicable: false, reason: 'embedded_has_no_http_workflow_streams'}}), {scope: 'workflow_stream_writes'}); + assert.throws(() => check({mode: 'embedded', workflow_stream_writes: {applicable: true}}), /embedded workflow streams/); +}); + +for (const [label, change, message] of [ + // Native answered 501 for both producer routes before this fixture. + ['an unimplemented append route', state => { + state.append_first = {status: 501, response: {reason: 'rust_capability_not_implemented'}}; + }, /append_first status/], + ['offsets that start at 1', state => { state.append_first.response.accepted_offsets = [1, 2]; }, /append_first receipt/], + ['a repeated idempotency key appended again', state => { + state.append_retry.response.accepted_offsets = [2, 3]; + state.append_retry.response.accepted = 2; + state.append_retry.response.deduped = 0; + }, /append_retry receipt/], + ['a partial window that clears pending items', state => { + state.show_after_window.response.stream.pending_items = 0; + }, /show_after_window stream/], + ['a read at the head that leaves pending items', state => { + state.items_all.response.stream.pending_items = 3; + }, /items_all stream/], + ['a cap that rejects the whole batch', state => { + state.show_after_full.response.stream.last_offset = 2; + state.show_after_full.response.stream.total_items = 3; + }, /show_after_full stream/], + ['a cap answered as accepted', state => { state.append_full.status = 200; }, /append_full status/], + ['a repeated close that replaces the first', state => { + state.close_again.response.stream.error_reason = 'late error'; + state.close_again.response.stream.status = 'errored'; + }, /close_again stream/], + ['a closed stream that accepts appends', state => { + state.append_closed = {status: 200, response: {}}; + }, /append_closed status/], + ['a closed tail that is not terminal', state => { state.items_closed_tail.response.terminal = false; }, /items_closed_tail terminal/], + ['an errored stream without its reason', state => { state.close_errored.response.stream.error_reason = null; }, /close_errored stream/], + ['the default retention missing', state => { state.close.response.stream.retention_seconds = null; }, /close stream/], + ['an invalid name refused with the read reason', state => { + state.refusals.bad_stream_append.response.reason = 'invalid_stream_name'; + }, /refusal bad_stream_append/], + ['a string reference accepted', state => { + state.refusals.unstored_reference = {status: 200, response: {}}; + }, /refusal unstored_reference/], + // The runner parses records losslessly; JSON.parse here cannot hold the + // int64 itself, so this checks any change to a stored payload value. + ['a changed payload value', state => { state.items_all.response.items[0].payload.token = 'Hellp'; }, /items_all item 0/], + ['microsecond times', state => { + state.append_first.response.stream.opened_at = '2026-10-25T06:42:31.123456Z'; + }, /append_first opened_at/], + ['a list out of name order', state => { state.list_final.response.streams.reverse(); }, /list_final is by name/], + ['a missing step', state => { delete state.items_window; }, /steps/], +]) { + test(`the contract refuses ${label}`, () => { + assert.throws(() => check(observation(change)), message); + }); +} From 576d995fb03a8b5d760522a30d250e02c1c48bf6 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Sun, 25 Oct 2026 07:30:49 +0000 Subject: [PATCH 2/6] Rust #325: serve durable workflow stream appends and closes like Server 2.11.1 Native now serves POST .../streams/{name}/items and .../close. The run's stream list, description and item reads now return real rows. - Appends open the stream on first use, assign offsets from 0, and return the existing offset for a repeated idempotency key. - The pending-items cap keeps the items before it, then answers 429. - Closing is final. Reads that reach the head clear the pending count, and a bounded wait polls every 250 ms, as PHP's readWithWait does. - Times are stored as whole seconds, as Eloquent writes them. - Laravel's trimming applies to payloads, and an empty object reads back as a list, as in PHP. - `offset` is reserved, so its statements are per-dialect constants. Continue-as-new no longer refuses an instance with durable streams: they stay with their run, as in PHP. Three differences are documented in the module: a batch with no item fields gets 400 instead of PHP's 500; a read at the head does not clear a later append's count; a reference object in payload_reference gets 501. Co-Authored-By: Claude Opus 5.5 --- docs/rust-server-port.md | 28 +- rust/src/runtime/activity_failures.rs | 2 +- rust/src/runtime/backend.rs | 7 + rust/src/runtime/continuations.rs | 9 +- rust/src/runtime/execution.rs | 18 + rust/src/runtime/http.rs | 12 +- rust/src/runtime/workflow_streams.rs | 731 ++++++++++++++++-- rust/tests/execution/waterline_reads.rs | 80 +- .../fixtures/unserved-php-routes.v2.11.1.txt | 2 - 9 files changed, 814 insertions(+), 75 deletions(-) diff --git a/docs/rust-server-port.md b/docs/rust-server-port.md index 0afd87729..60a535975 100644 --- a/docs/rust-server-port.md +++ b/docs/rust-server-port.md @@ -82,9 +82,10 @@ Updated October 24, 2026. ([#737](https://github.com/durable-workflow/server/issues/737)). No expectation is edited to make native pass. -- **Parity surface.** Native serves 133 of the 175 API route methods in - published 2.11.1's route table, with message streams, update validation - and activity cancellation acknowledgement. An execution test sends each +- **Parity surface.** Native serves 135 of the 175 API route methods in + published 2.11.1's route table, with message streams, durable workflow + stream writes, update validation and activity cancellation + acknowledgement. An execution test sends each one through the native router, and the ones still answering `501 rust_capability_not_implemented` must match [`unserved-php-routes.v2.11.1.txt`](../rust/tests/fixtures/unserved-php-routes.v2.11.1.txt). @@ -153,7 +154,6 @@ any other unknown path or method answers PHP's own 404 or 405 ([checkpoint](rust-server-port-checkpoints/2026-10-23-message-streams.md)), except external payload references as input, stream retention cleanup and takeover of PHP data with streams. - - Durable workflow stream items and close. - Service endpoints. (The service catalog reads from Server 2.11 are served: native never holds catalog rows, so every list is PHP's empty page and every identity its not-found answer; the `service-catalog` fixture checks 37 cases.) @@ -858,12 +858,20 @@ October 21, Waterline reads: - **Task queues.** `GET /api/task-queues` and `GET /api/task-queues/{name}` follow PHP's `TaskQueueController`: pollers, stats, current leases, repair candidates, recent task flow, activity attempt states and admission. -- **Durable streams.** The run's stream list, stream description and item - reads answer as PHP does, including the name pattern, the `from`, - `max_items` and `wait_seconds` rules and the refusal bodies. Native runs - never produce durable streams, and takeover refuses a database that holds - any, so every list is empty. Producer appends and closes are not - implemented. +- **Durable streams.** The run's stream list, stream description, item + reads, appends and closes answer as PHP does, with the + `workflow-stream-writes` fixture recorded from published 2.11.1: offsets + from 0, idempotent retries, the pending-items cap with its partial commit, + bounded waits, final closes, whole-second times and every refusal body. + A continued run keeps its streams, as in PHP. Native differs in three + places, documented in `rust/src/runtime/workflow_streams.rs`: + - a batch whose items hold no item fields gets 400, not PHP's 500; + - a read at the head never clears the count of a later append; + - a runtime external payload reference object in `payload_reference` + gets 501. + Still missing: the workflow-side `workflow_stream` command, so cluster + info does not advertise `workflow_streams_contract`, and takeover still + refuses a PHP database that holds streams. - **Last progress.** Runs record `last_progress_at` when they start, as PHP does at start and child start. - **Deliberate differences.** diff --git a/rust/src/runtime/activity_failures.rs b/rust/src/runtime/activity_failures.rs index 61b89964c..0e89d3084 100644 --- a/rust/src/runtime/activity_failures.rs +++ b/rust/src/runtime/activity_failures.rs @@ -219,7 +219,7 @@ fn resolve_details(details: &Value) -> Result<(Value, Option<&'static str>)> { /// A PHP array round trip: an object keyed 0..n, including an empty one, /// comes back as a list. -fn php_array(value: Value) -> Value { +pub(super) fn php_array(value: Value) -> Value { match value { Value::Array(items) => Value::Array(items.into_iter().map(php_array).collect()), Value::Object(fields) => { diff --git a/rust/src/runtime/backend.rs b/rust/src/runtime/backend.rs index 4b8660955..83475650a 100644 --- a/rust/src/runtime/backend.rs +++ b/rust/src/runtime/backend.rs @@ -78,6 +78,10 @@ pub(super) trait Backend: Database { const SEARCH_ATTRIBUTE_DELETE: &str = "DELETE FROM workflow_search_attributes WHERE workflow_run_id=$1 AND \"key\"=$2"; const SEARCH_ATTRIBUTE_KEY_ROWS: &str = "SELECT a.workflow_run_id,a.\"key\" AS attribute_key,a.type,a.value_string,a.value_keyword,a.value_int,a.value_float,CASE WHEN a.value_bool IS NULL THEN NULL WHEN a.value_bool THEN 1 ELSE 0 END AS value_bool,a.value_datetime,a.value_keyword_list FROM workflow_search_attributes a JOIN workflow_runs r ON r.id=a.workflow_run_id WHERE r.namespace=$1 AND a.\"key\"=$2"; const SEARCH_ATTRIBUTE_INHERIT: &str = "INSERT INTO workflow_search_attributes(workflow_run_id,workflow_instance_id,\"key\",type,value_string,value_keyword,value_keyword_list,value_int,value_float,value_bool,value_datetime,upserted_at_sequence,inherited_from_parent,created_at,updated_at) SELECT $1,$2,\"key\",type,value_string,value_keyword,value_keyword_list,value_int,value_float,value_bool,value_datetime,1,true,$3,$4 FROM workflow_search_attributes WHERE workflow_run_id=$5"; + // Durable workflow stream items; `offset` is a reserved word. + const STREAM_ITEM_BY_KEY: &str = "SELECT \"offset\" FROM workflow_durable_stream_items WHERE stream_id=$1 AND idempotency_key=$2 LIMIT 1"; + const STREAM_ITEM_INSERT: &str = "INSERT INTO workflow_durable_stream_items(stream_id,namespace,workflow_run_id,stream_name,\"offset\",idempotency_key,origin,origin_reference,payload,payload_reference,payload_codec,item_type,content_type,emitted_at,created_at,updated_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16)"; + const STREAM_ITEMS_WINDOW: &str = "SELECT * FROM workflow_durable_stream_items WHERE stream_id=$1 AND \"offset\">=$2 ORDER BY \"offset\" LIMIT $3"; fn statement(sql: &'static str) -> Cow<'static, str> { Cow::Borrowed(sql) } @@ -434,6 +438,9 @@ impl Backend for MySql { const SEARCH_ATTRIBUTE_DELETE: &str = "DELETE FROM workflow_search_attributes WHERE workflow_run_id=$1 AND `key`=$2"; const SEARCH_ATTRIBUTE_KEY_ROWS: &str = "SELECT a.workflow_run_id,a.`key` AS attribute_key,a.type,a.value_string,a.value_keyword,a.value_int,a.value_float,CASE WHEN a.value_bool IS NULL THEN NULL WHEN a.value_bool THEN 1 ELSE 0 END AS value_bool,a.value_datetime,a.value_keyword_list FROM workflow_search_attributes a JOIN workflow_runs r ON r.id=a.workflow_run_id WHERE r.namespace=$1 AND a.`key`=$2"; const SEARCH_ATTRIBUTE_INHERIT: &str = "INSERT INTO workflow_search_attributes(workflow_run_id,workflow_instance_id,`key`,type,value_string,value_keyword,value_keyword_list,value_int,value_float,value_bool,value_datetime,upserted_at_sequence,inherited_from_parent,created_at,updated_at) SELECT $1,$2,`key`,type,value_string,value_keyword,value_keyword_list,value_int,value_float,value_bool,value_datetime,1,true,$3,$4 FROM workflow_search_attributes WHERE workflow_run_id=$5"; + const STREAM_ITEM_BY_KEY: &str = "SELECT `offset` FROM workflow_durable_stream_items WHERE stream_id=$1 AND idempotency_key=$2 LIMIT 1"; + const STREAM_ITEM_INSERT: &str = "INSERT INTO workflow_durable_stream_items(stream_id,namespace,workflow_run_id,stream_name,`offset`,idempotency_key,origin,origin_reference,payload,payload_reference,payload_codec,item_type,content_type,emitted_at,created_at,updated_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16)"; + const STREAM_ITEMS_WINDOW: &str = "SELECT * FROM workflow_durable_stream_items WHERE stream_id=$1 AND `offset`>=$2 ORDER BY `offset` LIMIT $3"; const READ_SNAPSHOT_BEGIN: &str = "START TRANSACTION WITH CONSISTENT SNAPSHOT, READ ONLY"; const ACTIVITY_METADATA_HISTORY: &str = "SELECT h.sequence,h.event_type,CASE WHEN page.projection_bytes<=8388608 THEN h.payload ELSE NULL END AS payload,h.workflow_task_id,h.recorded_at,page.projection_count FROM (SELECT id,sequence,COUNT(*) OVER () AS projection_count,SUM(payload_bytes) OVER (ORDER BY sequence) AS projection_bytes FROM (SELECT id,sequence,octet_length(CAST(payload AS CHAR CHARACTER SET utf8mb4)) AS payload_bytes FROM workflow_history_events FORCE INDEX (workflow_history_events_workflow_run_id_sequence_unique) WHERE workflow_run_id=$1 AND event_type IN ('ActivityScheduled','ActivityStarted','ActivityHeartbeatRecorded','ActivityRetryScheduled','ActivityCompleted','ActivityFailed','ActivityTimedOut','ActivityCancelled')) AS sizes ORDER BY sequence LIMIT 10001) AS page JOIN workflow_history_events h ON h.id=page.id ORDER BY page.sequence"; const PHP_CONNECTION: &str = "mysql"; diff --git a/rust/src/runtime/continuations.rs b/rust/src/runtime/continuations.rs index e8590924a..5fec8b7f9 100644 --- a/rust/src/runtime/continuations.rs +++ b/rust/src/runtime/continuations.rs @@ -107,13 +107,10 @@ where // timers, children and waits stay with the closed predecessor, and no // parent-close policy applies. Message streams pass to the next run // when the completion records its stream acknowledgements. Durable - // output streams, service calls and live queries are not transferred - // by native yet. + // output streams are rows of their run and stay with the predecessor; + // PHP's continue-as-new does not touch them either. + // Service calls and live queries are not transferred by native yet. for (sql, selected) in [ - ( - "SELECT COUNT(*) FROM workflow_durable_streams WHERE workflow_instance_id=$1", - workflow.as_str(), - ), ( "SELECT COUNT(*) FROM workflow_service_calls WHERE caller_workflow_run_id=$1", old.as_str(), diff --git a/rust/src/runtime/execution.rs b/rust/src/runtime/execution.rs index 5088c39c6..2f5a83561 100644 --- a/rust/src/runtime/execution.rs +++ b/rust/src/runtime/execution.rs @@ -850,6 +850,24 @@ impl Runtime { } delegate!(run_in_namespace(workflow_id: &str, run_id: &str) -> Result); + delegate!(stream_run(workflow_id: &str, run_id: &str) -> Result>); + delegate!(list_workflow_streams(run_id: &str) -> Result>); + delegate!(show_workflow_stream(run_id: &str, name: &str) -> Result>); + delegate!(append_workflow_stream( + workflow_id: &str, + run_id: &str, + name: &str, + items: Vec, + max_pending: i64, + ) -> Result); + delegate!(read_workflow_stream(run_id: &str, name: &str, from: i64, max_items: i64) + -> Result>); + delegate!(close_workflow_stream( + run_id: &str, + name: &str, + error_reason: Option, + retention_seconds: Option, + ) -> Result>); pub(crate) async fn run_diagnostics( &self, workflow_id: &str, diff --git a/rust/src/runtime/http.rs b/rust/src/runtime/http.rs index 9bbb5882b..eeeea3918 100644 --- a/rust/src/runtime/http.rs +++ b/rust/src/runtime/http.rs @@ -140,7 +140,11 @@ pub fn router(runtime: Runtime) -> Router { ) .route( "/api/workflows/{workflow_id}/runs/{run_id}/streams/{stream_name}/items", - get(super::workflow_streams::items), + get(super::workflow_streams::items).post(super::workflow_streams::append), + ) + .route( + "/api/workflows/{workflow_id}/runs/{run_id}/streams/{stream_name}/close", + post(super::workflow_streams::close), ) .route( "/api/workflows/{workflow_id}/request-cancellation", @@ -747,8 +751,10 @@ fn workflow_route(rest: &str) -> Option { | [_, "runs", _, "debug"] | [_, "runs", _, "history", "export"] | [_, "runs", _, "streams"] - | [_, "runs", _, "streams", _] - | [_, "runs", _, "streams", _, "items"] => Some(READ), + | [_, "runs", _, "streams", _] => Some(READ), + // A run's durable streams are rows in the request's namespace. + [_, "runs", _, "streams", _, "items"] => Some(READ_WRITE), + [_, "runs", _, "streams", _, "close"] => Some(WRITE), [_, "signal" | "query" | "update", _] | [_, "runs", _, "signal" | "query" | "update", _] | [_, "repair" | "request-cancellation"] diff --git a/rust/src/runtime/workflow_streams.rs b/rust/src/runtime/workflow_streams.rs index efe6bc0d4..d755b50c8 100644 --- a/rust/src/runtime/workflow_streams.rs +++ b/rust/src/runtime/workflow_streams.rs @@ -1,18 +1,132 @@ -//! Durable workflow stream reads as published PHP's `WorkflowStreamController` -//! answers them. Native runs never produce durable streams, and takeover -//! refuses a PHP database that holds any, so every run has none: the list is -//! empty, a valid name is not found and an invalid one is refused. Producer -//! appends and closes are not implemented. +//! Durable workflow streams as published PHP's `WorkflowStreamController` +//! and `WorkflowStreamService` serve them. A producer appends ordered items +//! to a named, run-scoped stream, which the first append opens. Offsets +//! start at 0, a repeated idempotency key returns the item it already +//! names, and the pending-items cap keeps the items before it and then +//! answers 429. A close records a clean or errored end once. Readers page +//! by offset and may wait, and a read that reaches the head resets +//! `pending_items`. +//! +//! PHP's models write these times in Eloquent's `Y-m-d H:i:s` format, so +//! every stored time is a whole second. Native differs in three places: +//! - An append whose items each hold none of the item fields fails in PHP, +//! whose validated data then has no `items` (an undefined index, 500). +//! Native refuses it with the service's own 400. A batch with some such +//! items drops them, as PHP does. +//! - A read that reaches the head resets `pending_items` only while the +//! stream is still at that head. PHP's unlocked save can also clear the +//! count of an append committed after its read. +//! - A runtime external payload reference object in `payload_reference` is +//! refused with 501; PHP resolves and claims it. Other values are refused +//! as PHP's transport refuses them. +use super::laravel_rules::{self, Rule}; use super::laravel_validation::{failure, integer, normalized}; -use super::{Result, Runtime, RuntimeError, backend::Backend, store::Store}; +use super::{ + Result, Runtime, RuntimeError, + backend::Backend, + refuse, + store::{Store, now}, +}; use axum::{ Extension, Json, extract::{Path, Query}, http::StatusCode, }; +use chrono::{DateTime, SubsecRound, Utc}; use serde_json::{Value, json}; use sqlx::{ColumnIndex, Decode, Encode, Executor, IntoArguments, Type}; -use std::collections::HashMap; +use std::{collections::HashMap, time::Duration}; +use tokio::time::Instant; + +/// `WorkflowStreamService::DEFAULT_MAX_PENDING_ITEMS`. +const DEFAULT_MAX_PENDING_ITEMS: i64 = 10_000; +/// `WorkflowStreamService::DEFAULT_MAX_ITEMS_PER_SUBSCRIBE`. +const DEFAULT_MAX_ITEMS_PER_SUBSCRIBE: i64 = 100; +/// `WorkflowStreamService::SUBSCRIBE_MAX_ITEMS`. +const SUBSCRIBE_MAX_ITEMS: i64 = 500; +/// `WorkflowStreamService::DEFAULT_CLOSED_RETENTION_SECONDS`. +const DEFAULT_CLOSED_RETENTION_SECONDS: i64 = 3600; +/// `LegacyV1ProjectionContract::IMPORT_SOURCE`. +const PROJECTION_SOURCE: &str = "waterline_v1_projection"; +/// `readWithWait` checks again every 250 ms. +const WAIT_INTERVAL: Duration = Duration::from_millis(250); + +const STREAM_BY_NAME: &str = + "SELECT * FROM workflow_durable_streams WHERE namespace=$1 AND workflow_run_id=$2 AND stream_name=$3"; +const STREAM_BY_ID: &str = "SELECT * FROM workflow_durable_streams WHERE id=$1"; + +/// `append()`'s Laravel rules. +const APPEND_RULES: &laravel_rules::Rules = &[ + ("items", &[Rule::Required, Rule::Array, Rule::Min(1), Rule::Max(500)]), + ("items.*", &[Rule::Array]), + ("items.*.payload", &[Rule::Nullable]), + ("items.*.payload_reference", &[Rule::Nullable, Rule::String, Rule::Max(191)]), + ("items.*.payload_codec", &[Rule::Nullable, Rule::String, Rule::Max(64)]), + ("items.*.idempotency_key", &[Rule::Nullable, Rule::String, Rule::Max(191)]), + ("items.*.origin", &[Rule::Nullable, Rule::String, Rule::Max(64)]), + ("items.*.origin_reference", &[Rule::Nullable, Rule::String, Rule::Max(191)]), + ("items.*.item_type", &[Rule::Nullable, Rule::String, Rule::Max(64)]), + ("items.*.content_type", &[Rule::Nullable, Rule::String, Rule::Max(191)]), + ("max_pending_items", &[Rule::Nullable, Rule::Integer, Rule::Min(1)]), +]; + +/// `close()`'s Laravel rules. +const CLOSE_RULES: &laravel_rules::Rules = &[ + ("error_reason", &[Rule::Nullable, Rule::String, Rule::Max(191)]), + ("retention_seconds", &[Rule::Nullable, Rule::Integer, Rule::Min(1)]), +]; + +/// The item fields `append()` validates, which are all its validated data +/// keeps of an item. +const ITEM_FIELDS: [&str; 8] = [ + "payload", + "payload_reference", + "payload_codec", + "idempotency_key", + "origin", + "origin_reference", + "item_type", + "content_type", +]; + +/// One validated item as `WorkflowStreamService::append` reads it. +#[derive(Debug, Default, PartialEq)] +pub(crate) struct StreamItem { + payload: Value, + payload_reference: Option, + payload_codec: Option, + idempotency_key: Option, + origin: Option, + origin_reference: Option, + item_type: Option, + content_type: Option, +} + +pub(crate) enum Appended { + Accepted { + stream: Value, + offsets: Vec, + accepted: usize, + deduped: usize, + }, + /// The cap stopped the batch after committing the items before it. + Full { pending_items: i64 }, + Closed, + Errored, +} + +/// One `read()` window. +pub(crate) struct Window { + stream: Value, + items: Vec, + next_offset: i64, + terminal: bool, +} + +/// Carbon's `toJSON`. +fn carbon_json(at: DateTime) -> String { + at.format("%Y-%m-%dT%H:%M:%S%.6fZ").to_string() +} impl Store where @@ -39,6 +153,282 @@ where .await?; Ok(found > 0) } + + /// `locateRun`, then `LegacyV1Projection::isProjectedRun`: nothing when + /// the run is not in this namespace, else whether it is a read-only + /// Waterline v1 projection. + pub(super) async fn stream_run(&self, workflow_id: &str, run_id: &str) -> Result> { + let row = Self::query("SELECT import_source FROM workflow_runs WHERE namespace=$1 AND workflow_instance_id=$2 AND id=$3 AND EXISTS (SELECT 1 FROM workflow_instances WHERE id=$4 AND namespace=$5)") + .bind(self.namespace.as_ref()) + .bind(workflow_id) + .bind(run_id) + .bind(workflow_id) + .bind(self.namespace.as_ref()) + .fetch_optional(&self.pool) + .await?; + match row { + Some(row) => Ok(Some(DB::optional_string(&row, "import_source")?.as_deref() == Some(PROJECTION_SOURCE))), + None => Ok(None), + } + } + + /// `formatStream`. + fn stream_json(row: &DB::Row) -> Result { + let time = |field: &str| Ok::<_, RuntimeError>(DB::optional_instant(row, field)?.map(carbon_json)); + Ok(json!({ + "stream_name": DB::string(row, "stream_name")?, + "status": DB::string(row, "status")?, + "last_offset": DB::number(row, "last_offset")?, + "total_items": DB::number(row, "total_items")?, + "pending_items": DB::number(row, "pending_items")?, + "opened_at": time("opened_at")?, + "last_appended_at": time("last_appended_at")?, + "closed_at": time("closed_at")?, + "error_reason": DB::optional_string(row, "error_reason")?, + "retention_seconds": DB::optional_number(row, "retention_seconds")?, + })) + } + + /// One item as `items()` renders it. + fn stream_item_json(row: &DB::Row) -> Result { + Ok(json!({ + "offset": DB::number(row, "offset")?, + "idempotency_key": DB::optional_string(row, "idempotency_key")?, + "origin": DB::string(row, "origin")?, + "origin_reference": DB::optional_string(row, "origin_reference")?, + "item_type": DB::optional_string(row, "item_type")?, + "content_type": DB::optional_string(row, "content_type")?, + "payload": DB::optional_document_row(row, "payload")?.unwrap_or(Value::Null), + "payload_reference": DB::optional_string(row, "payload_reference")?, + "payload_codec": DB::optional_string(row, "payload_codec")?, + "emitted_at": carbon_json(DB::instant(row, "emitted_at")?), + })) + } + + /// `listForRun`: the run's streams by name, in the database's order. + pub(super) async fn list_workflow_streams(&self, run_id: &str) -> Result> { + Self::query("SELECT * FROM workflow_durable_streams WHERE namespace=$1 AND workflow_run_id=$2 ORDER BY stream_name") + .bind(self.namespace.as_ref()) + .bind(run_id) + .fetch_all(&self.pool) + .await? + .iter() + .map(Self::stream_json) + .collect() + } + + /// `describe` for a valid name. + pub(super) async fn show_workflow_stream(&self, run_id: &str, name: &str) -> Result> { + let row = Self::query(STREAM_BY_NAME) + .bind(self.namespace.as_ref()) + .bind(run_id) + .bind(name) + .fetch_optional(&self.pool) + .await?; + row.as_ref().map(Self::stream_json).transpose() + } + + /// `WorkflowStreamService::append` for a valid name and a non-empty + /// batch, in one transaction. + pub(super) async fn append_workflow_stream( + &self, + workflow_id: &str, + run_id: &str, + name: &str, + items: Vec, + max_pending: i64, + ) -> Result { + let at = now().trunc_subsecs(0); + let mut tx = self.begin().await?; + let found = Self::query(STREAM_BY_NAME) + .bind(self.namespace.as_ref()) + .bind(run_id) + .bind(name) + .fetch_optional(&mut *tx) + .await?; + let row = match found { + Some(row) => row, + None => { + Self::query("INSERT INTO workflow_durable_streams(namespace,workflow_instance_id,workflow_run_id,stream_name,status,last_offset,total_items,pending_items,opened_at,created_at,updated_at) VALUES ($1,$2,$3,$4,'open',-1,0,0,$5,$6,$7)") + .bind(self.namespace.as_ref()) + .bind(workflow_id) + .bind(run_id) + .bind(name) + .bind(DB::bind_time(at)) + .bind(DB::bind_time(at)) + .bind(DB::bind_time(at)) + .execute(&mut *tx) + .await?; + Self::query(STREAM_BY_NAME) + .bind(self.namespace.as_ref()) + .bind(run_id) + .bind(name) + .fetch_one(&mut *tx) + .await? + } + }; + match DB::string(&row, "status")?.as_str() { + "closed" => return Ok(Appended::Closed), + "errored" => return Ok(Appended::Errored), + _ => {} + } + let stream_id = DB::number(&row, "id")?; + let mut last_offset = DB::number(&row, "last_offset")?; + let mut total = DB::number(&row, "total_items")?; + let mut pending = DB::number(&row, "pending_items")?; + let mut offsets = Vec::with_capacity(items.len()); + let (mut accepted, mut deduped, mut full) = (0, 0, false); + for item in &items { + if let Some(key) = item.idempotency_key.as_deref() + && let Some(existing) = Self::scalar(DB::STREAM_ITEM_BY_KEY) + .bind(stream_id) + .bind(key) + .fetch_optional(&mut *tx) + .await? + { + offsets.push(existing); + deduped += 1; + continue; + } + if pending >= max_pending { + full = true; + break; + } + last_offset += 1; + Self::query(DB::STREAM_ITEM_INSERT) + .bind(stream_id) + .bind(self.namespace.as_ref()) + .bind(run_id) + .bind(name) + .bind(last_offset) + .bind(item.idempotency_key.as_deref()) + .bind(item.origin.as_deref().unwrap_or("workflow_command")) + .bind(item.origin_reference.as_deref()) + .bind(DB::optional_document(&item.payload)) + .bind(item.payload_reference.as_deref()) + .bind(item.payload_codec.as_deref()) + .bind(item.item_type.as_deref()) + .bind(item.content_type.as_deref()) + .bind(DB::bind_time(at)) + .bind(DB::bind_time(at)) + .bind(DB::bind_time(at)) + .execute(&mut *tx) + .await?; + total += 1; + pending += 1; + accepted += 1; + offsets.push(last_offset); + } + // Eloquent saves only changed attributes, so a batch that adds + // nothing leaves the row as it was. + if accepted > 0 { + Self::query("UPDATE workflow_durable_streams SET last_offset=$1,total_items=$2,pending_items=$3,last_appended_at=$4,updated_at=$5 WHERE id=$6") + .bind(last_offset) + .bind(total) + .bind(pending) + .bind(DB::bind_time(at)) + .bind(DB::bind_time(at)) + .bind(stream_id) + .execute(&mut *tx) + .await?; + } + let row = Self::query(STREAM_BY_ID).bind(stream_id).fetch_one(&mut *tx).await?; + tx.commit().await?; + if full { + return Ok(Appended::Full { pending_items: DB::number(&row, "pending_items")? }); + } + Ok(Appended::Accepted { stream: Self::stream_json(&row)?, offsets, accepted, deduped }) + } + + /// `WorkflowStreamService::read` for a valid name and offset. + pub(super) async fn read_workflow_stream( + &self, + run_id: &str, + name: &str, + from: i64, + max_items: i64, + ) -> Result> { + let Some(mut row) = Self::query(STREAM_BY_NAME) + .bind(self.namespace.as_ref()) + .bind(run_id) + .bind(name) + .fetch_optional(&self.pool) + .await? + else { + return Ok(None); + }; + let stream_id = DB::number(&row, "id")?; + let rows = Self::query(DB::STREAM_ITEMS_WINDOW) + .bind(stream_id) + .bind(from) + .bind(max_items.clamp(1, SUBSCRIBE_MAX_ITEMS)) + .fetch_all(&self.pool) + .await?; + let last = rows.last().map(|item| DB::number(item, "offset")).transpose()?; + let next_offset = last.map_or(from, |offset| offset + 1); + let head = DB::number(&row, "last_offset")?; + // The window reached the head: nothing is pending past it. + if last == Some(head) && DB::number(&row, "pending_items")? > 0 { + let mut tx = self.begin().await?; + Self::query("UPDATE workflow_durable_streams SET pending_items=0,updated_at=$1 WHERE id=$2 AND last_offset=$3") + .bind(DB::bind_time(now().trunc_subsecs(0))) + .bind(stream_id) + .bind(head) + .execute(&mut *tx) + .await?; + row = Self::query(STREAM_BY_ID).bind(stream_id).fetch_one(&mut *tx).await?; + tx.commit().await?; + } + let terminal = DB::string(&row, "status")? != "open" && next_offset > DB::number(&row, "last_offset")?; + Ok(Some(Window { + stream: Self::stream_json(&row)?, + items: rows.iter().map(Self::stream_item_json).collect::>()?, + next_offset, + terminal, + })) + } + + /// `WorkflowStreamService::close` for a valid name: a terminal stream is + /// returned unchanged. + pub(super) async fn close_workflow_stream( + &self, + run_id: &str, + name: &str, + error_reason: Option, + retention_seconds: Option, + ) -> Result> { + let mut tx = self.begin().await?; + let Some(row) = Self::query(STREAM_BY_NAME) + .bind(self.namespace.as_ref()) + .bind(run_id) + .bind(name) + .fetch_optional(&mut *tx) + .await? + else { + return Ok(None); + }; + if DB::string(&row, "status")? != "open" { + return Ok(Some(Self::stream_json(&row)?)); + } + let at = now().trunc_subsecs(0); + let retention = match retention_seconds { + Some(seconds) if seconds > 0 => seconds, + _ => DB::optional_number(&row, "retention_seconds")?.unwrap_or(DEFAULT_CLOSED_RETENTION_SECONDS), + }; + let stream_id = DB::number(&row, "id")?; + Self::query("UPDATE workflow_durable_streams SET status=$1,error_reason=$2,closed_at=$3,retention_seconds=$4,updated_at=$5 WHERE id=$6") + .bind(if error_reason.is_some() { "errored" } else { "closed" }) + .bind(error_reason.as_deref()) + .bind(DB::bind_time(at)) + .bind(retention) + .bind(DB::bind_time(at)) + .bind(stream_id) + .execute(&mut *tx) + .await?; + let row = Self::query(STREAM_BY_ID).bind(stream_id).fetch_one(&mut *tx).await?; + tx.commit().await?; + Ok(Some(Self::stream_json(&row)?)) + } } fn protocol(status: StatusCode, response: Value) -> RuntimeError { @@ -49,6 +439,10 @@ fn run_not_found() -> RuntimeError { protocol(StatusCode::NOT_FOUND, json!({"message":"Workflow run not found.","reason":"instance_not_found"})) } +fn stream_not_found() -> RuntimeError { + protocol(StatusCode::NOT_FOUND, json!({"message":"Stream not found.","reason":"stream_not_found"})) +} + /// `WorkflowStreamService::STREAM_NAME_PATTERN`. fn valid_name(name: &str) -> bool { let bytes = name.as_bytes(); @@ -58,31 +452,123 @@ fn valid_name(name: &str) -> bool { && bytes[1..].iter().all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b':' | b'-')) } -fn stream_refusal(name: &str) -> RuntimeError { - if valid_name(name) { - protocol(StatusCode::NOT_FOUND, json!({"message":"Stream not found.","reason":"stream_not_found"})) - } else { - protocol(StatusCode::BAD_REQUEST, json!({"message":"invalid stream name","reason":"invalid_stream_name"})) +/// The service's invalid-name refusal, which `append()` reports with its +/// general reason and the other routes with their own. +fn invalid_name(reason: &str) -> RuntimeError { + protocol(StatusCode::BAD_REQUEST, json!({"message":"invalid stream name","reason":reason})) +} + +/// The run lookup every route makes first. +async fn located(runtime: &Runtime, workflow_id: &str, run_id: &str) -> Result { + runtime.stream_run(workflow_id, run_id).await?.ok_or_else(run_not_found) +} + +/// `projectedRunReadOnly`. +fn projected_read_only(workflow_id: &str, run_id: &str) -> RuntimeError { + protocol(StatusCode::CONFLICT, json!({ + "message": "Durable streams cannot mutate a read-only v1 projection.", + "reason": "v1_projection_read_only", + "remediation": "Continue execution and stream production on the source v1 application.", + "workflow_id": workflow_id, + "run_id": run_id, + "execution_owner": "v1", + })) +} + +/// The items of a normalized append body, in data order, whether `items` +/// is a list or keyed. +fn body_items(body: &Value) -> Vec<&Value> { + match &body["items"] { + Value::Array(items) => items.iter().collect(), + Value::Object(items) => items.values().collect(), + _ => Vec::new(), + } +} + +/// `RuntimeExternalPayloadTransport::incoming` on `items.*.payload_reference`, +/// which runs before the controller: a value other than null or a reference +/// object is refused. Native does not resolve reference objects yet. +fn references(body: &Value) -> Result<()> { + for item in body_items(body) { + match item.get("payload_reference") { + None | Some(Value::Null) => {} + Some(Value::Array(_) | Value::Object(_)) => { + return Err(refuse(StatusCode::NOT_IMPLEMENTED, "rust_capability_not_implemented")); + } + Some(_) => { + return Err(super::external_transport::typed_error( + StatusCode::UNPROCESSABLE_ENTITY, + "external_payload_unsupported", + "Payload reference fields require an opaque runtime external payload reference object.", + )); + } + } + } + Ok(()) +} + +/// The validated items `append()` passes on: Laravel's validated data keeps +/// only the item fields, so an item with none of them is left out. A +/// present codec is canonicalized, and the first one refused fails the +/// request as `items.{index}.payload_codec`. +fn validated_items(body: &Value) -> Result> { + let keyed: Vec<(String, &Value)> = match &body["items"] { + Value::Array(items) => items.iter().enumerate().map(|(index, item)| (index.to_string(), item)).collect(), + Value::Object(items) => items.iter().map(|(key, item)| (key.clone(), item)).collect(), + _ => Vec::new(), + }; + let mut items = Vec::with_capacity(keyed.len()); + for (key, item) in keyed { + let Some(fields) = item.as_object() else { + continue; + }; + if !ITEM_FIELDS.iter().any(|field| fields.contains_key(*field)) { + continue; + } + let text = |field: &str| fields.get(field).and_then(Value::as_str).map(str::to_owned); + let payload_codec = match fields.get("payload_codec") { + None => None, + Some(codec) => match super::input_envelope::codec_refusal(codec) { + None => Some("avro".to_owned()), + Some(message) => { + return Err(super::laravel_validation::failure_messages(vec![( + format!("items.{key}.payload_codec"), + vec![message], + )]) + .expect("one message")); + } + }, + }; + items.push(StreamItem { + // The array cast stores a PHP array, so an empty object reads back + // as a list, as PHP's other array round trips do. + payload: super::activity_failures::php_array(fields.get("payload").cloned().unwrap_or(Value::Null)), + payload_reference: text("payload_reference"), + payload_codec, + // The service treats an empty key as none; normalization already + // turned empty strings into null. + idempotency_key: text("idempotency_key"), + origin: text("origin"), + origin_reference: text("origin_reference"), + item_type: text("item_type"), + content_type: text("content_type"), + }); } + Ok(items) } /// `items()`'s Laravel rules for `from`, `max_items` and `wait_seconds`. fn items_rules(query: &HashMap) -> Result<()> { - // Query values are strings; empty ones become null and pass `nullable`. - let integer = |field: &str| -> Option> { - let value = normalized(query.get(field).map(|text| json!(text)).as_ref())?; - (!value.is_null()).then(|| integer(&value).ok_or(())) - }; let mut errors: Vec<(&'static str, Vec<&'static str>)> = Vec::new(); let mut from = vec![]; - match integer("from") { + match query_integer(query, "from") { Some(Err(())) => from.push("The from field must be an integer."), Some(Ok(value)) if value < 0 => from.push("The from field must be at least 0."), _ => {} } errors.push(("from", from)); let mut max_items = vec![]; - match integer("max_items") { + match query_integer(query, "max_items") { Some(Err(())) => max_items.push("The max items field must be an integer."), Some(Ok(value)) if value < 1 => max_items.push("The max items field must be at least 1."), Some(Ok(value)) if value > 500 => max_items.push("The max items field must not be greater than 500."), @@ -90,7 +576,7 @@ fn items_rules(query: &HashMap) -> Result<()> { } errors.push(("max_items", max_items)); let mut wait = vec![]; - match integer("wait_seconds") { + match query_integer(query, "wait_seconds") { Some(Err(())) => wait.push("The wait seconds field must be an integer."), Some(Ok(value)) if value < 0 => wait.push("The wait seconds field must be at least 0."), Some(Ok(value)) if value > 60 => wait.push("The wait seconds field must not be greater than 60."), @@ -103,15 +589,21 @@ fn items_rules(query: &HashMap) -> Result<()> { } } +/// A query value as the `integer` rule reads it. Query values are strings; +/// empty ones become null and pass `nullable`. +fn query_integer(query: &HashMap, field: &str) -> Option> { + let value = normalized(query.get(field).map(|text| json!(text)).as_ref())?; + (!value.is_null()).then(|| integer(&value).ok_or(())) +} + /// `GET /api/workflows/{id}/runs/{run}/streams`. pub(super) async fn list( Extension(runtime): Extension, Path((workflow_id, run_id)): Path<(String, String)>, ) -> Result> { - if !runtime.run_in_namespace(&workflow_id, &run_id).await? { - return Err(run_not_found()); - } - Ok(Json(json!({"workflow_id":workflow_id,"workflow_run_id":run_id,"count":0,"streams":[]}))) + located(&runtime, &workflow_id, &run_id).await?; + let streams = runtime.list_workflow_streams(&run_id).await?; + Ok(Json(json!({"workflow_id":workflow_id,"workflow_run_id":run_id,"count":streams.len(),"streams":streams}))) } /// `GET /api/workflows/{id}/runs/{run}/streams/{name}`. @@ -119,29 +611,129 @@ pub(super) async fn show( Extension(runtime): Extension, Path((workflow_id, run_id, name)): Path<(String, String, String)>, ) -> Result> { - if !runtime.run_in_namespace(&workflow_id, &run_id).await? { - return Err(run_not_found()); + located(&runtime, &workflow_id, &run_id).await?; + if !valid_name(&name) { + return Err(invalid_name("invalid_stream_name")); } - Err(stream_refusal(&name)) + let stream = runtime.show_workflow_stream(&run_id, &name).await?.ok_or_else(stream_not_found)?; + Ok(Json(json!({"workflow_id":workflow_id,"workflow_run_id":run_id,"stream":stream}))) } -/// `GET /api/workflows/{id}/runs/{run}/streams/{name}/items`. +/// `GET /api/workflows/{id}/runs/{run}/streams/{name}/items`, with +/// `readWithWait`'s bounded wait for an open stream with nothing new. pub(super) async fn items( Extension(runtime): Extension, Path((workflow_id, run_id, name)): Path<(String, String, String)>, Query(query): Query>, ) -> Result> { - if !runtime.run_in_namespace(&workflow_id, &run_id).await? { - return Err(run_not_found()); - } + located(&runtime, &workflow_id, &run_id).await?; items_rules(&query)?; - Err(stream_refusal(&name)) + let value = |field: &str, default: i64| query_integer(&query, field).and_then(|value| value.ok()).unwrap_or(default); + let from = value("from", 0); + let max_items = value("max_items", DEFAULT_MAX_ITEMS_PER_SUBSCRIBE); + let wait_seconds = value("wait_seconds", 0); + if !valid_name(&name) { + return Err(invalid_name("invalid_stream_name")); + } + let (reader, run, stream) = (&runtime, run_id.as_str(), name.as_str()); + let read = move || async move { + reader.read_workflow_stream(run, stream, from, max_items).await?.ok_or_else(stream_not_found) + }; + let mut window = read().await?; + if wait_seconds > 0 && window.items.is_empty() && !window.terminal { + let deadline = Instant::now() + Duration::from_secs(wait_seconds as u64); + while Instant::now() < deadline { + tokio::time::sleep(WAIT_INTERVAL).await; + window = read().await?; + if !window.items.is_empty() || window.terminal { + break; + } + } + } + Ok(Json(json!({"workflow_id":workflow_id,"workflow_run_id":run_id,"stream":window.stream, + "items":window.items,"next_offset":window.next_offset,"terminal":window.terminal}))) +} + +/// `POST /api/workflows/{id}/runs/{run}/streams/{name}/items`. +pub(super) async fn append( + Extension(runtime): Extension, + Path((workflow_id, run_id, name)): Path<(String, String, String)>, + Json(body): Json, +) -> Result> { + // Laravel trims every string, payloads included, before anything reads it. + let body = super::start_request::normalized(body); + references(&body)?; + if located(&runtime, &workflow_id, &run_id).await? { + return Err(projected_read_only(&workflow_id, &run_id)); + } + laravel_rules::validate(&body, APPEND_RULES)?; + let items = validated_items(&body)?; + let max_pending = body.get("max_pending_items").and_then(integer).unwrap_or(DEFAULT_MAX_PENDING_ITEMS); + if !valid_name(&name) { + return Err(invalid_name("invalid_request")); + } + if items.is_empty() { + return Err(protocol(StatusCode::BAD_REQUEST, + json!({"message":"append requires at least one item","reason":"invalid_request"}))); + } + match runtime.append_workflow_stream(&workflow_id, &run_id, &name, items, max_pending).await? { + Appended::Accepted { stream, offsets, accepted, deduped } => Ok(Json(json!({ + "workflow_id": workflow_id, + "workflow_run_id": run_id, + "stream": stream, + "accepted_offsets": offsets, + "accepted": accepted, + "deduped": deduped, + }))), + Appended::Full { pending_items } => Err(protocol(StatusCode::TOO_MANY_REQUESTS, json!({ + "message": "Stream pending-items cap reached; consumer is draining too slowly.", + "reason": "stream_full", + "pending_items": pending_items, + "max_pending_items": max_pending, + }))), + Appended::Closed => Err(protocol(StatusCode::CONFLICT, json!({"message":"Stream is closed.","reason":"stream_closed"}))), + Appended::Errored => Err(protocol(StatusCode::CONFLICT, json!({"message":"Stream is errored.","reason":"stream_errored"}))), + } +} + +/// `POST /api/workflows/{id}/runs/{run}/streams/{name}/close`. +pub(super) async fn close( + Extension(runtime): Extension, + Path((workflow_id, run_id, name)): Path<(String, String, String)>, + Json(body): Json, +) -> Result> { + let body = super::start_request::normalized(body); + if located(&runtime, &workflow_id, &run_id).await? { + return Err(projected_read_only(&workflow_id, &run_id)); + } + laravel_rules::validate(&body, CLOSE_RULES)?; + let error_reason = body.get("error_reason").and_then(Value::as_str).map(str::to_owned); + let retention_seconds = body.get("retention_seconds").and_then(integer); + if !valid_name(&name) { + return Err(invalid_name("invalid_stream_name")); + } + let stream = runtime + .close_workflow_stream(&run_id, &name, error_reason, retention_seconds) + .await? + .ok_or_else(stream_not_found)?; + Ok(Json(json!({"workflow_id":workflow_id,"workflow_run_id":run_id,"stream":stream}))) } #[cfg(test)] mod tests { use super::*; + fn query(pairs: &[(&str, &str)]) -> HashMap { + pairs.iter().map(|(key, value)| ((*key).to_owned(), (*value).to_owned())).collect() + } + + fn refusal(result: Result) -> (StatusCode, Value) { + match result { + Err(RuntimeError::Protocol { status, response }) => (status, response), + other => panic!("{other:?}"), + } + } + #[test] fn stream_names_follow_php_pattern() { assert!(valid_name("orders.v1:main-2")); @@ -153,26 +745,61 @@ mod tests { #[test] fn item_rules_report_laravel_messages() { - let query: HashMap = [("from", "x"), ("max_items", "501"), ("wait_seconds", "61")] - .into_iter() - .map(|(key, value)| (key.to_owned(), value.to_owned())) - .collect(); - match items_rules(&query) { - Err(RuntimeError::Protocol { response, .. }) => { - assert_eq!(response["message"], "The from field must be an integer. (and 2 more errors)"); - } - other => panic!("{other:?}"), - } + let (_, response) = refusal(items_rules(&query(&[("from", "x"), ("max_items", "501"), ("wait_seconds", "61")]))); + assert_eq!(response["message"], "The from field must be an integer. (and 2 more errors)"); assert!(items_rules(&HashMap::new()).is_ok()); - let query: HashMap = [("from", "05"), ("max_items", "+5"), ("wait_seconds", " ")] - .into_iter() - .map(|(key, value)| (key.to_owned(), value.to_owned())) - .collect(); - match items_rules(&query) { - Err(RuntimeError::Protocol { response, .. }) => { - assert_eq!(response["errors"], json!({"from":["The from field must be an integer."]})); - } - other => panic!("{other:?}"), - } + let (_, response) = refusal(items_rules(&query(&[("from", "05"), ("max_items", "+5"), ("wait_seconds", " ")]))); + assert_eq!(response["errors"], json!({"from":["The from field must be an integer."]})); + } + + #[test] + fn append_rules_report_laravel_messages() { + let check = |body: Value| refusal(laravel_rules::validate(&super::super::start_request::normalized(body), APPEND_RULES)).1; + assert_eq!(check(json!({"items":[]}))["message"], "The items field is required."); + assert_eq!(check(json!({"items":["text"]}))["errors"], json!({"items.0":["The items.0 field must be an array."]})); + assert_eq!(check(json!({"items":[{"payload":1}],"max_pending_items":0}))["message"], + "The max pending items field must be at least 1."); + assert_eq!(check(json!({"items":[{"idempotency_key":"k".repeat(192)}]}))["message"], + "The items.0.idempotency_key field must not be greater than 191 characters."); + // Plain attributes report before the expanded item attributes. + assert_eq!(check(json!({"items":[{"origin":5}],"max_pending_items":"x"}))["message"], + "The max pending items field must be an integer. (and 1 more error)"); + } + + #[test] + fn validated_items_keep_only_item_fields_and_canonicalize_codecs() { + let body = super::super::start_request::normalized(json!({"items":[ + {"payload":{"token":" Hello ","empty":{}},"payload_codec":null,"idempotency_key":" "}, + {}, + {"unknown":1}, + {"payload":null}, + ]})); + let items = validated_items(&body).unwrap(); + assert_eq!(items.len(), 2); + assert_eq!(items[0].payload, json!({"token":"Hello","empty":[]})); + assert_eq!(items[0].payload_codec.as_deref(), Some("avro")); + assert_eq!(items[0].idempotency_key, None); + assert_eq!(items[1], StreamItem::default()); + let (status, response) = refusal(validated_items(&json!({"items":[{"payload":1},{"payload_codec":"json"}]}))); + assert_eq!(status, StatusCode::UNPROCESSABLE_ENTITY); + assert!(response["errors"]["items.1.payload_codec"][0].as_str().unwrap() + .starts_with("unsupported_payload_codec: workflow payload codec \"json\"")); + } + + #[test] + fn references_refuse_strings_and_defer_reference_objects() { + assert!(references(&json!({"items":[{"payload_reference":null},{"payload":1}]})).is_ok()); + assert!(references(&json!({"items":"text"})).is_ok()); + let (status, response) = refusal(references(&json!({"items":[{"payload_reference":"dw-ref"}]}))); + assert_eq!(status, StatusCode::UNPROCESSABLE_ENTITY); + assert_eq!(response["reason"], "external_payload_unsupported"); + assert!(matches!(references(&json!({"items":{"a":{"payload_reference":{"reference_id":"x"}}}})), + Err(RuntimeError::Refused { status: StatusCode::NOT_IMPLEMENTED, reason: "rust_capability_not_implemented" }))); + } + + #[test] + fn times_render_as_carbon_json() { + let at = DateTime::parse_from_rfc3339("2026-10-25T06:42:31.987Z").unwrap().with_timezone(&Utc); + assert_eq!(carbon_json(at.trunc_subsecs(0)), "2026-10-25T06:42:31.000000Z"); } } diff --git a/rust/tests/execution/waterline_reads.rs b/rust/tests/execution/waterline_reads.rs index 489655cd9..be9146b37 100644 --- a/rust/tests/execution/waterline_reads.rs +++ b/rust/tests/execution/waterline_reads.rs @@ -29,7 +29,7 @@ async fn task_queue_views_follow_registrations_and_tasks() { assert_eq!(other["task_queues"], json!([]), "{other}"); } -// Native runs have no durable streams: lists are empty, a valid name is not +// Durable stream reads: an empty run lists nothing, a valid name is not // found and an invalid name, bad item paging or a foreign run is refused. #[tokio::test] async fn durable_stream_reads_answer_like_php() { @@ -53,6 +53,84 @@ async fn durable_stream_reads_answer_like_php() { assert_eq!((status, body["reason"].clone()), (StatusCode::NOT_FOUND, json!("instance_not_found")), "{body}"); } +// Producer writes: appends open the stream and keep their offsets across a +// retry, reads at the head clear the pending count, a close is final, and +// another namespace can neither read nor write the run's streams. +#[tokio::test] +async fn durable_stream_writes_answer_like_php() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + namespaces::create(&app, "streams").await; + let started = start(&app, "producing").await; + let run = format!("/api/workflows/producing/runs/{}", started["run_id"].as_str().unwrap()); + let items = format!("{run}/streams/tokens/items"); + let (status, body) = request(&app, "POST", &items, json!({"items":[ + {"payload":{"token":" Hello ","empty":{}},"idempotency_key":"k1"},{"payload":"plain","payload_codec":null}]})).await; + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!((body["accepted_offsets"].clone(), body["accepted"].clone(), body["deduped"].clone()), (json!([0, 1]), json!(2), json!(0))); + assert_eq!(body["stream"]["pending_items"], 2); + let opened = body["stream"]["opened_at"].as_str().unwrap().to_owned(); + assert!(opened.ends_with(".000000Z"), "{opened}"); + let (status, body) = request(&app, "POST", &items, json!({"items":[{"payload":1,"idempotency_key":"k1"},{"payload":2}], + "max_pending_items":3})).await; + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!((body["accepted_offsets"].clone(), body["deduped"].clone()), (json!([0, 2]), json!(1))); + // At the cap, the batch keeps nothing past it and answers 429. + let (status, body) = request(&app, "POST", &items, json!({"items":[{"payload":3}],"max_pending_items":3})).await; + assert_eq!(status, StatusCode::TOO_MANY_REQUESTS, "{body}"); + assert_eq!((body["reason"].clone(), body["pending_items"].clone(), body["max_pending_items"].clone()), + (json!("stream_full"), json!(3), json!(3))); + let (status, body) = get(&app, "default", &format!("{items}?from=0")).await; + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!(body["items"][0]["payload"], json!({"token":"Hello","empty":[]})); + assert_eq!(body["items"][0]["origin"], "workflow_command"); + assert_eq!(body["items"][1]["payload_codec"], "avro"); + assert_eq!((body["next_offset"].clone(), body["terminal"].clone(), body["stream"]["pending_items"].clone()), + (json!(3), json!(false), json!(0))); + // Another namespace sees no such run. + let (status, body) = namespaces::scoped(&app, "streams", "POST", &items, json!({"items":[{"payload":1}]})).await; + assert_eq!((status, body["reason"].clone()), (StatusCode::NOT_FOUND, json!("instance_not_found")), "{body}"); + let close = format!("{run}/streams/tokens/close"); + let (status, body) = request(&app, "POST", &close, json!({})).await; + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!((body["stream"]["status"].clone(), body["stream"]["retention_seconds"].clone()), (json!("closed"), json!(3600))); + let (status, again) = request(&app, "POST", &close, json!({"error_reason":"late","retention_seconds":5})).await; + assert_eq!(status, StatusCode::OK, "{again}"); + assert_eq!(again["stream"], body["stream"]); + let (status, body) = request(&app, "POST", &items, json!({"items":[{"payload":1}]})).await; + assert_eq!((status, body["reason"].clone()), (StatusCode::CONFLICT, json!("stream_closed")), "{body}"); + let (status, body) = get(&app, "default", &format!("{items}?from=3&wait_seconds=5")).await; + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!((body["items"].clone(), body["terminal"].clone()), (json!([]), json!(true))); + let (status, body) = request(&app, "POST", &format!("{run}/streams/bad%20name/items"), json!({"items":[{"payload":1}]})).await; + assert_eq!((status, body["reason"].clone()), (StatusCode::BAD_REQUEST, json!("invalid_request")), "{body}"); + let (status, body) = request(&app, "POST", &items, json!({"items":[{"payload_reference":"dw-unstored"}]})).await; + assert_eq!((status, body["reason"].clone()), (StatusCode::UNPROCESSABLE_ENTITY, json!("external_payload_unsupported")), "{body}"); + let (status, body) = get(&app, "default", &format!("{run}/streams")).await; + assert_eq!((status, body["count"].clone()), (StatusCode::OK, json!(1)), "{body}"); +} + +// A run's durable streams stay with it when the workflow continues as new, +// as PHP keeps them; the continuation is not refused. +#[tokio::test] +async fn continue_as_new_keeps_durable_streams_with_the_predecessor() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + register(&app, "stream-worker", json!(["echo"]), json!([])).await; + let started = start(&app, "stream-continue").await; + let first = started["run_id"].as_str().unwrap().to_owned(); + let (status, body) = request(&app, "POST", &format!("/api/workflows/stream-continue/runs/{first}/streams/out/items"), + json!({"items":[{"payload":"before"}]})).await; + assert_eq!(status, StatusCode::OK, "{body}"); + let task = poll(&app, "stream-worker", "workflow").await; + let (status, body) = finish_task(&app, &task, json!([{"type":"continue_as_new"}])).await; + assert_eq!(status, StatusCode::OK, "{body}"); + let (_, runs) = request(&app, "GET", "/api/workflows/stream-continue/runs", Value::Null).await; + assert_eq!(runs["runs"].as_array().unwrap().len(), 2, "{runs}"); + let (status, body) = get(&app, "default", &format!("/api/workflows/stream-continue/runs/{first}/streams")).await; + assert_eq!((status, body["count"].clone()), (StatusCode::OK, json!(1)), "{body}"); +} + // The workflow list filters cancelled and terminated runs by their exact // status, as PHP does, instead of refusing those filters. #[tokio::test] diff --git a/rust/tests/fixtures/unserved-php-routes.v2.11.1.txt b/rust/tests/fixtures/unserved-php-routes.v2.11.1.txt index e62882383..53d977f92 100644 --- a/rust/tests/fixtures/unserved-php-routes.v2.11.1.txt +++ b/rust/tests/fixtures/unserved-php-routes.v2.11.1.txt @@ -41,5 +41,3 @@ POST /api/workflows/import/embedded-v2 POST /api/workflows/import/waterline-v1 GET /api/workflows/{workflowId}/nexus-operations GET /api/workflows/{workflowId}/runs/{runId}/nexus-operations -POST /api/workflows/{workflowId}/runs/{runId}/streams/{streamName}/close -POST /api/workflows/{workflowId}/runs/{runId}/streams/{streamName}/items From d25d98f0b7af33962c3925124c4ef364901b116a Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Mon, 26 Oct 2026 05:54:39 +0000 Subject: [PATCH 3/6] Rust #325: test durable stream writes refused on a Waterline projection An append and a clean or errored close against a projected v1 run answer 409 v1_projection_read_only, and no stream or item row is written. With the read-only checks removed from append and close, the append is accepted and the test fails. Co-Authored-By: Claude Opus 5.5 --- .../execution/waterline_projection_guards.rs | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/rust/tests/execution/waterline_projection_guards.rs b/rust/tests/execution/waterline_projection_guards.rs index 5edb30911..9a16513df 100644 --- a/rust/tests/execution/waterline_projection_guards.rs +++ b/rust/tests/execution/waterline_projection_guards.rs @@ -60,6 +60,33 @@ async fn appends_to_a_projection_are_refused_before_their_body_is_read() { assert_eq!(after, described); } +// Durable output streams on a projection are read-only too: an append and a +// close are refused as PHP's `projectedRunReadOnly` refuses them, and no +// stream or item is written. +#[tokio::test] +async fn durable_stream_writes_to_a_projection_are_refused() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + let (status, report) = import(&app, "stream-guard-src", running(1)).await; + assert_eq!(status, StatusCode::CREATED, "{report}"); + let workflow_id = report["identity"]["standalone"]["workflow_id"].as_str().unwrap().replace(':', "%3A"); + let run_id = report["identity"]["standalone"]["run_id"].as_str().unwrap().to_owned(); + let stream = format!("/api/workflows/{workflow_id}/runs/{run_id}/streams/tokens"); + let before = rows(&database).await; + for (path, body) in [ + (format!("{stream}/items"), json!({"items": [{"payload": "x", "idempotency_key": "k1"}]})), + (format!("{stream}/close"), json!({})), + (format!("{stream}/close"), json!({"error_reason": "failed"})), + ] { + let (status, refusal) = request(&app, "POST", &path, body.clone()).await; + assert_eq!((status, &refusal["reason"]), (StatusCode::CONFLICT, &json!("v1_projection_read_only")), "{path} {body}: {refusal}"); + assert_eq!((&refusal["run_id"], &refusal["execution_owner"]), (&json!(run_id), &json!("v1")), "{refusal}"); + } + let (status, listed) = request(&app, "GET", &format!("/api/workflows/{workflow_id}/runs/{run_id}/streams"), Value::Null).await; + assert_eq!((status, &listed["count"]), (StatusCode::OK, &json!(0)), "{listed}"); + assert_eq!(rows(&database).await, before); +} + #[tokio::test] async fn a_projection_past_the_event_limit_is_refused_before_any_row() { let database = TestDatabase::new().await; From eb389310c5fc124d82a0f0912895688157cb19a0 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Mon, 26 Oct 2026 07:45:17 +0000 Subject: [PATCH 4/6] Rust #325: native stream writes follow the #774 contract The native producer routes adopt the shared durable stream contract fixed in #774: - max_pending_items is at most 10000 and retention_seconds at most 2592000, refused with 422 by the append and close rules. - An append resolves stored and repeated idempotency keys first, then refuses the whole batch when its fresh items would pass the cap. The transaction is dropped, so nothing is written and a stream the append opened is not created. Appends run under the transition lock, so concurrent first appends open one stream and never pass the cap. - A closed or errored stream past closed_at plus its retention reads as not found: list omits it, and describe, subscribe and close answer 404. - prune_expired_streams reclaims expired streams in bounded, resumable steps (at most N streams per pass, oldest close first, and M items in transactions of at most 500), from the heartbeat inline retention pass under its own per-namespace throttle and from a general operator retention pass. The workflow-stream-writes contract accepts the whole-batch refusal and bounds published 2.11.1's partial commit as a published defect. New execution tests cover the whole-batch refusal, both bounds, concurrent near-cap and first-open appends through two pools, expiry and the bounded pass. Co-Authored-By: Claude Opus 5.5 --- docs/rust-server-port.md | 13 +- rust/src/runtime/backend.rs | 11 + rust/src/runtime/operator_metrics.rs | 13 ++ rust/src/runtime/retention.rs | 15 ++ rust/src/runtime/workflow_streams.rs | 159 ++++++++++++--- rust/tests/execution.rs | 2 + .../tests/execution/workflow_stream_bounds.rs | 188 ++++++++++++++++++ .../workflow-stream-writes-contract.mjs | 21 +- .../ServerParity/workflow-stream-writes.json | 2 +- ...ParityWorkflowStreamWritesContractTest.mjs | 33 ++- 10 files changed, 418 insertions(+), 39 deletions(-) create mode 100644 rust/tests/execution/workflow_stream_bounds.rs diff --git a/docs/rust-server-port.md b/docs/rust-server-port.md index 1088072ae..22623b503 100644 --- a/docs/rust-server-port.md +++ b/docs/rust-server-port.md @@ -912,8 +912,17 @@ October 21, Waterline reads: - **Durable streams.** The run's stream list, stream description, item reads, appends and closes answer as PHP does, with the `workflow-stream-writes` fixture recorded from published 2.11.1: offsets - from 0, idempotent retries, the pending-items cap with its partial commit, - bounded waits, final closes, whole-second times and every refusal body. + from 0, idempotent retries, bounded waits, final closes, whole-second times + and every refusal body. The shared contract fix of Server #774 is native + too: + - `max_pending_items` is at most 10000 and `retention_seconds` at most + 2592000, both refused with 422; + - a batch whose fresh items would pass the cap is refused whole, so + nothing of it is written (published 2.11.1 committed the items before the + cap; the fixture bounds that answer as a published defect); + - a closed stream past its retention reads as not found; + - a bounded pass reclaims expired streams from the worker-heartbeat + retention pass and from a general `POST /api/system/retention/pass`. A continued run keeps its streams, as in PHP. Native differs in three places, documented in `rust/src/runtime/workflow_streams.rs`: - a batch whose items hold no item fields gets 400, not PHP's 500; diff --git a/rust/src/runtime/backend.rs b/rust/src/runtime/backend.rs index 9fbfcf2ca..fc5fa85b2 100644 --- a/rust/src/runtime/backend.rs +++ b/rust/src/runtime/backend.rs @@ -86,6 +86,12 @@ pub(super) trait Backend: Database { const STREAM_ITEM_BY_KEY: &str = "SELECT \"offset\" FROM workflow_durable_stream_items WHERE stream_id=$1 AND idempotency_key=$2 LIMIT 1"; const STREAM_ITEM_INSERT: &str = "INSERT INTO workflow_durable_stream_items(stream_id,namespace,workflow_run_id,stream_name,\"offset\",idempotency_key,origin,origin_reference,payload,payload_reference,payload_codec,item_type,content_type,emitted_at,created_at,updated_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16)"; const STREAM_ITEMS_WINDOW: &str = "SELECT * FROM workflow_durable_stream_items WHERE stream_id=$1 AND \"offset\">=$2 ORDER BY \"offset\" LIMIT $3"; + // `WorkflowStreamRetention`: this namespace's closed or errored streams + // whose `closed_at` plus retention (3600 seconds when none was recorded) + // is no later than $2, oldest close first. + const EXPIRED_STREAMS: &str = "SELECT id FROM workflow_durable_streams WHERE namespace=$1 AND status IN ('closed','errored') AND closed_at IS NOT NULL AND closed_at<=$2-(COALESCE(retention_seconds,3600)*INTERVAL '1 second') ORDER BY closed_at,id LIMIT $3"; + const STREAM_SWEEP_BOUNDARY: &str = "SELECT \"offset\" FROM workflow_durable_stream_items WHERE stream_id=$1 ORDER BY \"offset\" LIMIT 1 OFFSET $2"; + const STREAM_SWEEP_THROUGH: &str = "DELETE FROM workflow_durable_stream_items WHERE stream_id=$1 AND \"offset\"<=$2"; fn statement(sql: &'static str) -> Cow<'static, str> { Cow::Borrowed(sql) } @@ -195,6 +201,8 @@ WHERE r.status IN ('pending','running','waiting') ORDER BY t.available_order,t.id LIMIT 100 "#; const SELECTION_RESOLUTION_SQL: &str = "SELECT id,payload FROM workflow_history_events WHERE workflow_run_id=$1 AND event_type='SelectionResolved' AND json_extract(payload,'$.selection_group_id')=$2 ORDER BY sequence LIMIT 1"; + // SQLite keeps times as text: closed_at against $2 less the retention. + const EXPIRED_STREAMS: &str = "SELECT id FROM workflow_durable_streams WHERE namespace=$1 AND status IN ('closed','errored') AND closed_at IS NOT NULL AND closed_at<=strftime('%Y-%m-%d %H:%M:%f',$2,'-'||COALESCE(retention_seconds,3600)||' seconds') ORDER BY closed_at,id LIMIT $3"; const REPAIR_COUNTER_MAX: i64 = i64::MAX; const RANGE_REPAIR_TASK_SQL: &str = "SELECT * FROM workflow_tasks WHERE workflow_run_id=$1 AND namespace=$2 AND task_type='workflow' AND status='failed' AND json_extract(payload,'$.replay_blocked')=1 ORDER BY CASE WHEN available_at IS NULL THEN 1 ELSE 0 END,available_at,created_at,id LIMIT 1"; const ACTIVITY_OPEN_TASK_SQL: &str = "SELECT * FROM workflow_tasks WHERE workflow_run_id=$1 AND task_type='activity' AND status IN ('ready','leased') AND json_extract(payload,'$.activity_execution_id')=$2 ORDER BY id LIMIT 1"; @@ -447,6 +455,9 @@ impl Backend for MySql { const STREAM_ITEM_BY_KEY: &str = "SELECT `offset` FROM workflow_durable_stream_items WHERE stream_id=$1 AND idempotency_key=$2 LIMIT 1"; const STREAM_ITEM_INSERT: &str = "INSERT INTO workflow_durable_stream_items(stream_id,namespace,workflow_run_id,stream_name,`offset`,idempotency_key,origin,origin_reference,payload,payload_reference,payload_codec,item_type,content_type,emitted_at,created_at,updated_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16)"; const STREAM_ITEMS_WINDOW: &str = "SELECT * FROM workflow_durable_stream_items WHERE stream_id=$1 AND `offset`>=$2 ORDER BY `offset` LIMIT $3"; + const EXPIRED_STREAMS: &str = "SELECT id FROM workflow_durable_streams WHERE namespace=$1 AND status IN ('closed','errored') AND closed_at IS NOT NULL AND closed_at<=DATE_SUB($2,INTERVAL COALESCE(retention_seconds,3600) SECOND) ORDER BY closed_at,id LIMIT $3"; + const STREAM_SWEEP_BOUNDARY: &str = "SELECT `offset` FROM workflow_durable_stream_items WHERE stream_id=$1 ORDER BY `offset` LIMIT 1 OFFSET $2"; + const STREAM_SWEEP_THROUGH: &str = "DELETE FROM workflow_durable_stream_items WHERE stream_id=$1 AND `offset`<=$2"; const READ_SNAPSHOT_BEGIN: &str = "START TRANSACTION WITH CONSISTENT SNAPSHOT, READ ONLY"; const ACTIVITY_METADATA_HISTORY: &str = "SELECT h.sequence,h.event_type,CASE WHEN page.projection_bytes<=8388608 THEN h.payload ELSE NULL END AS payload,h.workflow_task_id,h.recorded_at,page.projection_count FROM (SELECT id,sequence,COUNT(*) OVER () AS projection_count,SUM(payload_bytes) OVER (ORDER BY sequence) AS projection_bytes FROM (SELECT id,sequence,octet_length(CAST(payload AS CHAR CHARACTER SET utf8mb4)) AS payload_bytes FROM workflow_history_events FORCE INDEX (workflow_history_events_workflow_run_id_sequence_unique) WHERE workflow_run_id=$1 AND event_type IN ('ActivityScheduled','ActivityStarted','ActivityHeartbeatRecorded','ActivityRetryScheduled','ActivityCompleted','ActivityFailed','ActivityTimedOut','ActivityCancelled')) AS sizes ORDER BY sequence LIMIT 10001) AS page JOIN workflow_history_events h ON h.id=page.id ORDER BY page.sequence"; const PHP_CONNECTION: &str = "mysql"; diff --git a/rust/src/runtime/operator_metrics.rs b/rust/src/runtime/operator_metrics.rs index 8691494cd..86630486a 100644 --- a/rust/src/runtime/operator_metrics.rs +++ b/rust/src/runtime/operator_metrics.rs @@ -1234,6 +1234,12 @@ where /// `POST /api/system/retention/pass`: the report, and whether any run failed. pub(super) async fn retention_pass(&self, limit: i64, run_ids: Vec) -> Result<(Value, bool)> { + // A general pass also reclaims closed workflow streams past their own + // retention (Server #774), at most `limit` streams and `limit` * 100 + // items. The report is the run pass's. + if run_ids.is_empty() { + self.prune_expired_streams(limit, limit * 100).await?; + } let policy = self.retention_policy().await?; let report = self.prune_runs(&policy, limit, run_ids).await?; let failed = report["failed"].as_i64().is_some_and(|failed| failed > 0); @@ -1280,6 +1286,13 @@ where /// `runInlinePass`, reported on the worker heartbeat. pub(super) async fn retention_inline_pass(&self) -> Result { + // Closed workflow streams carry their own retention, whatever the + // history retention mode (Server #774); the stream pass has its own + // throttle and is not part of this report. + if retention::inline_stream_pass_due(self.namespace.as_ref()) { + let (streams, items) = super::workflow_streams::INLINE_SWEEP; + self.prune_expired_streams(streams, items).await?; + } let policy = self.retention_policy().await?; if policy.forever { return Ok(retention::inline_report(false, None)); diff --git a/rust/src/runtime/retention.rs b/rust/src/runtime/retention.rs index b1a8e1047..9ee1c054f 100644 --- a/rust/src/runtime/retention.rs +++ b/rust/src/runtime/retention.rs @@ -229,6 +229,21 @@ pub(super) fn inline_pass_due(namespace: &str) -> bool { } } +/// `WorkflowStreamRetention::runInlinePass`'s throttle: at most one stream +/// pass a minute per namespace in this process. +pub(super) fn inline_stream_pass_due(namespace: &str) -> bool { + static LAST: OnceLock>> = OnceLock::new(); + let mut last = LAST.get_or_init(Default::default).lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let now = Instant::now(); + match last.get(namespace) { + Some(at) if now.duration_since(*at).as_secs() < INLINE_THROTTLE_SECONDS => false, + _ => { + last.insert(namespace.to_owned(), now); + true + } + } +} + /// `runInlinePass`'s heartbeat report. pub(super) fn inline_report(throttled: bool, report: Option<&Map>) -> Value { let count = |key: &str| report.and_then(|report| report.get(key)).cloned().unwrap_or(json!(0)); diff --git a/rust/src/runtime/workflow_streams.rs b/rust/src/runtime/workflow_streams.rs index d755b50c8..e8add3010 100644 --- a/rust/src/runtime/workflow_streams.rs +++ b/rust/src/runtime/workflow_streams.rs @@ -2,10 +2,11 @@ //! and `WorkflowStreamService` serve them. A producer appends ordered items //! to a named, run-scoped stream, which the first append opens. Offsets //! start at 0, a repeated idempotency key returns the item it already -//! names, and the pending-items cap keeps the items before it and then -//! answers 429. A close records a clean or errored end once. Readers page -//! by offset and may wait, and a read that reaches the head resets -//! `pending_items`. +//! names, and a batch whose fresh items would pass the pending-items cap is +//! refused whole with 429 (Server #774). A close records a clean or errored +//! end once. Readers page by offset and may wait, and a read that reaches +//! the head resets `pending_items`. A closed stream past its retention +//! reads as not found, and a bounded pass reclaims its rows. //! //! PHP's models write these times in Eloquent's `Y-m-d H:i:s` format, so //! every stored time is a whole second. Native differs in three places: @@ -46,6 +47,15 @@ const DEFAULT_MAX_ITEMS_PER_SUBSCRIBE: i64 = 100; const SUBSCRIBE_MAX_ITEMS: i64 = 500; /// `WorkflowStreamService::DEFAULT_CLOSED_RETENTION_SECONDS`. const DEFAULT_CLOSED_RETENTION_SECONDS: i64 = 3600; +/// `WorkflowStreamService::MAX_PENDING_ITEMS_MAX`: a producer may lower its +/// cap, not raise it. +const MAX_PENDING_ITEMS_MAX: i64 = 10_000; +/// `WorkflowStreamService::RETENTION_SECONDS_MAX` (30 days). +const RETENTION_SECONDS_MAX: i64 = 2_592_000; +/// `WorkflowStreamRetention::ITEM_BATCH`: items deleted per transaction. +const SWEEP_ITEM_BATCH: i64 = 500; +/// `WorkflowStreamRetention::INLINE_STREAM_LIMIT` and `INLINE_ITEM_BUDGET`. +pub(super) const INLINE_SWEEP: (i64, i64) = (10, 1000); /// `LegacyV1ProjectionContract::IMPORT_SOURCE`. const PROJECTION_SOURCE: &str = "waterline_v1_projection"; /// `readWithWait` checks again every 250 ms. @@ -67,13 +77,13 @@ const APPEND_RULES: &laravel_rules::Rules = &[ ("items.*.origin_reference", &[Rule::Nullable, Rule::String, Rule::Max(191)]), ("items.*.item_type", &[Rule::Nullable, Rule::String, Rule::Max(64)]), ("items.*.content_type", &[Rule::Nullable, Rule::String, Rule::Max(191)]), - ("max_pending_items", &[Rule::Nullable, Rule::Integer, Rule::Min(1)]), + ("max_pending_items", &[Rule::Nullable, Rule::Integer, Rule::Min(1), Rule::Max(MAX_PENDING_ITEMS_MAX)]), ]; /// `close()`'s Laravel rules. const CLOSE_RULES: &laravel_rules::Rules = &[ ("error_reason", &[Rule::Nullable, Rule::String, Rule::Max(191)]), - ("retention_seconds", &[Rule::Nullable, Rule::Integer, Rule::Min(1)]), + ("retention_seconds", &[Rule::Nullable, Rule::Integer, Rule::Min(1), Rule::Max(RETENTION_SECONDS_MAX)]), ]; /// The item fields `append()` validates, which are all its validated data @@ -173,6 +183,19 @@ where } /// `formatStream`. + /// `WorkflowStreamService::isExpired`: a closed or errored stream past + /// `closed_at` plus its retention (3600 seconds when none was recorded). + fn stream_expired(row: &DB::Row, at: DateTime) -> Result { + if DB::string(row, "status")? == "open" { + return Ok(false); + } + let Some(closed) = DB::optional_instant(row, "closed_at")? else { + return Ok(false); + }; + let retention = DB::optional_number(row, "retention_seconds")?.unwrap_or(DEFAULT_CLOSED_RETENTION_SECONDS); + Ok(closed + chrono::Duration::seconds(retention) <= at) + } + fn stream_json(row: &DB::Row) -> Result { let time = |field: &str| Ok::<_, RuntimeError>(DB::optional_instant(row, field)?.map(carbon_json)); Ok(json!({ @@ -207,14 +230,19 @@ where /// `listForRun`: the run's streams by name, in the database's order. pub(super) async fn list_workflow_streams(&self, run_id: &str) -> Result> { - Self::query("SELECT * FROM workflow_durable_streams WHERE namespace=$1 AND workflow_run_id=$2 ORDER BY stream_name") + let at = now(); + let mut streams = Vec::new(); + for row in Self::query("SELECT * FROM workflow_durable_streams WHERE namespace=$1 AND workflow_run_id=$2 ORDER BY stream_name") .bind(self.namespace.as_ref()) .bind(run_id) .fetch_all(&self.pool) .await? - .iter() - .map(Self::stream_json) - .collect() + { + if !Self::stream_expired(&row, at)? { + streams.push(Self::stream_json(&row)?); + } + } + Ok(streams) } /// `describe` for a valid name. @@ -225,7 +253,10 @@ where .bind(name) .fetch_optional(&self.pool) .await?; - row.as_ref().map(Self::stream_json).transpose() + match row { + Some(row) if !Self::stream_expired(&row, now())? => Ok(Some(Self::stream_json(&row)?)), + _ => Ok(None), + } } /// `WorkflowStreamService::append` for a valid name and a non-empty @@ -276,24 +307,38 @@ where let mut last_offset = DB::number(&row, "last_offset")?; let mut total = DB::number(&row, "total_items")?; let mut pending = DB::number(&row, "pending_items")?; + // The whole batch is admitted or refused before any item is written + // (Server #774). An item whose key is already stored, or earlier in + // this batch, takes no capacity. + let mut stored: HashMap<&str, Option> = HashMap::new(); + let mut fresh = 0; + for item in &items { + match item.idempotency_key.as_deref() { + Some(key) if stored.contains_key(key) => {} + Some(key) => { + let existing = Self::scalar(DB::STREAM_ITEM_BY_KEY).bind(stream_id).bind(key).fetch_optional(&mut *tx).await?; + if existing.is_none() { + fresh += 1; + } + stored.insert(key, existing); + } + None => fresh += 1, + } + } + if fresh > 0 && pending + fresh > max_pending { + // Dropping the transaction rolls back a stream this append opened. + return Ok(Appended::Full { pending_items: pending }); + } let mut offsets = Vec::with_capacity(items.len()); - let (mut accepted, mut deduped, mut full) = (0, 0, false); + let (mut accepted, mut deduped) = (0, 0); for item in &items { if let Some(key) = item.idempotency_key.as_deref() - && let Some(existing) = Self::scalar(DB::STREAM_ITEM_BY_KEY) - .bind(stream_id) - .bind(key) - .fetch_optional(&mut *tx) - .await? + && let Some(Some(existing)) = stored.get(key) { - offsets.push(existing); + offsets.push(*existing); deduped += 1; continue; } - if pending >= max_pending { - full = true; - break; - } last_offset += 1; Self::query(DB::STREAM_ITEM_INSERT) .bind(stream_id) @@ -314,6 +359,9 @@ where .bind(DB::bind_time(at)) .execute(&mut *tx) .await?; + if let Some(key) = item.idempotency_key.as_deref() { + stored.insert(key, Some(last_offset)); + } total += 1; pending += 1; accepted += 1; @@ -334,12 +382,67 @@ where } let row = Self::query(STREAM_BY_ID).bind(stream_id).fetch_one(&mut *tx).await?; tx.commit().await?; - if full { - return Ok(Appended::Full { pending_items: DB::number(&row, "pending_items")? }); - } Ok(Appended::Accepted { stream: Self::stream_json(&row)?, offsets, accepted, deduped }) } + /// `WorkflowStreamRetention::pruneExpired` for this namespace: at most + /// `stream_limit` expired streams, oldest close first, and at most + /// `item_budget` items, deleted in transactions of at most 500 under the + /// transition lock. A stream row goes in the transaction that finds it + /// empty, and the next pass resumes with the same oldest streams. Returns + /// the streams removed, the items deleted and whether the budget ran out. + pub(super) async fn prune_expired_streams(&self, stream_limit: i64, item_budget: i64) -> Result<(i64, i64, bool)> { + let at = now(); + let candidates = Self::query(DB::EXPIRED_STREAMS) + .bind(self.namespace.as_ref()) + .bind(DB::bind_time(at)) + .bind(stream_limit.max(1)) + .fetch_all(&self.pool) + .await? + .iter() + .map(|row| DB::number(row, "id")) + .collect::>>()?; + let budget = item_budget.max(1); + let (mut streams, mut items) = (0, 0); + for stream_id in candidates { + loop { + if items >= budget { + return Ok((streams, items, true)); + } + let take = SWEEP_ITEM_BATCH.min(budget - items); + let mut tx = self.begin().await?; + let Some(row) = Self::query(STREAM_BY_ID).bind(stream_id).fetch_optional(&mut *tx).await? else { + break; + }; + if !Self::stream_expired(&row, at)? { + break; + } + // The offset of the `take`th item, when that many remain. + let through = Self::scalar(DB::STREAM_SWEEP_BOUNDARY) + .bind(stream_id) + .bind(take - 1) + .fetch_optional(&mut *tx) + .await?; + if let Some(offset) = through { + let deleted = Self::query(DB::STREAM_SWEEP_THROUGH).bind(stream_id).bind(offset).execute(&mut *tx).await?; + tx.commit().await?; + items += DB::affected(deleted) as i64; + continue; + } + let deleted = Self::query("DELETE FROM workflow_durable_stream_items WHERE stream_id=$1") + .bind(stream_id) + .execute(&mut *tx) + .await?; + Self::query("DELETE FROM workflow_durable_streams WHERE id=$1").bind(stream_id).execute(&mut *tx).await?; + tx.commit().await?; + items += DB::affected(deleted) as i64; + streams += 1; + break; + } + } + Ok((streams, items, false)) + } + /// `WorkflowStreamService::read` for a valid name and offset. pub(super) async fn read_workflow_stream( &self, @@ -357,6 +460,9 @@ where else { return Ok(None); }; + if Self::stream_expired(&row, now())? { + return Ok(None); + } let stream_id = DB::number(&row, "id")?; let rows = Self::query(DB::STREAM_ITEMS_WINDOW) .bind(stream_id) @@ -407,6 +513,9 @@ where else { return Ok(None); }; + if Self::stream_expired(&row, now())? { + return Ok(None); + } if DB::string(&row, "status")? != "open" { return Ok(Some(Self::stream_json(&row)?)); } diff --git a/rust/tests/execution.rs b/rust/tests/execution.rs index 816707a7d..c9c65c82a 100644 --- a/rust/tests/execution.rs +++ b/rust/tests/execution.rs @@ -210,6 +210,8 @@ mod describe_actions; mod run_deadlines; #[path = "execution/waterline_projection_guards.rs"] mod waterline_projection_guards; +#[path = "execution/workflow_stream_bounds.rs"] +mod workflow_stream_bounds; #[path = "execution/operator_dashboard.rs"] mod operator_dashboard; #[path = "execution/waterline_reads.rs"] diff --git a/rust/tests/execution/workflow_stream_bounds.rs b/rust/tests/execution/workflow_stream_bounds.rs new file mode 100644 index 000000000..23949dd9d --- /dev/null +++ b/rust/tests/execution/workflow_stream_bounds.rs @@ -0,0 +1,188 @@ +use super::*; + +// The durable stream producer contract of Server #774, natively: bounded +// producer controls, whole-batch capacity under contention, expiry at read +// and a bounded, resumable retention pass. + +fn route(workflow: &Value, run: &Value, stream: &str, tail: &str) -> String { + format!("/api/workflows/{}/runs/{}/streams/{stream}{tail}", workflow.as_str().unwrap(), run.as_str().unwrap()) +} + +fn items(count: usize) -> Value { + json!((0..count).map(|n| json!({"payload": n})).collect::>()) +} + +async fn usage(app: &Router, resource: &str) -> i64 { + let (status, body) = request(app, "GET", "/api/system/metrics", Value::Null).await; + assert_eq!(status, StatusCode::OK, "{body}"); + body["metrics"]["dw_namespace_durable_state_usage"]["usage"][resource].as_i64().unwrap() +} + +#[tokio::test] +async fn a_capacity_refusal_writes_nothing_of_the_batch() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + let run = start(&app, "stream-batch").await; + let (workflow, run) = (&run["workflow_id"], &run["run_id"]); + let append = |body: Value| { + let app = app.clone(); + let path = route(workflow, run, "tokens", "/items"); + async move { request(&app, "POST", &path, body).await } + }; + // Three fresh items past a cap of two: nothing is written and the stream + // the append would have opened does not exist. + let (status, refused) = append(json!({"max_pending_items":2,"items":items(3)})).await; + assert_eq!((status, &refused["reason"]), (StatusCode::TOO_MANY_REQUESTS, &json!("stream_full")), "{refused}"); + let (_, listed) = request(&app, "GET", &route(workflow, run, "", "").trim_end_matches('/').to_owned(), Value::Null).await; + assert_eq!(listed["streams"], json!([]), "{listed}"); + assert_eq!(usage(&app, "workflow_stream_items").await, 0); + + let (status, body) = append(json!({"max_pending_items":3,"items":[{"payload":0,"idempotency_key":"k0"},{"payload":1},{"payload":2}]})).await; + assert_eq!(status, StatusCode::OK, "{body}"); + // At the cap one fresh item is refused, and a stored key with it is not + // answered either; duplicates alone take no capacity. + let (status, _) = append(json!({"max_pending_items":3,"items":[{"payload":3}]})).await; + assert_eq!(status, StatusCode::TOO_MANY_REQUESTS); + let (status, _) = append(json!({"max_pending_items":3,"items":[{"payload":0,"idempotency_key":"k0"},{"payload":4}]})).await; + assert_eq!(status, StatusCode::TOO_MANY_REQUESTS); + let (status, body) = append(json!({"max_pending_items":3,"items":[{"payload":0,"idempotency_key":"k0"},{"payload":0,"idempotency_key":"k0"}]})).await; + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!((&body["accepted"], &body["deduped"], &body["accepted_offsets"]), (&json!(0), &json!(2), &json!([0, 0])), "{body}"); + assert_eq!(usage(&app, "workflow_stream_items").await, 3); +} + +#[tokio::test] +async fn producer_controls_are_bounded() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + let run = start(&app, "stream-bounds").await; + let (workflow, run) = (&run["workflow_id"], &run["run_id"]); + let (status, refused) = request(&app, "POST", &route(workflow, run, "tokens", "/items"), + json!({"max_pending_items":10_001,"items":items(1)})).await; + assert_eq!(status, StatusCode::UNPROCESSABLE_ENTITY, "{refused}"); + assert_eq!(refused["errors"]["max_pending_items"], json!(["The max pending items field must not be greater than 10000."])); + assert_eq!(usage(&app, "workflow_streams").await, 0); + let (status, _) = request(&app, "POST", &route(workflow, run, "tokens", "/items"), + json!({"max_pending_items":10_000,"items":items(1)})).await; + assert_eq!(status, StatusCode::OK); + let (status, refused) = request(&app, "POST", &route(workflow, run, "tokens", "/close"), json!({"retention_seconds":2_592_001})).await; + assert_eq!(status, StatusCode::UNPROCESSABLE_ENTITY, "{refused}"); + assert_eq!(refused["errors"]["retention_seconds"], json!(["The retention seconds field must not be greater than 2592000."])); + let (status, closed) = request(&app, "POST", &route(workflow, run, "tokens", "/close"), json!({"retention_seconds":2_592_000})).await; + assert_eq!(status, StatusCode::OK, "{closed}"); + assert_eq!(closed["stream"]["retention_seconds"], 2_592_000); +} + +/// Eight appends of three items at once through two pools, against a cap of +/// five: `stored` items exist first. Returns the accepted and refused counts. +async fn contend(stored: usize) -> (usize, usize, i64) { + let database = TestDatabase::new().await; + let apps = [router(database.open().await.unwrap()), router(database.open().await.unwrap())]; + let run = start(&apps[0], "stream-race").await; + let path = route(&run["workflow_id"], &run["run_id"], "tokens", "/items"); + if stored > 0 { + let (status, _) = request(&apps[0], "POST", &path, json!({"items":items(stored)})).await; + assert_eq!(status, StatusCode::OK); + } + let mut tasks = tokio::task::JoinSet::new(); + for index in 0..8 { + let app = apps[index % 2].clone(); + let path = path.clone(); + tasks.spawn(async move { request(&app, "POST", &path, json!({"max_pending_items":5,"items":items(3)})).await }); + } + let (mut accepted, mut refused) = (0, 0); + while let Some(answer) = tasks.join_next().await { + let (status, body) = answer.unwrap(); + match status { + StatusCode::OK => accepted += 1, + StatusCode::TOO_MANY_REQUESTS => refused += 1, + other => panic!("{other}: {body}"), + } + } + (accepted, refused, usage(&apps[0], "workflow_stream_items").await) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn concurrent_appends_never_pass_the_cap() { + // Concurrent first appends open one stream; the first batch fits, every + // other one is refused whole. + assert_eq!(contend(0).await, (1, 7, 3)); + // With one item stored, one batch reaches four and the rest are refused. + assert_eq!(contend(1).await, (1, 7, 4)); +} + +#[tokio::test] +async fn an_expired_stream_reads_as_not_found() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + let run = start(&app, "stream-expiry").await; + let (workflow, run) = (&run["workflow_id"], &run["run_id"]); + for name in ["tokens", "kept"] { + let (status, _) = request(&app, "POST", &route(workflow, run, name, "/items"), json!({"items":items(1)})).await; + assert_eq!(status, StatusCode::OK); + } + let (status, _) = request(&app, "POST", &route(workflow, run, "tokens", "/close"), json!({"retention_seconds":1})).await; + assert_eq!(status, StatusCode::OK); + tokio::time::sleep(std::time::Duration::from_millis(2100)).await; + + let (status, body) = request(&app, "GET", &route(workflow, run, "tokens", ""), Value::Null).await; + assert_eq!((status, &body["reason"]), (StatusCode::NOT_FOUND, &json!("stream_not_found")), "{body}"); + let (status, body) = request(&app, "GET", &route(workflow, run, "tokens", "/items"), Value::Null).await; + assert_eq!((status, &body["reason"]), (StatusCode::NOT_FOUND, &json!("stream_not_found")), "{body}"); + let (status, body) = request(&app, "POST", &route(workflow, run, "tokens", "/close"), json!({})).await; + assert_eq!((status, &body["reason"]), (StatusCode::NOT_FOUND, &json!("stream_not_found")), "{body}"); + let list = format!("/api/workflows/{}/runs/{}/streams", workflow.as_str().unwrap(), run.as_str().unwrap()); + let (_, listed) = request(&app, "GET", &list, Value::Null).await; + let names: Vec<_> = listed["streams"].as_array().unwrap().iter().map(|stream| stream["stream_name"].clone()).collect(); + assert_eq!(names, vec![json!("kept")], "{listed}"); + let (status, body) = request(&app, "POST", &route(workflow, run, "tokens", "/items"), json!({"items":items(1)})).await; + assert_eq!((status, &body["reason"]), (StatusCode::CONFLICT, &json!("stream_closed")), "{body}"); +} + +#[tokio::test] +async fn the_retention_pass_reclaims_expired_streams_in_bounded_resumable_steps() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + let run = start(&app, "stream-sweep").await; + let (workflow, run) = (&run["workflow_id"], &run["run_id"]); + let fill = |name: &'static str, count: usize| { + let app = app.clone(); + let path = route(workflow, run, name, "/items"); + async move { + let mut left = count; + while left > 0 { + let batch = left.min(500); + let (status, body) = request(&app, "POST", &path, json!({"items":items(batch)})).await; + assert_eq!(status, StatusCode::OK, "{body}"); + left -= batch; + } + } + }; + fill("large", 150).await; + fill("small", 1).await; + fill("long", 1).await; + fill("open", 1).await; + for (name, retention) in [("large", 1), ("small", 1), ("long", 3600)] { + let (status, _) = request(&app, "POST", &route(workflow, run, name, "/close"), json!({"retention_seconds":retention})).await; + assert_eq!(status, StatusCode::OK); + } + tokio::time::sleep(std::time::Duration::from_millis(2100)).await; + let pass = |limit: i64| { + let app = app.clone(); + async move { + let (status, body) = request(&app, "POST", "/api/system/retention/pass", json!({"limit":limit})).await; + assert!(status == StatusCode::OK, "{status}: {body}"); + } + }; + // limit 1: one stream and at most 100 items per pass; the next pass + // resumes where it stopped. + pass(1).await; + assert_eq!((usage(&app, "workflow_streams").await, usage(&app, "workflow_stream_items").await), (4, 53)); + pass(1).await; + assert_eq!((usage(&app, "workflow_streams").await, usage(&app, "workflow_stream_items").await), (3, 3)); + pass(1).await; + assert_eq!((usage(&app, "workflow_streams").await, usage(&app, "workflow_stream_items").await), (2, 2)); + // The long-retention and open streams stay. + pass(100).await; + assert_eq!((usage(&app, "workflow_streams").await, usage(&app, "workflow_stream_items").await), (2, 2)); +} diff --git a/scripts/conformance/server-parity/workflow-stream-writes-contract.mjs b/scripts/conformance/server-parity/workflow-stream-writes-contract.mjs index e567e3330..7b77b4e18 100644 --- a/scripts/conformance/server-parity/workflow-stream-writes-contract.mjs +++ b/scripts/conformance/server-parity/workflow-stream-writes-contract.mjs @@ -121,15 +121,21 @@ export function checkWorkflowStreamWrites(fixture, observation, workflowId) { items(state.items_wait_empty, ids, {...three, pending_items: 0}, [], 3, false, label('items_wait_empty')); assert.equal(state.items_wait_returned_after_wait, true, label('a bounded wait on an open stream waits')); - // The cap commits the items before it and then refuses the rest. + // Server #774 refuses the over-cap batch whole: nothing is written, so the + // stream keeps its three items. Published 2.11.1 committed the items before + // the cap and then refused the rest (a published defect, bounded here): its + // answers are accepted as a set, and every later step follows the branch + // the observation took. + const whole = state.show_after_full?.response?.stream?.total_items === 3; exact(state.append_full, 429, {message: 'Stream pending-items cap reached; consumer is draining too slowly.', - reason: 'stream_full', pending_items: spec.max_pending_items, max_pending_items: spec.max_pending_items}, + reason: 'stream_full', pending_items: whole ? 0 : spec.max_pending_items, max_pending_items: spec.max_pending_items}, label('append_full')); - const five = {...open, last_offset: 4, total_items: 5}; - const full = streamRead(state.show_after_full, ids, {...five, pending_items: 2}, label('show_after_full')); - assert.ok(full.appended >= first.appended, label('the capped append moved last_appended_at forward')); + const five = whole ? three : {...open, last_offset: 4, total_items: 5}; + const full = streamRead(state.show_after_full, ids, {...five, pending_items: whole ? 0 : 2}, label('show_after_full')); + assert.ok(full.appended >= first.appended, label('the capped append kept last_appended_at in order')); items(state.items_after_full, ids, {...five, pending_items: 0}, - [item(3, {...batch.over_cap[0]}), item(4, {...batch.over_cap[1]})], 5, false, label('items_after_full')); + whole ? [] : [item(3, {...batch.over_cap[0]}), item(4, {...batch.over_cap[1]})], whole ? 3 : 5, false, + label('items_after_full')); const r = state.refusals; const refusal = name => label(`refusal ${name}`); @@ -179,7 +185,8 @@ export function checkWorkflowStreamWrites(fixture, observation, workflowId) { assert.equal(again.closed, firstClose.closed, label('a repeated close keeps closed_at')); exact(state.append_closed, 409, {message: 'Stream is closed.', reason: 'stream_closed'}, label('append_closed')); items(state.items_closed_tail, ids, closed, [], 5, true, label('items_closed_tail')); - items(state.items_closed_head, ids, closed, [item(4, {...batch.over_cap[1]})], 5, true, label('items_closed_head')); + items(state.items_closed_head, ids, closed, whole ? [] : [item(4, {...batch.over_cap[1]})], whole ? 4 : 5, true, + label('items_closed_head')); const other = {stream_name: audit, status: 'open', last_offset: 0, total_items: 1, pending_items: 1}; appended(state.append_other, ids, other, [0], 1, 0, label('append_other')); diff --git a/tests/Fixtures/ServerParity/workflow-stream-writes.json b/tests/Fixtures/ServerParity/workflow-stream-writes.json index 5fbe11aaa..4670776ff 100644 --- a/tests/Fixtures/ServerParity/workflow-stream-writes.json +++ b/tests/Fixtures/ServerParity/workflow-stream-writes.json @@ -2,7 +2,7 @@ "schema": "durable-workflow.server-parity-fixture/v1", "id": "workflow-stream-writes", "review_state": "reviewed", - "contract": "Published Server 2.11.1 durable workflow stream producer routes: POST runs/{run}/streams/{name}/items opens a stream on first append, assigns offsets from 0, returns an existing item for a repeated idempotency key, commits the items before the pending-items cap and then answers 429 stream_full, and refuses closed and errored streams with 409. POST .../close closes or errors a stream once, keeps a terminal stream unchanged and defaults retention to 3600 seconds. Reads report the rows, reset pending_items when a read reaches the head, and a bounded wait on an open stream returns empty. Every validation, name and unknown-run refusal is recorded.", + "contract": "Published Server 2.11.1 durable workflow stream producer routes: POST runs/{run}/streams/{name}/items opens a stream on first append, assigns offsets from 0, returns an existing item for a repeated idempotency key, refuses a batch whose fresh items would pass the pending-items cap whole with 429 stream_full (Server #774; published 2.11.1 committed the items before the cap first, a bounded published defect), and refuses closed and errored streams with 409. POST .../close closes or errors a stream once, keeps a terminal stream unchanged and defaults retention to 3600 seconds. Reads report the rows, reset pending_items when a read reaches the head, and a bounded wait on an open stream returns empty. Every validation, name and unknown-run refusal is recorded.", "workflow_type": "parity.v1.echo", "input": { "message": "message streams λ", diff --git a/tests/Unit/ServerParityWorkflowStreamWritesContractTest.mjs b/tests/Unit/ServerParityWorkflowStreamWritesContractTest.mjs index 5942455b5..14a1035b6 100644 --- a/tests/Unit/ServerParityWorkflowStreamWritesContractTest.mjs +++ b/tests/Unit/ServerParityWorkflowStreamWritesContractTest.mjs @@ -19,6 +19,27 @@ test('the published 2.11.1 recording satisfies the contract', () => { assert.deepStrictEqual(check(observation()), {scope: 'workflow_stream_writes'}); }); +// The same run under Server #774: the over-cap batch is refused whole, so the +// tokens stream keeps its three items through every later step. +const three = stream => Object.assign(stream, {last_offset: 2, total_items: 3}); +const wholeBatch = state => { + state.append_full.response.pending_items = 0; + three(state.show_after_full.response.stream).pending_items = 0; + Object.assign(state.items_after_full.response, {items: [], next_offset: 3}); + three(state.items_after_full.response.stream); + three(state.list_after_refusals.response.streams[0]); + three(state.close.response.stream); + three(state.close_again.response.stream); + three(state.items_closed_tail.response.stream); + Object.assign(state.items_closed_head.response, {items: [], next_offset: 4}); + three(state.items_closed_head.response.stream); + three(state.list_final.response.streams.find(entry => entry.stream_name === 'tokens')); +}; + +test('a whole-batch capacity refusal (Server #774) satisfies the contract', () => { + assert.deepStrictEqual(check(observation(wholeBatch)), {scope: 'workflow_stream_writes'}); +}); + test('embedded applications are explicitly inapplicable', () => { assert.deepStrictEqual(check({mode: 'embedded', workflow_stream_writes: {applicable: false, reason: 'embedded_has_no_http_workflow_streams'}}), {scope: 'workflow_stream_writes'}); @@ -42,10 +63,14 @@ for (const [label, change, message] of [ ['a read at the head that leaves pending items', state => { state.items_all.response.stream.pending_items = 3; }, /items_all stream/], - ['a cap that rejects the whole batch', state => { - state.show_after_full.response.stream.last_offset = 2; - state.show_after_full.response.stream.total_items = 3; - }, /show_after_full stream/], + ['a whole refusal that reports the published pending count', state => { + wholeBatch(state); + state.append_full.response.pending_items = 2; + }, /append_full body/], + ['a whole refusal whose later reads still show the capped items', state => { + wholeBatch(state); + state.items_closed_head.response.next_offset = 5; + }, /items_closed_head next_offset/], ['a cap answered as accepted', state => { state.append_full.status = 200; }, /append_full status/], ['a repeated close that replaces the first', state => { state.close_again.response.stream.error_reason = 'late error'; From f56bc6327221fb52c42fd5a050773f7098f8860b Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Mon, 26 Oct 2026 17:48:22 +0000 Subject: [PATCH 5/6] Rust #325: native stream expiry, indexed reclamation and pruned-run refusal (#774) Native follows #785's stream lifecycle: - A closed stream past its retention reads as not found, append included, and keeps its row until its run is pruned. - Its deadline lives in the native-owned dw_stream_reclaims relation (schema 6 on all four backends; catalogs, upgrade lists and parity scripts updated), indexed (namespace, reclaim_at, stream_id). - The retention pass reads due streams from that index in deadline order, deletes items in bounded batches under the transition lock, and stops blocked when items hold external references, which native does not delete yet. - Run retention deletes a pruned run's streams unless they hold external references, and an append to a run whose details are pruned answers 404 stream_not_found instead of opening a stream. Co-Authored-By: Claude Opus 5.5 --- .../{0005_catalog.json => 0006_catalog.json} | 4 + .../mariadb/0006_stream_reclaims.sql | 13 ++ .../{0005_catalog.json => 0006_catalog.json} | 4 + .../migrations/mysql/0006_stream_reclaims.sql | 13 ++ .../{0005_catalog.json => 0006_catalog.json} | 99 ++++++++++ .../postgres/0006_stream_reclaims.sql | 13 ++ .../sqlite/0006_stream_reclaims.sql | 13 ++ rust/src/runtime/backend.rs | 25 ++- rust/src/runtime/mysql.rs | 28 ++- rust/src/runtime/namespace_cleanup.rs | 1 + rust/src/runtime/operator_metrics.rs | 22 ++- rust/src/runtime/postgres.rs | 18 +- rust/src/runtime/retention.rs | 19 +- rust/src/runtime/schema.rs | 12 +- rust/src/runtime/upgrade.rs | 3 + rust/src/runtime/upgrade_tests.rs | 5 +- rust/src/runtime/workflow_streams.rs | 102 +++++++--- .../tests/execution/workflow_stream_bounds.rs | 186 +++++++++++++++++- .../server-parity/mysql-schema.php | 2 +- .../server-parity/postgres-schema.php | 4 +- scripts/conformance/server-parity/schema.php | 2 +- 21 files changed, 519 insertions(+), 69 deletions(-) rename rust/migrations/mariadb/{0005_catalog.json => 0006_catalog.json} (99%) create mode 100644 rust/migrations/mariadb/0006_stream_reclaims.sql rename rust/migrations/mysql/{0005_catalog.json => 0006_catalog.json} (99%) create mode 100644 rust/migrations/mysql/0006_stream_reclaims.sql rename rust/migrations/postgres/{0005_catalog.json => 0006_catalog.json} (99%) create mode 100644 rust/migrations/postgres/0006_stream_reclaims.sql create mode 100644 rust/migrations/sqlite/0006_stream_reclaims.sql diff --git a/rust/migrations/mariadb/0005_catalog.json b/rust/migrations/mariadb/0006_catalog.json similarity index 99% rename from rust/migrations/mariadb/0005_catalog.json rename to rust/migrations/mariadb/0006_catalog.json index 57d64e99a..737a3c180 100644 --- a/rust/migrations/mariadb/0005_catalog.json +++ b/rust/migrations/mariadb/0006_catalog.json @@ -19,6 +19,10 @@ "dw_server_schema", "CREATE TABLE `dw_server_schema` (\n `engine` varchar(64) NOT NULL,\n `version` bigint(20) NOT NULL,\n `bootstrap_checksum` varbinary(48) NOT NULL,\n PRIMARY KEY (`engine`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin" ], + [ + "dw_stream_reclaims", + "CREATE TABLE `dw_stream_reclaims` (\n `stream_id` bigint(20) unsigned NOT NULL,\n `namespace` varchar(128) NOT NULL,\n `workflow_run_id` varchar(26) NOT NULL,\n `reclaim_at` datetime(6) NOT NULL,\n PRIMARY KEY (`stream_id`),\n KEY `dw_stream_reclaims_due` (`namespace`,`reclaim_at`,`stream_id`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci" + ], [ "dw_task_completions", "CREATE TABLE `dw_task_completions` (\n `task_id` varchar(26) NOT NULL,\n `receipt` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL CHECK (json_valid(`receipt`)),\n PRIMARY KEY (`task_id`),\n CONSTRAINT `dw_task_completions_task` FOREIGN KEY (`task_id`) REFERENCES `workflow_tasks` (`id`) ON DELETE CASCADE\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci" diff --git a/rust/migrations/mariadb/0006_stream_reclaims.sql b/rust/migrations/mariadb/0006_stream_reclaims.sql new file mode 100644 index 000000000..2adaf3ba0 --- /dev/null +++ b/rust/migrations/mariadb/0006_stream_reclaims.sql @@ -0,0 +1,13 @@ +-- Native development bootstrap version 6: the reclaim deadline of each closed +-- or errored durable workflow stream (Server #774), closed_at plus its +-- retention. The stream retention pass reads a namespace's due streams in +-- deadline order from (namespace, reclaim_at, stream_id). A stream leaves the +-- table once its items are reclaimed; its row stays until its run is pruned. +CREATE TABLE dw_stream_reclaims ( + stream_id BIGINT UNSIGNED NOT NULL PRIMARY KEY, + namespace VARCHAR(128) NOT NULL, + workflow_run_id VARCHAR(26) NOT NULL, + reclaim_at DATETIME(6) NOT NULL, + KEY dw_stream_reclaims_due (namespace, reclaim_at, stream_id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +UPDATE dw_server_schema SET version=6 WHERE engine='rust-development' AND version=5; diff --git a/rust/migrations/mysql/0005_catalog.json b/rust/migrations/mysql/0006_catalog.json similarity index 99% rename from rust/migrations/mysql/0005_catalog.json rename to rust/migrations/mysql/0006_catalog.json index 4f67db5a6..b57c5f919 100644 --- a/rust/migrations/mysql/0005_catalog.json +++ b/rust/migrations/mysql/0006_catalog.json @@ -23,6 +23,10 @@ "dw_server_schema", "CREATE TABLE `dw_server_schema` (\n `engine` varchar(64) COLLATE utf8mb4_bin NOT NULL,\n `version` bigint NOT NULL,\n `bootstrap_checksum` varbinary(48) NOT NULL,\n PRIMARY KEY (`engine`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin" ], + [ + "dw_stream_reclaims", + "CREATE TABLE `dw_stream_reclaims` (\n `stream_id` bigint unsigned NOT NULL,\n `namespace` varchar(128) COLLATE utf8mb4_unicode_ci NOT NULL,\n `workflow_run_id` varchar(26) COLLATE utf8mb4_unicode_ci NOT NULL,\n `reclaim_at` datetime(6) NOT NULL,\n PRIMARY KEY (`stream_id`),\n KEY `dw_stream_reclaims_due` (`namespace`,`reclaim_at`,`stream_id`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci" + ], [ "dw_task_completions", "CREATE TABLE `dw_task_completions` (\n `task_id` varchar(26) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL,\n `receipt` json NOT NULL,\n PRIMARY KEY (`task_id`),\n CONSTRAINT `dw_task_completions_task` FOREIGN KEY (`task_id`) REFERENCES `workflow_tasks` (`id`) ON DELETE CASCADE\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci" diff --git a/rust/migrations/mysql/0006_stream_reclaims.sql b/rust/migrations/mysql/0006_stream_reclaims.sql new file mode 100644 index 000000000..2adaf3ba0 --- /dev/null +++ b/rust/migrations/mysql/0006_stream_reclaims.sql @@ -0,0 +1,13 @@ +-- Native development bootstrap version 6: the reclaim deadline of each closed +-- or errored durable workflow stream (Server #774), closed_at plus its +-- retention. The stream retention pass reads a namespace's due streams in +-- deadline order from (namespace, reclaim_at, stream_id). A stream leaves the +-- table once its items are reclaimed; its row stays until its run is pruned. +CREATE TABLE dw_stream_reclaims ( + stream_id BIGINT UNSIGNED NOT NULL PRIMARY KEY, + namespace VARCHAR(128) NOT NULL, + workflow_run_id VARCHAR(26) NOT NULL, + reclaim_at DATETIME(6) NOT NULL, + KEY dw_stream_reclaims_due (namespace, reclaim_at, stream_id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +UPDATE dw_server_schema SET version=6 WHERE engine='rust-development' AND version=5; diff --git a/rust/migrations/postgres/0005_catalog.json b/rust/migrations/postgres/0006_catalog.json similarity index 99% rename from rust/migrations/postgres/0005_catalog.json rename to rust/migrations/postgres/0006_catalog.json index 4dd16a521..38675b901 100644 --- a/rust/migrations/postgres/0005_catalog.json +++ b/rust/migrations/postgres/0006_catalog.json @@ -766,6 +766,58 @@ "generated": "" } ], + [ + "column", + "dw_stream_reclaims.1", + { + "collation": null, + "default": null, + "generated": "", + "identity": "", + "name": "stream_id", + "not_null": true, + "type": "bigint" + } + ], + [ + "column", + "dw_stream_reclaims.2", + { + "collation": "default", + "default": null, + "generated": "", + "identity": "", + "name": "namespace", + "not_null": true, + "type": "text" + } + ], + [ + "column", + "dw_stream_reclaims.3", + { + "collation": "default", + "default": null, + "generated": "", + "identity": "", + "name": "workflow_run_id", + "not_null": true, + "type": "text" + } + ], + [ + "column", + "dw_stream_reclaims.4", + { + "collation": null, + "default": null, + "generated": "", + "identity": "", + "name": "reclaim_at", + "not_null": true, + "type": "timestamp(6) without time zone" + } + ], [ "column", "dw_task_completions.1", @@ -11987,6 +12039,16 @@ "definition": "PRIMARY KEY (key)" } ], + [ + "constraint", + "dw_stream_reclaims.dw_stream_reclaims_pkey", + { + "deferrable": false, + "deferred": false, + "definition": "PRIMARY KEY (stream_id)", + "validated": true + } + ], [ "constraint", "dw_task_completions.dw_task_completions_pkey", @@ -13082,6 +13144,28 @@ "definition": "CREATE UNIQUE INDEX dw_query_cache_pkey ON public.dw_query_cache USING btree (key)" } ], + [ + "index", + "dw_stream_reclaims_due", + { + "definition": "CREATE INDEX dw_stream_reclaims_due ON public.dw_stream_reclaims USING btree (namespace, reclaim_at, stream_id)", + "primary": false, + "ready": true, + "unique": false, + "valid": true + } + ], + [ + "index", + "dw_stream_reclaims_pkey", + { + "definition": "CREATE UNIQUE INDEX dw_stream_reclaims_pkey ON public.dw_stream_reclaims USING btree (stream_id)", + "primary": true, + "ready": true, + "unique": true, + "valid": true + } + ], [ "index", "dw_task_completions_pkey", @@ -17308,6 +17392,21 @@ "force_row_security": false } ], + [ + "relation", + "dw_stream_reclaims", + { + "access_method": "heap", + "force_row_security": false, + "kind": "r", + "options": null, + "partition": false, + "partition_key": null, + "persistence": "p", + "row_security": false, + "view": null + } + ], [ "relation", "dw_task_completions", diff --git a/rust/migrations/postgres/0006_stream_reclaims.sql b/rust/migrations/postgres/0006_stream_reclaims.sql new file mode 100644 index 000000000..bf7114bdc --- /dev/null +++ b/rust/migrations/postgres/0006_stream_reclaims.sql @@ -0,0 +1,13 @@ +-- Native development bootstrap version 6: the reclaim deadline of each closed +-- or errored durable workflow stream (Server #774), closed_at plus its +-- retention. The stream retention pass reads a namespace's due streams in +-- deadline order from (namespace, reclaim_at, stream_id). A stream leaves the +-- table once its items are reclaimed; its row stays until its run is pruned. +CREATE TABLE dw_stream_reclaims ( + stream_id bigint NOT NULL PRIMARY KEY, + namespace text NOT NULL, + workflow_run_id text NOT NULL, + reclaim_at timestamp(6) without time zone NOT NULL +); +CREATE INDEX dw_stream_reclaims_due ON dw_stream_reclaims (namespace, reclaim_at, stream_id); +UPDATE dw_server_schema SET version=6 WHERE engine='rust-development' AND version=5; diff --git a/rust/migrations/sqlite/0006_stream_reclaims.sql b/rust/migrations/sqlite/0006_stream_reclaims.sql new file mode 100644 index 000000000..481e73c9f --- /dev/null +++ b/rust/migrations/sqlite/0006_stream_reclaims.sql @@ -0,0 +1,13 @@ +-- Native development bootstrap version 6: the reclaim deadline of each closed +-- or errored durable workflow stream (Server #774), closed_at plus its +-- retention. The stream retention pass reads a namespace's due streams in +-- deadline order from (namespace, reclaim_at, stream_id). A stream leaves the +-- table once its items are reclaimed; its row stays until its run is pruned. +CREATE TABLE dw_stream_reclaims ( + stream_id INTEGER NOT NULL PRIMARY KEY, + namespace TEXT NOT NULL, + workflow_run_id TEXT NOT NULL, + reclaim_at TEXT NOT NULL +); +CREATE INDEX dw_stream_reclaims_due ON dw_stream_reclaims (namespace, reclaim_at, stream_id); +UPDATE dw_server_schema SET version=6 WHERE engine='rust-development' AND version=5; diff --git a/rust/src/runtime/backend.rs b/rust/src/runtime/backend.rs index c91892bed..6cd8306a9 100644 --- a/rust/src/runtime/backend.rs +++ b/rust/src/runtime/backend.rs @@ -86,12 +86,16 @@ pub(super) trait Backend: Database { const STREAM_ITEM_BY_KEY: &str = "SELECT \"offset\" FROM workflow_durable_stream_items WHERE stream_id=$1 AND idempotency_key=$2 LIMIT 1"; const STREAM_ITEM_INSERT: &str = "INSERT INTO workflow_durable_stream_items(stream_id,namespace,workflow_run_id,stream_name,\"offset\",idempotency_key,origin,origin_reference,payload,payload_reference,payload_codec,item_type,content_type,emitted_at,created_at,updated_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16)"; const STREAM_ITEMS_WINDOW: &str = "SELECT * FROM workflow_durable_stream_items WHERE stream_id=$1 AND \"offset\">=$2 ORDER BY \"offset\" LIMIT $3"; - // `WorkflowStreamRetention`: this namespace's closed or errored streams - // whose `closed_at` plus retention (3600 seconds when none was recorded) - // is no later than $2, oldest close first. - const EXPIRED_STREAMS: &str = "SELECT id FROM workflow_durable_streams WHERE namespace=$1 AND status IN ('closed','errored') AND closed_at IS NOT NULL AND closed_at<=$2-(COALESCE(retention_seconds,3600)*INTERVAL '1 second') ORDER BY closed_at,id LIMIT $3"; const STREAM_SWEEP_BOUNDARY: &str = "SELECT \"offset\" FROM workflow_durable_stream_items WHERE stream_id=$1 ORDER BY \"offset\" LIMIT 1 OFFSET $2"; const STREAM_SWEEP_THROUGH: &str = "DELETE FROM workflow_durable_stream_items WHERE stream_id=$1 AND \"offset\"<=$2"; + // Durable stream items, and streams, holding an external payload + // reference, which native does not settle (Server #774). PostgreSQL + // stores these payloads as `json`, matched through its text. + const STREAM_ITEM_REFERENCES: &str = concat!("SELECT COUNT(*) FROM workflow_durable_stream_items WHERE stream_id=$1 AND (payload_reference IS NOT NULL OR ", super::retention::reference!("payload::text"), ")"); + const RUN_STREAM_REFERENCES: &str = concat!( + "SELECT (SELECT COUNT(*) FROM workflow_durable_stream_items WHERE workflow_run_id=$1 AND (payload_reference IS NOT NULL OR ", super::retention::reference!("payload::text"), "))", + "+(SELECT COUNT(*) FROM workflow_durable_streams WHERE workflow_run_id=$2 AND ", super::retention::reference!("metadata::text"), ")" + ); // `duplicateBridgeCommand`: the latest accepted command PHP recorded for // this adapter, action, key and target name ($6 names the target field), // filtered in the database as PHP's JSON-path query is. PHP's bridge @@ -207,8 +211,11 @@ WHERE r.status IN ('pending','running','waiting') ORDER BY t.available_order,t.id LIMIT 100 "#; const SELECTION_RESOLUTION_SQL: &str = "SELECT id,payload FROM workflow_history_events WHERE workflow_run_id=$1 AND event_type='SelectionResolved' AND json_extract(payload,'$.selection_group_id')=$2 ORDER BY sequence LIMIT 1"; - // SQLite keeps times as text: closed_at against $2 less the retention. - const EXPIRED_STREAMS: &str = "SELECT id FROM workflow_durable_streams WHERE namespace=$1 AND status IN ('closed','errored') AND closed_at IS NOT NULL AND closed_at<=strftime('%Y-%m-%d %H:%M:%f',$2,'-'||COALESCE(retention_seconds,3600)||' seconds') ORDER BY closed_at,id LIMIT $3"; + const STREAM_ITEM_REFERENCES: &str = concat!("SELECT COUNT(*) FROM workflow_durable_stream_items WHERE stream_id=$1 AND (payload_reference IS NOT NULL OR ", super::retention::reference!("payload"), ")"); + const RUN_STREAM_REFERENCES: &str = concat!( + "SELECT (SELECT COUNT(*) FROM workflow_durable_stream_items WHERE workflow_run_id=$1 AND (payload_reference IS NOT NULL OR ", super::retention::reference!("payload"), "))", + "+(SELECT COUNT(*) FROM workflow_durable_streams WHERE workflow_run_id=$2 AND ", super::retention::reference!("metadata"), ")" + ); const BRIDGE_COMMAND: &str = "SELECT workflow_run_id,resolved_workflow_run_id FROM workflow_commands WHERE workflow_instance_id=$1 AND command_type=$2 AND status='accepted' AND json_extract(context,'$.server.metadata.adapter')=$3 AND json_extract(context,'$.server.metadata.action')=$4 AND json_extract(context,'$.server.metadata.idempotency_key')=$5 AND json_extract(context,'$.server.metadata.'||$6)=$7 AND (request_id IS NULL OR request_id=$8) ORDER BY accepted_at DESC,command_sequence DESC LIMIT 1"; const REPAIR_COUNTER_MAX: i64 = i64::MAX; const RANGE_REPAIR_TASK_SQL: &str = "SELECT * FROM workflow_tasks WHERE workflow_run_id=$1 AND namespace=$2 AND task_type='workflow' AND status='failed' AND json_extract(payload,'$.replay_blocked')=1 ORDER BY CASE WHEN available_at IS NULL THEN 1 ELSE 0 END,available_at,created_at,id LIMIT 1"; @@ -462,9 +469,13 @@ impl Backend for MySql { const STREAM_ITEM_BY_KEY: &str = "SELECT `offset` FROM workflow_durable_stream_items WHERE stream_id=$1 AND idempotency_key=$2 LIMIT 1"; const STREAM_ITEM_INSERT: &str = "INSERT INTO workflow_durable_stream_items(stream_id,namespace,workflow_run_id,stream_name,`offset`,idempotency_key,origin,origin_reference,payload,payload_reference,payload_codec,item_type,content_type,emitted_at,created_at,updated_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16)"; const STREAM_ITEMS_WINDOW: &str = "SELECT * FROM workflow_durable_stream_items WHERE stream_id=$1 AND `offset`>=$2 ORDER BY `offset` LIMIT $3"; - const EXPIRED_STREAMS: &str = "SELECT id FROM workflow_durable_streams WHERE namespace=$1 AND status IN ('closed','errored') AND closed_at IS NOT NULL AND closed_at<=DATE_SUB($2,INTERVAL COALESCE(retention_seconds,3600) SECOND) ORDER BY closed_at,id LIMIT $3"; const STREAM_SWEEP_BOUNDARY: &str = "SELECT `offset` FROM workflow_durable_stream_items WHERE stream_id=$1 ORDER BY `offset` LIMIT 1 OFFSET $2"; const STREAM_SWEEP_THROUGH: &str = "DELETE FROM workflow_durable_stream_items WHERE stream_id=$1 AND `offset`<=$2"; + const STREAM_ITEM_REFERENCES: &str = concat!("SELECT COUNT(*) FROM workflow_durable_stream_items WHERE stream_id=$1 AND (payload_reference IS NOT NULL OR ", super::retention::reference!("payload"), ")"); + const RUN_STREAM_REFERENCES: &str = concat!( + "SELECT (SELECT COUNT(*) FROM workflow_durable_stream_items WHERE workflow_run_id=$1 AND (payload_reference IS NOT NULL OR ", super::retention::reference!("payload"), "))", + "+(SELECT COUNT(*) FROM workflow_durable_streams WHERE workflow_run_id=$2 AND ", super::retention::reference!("metadata"), ")" + ); const READ_SNAPSHOT_BEGIN: &str = "START TRANSACTION WITH CONSISTENT SNAPSHOT, READ ONLY"; const ACTIVITY_METADATA_HISTORY: &str = "SELECT h.sequence,h.event_type,CASE WHEN page.projection_bytes<=8388608 THEN h.payload ELSE NULL END AS payload,h.workflow_task_id,h.recorded_at,page.projection_count FROM (SELECT id,sequence,COUNT(*) OVER () AS projection_count,SUM(payload_bytes) OVER (ORDER BY sequence) AS projection_bytes FROM (SELECT id,sequence,octet_length(CAST(payload AS CHAR CHARACTER SET utf8mb4)) AS payload_bytes FROM workflow_history_events FORCE INDEX (workflow_history_events_workflow_run_id_sequence_unique) WHERE workflow_run_id=$1 AND event_type IN ('ActivityScheduled','ActivityStarted','ActivityHeartbeatRecorded','ActivityRetryScheduled','ActivityCompleted','ActivityFailed','ActivityTimedOut','ActivityCancelled')) AS sizes ORDER BY sequence LIMIT 10001) AS page JOIN workflow_history_events h ON h.id=page.id ORDER BY page.sequence"; const PHP_CONNECTION: &str = "mysql"; diff --git a/rust/src/runtime/mysql.rs b/rust/src/runtime/mysql.rs index 53d7ae4d8..81bc63261 100644 --- a/rust/src/runtime/mysql.rs +++ b/rust/src/runtime/mysql.rs @@ -9,7 +9,7 @@ use sqlx::{ }; use std::{collections::BTreeMap, sync::OnceLock, time::Duration}; -pub const VERSION: i64 = 5; +pub const VERSION: i64 = 6; type Catalog = Vec<(String, String)>; pub const MARKER_SQL: &str = "CREATE TABLE dw_server_schema (engine VARCHAR(64) NOT NULL PRIMARY KEY,version BIGINT NOT NULL,bootstrap_checksum VARBINARY(48) NOT NULL) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin"; @@ -46,7 +46,9 @@ impl Flavor { "\n", include_str!("../../migrations/mysql/0004_worker_incarnations.sql"), "\n", - include_str!("../../migrations/mysql/0005_worker_session_holders.sql") + include_str!("../../migrations/mysql/0005_worker_session_holders.sql"), + "\n", + include_str!("../../migrations/mysql/0006_stream_reclaims.sql") ), ), Self::MariaDb => ( @@ -56,14 +58,16 @@ impl Flavor { "\n", include_str!("../../migrations/mariadb/0004_worker_incarnations.sql"), "\n", - include_str!("../../migrations/mariadb/0005_worker_session_holders.sql") + include_str!("../../migrations/mariadb/0005_worker_session_holders.sql"), + "\n", + include_str!("../../migrations/mariadb/0006_stream_reclaims.sql") ), ), }; slot.get_or_init(|| { let mut migrations = Migrator::with_migrations(vec![Migration::new( VERSION, - "full frozen PHP schema, native worker incarnations and session holders".into(), + "full frozen PHP schema, native worker incarnations, session holders and stream reclaim deadlines".into(), MigrationType::Simple, sql.into_sql_str(), false, @@ -76,8 +80,8 @@ impl Flavor { } fn expected(self) -> Result { Ok(serde_json::from_str(&canonical(match self { - Self::MySql => include_str!("../../migrations/mysql/0005_catalog.json"), - Self::MariaDb => include_str!("../../migrations/mariadb/0005_catalog.json"), + Self::MySql => include_str!("../../migrations/mysql/0006_catalog.json"), + Self::MariaDb => include_str!("../../migrations/mariadb/0006_catalog.json"), }))?) } /// The single native migration, which a takeover records as applied. @@ -94,7 +98,9 @@ impl Flavor { concat!( include_str!("../../migrations/mysql/0004_worker_incarnations.sql"), "\n", - include_str!("../../migrations/mysql/0005_worker_session_holders.sql") + include_str!("../../migrations/mysql/0005_worker_session_holders.sql"), + "\n", + include_str!("../../migrations/mysql/0006_stream_reclaims.sql") ), ), Self::MariaDb => ( @@ -102,7 +108,9 @@ impl Flavor { concat!( include_str!("../../migrations/mariadb/0004_worker_incarnations.sql"), "\n", - include_str!("../../migrations/mariadb/0005_worker_session_holders.sql") + include_str!("../../migrations/mariadb/0005_worker_session_holders.sql"), + "\n", + include_str!("../../migrations/mariadb/0006_stream_reclaims.sql") ), ), }; @@ -118,6 +126,10 @@ impl Flavor { .replace( "CREATE TABLE dw_worker_session_holders", "CREATE TABLE IF NOT EXISTS dw_worker_session_holders", + ) + .replace( + "CREATE TABLE dw_stream_reclaims", + "CREATE TABLE IF NOT EXISTS dw_stream_reclaims", )) } pub(super) fn checksum(self) -> &'static [u8] { diff --git a/rust/src/runtime/namespace_cleanup.rs b/rust/src/runtime/namespace_cleanup.rs index 780382dc0..40aeb114b 100644 --- a/rust/src/runtime/namespace_cleanup.rs +++ b/rust/src/runtime/namespace_cleanup.rs @@ -149,6 +149,7 @@ const NATIVE_CLEANUP: &[&str] = &[ "DELETE FROM dw_poll_receipts WHERE namespace=$1", "DELETE FROM dw_worker_registration_incarnations WHERE namespace=$1", "DELETE FROM dw_worker_session_holders WHERE namespace=$1", + "DELETE FROM dw_stream_reclaims WHERE namespace=$1", ]; /// Registered external payloads and payload columns holding a reference. diff --git a/rust/src/runtime/operator_metrics.rs b/rust/src/runtime/operator_metrics.rs index 86630486a..815835b3c 100644 --- a/rust/src/runtime/operator_metrics.rs +++ b/rust/src/runtime/operator_metrics.rs @@ -1236,9 +1236,12 @@ where pub(super) async fn retention_pass(&self, limit: i64, run_ids: Vec) -> Result<(Value, bool)> { // A general pass also reclaims closed workflow streams past their own // retention (Server #774), at most `limit` streams and `limit` * 100 - // items. The report is the run pass's. - if run_ids.is_empty() { - self.prune_expired_streams(limit, limit * 100).await?; + // items. The report is the run pass's; a stream failure leaves its + // items for the next pass and does not stop the run pass. + if run_ids.is_empty() + && let Err(error) = self.prune_expired_streams(limit, limit * 100).await + { + eprintln!("workflow_stream_retention_failed: {error}"); } let policy = self.retention_policy().await?; let report = self.prune_runs(&policy, limit, run_ids).await?; @@ -1291,7 +1294,9 @@ where // throttle and is not part of this report. if retention::inline_stream_pass_due(self.namespace.as_ref()) { let (streams, items) = super::workflow_streams::INLINE_SWEEP; - self.prune_expired_streams(streams, items).await?; + if let Err(error) = self.prune_expired_streams(streams, items).await { + eprintln!("workflow_stream_retention_failed: {error}"); + } } let policy = self.retention_policy().await?; if policy.forever { @@ -1372,6 +1377,12 @@ where } deleted.insert((*key).into(), json!(DB::affected(query.execute(&mut *tx).await?))); } + for (key, sql) in retention::DURABLE_STREAMS { + let affected = DB::affected(Self::query(sql).bind(run_id).execute(&mut *tx).await?); + if let Some(key) = key { + deleted.insert((*key).into(), json!(affected)); + } + } // The run's history is gone; no cached read of it may survive. tx.scope_history = None; Self::query("UPDATE workflow_runs SET details_pruned_at=$1 WHERE id=$2 AND details_pruned_at IS NULL") @@ -1399,6 +1410,9 @@ where return Ok(Some(reason)); } } + if Self::scalar(DB::RUN_STREAM_REFERENCES).bind(run_id).bind(run_id).fetch_one(&self.pool).await? > 0 { + return Ok(Some("external_payload_cleanup_unavailable")); + } Ok(None) } diff --git a/rust/src/runtime/postgres.rs b/rust/src/runtime/postgres.rs index fc3077fd6..66553bb59 100644 --- a/rust/src/runtime/postgres.rs +++ b/rust/src/runtime/postgres.rs @@ -13,7 +13,7 @@ use sqlx::{ use super::{Result, refuse}; -pub const VERSION: i64 = 5; +pub const VERSION: i64 = 6; pub const CATALOG_QUERY: &str = include_str!("postgres-catalog.sql"); type Catalog = Vec<(String, String, Json)>; @@ -22,14 +22,16 @@ fn migrator() -> &'static Migrator { MIGRATOR.get_or_init(|| { let mut migrations = Migrator::with_migrations(vec![Migration::new( VERSION, - "full frozen PHP PostgreSQL schema, native worker incarnations and session holders".into(), + "full frozen PHP PostgreSQL schema, native worker incarnations, session holders and stream reclaim deadlines".into(), MigrationType::Simple, concat!( include_str!("../../migrations/postgres/0003_full_schema.sql"), "\n", include_str!("../../migrations/postgres/0004_worker_incarnations.sql"), "\n", - include_str!("../../migrations/postgres/0005_worker_session_holders.sql") + include_str!("../../migrations/postgres/0005_worker_session_holders.sql"), + "\n", + include_str!("../../migrations/postgres/0006_stream_reclaims.sql") ) .into_sql_str(), false, @@ -57,7 +59,9 @@ pub(super) fn native_relations_sql() -> Result { concat!( include_str!("../../migrations/postgres/0004_worker_incarnations.sql"), "\n", - include_str!("../../migrations/postgres/0005_worker_session_holders.sql") + include_str!("../../migrations/postgres/0005_worker_session_holders.sql"), + "\n", + include_str!("../../migrations/postgres/0006_stream_reclaims.sql") ), ) } @@ -91,7 +95,7 @@ pub(super) async fn php_source_catalog(connection: &mut PgConnection) -> Result< )); } let native: Catalog = - serde_json::from_str(include_str!("../../migrations/postgres/0005_catalog.json"))?; + serde_json::from_str(include_str!("../../migrations/postgres/0006_catalog.json"))?; let expected: Catalog = native .into_iter() .filter(|(kind, name, _)| { @@ -186,7 +190,7 @@ pub(super) async fn verify(connection: &mut PgConnection) -> Result<()> { // from SQL text. It retains types, precision, defaults, indexes, foreign // keys and sequence ownership while excluding live rows/sequence values. let expected: Catalog = - serde_json::from_str(include_str!("../../migrations/postgres/0005_catalog.json"))?; + serde_json::from_str(include_str!("../../migrations/postgres/0006_catalog.json"))?; let actual = catalog(connection).await?; if actual != expected { return Err(refuse( @@ -398,7 +402,7 @@ mod tests { #[test] fn compiled_catalog_preserves_long_object_identities_and_int64_limits() { let rows: Catalog = - serde_json::from_str(include_str!("../../migrations/postgres/0005_catalog.json")) + serde_json::from_str(include_str!("../../migrations/postgres/0006_catalog.json")) .unwrap(); let identities: std::collections::BTreeSet<_> = rows.iter().map(|(kind, name, _)| (kind, name)).collect(); diff --git a/rust/src/runtime/retention.rs b/rust/src/runtime/retention.rs index 9ee1c054f..3053e0c8b 100644 --- a/rust/src/runtime/retention.rs +++ b/rust/src/runtime/retention.rs @@ -73,17 +73,28 @@ const REFERENCES_SQL: &str = concat!( "+(SELECT COUNT(*) FROM activity_executions WHERE workflow_run_id=$5 AND (", reference!("arguments"), " OR ", reference!("result"), "))", ); +/// The run's durable workflow streams, deleted with it as PHP's +/// `HistoryRetentionEnforcer::pruneRun` deletes them (Server #774): report +/// key, if any, and statement binding the run id once. +pub(super) const DURABLE_STREAMS: &[(Option<&str>, &str)] = &[ + (Some("durable_stream_items_deleted"), "DELETE FROM workflow_durable_stream_items WHERE workflow_run_id=$1"), + (None, "DELETE FROM dw_stream_reclaims WHERE workflow_run_id=$1"), + (Some("durable_streams_deleted"), "DELETE FROM workflow_durable_streams WHERE workflow_run_id=$1"), +]; + /// `MessageStreamRetentionCleanup::apply`'s counts in the prune report. pub(super) const STREAM_COUNTS: [&str; 3] = ["message_streams_deleted", "message_stream_items_deleted", "message_stream_items_compacted"]; /// Counts that hold a run's details, the run-id binds of each, and the /// skip reason. An abandoned activity's hold is read from history. A run -/// whose instance has inbound message streams is held too: native does not -/// compact or remove stream state as PHP's MessageStreamRetentionCleanup -/// does yet, so it keeps the run rather than orphan the streams. +/// with messages or whose instance has inbound message streams is held too: +/// native does not compact or remove that state as PHP's +/// MessageStreamRetentionCleanup does yet, so it keeps the run rather than +/// orphan it. Durable workflow streams go with the run (`DURABLE_STREAMS`) +/// unless they hold an external reference (`Backend::RUN_STREAM_REFERENCES`). pub(super) const HOLDS: &[(&str, usize, &str)] = &[ - ("SELECT (SELECT COUNT(*) FROM workflow_durable_streams WHERE workflow_run_id=$1)+(SELECT COUNT(*) FROM workflow_messages WHERE workflow_run_id=$2)+(SELECT COUNT(*) FROM workflow_inbound_streams WHERE workflow_instance_id IN (SELECT workflow_instance_id FROM workflow_runs WHERE id=$3))", 3, "message_stream_cleanup_unavailable"), + ("SELECT (SELECT COUNT(*) FROM workflow_messages WHERE workflow_run_id=$1)+(SELECT COUNT(*) FROM workflow_inbound_streams WHERE workflow_instance_id IN (SELECT workflow_instance_id FROM workflow_runs WHERE id=$2))", 2, "message_stream_cleanup_unavailable"), (REFERENCES_SQL, 5, "external_payload_cleanup_unavailable"), ]; diff --git a/rust/src/runtime/schema.rs b/rust/src/runtime/schema.rs index 55db13331..62fd112b4 100644 --- a/rust/src/runtime/schema.rs +++ b/rust/src/runtime/schema.rs @@ -11,21 +11,23 @@ use tokio::sync::OnceCell; use super::{Result, refuse}; -pub(super) const VERSION: i64 = 5; +pub(super) const VERSION: i64 = 6; fn migrator() -> &'static Migrator { static MIGRATOR: OnceLock = OnceLock::new(); MIGRATOR.get_or_init(|| { Migrator::with_migrations(vec![Migration::new( VERSION, - "full frozen PHP schema, native worker incarnations and session holders".into(), + "full frozen PHP schema, native worker incarnations, session holders and stream reclaim deadlines".into(), MigrationType::Simple, concat!( include_str!("../../migrations/sqlite/0003_full_schema.sql"), "\n", include_str!("../../migrations/sqlite/0004_worker_incarnations.sql"), "\n", - include_str!("../../migrations/sqlite/0005_worker_session_holders.sql") + include_str!("../../migrations/sqlite/0005_worker_session_holders.sql"), + "\n", + include_str!("../../migrations/sqlite/0006_stream_reclaims.sql") ) .into_sql_str(), false, @@ -43,7 +45,9 @@ pub(super) fn native_relations_sql() -> Result { concat!( include_str!("../../migrations/sqlite/0004_worker_incarnations.sql"), "\n", - include_str!("../../migrations/sqlite/0005_worker_session_holders.sql") + include_str!("../../migrations/sqlite/0005_worker_session_holders.sql"), + "\n", + include_str!("../../migrations/sqlite/0006_stream_reclaims.sql") ), ) } diff --git a/rust/src/runtime/upgrade.rs b/rust/src/runtime/upgrade.rs index 612e0977c..eabf0db5a 100644 --- a/rust/src/runtime/upgrade.rs +++ b/rust/src/runtime/upgrade.rs @@ -106,6 +106,7 @@ pub(super) fn native_relation(name: &str) -> bool { | "dw_query_cache" | "dw_worker_registration_incarnations" | "dw_worker_session_holders" + | "dw_stream_reclaims" ) } @@ -122,6 +123,8 @@ pub(super) fn native_object(name: &str) -> bool { | "dw_worker_incarnation_authority" | "dw_worker_incarnation_retention" | "dw_worker_session_holders_pkey" + | "dw_stream_reclaims_pkey" + | "dw_stream_reclaims_due" ) } diff --git a/rust/src/runtime/upgrade_tests.rs b/rust/src/runtime/upgrade_tests.rs index a76db6497..c2953c883 100644 --- a/rust/src/runtime/upgrade_tests.rs +++ b/rust/src/runtime/upgrade_tests.rs @@ -27,12 +27,15 @@ async fn php_layout() -> (tempfile::TempDir, String) { "\n", include_str!("../../migrations/sqlite/0004_worker_incarnations.sql"), "\n", - include_str!("../../migrations/sqlite/0005_worker_session_holders.sql") + include_str!("../../migrations/sqlite/0005_worker_session_holders.sql"), + "\n", + include_str!("../../migrations/sqlite/0006_stream_reclaims.sql") )) .execute(&mut connection) .await .unwrap(); for table in [ + "dw_stream_reclaims", "dw_worker_session_holders", "dw_worker_registration_incarnations", "dw_task_completions", diff --git a/rust/src/runtime/workflow_streams.rs b/rust/src/runtime/workflow_streams.rs index e8add3010..a9c69e59f 100644 --- a/rust/src/runtime/workflow_streams.rs +++ b/rust/src/runtime/workflow_streams.rs @@ -53,9 +53,13 @@ const MAX_PENDING_ITEMS_MAX: i64 = 10_000; /// `WorkflowStreamService::RETENTION_SECONDS_MAX` (30 days). const RETENTION_SECONDS_MAX: i64 = 2_592_000; /// `WorkflowStreamRetention::ITEM_BATCH`: items deleted per transaction. -const SWEEP_ITEM_BATCH: i64 = 500; +const SWEEP_ITEM_BATCH: i64 = 100; /// `WorkflowStreamRetention::INLINE_STREAM_LIMIT` and `INLINE_ITEM_BUDGET`. -pub(super) const INLINE_SWEEP: (i64, i64) = (10, 1000); +pub(super) const INLINE_SWEEP: (i64, i64) = (10, 500); +/// A namespace's due streams in deadline order, read from the +/// `(namespace, reclaim_at, stream_id)` index of `dw_stream_reclaims`. +pub(super) const DUE_STREAMS: &str = + "SELECT stream_id FROM dw_stream_reclaims WHERE namespace=$1 AND reclaim_at<=$2 ORDER BY reclaim_at,stream_id LIMIT $3"; /// `LegacyV1ProjectionContract::IMPORT_SOURCE`. const PROJECTION_SOURCE: &str = "waterline_v1_projection"; /// `readWithWait` checks again every 250 ms. @@ -119,10 +123,25 @@ pub(crate) enum Appended { accepted: usize, deduped: usize, }, - /// The cap stopped the batch after committing the items before it. + /// The batch would pass the cap, so none of it was written. Full { pending_items: i64 }, Closed, Errored, + /// Past its retention the stream reads as not found, append included. + Expired, + /// The run is gone, or retention has pruned its details: it has no + /// streams to write (Server #774). + RunPruned, +} + +/// One retention pass over a namespace's due streams. +#[derive(Default)] +pub(crate) struct StreamReclaim { + pub(crate) streams_reclaimed: i64, + pub(crate) items_removed: i64, + pub(crate) budget_exhausted: bool, + /// Why the pass stopped short of a due stream. + pub(crate) blocked_reason: Option<&'static str>, } /// One `read()` window. @@ -271,6 +290,18 @@ where ) -> Result { let at = now().trunc_subsecs(0); let mut tx = self.begin().await?; + // Run retention deletes a run's streams and marks its details pruned + // under the same transition lock, so an append after it must not open + // a stream on the pruned run, as PHP's `lockStream` refuses it. + let pruned = Self::query("SELECT details_pruned_at FROM workflow_runs WHERE id=$1 AND namespace=$2") + .bind(run_id) + .bind(self.namespace.as_ref()) + .fetch_optional(&mut *tx) + .await?; + match pruned { + Some(row) if DB::optional_instant(&row, "details_pruned_at")?.is_none() => {} + _ => return Ok(Appended::RunPruned), + } let found = Self::query(STREAM_BY_NAME) .bind(self.namespace.as_ref()) .bind(run_id) @@ -298,6 +329,12 @@ where .await? } }; + // From the expiry instant a stream reads as not found for append + // too. Its row stays until its run is pruned, so the name is never + // opened again at offset 0 (Server #774). + if Self::stream_expired(&row, at)? { + return Ok(Appended::Expired); + } match DB::string(&row, "status")?.as_str() { "closed" => return Ok(Appended::Closed), "errored" => return Ok(Appended::Errored), @@ -385,38 +422,48 @@ where Ok(Appended::Accepted { stream: Self::stream_json(&row)?, offsets, accepted, deduped }) } - /// `WorkflowStreamRetention::pruneExpired` for this namespace: at most - /// `stream_limit` expired streams, oldest close first, and at most - /// `item_budget` items, deleted in transactions of at most 500 under the - /// transition lock. A stream row goes in the transaction that finds it - /// empty, and the next pass resumes with the same oldest streams. Returns - /// the streams removed, the items deleted and whether the budget ran out. - pub(super) async fn prune_expired_streams(&self, stream_limit: i64, item_budget: i64) -> Result<(i64, i64, bool)> { + /// `WorkflowStreamRetention::pruneExpired` for this namespace (Server + /// #774). It reads at most `stream_limit` due streams in deadline order + /// from `dw_stream_reclaims` and deletes at most `item_budget` of their + /// items, in transactions of at most SWEEP_ITEM_BATCH under the + /// transition lock. A stream with no items left leaves + /// `dw_stream_reclaims`; its row stays until its run is pruned. Native + /// settles no external payloads, so a stream whose items hold an + /// external reference stops the pass, blocked, with its items kept. + pub(super) async fn prune_expired_streams(&self, stream_limit: i64, item_budget: i64) -> Result { let at = now(); - let candidates = Self::query(DB::EXPIRED_STREAMS) + let due = Self::query(DUE_STREAMS) .bind(self.namespace.as_ref()) .bind(DB::bind_time(at)) .bind(stream_limit.max(1)) .fetch_all(&self.pool) .await? .iter() - .map(|row| DB::number(row, "id")) + .map(|row| DB::number(row, "stream_id")) .collect::>>()?; let budget = item_budget.max(1); - let (mut streams, mut items) = (0, 0); - for stream_id in candidates { + let mut report = StreamReclaim::default(); + for stream_id in due { loop { - if items >= budget { - return Ok((streams, items, true)); + if report.items_removed >= budget { + report.budget_exhausted = true; + return Ok(report); } - let take = SWEEP_ITEM_BATCH.min(budget - items); + let take = SWEEP_ITEM_BATCH.min(budget - report.items_removed); let mut tx = self.begin().await?; let Some(row) = Self::query(STREAM_BY_ID).bind(stream_id).fetch_optional(&mut *tx).await? else { + // The stream went with its run; its deadline goes too. + Self::query("DELETE FROM dw_stream_reclaims WHERE stream_id=$1").bind(stream_id).execute(&mut *tx).await?; + tx.commit().await?; break; }; if !Self::stream_expired(&row, at)? { break; } + if Self::scalar(DB::STREAM_ITEM_REFERENCES).bind(stream_id).fetch_one(&mut *tx).await? > 0 { + report.blocked_reason = Some("external_payload_cleanup_unavailable"); + return Ok(report); + } // The offset of the `take`th item, when that many remain. let through = Self::scalar(DB::STREAM_SWEEP_BOUNDARY) .bind(stream_id) @@ -426,21 +473,23 @@ where if let Some(offset) = through { let deleted = Self::query(DB::STREAM_SWEEP_THROUGH).bind(stream_id).bind(offset).execute(&mut *tx).await?; tx.commit().await?; - items += DB::affected(deleted) as i64; + report.items_removed += DB::affected(deleted) as i64; continue; } let deleted = Self::query("DELETE FROM workflow_durable_stream_items WHERE stream_id=$1") .bind(stream_id) .execute(&mut *tx) .await?; - Self::query("DELETE FROM workflow_durable_streams WHERE id=$1").bind(stream_id).execute(&mut *tx).await?; + // The terminal row keeps the name; without a deadline the + // pass no longer reads it. + Self::query("DELETE FROM dw_stream_reclaims WHERE stream_id=$1").bind(stream_id).execute(&mut *tx).await?; tx.commit().await?; - items += DB::affected(deleted) as i64; - streams += 1; + report.items_removed += DB::affected(deleted) as i64; + report.streams_reclaimed += 1; break; } } - Ok((streams, items, false)) + Ok(report) } /// `WorkflowStreamService::read` for a valid name and offset. @@ -534,6 +583,14 @@ where .bind(stream_id) .execute(&mut *tx) .await?; + // The retention pass's deadline: the instant the stream expires. + Self::query("INSERT INTO dw_stream_reclaims(stream_id,namespace,workflow_run_id,reclaim_at) VALUES ($1,$2,$3,$4)") + .bind(stream_id) + .bind(self.namespace.as_ref()) + .bind(run_id) + .bind(DB::bind_time(at + chrono::Duration::seconds(retention))) + .execute(&mut *tx) + .await?; let row = Self::query(STREAM_BY_ID).bind(stream_id).fetch_one(&mut *tx).await?; tx.commit().await?; Ok(Some(Self::stream_json(&row)?)) @@ -800,6 +857,7 @@ pub(super) async fn append( "pending_items": pending_items, "max_pending_items": max_pending, }))), + Appended::Expired | Appended::RunPruned => Err(stream_not_found()), Appended::Closed => Err(protocol(StatusCode::CONFLICT, json!({"message":"Stream is closed.","reason":"stream_closed"}))), Appended::Errored => Err(protocol(StatusCode::CONFLICT, json!({"message":"Stream is errored.","reason":"stream_errored"}))), } diff --git a/rust/tests/execution/workflow_stream_bounds.rs b/rust/tests/execution/workflow_stream_bounds.rs index 23949dd9d..8beafc128 100644 --- a/rust/tests/execution/workflow_stream_bounds.rs +++ b/rust/tests/execution/workflow_stream_bounds.rs @@ -1,8 +1,9 @@ use super::*; // The durable stream producer contract of Server #774, natively: bounded -// producer controls, whole-batch capacity under contention, expiry at read -// and a bounded, resumable retention pass. +// producer controls, whole-batch capacity under contention, expiry for every +// operation that keeps the stream's identity, and a bounded, resumable +// retention pass that reads due streams from an index. fn route(workflow: &Value, run: &Value, stream: &str, tail: &str) -> String { format!("/api/workflows/{}/runs/{}/streams/{stream}{tail}", workflow.as_str().unwrap(), run.as_str().unwrap()) @@ -135,8 +136,10 @@ async fn an_expired_stream_reads_as_not_found() { let (_, listed) = request(&app, "GET", &list, Value::Null).await; let names: Vec<_> = listed["streams"].as_array().unwrap().iter().map(|stream| stream["stream_name"].clone()).collect(); assert_eq!(names, vec![json!("kept")], "{listed}"); + // Append is not found too, and writes nothing. let (status, body) = request(&app, "POST", &route(workflow, run, "tokens", "/items"), json!({"items":items(1)})).await; - assert_eq!((status, &body["reason"]), (StatusCode::CONFLICT, &json!("stream_closed")), "{body}"); + assert_eq!((status, &body["reason"]), (StatusCode::NOT_FOUND, &json!("stream_not_found")), "{body}"); + assert_eq!((usage(&app, "workflow_streams").await, usage(&app, "workflow_stream_items").await), (2, 2)); } #[tokio::test] @@ -174,15 +177,182 @@ async fn the_retention_pass_reclaims_expired_streams_in_bounded_resumable_steps( assert!(status == StatusCode::OK, "{status}: {body}"); } }; + let deadlines = || task_retry::rows(&database, "SELECT COUNT(*) FROM dw_stream_reclaims"); + assert_eq!(deadlines().await, 3); // limit 1: one stream and at most 100 items per pass; the next pass - // resumes where it stopped. + // resumes where it stopped. Reclaimed streams keep their rows. pass(1).await; assert_eq!((usage(&app, "workflow_streams").await, usage(&app, "workflow_stream_items").await), (4, 53)); pass(1).await; - assert_eq!((usage(&app, "workflow_streams").await, usage(&app, "workflow_stream_items").await), (3, 3)); + assert_eq!((usage(&app, "workflow_streams").await, usage(&app, "workflow_stream_items").await), (4, 3)); pass(1).await; - assert_eq!((usage(&app, "workflow_streams").await, usage(&app, "workflow_stream_items").await), (2, 2)); - // The long-retention and open streams stay. + assert_eq!((usage(&app, "workflow_streams").await, usage(&app, "workflow_stream_items").await), (4, 2)); + // The long-retention and open streams stay; only the long one still has + // a deadline. pass(100).await; - assert_eq!((usage(&app, "workflow_streams").await, usage(&app, "workflow_stream_items").await), (2, 2)); + assert_eq!((usage(&app, "workflow_streams").await, usage(&app, "workflow_stream_items").await), (4, 2)); + assert_eq!(deadlines().await, 1); + // A reclaimed stream's name is never opened again. + let (status, body) = request(&app, "POST", &route(workflow, run, "large", "/items"), json!({"items":items(1)})).await; + assert_eq!((status, &body["reason"]), (StatusCode::NOT_FOUND, &json!("stream_not_found")), "{body}"); + assert_eq!(usage(&app, "workflow_stream_items").await, 2); +} + +#[tokio::test] +async fn a_stream_whose_items_hold_external_references_is_left_with_its_items() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + let run = start(&app, "stream-references").await; + let (workflow, run) = (&run["workflow_id"], &run["run_id"]); + let (status, _) = request(&app, "POST", &route(workflow, run, "tokens", "/items"), json!({"items":items(2)})).await; + assert_eq!(status, StatusCode::OK); + let (status, _) = request(&app, "POST", &route(workflow, run, "tokens", "/close"), json!({"retention_seconds":1})).await; + assert_eq!(status, StatusCode::OK); + // Native admits no external reference and settles none, so a row + // holding one, written here directly, keeps its stream unreclaimed. + database.execute("UPDATE workflow_durable_stream_items SET payload_reference='file:///php/stream-item.bin'").await; + tokio::time::sleep(std::time::Duration::from_millis(2100)).await; + let (status, body) = request(&app, "POST", "/api/system/retention/pass", json!({})).await; + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!(usage(&app, "workflow_stream_items").await, 2); + assert_eq!(task_retry::rows(&database, "SELECT COUNT(*) FROM dw_stream_reclaims").await, 1); +} + +async fn completed_run(app: &Router, workflow: &str, stream_items: usize) -> String { + let started = start(app, workflow).await; + let (status, _) = request(app, "POST", &route(&started["workflow_id"], &started["run_id"], "tokens", "/items"), json!({"items":items(stream_items)})).await; + assert_eq!(status, StatusCode::OK); + let (status, _) = request(app, "POST", &route(&started["workflow_id"], &started["run_id"], "tokens", "/close"), json!({})).await; + assert_eq!(status, StatusCode::OK); + let task = poll(app, "worker", "workflow").await; + let complete = json!({"type":"complete_workflow","result":envelope(Payload::Long(1))}); + assert_eq!(finish_task(app, &task, json!([complete])).await.0, StatusCode::OK); + started["run_id"].as_str().unwrap().to_owned() +} + +#[tokio::test] +async fn run_retention_removes_a_pruned_runs_streams_unless_they_hold_references() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + register(&app, "worker", json!(["echo"]), json!([])).await; + let pruned = completed_run(&app, "stream-run-pruned", 2).await; + let held = completed_run(&app, "stream-run-held", 1).await; + for run in [&pruned, &held] { + for table in ["workflow_runs", "workflow_run_summaries"] { + database + .execute(sqlx::AssertSqlSafe(format!("UPDATE {table} SET closed_at='2020-01-01 00:00:00.000000' WHERE id='{run}'"))) + .await; + } + } + database + .execute(sqlx::AssertSqlSafe(format!("UPDATE workflow_durable_stream_items SET payload_reference='file:///php/held.bin' WHERE workflow_run_id='{held}'"))) + .await; + + let (status, body) = request(&app, "POST", "/api/system/retention/pass", json!({"run_ids":[pruned, held]})).await; + assert_eq!(status, StatusCode::OK, "{body}"); + let result = &body["results"][0]; + assert_eq!(result["outcome"], "pruned", "{body}"); + assert_eq!(result["deleted"]["durable_stream_items_deleted"], 2, "{body}"); + assert_eq!(result["deleted"]["durable_streams_deleted"], 1, "{body}"); + assert_eq!(body["results"][1]["reason"], "external_payload_cleanup_unavailable", "{body}"); + // A pruned run has no streams to write: a later append opens none. + let (status, body) = request(&app, "POST", &route(&json!("stream-run-pruned"), &json!(pruned), "late", "/items"), json!({"items":items(1)})).await; + assert_eq!((status, body["reason"].clone()), (StatusCode::NOT_FOUND, json!("stream_not_found")), "{body}"); + for (sql, expected) in [ + (format!("SELECT COUNT(*) FROM workflow_durable_streams WHERE workflow_run_id='{pruned}'"), 0), + (format!("SELECT COUNT(*) FROM dw_stream_reclaims WHERE workflow_run_id='{pruned}'"), 0), + (format!("SELECT COUNT(*) FROM workflow_durable_stream_items WHERE workflow_run_id='{held}'"), 1), + (format!("SELECT COUNT(*) FROM dw_stream_reclaims WHERE workflow_run_id='{held}'"), 1), + ] { + let sql: &'static str = Box::leak(sql.into_boxed_str()); + assert_eq!(task_retry::rows(&database, sql).await, expected, "{sql}"); + } +} + +/// The retention pass reads a namespace's due streams from +/// `dw_stream_reclaims_due`: an index range in deadline order that stops at +/// its limit, whatever else the namespaces hold. +#[tokio::test] +async fn due_stream_discovery_reads_the_deadline_index() { + use sqlx::{Connection, Row}; + let database = TestDatabase::new().await; + let _app = router(database.open().await.unwrap()); + // 2,000 deadlines in two namespaces; 30 of `default`'s are due. + let mut values = Vec::new(); + for (n, namespace) in ["default", "elsewhere"].into_iter().enumerate() { + for i in 0..1000 { + let id = n * 1000 + i + 1; + let at = if namespace == "default" && i < 30 { "2020-01-01 00:00:00" } else { "2099-01-01 00:00:00" }; + values.push(format!("({id},'{namespace}','run-{id}','{at}')")); + } + } + for chunk in values.chunks(500) { + database + .execute(sqlx::AssertSqlSafe(format!( + "INSERT INTO dw_stream_reclaims(stream_id,namespace,workflow_run_id,reclaim_at) VALUES {}", + chunk.join(",") + ))) + .await; + } + let sql = "SELECT stream_id FROM dw_stream_reclaims WHERE namespace='default' AND reclaim_at<='2026-01-01 00:00:00' ORDER BY reclaim_at,stream_id LIMIT 10"; + let plan = match &database { + TestDatabase::Sqlite(dir) => { + let pool = sqlx::SqlitePool::connect(&format!("sqlite://{}", dir.path().join("runtime.sqlite").display())).await.unwrap(); + let rows: Vec<(i64, i64, i64, String)> = + sqlx::query_as(sqlx::AssertSqlSafe(format!("EXPLAIN QUERY PLAN {sql}"))).fetch_all(&pool).await.unwrap(); + pool.close().await; + let plan = rows.into_iter().map(|row| row.3).collect::>().join("\n"); + assert!(plan.contains("USING COVERING INDEX dw_stream_reclaims_due (namespace=? AND reclaim_at { + let mut connection = sqlx::PgConnection::connect_with(options).await.unwrap(); + sqlx::query("ANALYZE dw_stream_reclaims").execute(&mut connection).await.unwrap(); + // A small table plans as a sequential scan; without one, the plan + // shows the index serves both the predicate and the order. + let mut tx = connection.begin().await.unwrap(); + sqlx::query("SET LOCAL enable_seqscan=off").execute(&mut *tx).await.unwrap(); + let rows: Vec<(String,)> = sqlx::query_as(sqlx::AssertSqlSafe(format!( + "EXPLAIN (ANALYZE, COSTS OFF, TIMING OFF, SUMMARY OFF) {sql}" + ))) + .fetch_all(&mut *tx) + .await + .unwrap(); + tx.rollback().await.unwrap(); + let plan = rows.into_iter().map(|row| row.0).collect::>().join("\n"); + assert!(plan.contains("Index Only Scan using dw_stream_reclaims_due"), "{plan}"); + assert!(plan.contains("actual rows=10 loops=1"), "{plan}"); + assert!(!plan.contains("Sort"), "{plan}"); + plan + } + TestDatabase::MySql { options, .. } => { + let mut connection = sqlx::MySqlConnection::connect_with(options).await.unwrap(); + sqlx::raw_sql("ANALYZE TABLE dw_stream_reclaims").execute(&mut connection).await.unwrap(); + let explained = sqlx::raw_sql(sqlx::AssertSqlSafe(format!("EXPLAIN {sql}"))).fetch_all(&mut connection).await.unwrap(); + assert_eq!(explained.len(), 1); + let field = |name: &str| explained[0].try_get::, _>(name).unwrap().unwrap_or_default(); + let plan = format!("type={} key={} extra={}", field("type"), field("key"), field("Extra")); + assert_eq!((field("type"), field("key")), ("range".into(), "dw_stream_reclaims_due".into()), "{plan}"); + assert!(!plan.contains("filesort"), "{plan}"); + // The first use of a table or status view reads the data + // dictionary, which MySQL counts too; measure a second read. + sqlx::raw_sql(sql).fetch_all(&mut connection).await.unwrap(); + sqlx::raw_sql("SHOW SESSION STATUS LIKE 'Handler_read%'").fetch_all(&mut connection).await.unwrap(); + sqlx::raw_sql("FLUSH STATUS").execute(&mut connection).await.unwrap(); + assert_eq!(sqlx::raw_sql(sql).fetch_all(&mut connection).await.unwrap().len(), 10); + let reads: std::collections::BTreeMap = sqlx::raw_sql("SHOW SESSION STATUS LIKE 'Handler_read%'") + .fetch_all(&mut connection) + .await + .unwrap() + .iter() + .map(|row| (row.try_get::(0).unwrap(), row.try_get::(1).unwrap().parse().unwrap())) + .collect(); + // One seek and nine index steps: the limit, not the namespace. + assert!(reads["Handler_read_key"] + reads["Handler_read_next"] <= 11, "{reads:?}"); + assert_eq!(reads["Handler_read_rnd_next"], 0, "{reads:?}"); + format!("{plan} reads={reads:?}") + } + }; + eprintln!("[due-stream plan]\n{plan}"); } diff --git a/scripts/conformance/server-parity/mysql-schema.php b/scripts/conformance/server-parity/mysql-schema.php index 1a71793d7..0d828a2cf 100644 --- a/scripts/conformance/server-parity/mysql-schema.php +++ b/scripts/conformance/server-parity/mysql-schema.php @@ -106,7 +106,7 @@ function catalog(PDO $pdo): array } $pdo->exec('ROLLBACK'); $pdo = connection($argv[3]); -$native = array_values(array_filter(catalog($pdo), static fn (array $row): bool => ! in_array($row[0], ['dw_server_schema', '_sqlx_migrations', 'dw_task_completions', 'dw_poll_receipts', 'dw_query_cache', 'dw_worker_registration_incarnations', 'dw_worker_session_holders'], true))); +$native = array_values(array_filter(catalog($pdo), static fn (array $row): bool => ! in_array($row[0], ['dw_server_schema', '_sqlx_migrations', 'dw_task_completions', 'dw_poll_receipts', 'dw_query_cache', 'dw_worker_registration_incarnations', 'dw_worker_session_holders', 'dw_stream_reclaims'], true))); $pdo->exec('ROLLBACK'); // Native polling uses an additional index on an existing table. Retain all // original column, index and constraint definitions in the comparison. diff --git a/scripts/conformance/server-parity/postgres-schema.php b/scripts/conformance/server-parity/postgres-schema.php index 7ea05ff49..236486e51 100644 --- a/scripts/conformance/server-parity/postgres-schema.php +++ b/scripts/conformance/server-parity/postgres-schema.php @@ -70,8 +70,8 @@ function catalog(string $database): array throw new InvalidArgumentException('Compare requires two independent databases.'); } $native = catalog($argv[3]); -$tables = ['_sqlx_migrations', 'dw_server_schema', 'dw_task_completions', 'dw_poll_receipts', 'dw_query_cache', 'dw_worker_registration_incarnations', 'dw_worker_session_holders']; -$indexes = ['_sqlx_migrations_pkey', 'dw_task_completions_pkey', 'dw_poll_receipts_pkey', 'dw_workflow_tasks_poll', 'dw_query_cache_pkey', 'dw_worker_registration_incarnations_pkey', 'dw_worker_incarnation_authority', 'dw_worker_incarnation_retention', 'dw_worker_session_holders_pkey']; +$tables = ['_sqlx_migrations', 'dw_server_schema', 'dw_task_completions', 'dw_poll_receipts', 'dw_query_cache', 'dw_worker_registration_incarnations', 'dw_worker_session_holders', 'dw_stream_reclaims']; +$indexes = ['_sqlx_migrations_pkey', 'dw_task_completions_pkey', 'dw_poll_receipts_pkey', 'dw_workflow_tasks_poll', 'dw_query_cache_pkey', 'dw_worker_registration_incarnations_pkey', 'dw_worker_incarnation_authority', 'dw_worker_incarnation_retention', 'dw_worker_session_holders_pkey', 'dw_stream_reclaims_pkey', 'dw_stream_reclaims_due']; $native = array_values(array_filter($native, static function (array $row) use ($tables, $indexes): bool { if (in_array($row[0], ['column', 'constraint'], true)) { return ! in_array(explode('.', $row[1], 2)[0], $tables, true); diff --git a/scripts/conformance/server-parity/schema.php b/scripts/conformance/server-parity/schema.php index 5929ce8db..f52b18e84 100644 --- a/scripts/conformance/server-parity/schema.php +++ b/scripts/conformance/server-parity/schema.php @@ -38,7 +38,7 @@ function catalog(string $filename): array $php = catalog($argv[1]); $native = catalog($argv[2]); -$nativeOnly = ['_sqlx_migrations', 'dw_server_schema', 'dw_task_completions', 'dw_poll_receipts', 'dw_workflow_tasks_poll', 'dw_query_cache', 'dw_worker_registration_incarnations', 'dw_worker_incarnation_authority', 'dw_worker_incarnation_retention', 'dw_worker_session_holders']; +$nativeOnly = ['_sqlx_migrations', 'dw_server_schema', 'dw_task_completions', 'dw_poll_receipts', 'dw_workflow_tasks_poll', 'dw_query_cache', 'dw_worker_registration_incarnations', 'dw_worker_incarnation_authority', 'dw_worker_incarnation_retention', 'dw_worker_session_holders', 'dw_stream_reclaims', 'dw_stream_reclaims_due']; $native = array_values(array_filter($native, static fn (array $row): bool => ! in_array($row['name'], $nativeOnly, true))); if ($php !== $native) { $byName = static fn (array $rows): array => array_column($rows, null, 'name'); From 0b39fd74c1955e92a75578f1738e47353e9897c8 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Mon, 26 Oct 2026 18:26:56 +0000 Subject: [PATCH 6/6] Rust #325: log native stream reclamation, run retention and pruned-run refusal Co-Authored-By: Claude Opus 5.5 --- docs/rust-server-port.md | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/docs/rust-server-port.md b/docs/rust-server-port.md index a5b9ed946..a82e4a1dc 100644 --- a/docs/rust-server-port.md +++ b/docs/rust-server-port.md @@ -942,9 +942,18 @@ October 21, Waterline reads: - a batch whose fresh items would pass the cap is refused whole, so nothing of it is written (published 2.11.1 committed the items before the cap; the fixture bounds that answer as a published defect); - - a closed stream past its retention reads as not found; - - a bounded pass reclaims expired streams from the worker-heartbeat - retention pass and from a general `POST /api/system/retention/pass`. + - a closed stream past its retention reads as not found, append + included, and keeps its row until its run is pruned, so the name is + never opened again; + - each deadline is a row in the native-owned `dw_stream_reclaims` + (schema 6), indexed `(namespace, reclaim_at, stream_id)`. A bounded pass + reads due streams from that index, from the worker-heartbeat retention + pass and from a general `POST /api/system/retention/pass`. It deletes + items in batches under the transition lock and stops blocked when + items hold external references, which native does not delete yet; + - run retention deletes a pruned run's streams unless they hold external + references, and an append to a run whose details are pruned answers + 404 `stream_not_found`. A continued run keeps its streams, as in PHP. Native differs in three places, documented in `rust/src/runtime/workflow_streams.rs`: - a batch whose items hold no item fields gets 400, not PHP's 500;