From 41a7a5f8eb2e590b47228bcac8eebb543c7db1ad Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Sat, 24 Oct 2026 15:02:16 +0000 Subject: [PATCH 1/9] Server parity: record_local_activity fixture, recorded from published 2.11.1 Raw workers complete workflow tasks with record_local_activity commands: the advertised local_activities contract and registration agreement, the protocol floor, capability and ownership gates, transport, scope and normalizer refusals, a success after a retried attempt, a failure after retries and a timeout, and the protocol 1.18 shape without attempts. History payloads, the run's activity view and status are recorded with identities as case-keeping labels and attempt times as offsets from each close. Recorded three times, identical. One published defect is bounded: transport validation reorders a nested report whose first entry lacks an optional field, so the normalizer refuses it as unordered. --- .github/workflows/server-parity.yml | 4 +- .../conformance/server-parity/contract.mjs | 3 + scripts/conformance/server-parity/probe.php | 6 + .../record-local-activity-contract.mjs | 44 + .../record-local-activity-probe.php | 404 +++ .../ServerParity/record-local-activity.json | 3231 +++++++++++++++++ .../record-local-activity-observation.json | 3070 ++++++++++++++++ ...rParityRecordLocalActivityContractTest.mjs | 99 + 8 files changed, 6860 insertions(+), 1 deletion(-) create mode 100644 scripts/conformance/server-parity/record-local-activity-contract.mjs create mode 100644 scripts/conformance/server-parity/record-local-activity-probe.php create mode 100644 tests/Fixtures/ServerParity/record-local-activity.json create mode 100644 tests/Fixtures/ServerParityExpected/record-local-activity-observation.json create mode 100644 tests/Unit/ServerParityRecordLocalActivityContractTest.mjs diff --git a/.github/workflows/server-parity.yml b/.github/workflows/server-parity.yml index 9960470e7..a1423d39b 100644 --- a/.github/workflows/server-parity.yml +++ b/.github/workflows/server-parity.yml @@ -35,6 +35,7 @@ on: - tests/Unit/ServerParityWorkerReplacementContractTest.mjs - tests/Unit/ServerParityOperatorMetricsContractTest.mjs - tests/Unit/ServerParityActivityLeaseRenewalContractTest.mjs + - tests/Unit/ServerParityRecordLocalActivityContractTest.mjs - tests/Unit/ServerParityServiceCatalogContractTest.mjs - tests/Unit/ServerParitySystemMetricsContractTest.mjs - tests/Unit/ServerParityOperatorDashboardTypesContractTest.mjs @@ -91,6 +92,7 @@ on: - tests/Unit/ServerParityWorkerReplacementContractTest.mjs - tests/Unit/ServerParityOperatorMetricsContractTest.mjs - tests/Unit/ServerParityActivityLeaseRenewalContractTest.mjs + - tests/Unit/ServerParityRecordLocalActivityContractTest.mjs - tests/Unit/ServerParityServiceCatalogContractTest.mjs - tests/Unit/ServerParitySystemMetricsContractTest.mjs - tests/Unit/ServerParityOperatorDashboardTypesContractTest.mjs @@ -1529,7 +1531,7 @@ jobs: -e COMPOSER_HOME=/tmp/composer -v "$PWD:/source" -w /source "$image" \ install --working-dir=scripts/conformance/server-parity --no-interaction --no-progress docker run --rm --user=1000:1000 --entrypoint node -v "$PWD:/source:ro" -w /source \ - "$image" --test tests/Unit/ServerParityRunnerTest.mjs tests/Unit/ServerParityCooperativeContractTest.mjs tests/Unit/ServerParityChildCancellationContractTest.mjs tests/Unit/ServerParityScheduleContractTest.mjs tests/Unit/ServerParityVisibilityContractTest.mjs tests/Unit/ServerParityAdmissionContractTest.mjs tests/Unit/ServerParityRoleTokenContractTest.mjs tests/Unit/ServerParityWorkerDeregistrationContractTest.mjs tests/Unit/ServerParitySearchAttributeContractTest.mjs tests/Unit/ServerParityWorkerListingContractTest.mjs tests/Unit/ServerParityWorkerReplacementContractTest.mjs tests/Unit/ServerParityPatchDeploymentContractTest.mjs tests/Unit/ServerParityWaitingHistoryContractTest.mjs tests/Unit/ServerParityTaskRetryContractTest.mjs tests/Unit/ServerParityPublishedPatchContractTest.mjs tests/Unit/ServerParityPublishedPatchTransportTest.mjs tests/Unit/ServerParityRecordFileTest.mjs tests/Unit/ServerParityLateWorkflowCompletionContractTest.mjs tests/Unit/ServerParityStorageTestContractTest.mjs tests/Unit/ServerParitySchedulePoliciesContractTest.mjs tests/Unit/ServerParitySignalsBeforeWorkerContractTest.mjs tests/Unit/ServerParityScheduleCatalogContractTest.mjs tests/Unit/ServerParityOperatorMetricsContractTest.mjs tests/Unit/ServerParityActivityLeaseRenewalContractTest.mjs tests/Unit/ServerParityServiceCatalogContractTest.mjs tests/Unit/ServerParitySystemMetricsContractTest.mjs tests/Unit/ServerParityOperatorDashboardTypesContractTest.mjs tests/Unit/ServerParityPrometheusMetricsContractTest.mjs tests/Unit/ServerParityPayloadCleanupContractTest.mjs tests/Unit/ServerParityRepairPassContractTest.mjs tests/Unit/ServerParityClusterInfoContractTest.mjs tests/Unit/ServerParityNamespaceDeleteContractTest.mjs > parity-evidence/comparator-tests.txt + "$image" --test tests/Unit/ServerParityRunnerTest.mjs tests/Unit/ServerParityCooperativeContractTest.mjs tests/Unit/ServerParityChildCancellationContractTest.mjs tests/Unit/ServerParityScheduleContractTest.mjs tests/Unit/ServerParityVisibilityContractTest.mjs tests/Unit/ServerParityAdmissionContractTest.mjs tests/Unit/ServerParityRoleTokenContractTest.mjs tests/Unit/ServerParityWorkerDeregistrationContractTest.mjs tests/Unit/ServerParitySearchAttributeContractTest.mjs tests/Unit/ServerParityWorkerListingContractTest.mjs tests/Unit/ServerParityWorkerReplacementContractTest.mjs tests/Unit/ServerParityPatchDeploymentContractTest.mjs tests/Unit/ServerParityWaitingHistoryContractTest.mjs tests/Unit/ServerParityTaskRetryContractTest.mjs tests/Unit/ServerParityPublishedPatchContractTest.mjs tests/Unit/ServerParityPublishedPatchTransportTest.mjs tests/Unit/ServerParityRecordFileTest.mjs tests/Unit/ServerParityLateWorkflowCompletionContractTest.mjs tests/Unit/ServerParityStorageTestContractTest.mjs tests/Unit/ServerParitySchedulePoliciesContractTest.mjs tests/Unit/ServerParitySignalsBeforeWorkerContractTest.mjs tests/Unit/ServerParityScheduleCatalogContractTest.mjs tests/Unit/ServerParityOperatorMetricsContractTest.mjs tests/Unit/ServerParityActivityLeaseRenewalContractTest.mjs tests/Unit/ServerParityRecordLocalActivityContractTest.mjs tests/Unit/ServerParityServiceCatalogContractTest.mjs tests/Unit/ServerParitySystemMetricsContractTest.mjs tests/Unit/ServerParityOperatorDashboardTypesContractTest.mjs tests/Unit/ServerParityPrometheusMetricsContractTest.mjs tests/Unit/ServerParityPayloadCleanupContractTest.mjs tests/Unit/ServerParityRepairPassContractTest.mjs tests/Unit/ServerParityClusterInfoContractTest.mjs tests/Unit/ServerParityNamespaceDeleteContractTest.mjs > parity-evidence/comparator-tests.txt docker run --rm --user=1000:1000 --cpus=1 --memory=256m --memory-swap=256m \ --network=none --entrypoint node -v "$PWD:/source:ro" -w /source \ "$image" --test tests/Unit/ServerParityCodecDeadlineLoadTest.mjs > parity-evidence/codec-load-tests.txt diff --git a/scripts/conformance/server-parity/contract.mjs b/scripts/conformance/server-parity/contract.mjs index 69b947698..46c028f8a 100644 --- a/scripts/conformance/server-parity/contract.mjs +++ b/scripts/conformance/server-parity/contract.mjs @@ -13,6 +13,7 @@ import {checkLateWorkflowCompletion} from './late-workflow-completion-contract.m import {checkSearchAttributes} from './search-attribute-contract.mjs'; import {checkStorageTest} from './storage-test-contract.mjs'; import {checkActivityLeaseRenewal} from './activity-lease-renewal-contract.mjs'; +import {checkRecordLocalActivity} from './record-local-activity-contract.mjs'; import {checkServiceCatalog} from './service-catalog-contract.mjs'; import {checkSystemMetrics} from './system-metrics-contract.mjs'; import {checkOperatorDashboardTypes} from './operator-dashboard-types-contract.mjs'; @@ -151,6 +152,7 @@ export function checkObservation(fixture, observation, workflowId, artifacts) { const searchAttributes = fixture.search_attributes ? checkSearchAttributes(fixture, observation, workflowId) : null; const storageTest = fixture.storage_test ? checkStorageTest(fixture, observation, workflowId) : null; const activityLeaseRenewal = fixture.activity_lease_renewal ? checkActivityLeaseRenewal(fixture, observation, workflowId) : null; + const recordLocalActivity = fixture.record_local_activity ? checkRecordLocalActivity(fixture, observation, workflowId) : null; const serviceCatalog = fixture.service_catalog ? checkServiceCatalog(fixture, observation, workflowId) : null; const systemMetrics = fixture.system_metrics ? checkSystemMetrics(fixture, observation, workflowId) : null; const operatorDashboardTypes = fixture.operator_dashboard_types ? checkOperatorDashboardTypes(fixture, observation, workflowId) : null; @@ -706,6 +708,7 @@ export function checkObservation(fixture, observation, workflowId, artifacts) { ...(searchAttributes ? {search_attributes: searchAttributes} : {}), ...(storageTest ? {storage_test: storageTest} : {}), ...(activityLeaseRenewal ? {activity_lease_renewal: activityLeaseRenewal} : {}), + ...(recordLocalActivity ? {record_local_activity: recordLocalActivity} : {}), ...(serviceCatalog ? {service_catalog: serviceCatalog} : {}), ...(systemMetrics ? {system_metrics: systemMetrics} : {}), ...(operatorDashboardTypes ? {operator_dashboard_types: operatorDashboardTypes} : {}), diff --git a/scripts/conformance/server-parity/probe.php b/scripts/conformance/server-parity/probe.php index 4df73a65f..b234ec993 100644 --- a/scripts/conformance/server-parity/probe.php +++ b/scripts/conformance/server-parity/probe.php @@ -431,6 +431,7 @@ function httpObservation(array $fixture, string $workflowId, string $namespace, $searchAttributes = isset($fixture['search_attributes']) ? finishHttpSearchAttributes($client, $fixture, $workflowId, $url) : null; $storageTest = isset($fixture['storage_test']) ? finishHttpStorageTest($client, $fixture, $workflowId, $url) : null; $activityLeaseRenewal = isset($fixture['activity_lease_renewal']) ? finishHttpActivityLeaseRenewal($client, $fixture, $workflowId, $url) : null; + $recordLocalActivity = isset($fixture['record_local_activity']) ? finishHttpRecordLocalActivity($client, $fixture, $workflowId, $url) : null; $serviceCatalog = isset($fixture['service_catalog']) ? finishHttpServiceCatalog($client, $fixture, $workflowId, $url) : null; $systemMetrics = isset($fixture['system_metrics']) ? finishHttpSystemMetrics($client, $fixture, $workflowId, $url) : null; $operatorDashboardTypes = isset($fixture['operator_dashboard_types']) ? finishHttpOperatorDashboardTypes($client, $fixture, $workflowId, $url) : null; @@ -539,6 +540,7 @@ function httpObservation(array $fixture, string $workflowId, string $namespace, 'search_attributes' => $searchAttributes, 'storage_test' => $storageTest, 'activity_lease_renewal' => $activityLeaseRenewal, + 'record_local_activity' => $recordLocalActivity, 'service_catalog' => $serviceCatalog, 'system_metrics' => $systemMetrics, 'operator_dashboard_types' => $operatorDashboardTypes, @@ -851,6 +853,7 @@ function embeddedObservation(array $fixture, string $workflowId, string $namespa require __DIR__.'/search-attribute-probe.php'; require __DIR__.'/storage-test-probe.php'; require __DIR__.'/activity-lease-renewal-probe.php'; + require __DIR__.'/record-local-activity-probe.php'; require __DIR__.'/service-catalog-probe.php'; require __DIR__.'/system-metrics-probe.php'; require __DIR__.'/operator-dashboard-types-probe.php'; @@ -1020,6 +1023,9 @@ function embeddedObservation(array $fixture, string $workflowId, string $namespa if ($mode === 'embedded' && isset($fixture['activity_lease_renewal'])) { $observation['activity_lease_renewal'] = ['applicable' => false, 'reason' => 'embedded_has_no_worker_protocol']; } + if ($mode === 'embedded' && isset($fixture['record_local_activity'])) { + $observation['record_local_activity'] = ['applicable' => false, 'reason' => 'embedded_has_no_worker_protocol']; + } if ($mode === 'embedded' && isset($fixture['service_catalog'])) { $observation['service_catalog'] = ['applicable' => false, 'reason' => 'embedded_has_no_http_service_catalog']; } diff --git a/scripts/conformance/server-parity/record-local-activity-contract.mjs b/scripts/conformance/server-parity/record-local-activity-contract.mjs new file mode 100644 index 000000000..dca18c5ba --- /dev/null +++ b/scripts/conformance/server-parity/record-local-activity-contract.mjs @@ -0,0 +1,44 @@ +import assert from 'node:assert/strict'; + +// Reviewed record_local_activity completions (worker protocol 1.18+), +// recorded against published Server 2.11.1. The fixture's `expected` holds +// the whole normalized observation, with the probe's workflow prefix as +// `{id}`, run identities as labels that keep their letter case, attempt +// times as offsets from each local activity's close and other times as +// `{time}`. Every step compares exactly. +// +// Published defect boundary: published PHP validates a completion's +// commands through Laravel's validated(), which rebuilds nested lists in +// rule order, so a report whose first attempt lacks attempt_id, or whose +// first heartbeat lacks details, while a later entry has it is refused as +// unordered (fixture `published_defects`). At those steps the published +// refusals, all of them, or the normalizer's own answers on the reports as +// sent are accepted; nothing else is relaxed. +export function checkRecordLocalActivity(fixture, observation, workflowId) { + const state = observation.record_local_activity; + const projection = {scope: 'record_local_activity'}; + assert.ok(state && typeof state === 'object', 'actual record local activity observation'); + if (observation.mode === 'embedded') { + assert.deepStrictEqual(state, {applicable: false, reason: 'embedded_has_no_worker_protocol'}, + 'embedded record local activity explicitly inapplicable'); + return projection; + } + const definition = fixture.record_local_activity; + const actual = JSON.parse(JSON.stringify(state).replaceAll(workflowId, '{id}')); + const defect = definition.published_defects.validated_reorders_nested_reports.published; + const steps = Object.keys(defect); + let published = true; + try { + for (const step of steps) assert.deepStrictEqual(actual[step], defect[step]); + } catch { + published = false; + } + if (published) { + for (const step of steps) actual[step] = definition.expected[step]; + } + for (const [step, expected] of Object.entries(definition.expected)) { + assert.deepStrictEqual(actual[step], expected, `record local activity ${step}`); + } + assert.deepStrictEqual(Object.keys(actual).sort(), Object.keys(definition.expected).sort(), 'record local activity steps'); + return projection; +} diff --git a/scripts/conformance/server-parity/record-local-activity-probe.php b/scripts/conformance/server-parity/record-local-activity-probe.php new file mode 100644 index 000000000..7bcec812e --- /dev/null +++ b/scripts/conformance/server-parity/record-local-activity-probe.php @@ -0,0 +1,404 @@ +createNamespace($namespace, 'record local activity λ', 30); + $token = getenv('DW_PARITY_TOKEN') ?: null; + // The SQLite reference shares its database with a one-second maintenance + // loop; retry its retryable lock-pressure refusals, as the other probes do. + $stack = HandlerStack::create(); + $stack->push(LockPressurePollRetries::middleware(), 'lock_pressure_poll_retry'); + $http = new RecordLocalActivityHttp(['base_uri' => rtrim($url, '/'), 'http_errors' => false, + 'connect_timeout' => 1, 'timeout' => 30, 'handler' => $stack]); + // $protocol null is a control-plane request; otherwise the worker header. + $raw = static function (string $method, string $path, ?array $body = null, ?string $protocol = null) use ($http, $token, $namespace): array { + $headers = array_filter(['Authorization' => $token === null ? null : 'Bearer '.$token, + 'Accept' => 'application/json', 'X-Namespace' => $namespace, + $protocol === null ? 'X-Durable-Workflow-Control-Plane-Version' : 'X-Durable-Workflow-Protocol-Version' => $protocol ?? '2']); + $response = $http->request($method, $path, ['headers' => $headers] + ($body === null ? [] : ['json' => $body])); + $decoded = json_decode((string) $response->getBody(), true); + $capabilities = is_array($decoded) ? ($decoded['server_capabilities'] ?? null) : null; + if (is_array($decoded)) { + unset($decoded['server_capabilities'], $decoded['control_plane'], $decoded['protocol_version']); + } + + return ['status' => $response->getStatusCode(), 'response' => $decoded, 'capabilities' => $capabilities]; + }; + $answer = static fn (array $recorded): array => ['status' => $recorded['status'], 'response' => $recorded['response']]; + $ok = static function (array $recorded, int $status, string $operation): array { + if ($recorded['status'] !== $status) { + throw new RuntimeException('Record local activity '.$operation.': '.json_encode($recorded)); + } + + return $recorded['response']; + }; + $type = $definition['workflow_type']; + $empty = ['codec' => 'avro', 'blob' => $definition['empty_blob']]; + $done = ['codec' => 'avro', 'blob' => $definition['result_blob']]; + $contract = ['signals' => ['resume'], 'signal_contracts' => [['name' => 'resume', 'parameters' => []]], + 'queries' => [], 'updates' => [], 'update_contracts' => []]; + $register = static function (string $suffix, array $capabilities, ?array $manifest = null) use ($raw, $type, $contract, $workflowId): array { + $worker = $workflowId.'-rla-'.$suffix; + + return $raw('POST', '/api/worker/register', ['worker_id' => $worker, 'task_queue' => $worker.'-queue', + 'runtime' => 'python', 'sdk_version' => 'raw-http', 'supported_workflow_types' => [$type], + 'supported_activity_types' => [], 'workflow_command_contracts' => [$type => $contract], + 'capabilities' => $capabilities, + 'capability_manifest' => $manifest ?? CapabilityManifest::forCapabilities($capabilities)], '1.20'); + }; + // The local-activity part of an advertised capability map. The prepared + // flags' values belong to the prepared-local routes, which have their own + // fixture; here only their presence, type and agreement are observed. + $advertised = static function (?array $capabilities): array { + $prepared = $capabilities['prepared_local_activities'] ?? null; + $groups = $capabilities['prepared_local_activity_groups'] ?? null; + $policies = $capabilities['prepared_local_activity_cancellation_policies'] ?? null; + $known = ['try_cancel', 'wait_cancellation_completed']; + + return [ + 'local_activities' => is_array($capabilities) && array_key_exists('local_activities', $capabilities) + ? $capabilities['local_activities'] : '(absent)', + 'record_local_activity_command' => in_array('record_local_activity', + (array) ($capabilities['supported_workflow_task_commands'] ?? []), true), + 'prepared_flags' => [ + 'prepared_local_activities' => get_debug_type($prepared), + 'prepared_local_activity_groups' => get_debug_type($groups), + 'prepared_local_activity_cancellation_policies' => is_array($policies) && array_is_list($policies) + && array_values(array_intersect($known, $policies)) === $policies ? 'ordered subset' : get_debug_type($policies), + 'agree' => is_bool($prepared) && is_bool($groups) && is_array($policies) + && ($prepared || (! $groups && $policies === [])) && (! $groups || $prepared), + ], + ]; + }; + // Starts a run on a worker's queue and claims its first workflow task. + $claim = static function (string $suffix) use ($raw, $ok, $empty, $type, $workflowId): array { + $worker = $workflowId.'-rla-'.$suffix; + $id = $workflowId.'-rla-run-'.$suffix; + $started = $ok($raw('POST', '/api/workflows', ['workflow_id' => $id, 'workflow_type' => $type, + 'task_queue' => $worker.'-queue', 'input' => $empty]), 201, 'start '.$suffix); + $task = $ok($raw('POST', '/api/worker/workflow-tasks/poll', ['worker_id' => $worker, 'task_queue' => $worker.'-queue', + 'timeout_seconds' => 5], '1.20'), 200, 'claim '.$suffix)['task'] ?? null; + if (! is_array($task) || ($task['run_id'] ?? null) !== $started['run_id']) { + throw new RuntimeException('Record local activity claim '.$suffix.': '.json_encode($task)); + } + + return ['workflow_id' => $id, 'run_id' => $started['run_id'], 'task_id' => $task['task_id'], + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt']]; + }; + $complete = static fn (array $claimed, array $commands, string $protocol = '1.20', ?string $owner = null): array => $answer($raw('POST', + '/api/worker/workflow-tasks/'.$claimed['task_id'].'/complete', ['lease_owner' => $owner ?? $claimed['lease_owner'], + 'workflow_task_attempt' => $claimed['workflow_task_attempt'], 'commands' => $commands], $protocol)); + $wait = ['type' => 'open_signal_wait', 'signal_name' => 'resume']; + // A minimal valid record: one completed attempt. + $record = static fn (array $fields = []): array => array_merge(['type' => 'record_local_activity', + 'activity_type' => $definition['activity_type'], 'arguments' => $empty, 'result' => $done, 'outcome' => 'completed', + 'attempts' => [['attempt_id' => 'w-1', 'attempt_number' => 1, 'outcome' => 'completed', 'duration_ms' => 25]]], $fields); + $events = static fn (array $claimed): array => $ok($raw('GET', '/api/workflows/'.$claimed['workflow_id'].'/runs/' + .$claimed['run_id'].'/history?page_size=1000'), 200, 'history')['events'] ?? []; + // The run after its completion: every event type, every Activity* event + // in full, the run's activity view and its status. + $observeRun = static function (array $claimed) use ($raw, $ok, $events): array { + $history = $events($claimed); + $activities = $ok($raw('GET', '/api/workflows/'.$claimed['workflow_id'].'/runs/'.$claimed['run_id'].'/activities'), + 200, 'activities')['activities'] ?? null; + $run = $ok($raw('GET', '/api/workflows/'.$claimed['workflow_id'].'/runs/'.$claimed['run_id']), 200, 'describe'); + + $activityEvents = array_values(array_filter($history, + static fn (array $event): bool => str_starts_with((string) ($event['event_type'] ?? ''), 'Activity'))); + // Every event of one completion snapshots the same hosting task: an + // identical snapshot after the first is recorded by reference. + foreach ($activityEvents as $index => $event) { + if ($index > 0 && ($event['payload']['task'] ?? null) === ($activityEvents[0]['payload']['task'] ?? null)) { + $activityEvents[$index]['payload']['task'] = '(the first event\'s task)'; + } + } + + return [ + 'event_types' => array_column($history, 'event_type'), + 'activity_events' => $activityEvents, + 'activities' => $activities, + 'run' => array_intersect_key($run, array_flip(['status', 'status_bucket', 'is_terminal', 'closed_reason', + 'wait_kind', 'wait_reason'])), + ]; + }; + + $state = ['applicable' => true, 'namespace' => $namespace]; + + // The advertised contract. + $cluster = $ok($raw('GET', '/api/cluster/info'), 200, 'cluster info'); + $clusterCapabilities = $cluster['worker_protocol']['server_capabilities'] ?? null; + $state['cluster_info'] = $advertised($clusterCapabilities) + [ + 'cluster_capabilities_local_keys' => array_values(array_filter(array_keys((array) ($cluster['capabilities'] ?? [])), + static fn (string $key): bool => str_contains($key, 'local'))), + ]; + + // Registration: the local capability and its manifest entry agree. + $registered = $register('a', ['local_activities']); + $state['register_local'] = [ + 'status' => $registered['status'], + 'response' => array_intersect_key((array) $registered['response'], + array_flip(['registered', 'status', 'capabilities', 'capability_manifest'])), + 'advertised_as_cluster_info' => $advertised($registered['capabilities']) === $advertised($clusterCapabilities), + ]; + $state['register_flat_only'] = $answer($register('flat-only', ['local_activities'], CapabilityManifest::forCapabilities([]))); + $state['register_manifest_only'] = $answer($register('manifest-only', [], CapabilityManifest::forCapabilities(['local_activities']))); + + // Run A: refusals on one claim, then a success after a retried attempt. + $a = $claim('a'); + $state['refused_protocol_floor'] = $complete($a, [$record(), $wait], '1.17'); + $state['refused_wrong_owner'] = $complete($a, [$record(), $wait], '1.20', 'someone-else'); + $semantic = $record(['non_retryable' => false, 'result' => null, + 'attempts' => [['attempt_number' => 2, 'outcome' => 'failed', 'extra' => 1]]]); + $state['refused_wrong_owner_invalid_report'] = $complete($a, [$semantic], '1.20', 'someone-else'); + $state['invalid_transport'] = $complete($a, [$record(['outcome' => 'weird', 'attempts' => [['outcome' => 'bogus']]])]); + $state['invalid_worker_session'] = $complete($a, [$record(['worker_session' => ['session_id' => 's-1']])]); + $state['invalid_cancellation_policy'] = $complete($a, [$record(['cancellation_policy' => 'try_cancel'])]); + $state['invalid_timeout_envelope'] = $complete($a, [$record(['start_to_close_timeout' => 10, 'heartbeat_timeout' => 20])]); + $state['invalid_field_scope'] = $complete($a, [$record(['schedule_to_start_timeout' => 5])]); + $state['invalid_activity_type'] = $complete($a, [$record(['activity_type' => ' ', 'outcome' => null])]); + $state['invalid_outcome'] = $complete($a, [$record(['outcome' => null, 'result' => null])]); + $state['invalid_report'] = $complete($a, [$semantic]); + $missing = $record(); + unset($missing['attempts']); + $state['invalid_missing_attempts'] = $complete($a, [$missing]); + $state['invalid_attempt_budget'] = $complete($a, [$record(['attempts' => [ + ['attempt_number' => 1, 'outcome' => 'failed', 'message' => 'one', 'retry_reason' => 'failure', 'backoff_seconds' => 0], + ['attempt_number' => 2, 'outcome' => 'completed'], + ]])]); + $state['invalid_retry_policy'] = $complete($a, [$record(['retry_policy' => ['max_attempts' => 101, 'backoff_seconds' => [1, 2], + 'non_retryable_error_types' => ['A', 'A'], 'jitter' => 1]])]); + // Each report also once with an optional field missing from its first + // entry only, which published PHP's transport validation reorders. + $retries = static fn (?string $first): array => $record(['retry_policy' => ['max_attempts' => 3, 'backoff_seconds' => [4]], + 'attempts' => [ + array_filter(['attempt_id' => $first, 'attempt_number' => 1, 'outcome' => 'completed', 'non_retryable' => true], + static fn (mixed $value): bool => $value !== null), + ['attempt_id' => 'w-2', 'attempt_number' => 2, 'outcome' => 'failed', 'message' => 'two', 'retry_reason' => 'timeout', + 'backoff_seconds' => 5, 'non_retryable' => true, 'timeout_kind' => 'heartbeat'], + ['attempt_id' => 'w-2', 'attempt_number' => 3, 'outcome' => 'completed', 'retry_reason' => 'failure'], + ]]); + $state['invalid_retry_attempts'] = $complete($a, [$retries('w-1')]); + $state['reordered_attempts_report'] = $complete($a, [$retries(null)]); + $state['invalid_timed_out'] = $complete($a, [$record(['outcome' => 'timed_out', 'result' => null, 'message' => ' ', + 'non_retryable' => true, 'attempts' => [['attempt_number' => 1, 'outcome' => 'timed_out', 'message' => 'late', + 'non_retryable' => false]]])]); + $beats = static fn (array $heartbeats): array => $record(['attempts' => [['attempt_number' => 1, 'outcome' => 'completed', + 'heartbeats' => $heartbeats]]]); + $state['invalid_heartbeats'] = $complete($a, [$beats([['elapsed_ms' => 500, 'details' => ['a' => 1]], + ['elapsed_ms' => 100, 'details' => [1, 2]], ['elapsed_ms' => 600, 'note' => 'x']])]); + $state['reordered_heartbeats_report'] = $complete($a, [$beats([['elapsed_ms' => 500], + ['elapsed_ms' => 100, 'details' => [1, 2]], ['elapsed_ms' => 600, 'note' => 'x'], 'beat'])]); + $state['invalid_second_command'] = $complete($a, [$record(), $record(['activity_type' => null]), $wait]); + $success = $record(['arguments' => $definition['empty_blob'], 'payload_codec' => 'avro', 'queue' => 'ignored-queue', + 'connection' => 'ignored-connection', 'retry_policy' => ['max_attempts' => 2, 'backoff_seconds' => [1]], + 'start_to_close_timeout' => 300, 'schedule_to_close_timeout' => 600, 'heartbeat_timeout' => 120, + 'attempts' => [ + ['attempt_id' => 'w-1', 'attempt_number' => 1, 'outcome' => 'failed', 'duration_ms' => 1500, 'message' => ' first λ ', + 'exception_type' => 'RuntimeException', 'retry_reason' => 'failure', 'backoff_seconds' => 1, + 'heartbeats' => [['elapsed_ms' => 500, 'details' => ['step' => ' one ']]]], + ['attempt_id' => 'w-2', 'attempt_number' => 2, 'outcome' => 'completed', 'duration_ms' => 200, 'heartbeats' => []], + ]]); + $state['completed'] = $complete($a, [$success, $wait]); + $state['completed_run'] = $observeRun($a); + + // Run B: a failure after retries and a timeout, then the run completes. + $register('b', ['local_activities']); + $b = $claim('b'); + $state['failed'] = $complete($b, [ + $record(['outcome' => 'failed', 'result' => null, 'message' => 'gave up', 'exception_type' => 'App\\Errors\\Fatal', + 'non_retryable' => true, 'retry_policy' => ['max_attempts' => 3, 'backoff_seconds' => [2, 0], + 'non_retryable_error_types' => [' App\\Errors\\Fatal ']], + 'attempts' => [ + ['attempt_id' => 'b-1', 'attempt_number' => 1, 'outcome' => 'timed_out', 'timeout_kind' => 'start_to_close', + 'message' => 'slow', 'duration_ms' => 3000, 'retry_reason' => 'timeout', 'backoff_seconds' => 2, + 'heartbeats' => [['elapsed_ms' => 1000, 'details' => []], ['elapsed_ms' => 1000, 'details' => new stdClass()], + ['elapsed_ms' => 2500, 'details' => ['p' => 2]]]], + ['attempt_id' => 'b-2', 'attempt_number' => 2, 'outcome' => 'failed', 'message' => 'flaky', + 'exception_type' => 'RuntimeException', 'retry_reason' => 'failure', 'backoff_seconds' => 0], + ['attempt_id' => 'b-3', 'attempt_number' => 3, 'outcome' => 'failed', 'message' => 'fatal', + 'exception_type' => 'App\\Errors\\Fatal', 'non_retryable' => true, 'duration_ms' => 40], + ]]), + $record(['activity_type' => $definition['activity_type'].'.slow', 'outcome' => 'timed_out', 'result' => null, + 'message' => 'deadline', 'timeout_kind' => 'schedule_to_close', 'schedule_to_close_timeout' => 30, + 'attempts' => [['attempt_number' => 1, 'outcome' => 'timed_out', 'timeout_kind' => 'schedule_to_close', + 'message' => 'deadline', 'duration_ms' => 30000]]]), + ['type' => 'complete_workflow', 'result' => $done], + ]); + $state['failed_run'] = $observeRun($b); + + // Run C: the protocol 1.18 shape, where attempts are optional and unknown + // nested fields are ignored. + $register('c', ['local_activities']); + $c = $claim('c'); + $bare = static function (array $fields) use ($record): array { + $command = $record(['result' => null] + $fields); + unset($command['attempts']); + + return $command; + }; + $state['protocol_1_18_missing_result'] = $complete($c, [$bare(['outcome' => 'completed'])], '1.18'); + $state['protocol_1_18'] = $complete($c, [ + $bare(['outcome' => 'failed', 'message' => 'no attempts', 'exception_type' => 'RuntimeException', 'non_retryable' => false]), + $bare(['outcome' => 'timed_out', 'message' => 'no attempts', 'timeout_kind' => 'heartbeat']), + $bare(['outcome' => 'cancelled', 'message' => 'no attempts']), + $record(['retry_policy' => ['max_attempts' => 1, 'jitter' => 1], 'attempts' => [['attempt_number' => 1, + 'outcome' => 'completed', 'shard' => 'x', 'heartbeats' => [['elapsed_ms' => 5, 'note' => 'x']]]]]), + $wait, + ], '1.18'); + $state['protocol_1_18_run'] = $observeRun($c); + + // Run E: a valid report whose first attempt alone has no attempt_id. + $register('e', ['local_activities']); + $e = $claim('e'); + $state['reordered_valid_report'] = $complete($e, [$record(['retry_policy' => ['max_attempts' => 2, 'backoff_seconds' => [0]], + 'attempts' => [ + ['attempt_number' => 1, 'outcome' => 'failed', 'message' => 'one', 'retry_reason' => 'failure', 'backoff_seconds' => 0], + ['attempt_id' => 'e-2', 'attempt_number' => 2, 'outcome' => 'completed', + 'heartbeats' => [['elapsed_ms' => 5], ['elapsed_ms' => 9, 'details' => ['k' => 'v']]]], + ]]), $wait]); + + // Run D: a worker without the capability. + $register('d', []); + $d = $claim('d'); + $state['refused_capability'] = $complete($d, [$record(), $wait]); + $state['refused_capability_below_floor'] = $complete($d, [$record(), $wait], '1.17'); + $state['released_without_local'] = $complete($d, [$wait])['status']; + + return recordLocalActivityLabels($state, [$a, $b, $c, $e, $d]); +} + +/** + * Replaces run identities with labels and synthesized times with offsets. + * + * Identities are labelled in a fixed order that does not depend on either + * server's key order: claimed runs and tasks, then for each run's history in + * sequence order the execution, attempt and failure identities. An attempt + * identity's label keeps its letter case (`:upper`, `:lower`): published + * attempt identities are uppercase ULIDs, unlike its other identities. Inside each + * Activity* event and activity view entry, attempt and execution times + * become `@` from that local activity's close; every other time + * becomes `{time}`. The raw order of the attempt identities is recorded too. + * + * @param list> $claims + */ +function recordLocalActivityLabels(array $state, array $claims): array +{ + $labels = []; + $counters = []; + $label = static function (?string $raw, string $kind) use (&$labels, &$counters): void { + if ($raw === null || $raw === '' || isset($labels[$raw])) { + return; + } + $counters[$kind] = ($counters[$kind] ?? 0) + 1; + $labels[$raw] = $kind.'-'.$counters[$kind]; + if ($kind === 'attempt') { + $labels[$raw] .= preg_match('/^[0-9A-HJKMNP-TV-Z]{26}$/D', $raw) === 1 ? ':upper' + : (preg_match('/^[0-9a-hjkmnp-tv-z]{26}$/D', $raw) === 1 ? ':lower' : ':other'); + } + }; + foreach ($claims as $claim) { + $label($claim['run_id'], 'run'); + $label($claim['task_id'], 'task'); + } + $time = '/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,9})?(?:Z|[+-]\d{2}:\d{2})$/D'; + $instant = static fn (string $value): float => (float) (new DateTimeImmutable($value))->format('U.u'); + $offset = static function (mixed $value, ?float $anchor) use ($time, $instant): mixed { + if (! is_string($value) || preg_match($time, $value) !== 1 || $anchor === null) { + return $value; + } + $micros = (int) round(($instant($value) - $anchor) * 1_000_000); + + return $micros % 1000 === 0 ? '@'.intdiv($micros, 1000).'ms' : '@'.$micros.'us'; + }; + $attemptOrder = []; + foreach ($state as $step => $value) { + if (! is_array($value) || ! array_key_exists('activity_events', $value)) { + continue; + } + $anchors = []; + foreach ($value['activity_events'] as $event) { + $payload = $event['payload'] ?? []; + $label($payload['activity_execution_id'] ?? null, 'execution'); + $label($payload['activity_attempt_id'] ?? null, 'attempt'); + $label($payload['failure_id'] ?? null, 'failure'); + if (in_array($event['event_type'] ?? null, ['ActivityCompleted', 'ActivityFailed', 'ActivityTimedOut', 'ActivityCancelled'], true) + && is_string($payload['activity']['closed_at'] ?? null)) { + $anchors[$payload['activity_execution_id'] ?? ''] = $instant($payload['activity']['closed_at']); + } + if (is_string($payload['activity_attempt_id'] ?? null)) { + $attemptOrder[$step][$payload['activity_attempt_id']] = true; + } + } + foreach ($value['activity_events'] as $index => $event) { + $anchor = $anchors[$event['payload']['activity_execution_id'] ?? ''] ?? null; + foreach (['heartbeat_at', 'cancelled_at'] as $field) { + if (array_key_exists($field, $event['payload'] ?? [])) { + $state[$step]['activity_events'][$index]['payload'][$field] = $offset($event['payload'][$field], $anchor); + } + } + foreach (['activity', 'activity_attempt'] as $snapshot) { + foreach (['started_at', 'closed_at', 'last_heartbeat_at'] as $field) { + if (is_array($event['payload'][$snapshot] ?? null) && array_key_exists($field, $event['payload'][$snapshot])) { + $state[$step]['activity_events'][$index]['payload'][$snapshot][$field] + = $offset($event['payload'][$snapshot][$field], $anchor); + } + } + } + } + foreach ((array) ($value['activities'] ?? []) as $index => $activity) { + $anchor = is_array($activity) ? ($anchors[$activity['id'] ?? ''] ?? null) : null; + $walk = static function (mixed $node, string $key = '') use (&$walk, $offset, $anchor): mixed { + if (is_array($node)) { + foreach ($node as $field => $child) { + $node[$field] = $walk($child, (string) $field); + } + + return $node; + } + + return in_array($key, ['created_at', 'scheduled_at', 'lease_expires_at'], true) ? $node : $offset($node, $anchor); + }; + $state[$step]['activities'][$index] = $walk($activity); + } + } + // The raw identities' sort order, as attempt labels. + foreach ($attemptOrder as $step => $ids) { + $ids = array_keys($ids); + sort($ids, SORT_STRING); + $state[$step]['attempt_identity_order'] = array_map(static fn (string $id): string => $labels[$id] ?? $id, $ids); + } + $replace = static function (mixed $node) use (&$replace, $labels, $time): mixed { + if (is_array($node)) { + return array_map($replace, $node); + } + if (! is_string($node)) { + return $node; + } + if (isset($labels[$node])) { + return $labels[$node]; + } + + return preg_match($time, $node) === 1 ? (str_ends_with($node, 'Z') ? '{time}' : '{time+offset}') : $node; + }; + + return $replace($state); +} diff --git a/tests/Fixtures/ServerParity/record-local-activity.json b/tests/Fixtures/ServerParity/record-local-activity.json new file mode 100644 index 000000000..9715a31e9 --- /dev/null +++ b/tests/Fixtures/ServerParity/record-local-activity.json @@ -0,0 +1,3231 @@ +{ + "schema": "durable-workflow.server-parity-fixture/v1", + "id": "record-local-activity", + "review_state": "reviewed", + "workflow_type": "parity.v1.echo", + "input": { + "message": "record local activity λ", + "count": 9007199254740993, + "optional": null + }, + "typed_value": { + "type": "map", + "value": { + "message": { + "type": "string", + "value": "record local activity λ" + }, + "count": { + "type": "int64", + "value": "9007199254740993" + }, + "optional": { + "type": "null", + "value": null + } + } + }, + "activity": false, + "expected_events": [ + "StartAccepted", + "WorkflowStarted", + "WorkflowCompleted" + ], + "record_local_activity": { + "workflow_type": "parity.v1.local_host", + "activity_type": "parity.v1.local", + "empty_blob": "wwHioz3/VYAiNwwA", + "result_blob": "wwHioz3/VYAiNwoIZG9uZQ==", + "expected": { + "applicable": true, + "namespace": "{id}-rla", + "cluster_info": { + "local_activities": { + "schema": "durable-workflow.v2.local-activity.contract", + "version": 1, + "supported": true, + "worker_capability": "local_activities", + "minimum_worker_protocol_version": "1.18", + "command_type": "record_local_activity", + "command": { + "type": "record_local_activity", + "minimum_protocol_version": "1.18", + "required_fields": [ + "activity_type", + "outcome", + "attempts" + ], + "attempt_reports": { + "field": "attempts", + "shape": "ordered_attempt_reports", + "required_fields": [ + "attempt_number", + "outcome" + ], + "maximum": 100, + "required_from_protocol_version": "1.19", + "worker_attempt_identity": { + "report_field": "attempt_id", + "persistence_field": "worker_attempt_id", + "history_field": "worker_attempt_id", + "durable_field": "activity_attempt_id" + } + }, + "heartbeat_reports": { + "field": "heartbeats", + "shape": "ordered_heartbeat_reports", + "required_fields": [ + "elapsed_ms" + ], + "maximum_per_attempt": 1000, + "maximum_per_command": 1000, + "elapsed_ms_origin": "attempt_started_at", + "persisted_last_heartbeat_at": "attempt_started_at_plus_last_elapsed_ms" + }, + "rolling_upgrade": { + "retained_protocol_version": "1.18", + "missing_attempts": "normalize_to_one_terminal_attempt_without_worker_identity_or_heartbeats", + "strict_shape_selected_by": "negotiated_worker_protocol_version", + "retained_nested_object_unknown_fields": "ignore", + "strict_nested_object_shape_from_protocol_version": "1.19" + } + }, + "api": { + "functions": [ + "Workflow\\V2\\localActivity" + ], + "workflow_facade": [ + "Workflow\\V2\\Workflow::localActivity", + "Workflow\\V2\\Workflow::executeLocalActivity" + ], + "options": "Workflow\\V2\\Support\\LocalActivityOptions" + }, + "execution": { + "mode": "local", + "same_process": true, + "ordinary_activity_task_created": false, + "history_events": [ + "ActivityScheduled", + "ActivityStarted", + "ActivityHeartbeatRecorded", + "ActivityRetryScheduled", + "ActivityCompleted", + "ActivityFailed", + "ActivityTimedOut", + "ActivityCancelled" + ], + "history_marker": { + "execution_mode": "local", + "local_activity": true + } + }, + "heartbeating": { + "lease": "workflow_task", + "automatic_points": [ + "before_attempt_start", + "after_activity_heartbeat" + ], + "long_running_requirement": "activity_code_calls_heartbeat_before_the_workflow_task_lease_can_expire" + }, + "cancellation": { + "preemption": "cooperative", + "observed_at": [ + "heartbeat", + "timeout_enforcement", + "attempt_completion" + ], + "terminal_event": "ActivityCancelled" + }, + "shutdown": { + "graceful": "worker_stops_claiming_new_workflow_tasks_and_allows_in_process_attempt_to_finish_or_heartbeat", + "crash": "workflow_task_lease_expires_and_task_repair_reclaims_the_workflow_task", + "cold_replay": "committed_terminal_event_is_replayed_or_started_attempt_without_terminal_event_schedules_local_retry" + }, + "routing": { + "admission": "activity_class_must_resolve_in_the_workflow_worker_process", + "queue_bypassed": true, + "rejected_options": [ + "connection", + "queue", + "worker_session", + "schedule_to_start_timeout" + ] + }, + "retry": { + "new_attempt_when": [ + "a local attempt starts", + "a retry workflow task replays after backoff", + "cold replay finds a previously started local attempt without a terminal event" + ], + "backoff": "durable_workflow_task_available_at", + "cold_replay_reason": "cold_replay" + }, + "timeouts": { + "start_to_close_timeout": "one_local_attempt", + "schedule_to_close_timeout": "entire_local_execution_including_retries", + "heartbeat_timeout": "gap_between_recorded_local_activity_heartbeats", + "enforcement": "cooperative_at_heartbeat_attempt_start_and_attempt_completion_boundaries" + }, + "visibility": { + "activity_execution_marker": "activity_options.execution_mode", + "history_marker": "payload.execution_mode", + "metrics_marker": "activities.local_*", + "history_export": "same_activity_event_payloads_with_execution_mode_local" + } + }, + "record_local_activity_command": true, + "prepared_flags": { + "prepared_local_activities": "bool", + "prepared_local_activity_groups": "bool", + "prepared_local_activity_cancellation_policies": "ordered subset", + "agree": true + }, + "cluster_capabilities_local_keys": [] + }, + "register_local": { + "status": 201, + "response": { + "registered": true, + "capabilities": [ + "local_activities" + ], + "capability_manifest": { + "local_activities": { + "supported": true, + "minimum_protocol_version": "1.18", + "implementation": "record_local_activity" + }, + "worker_sessions": { + "supported": false, + "minimum_protocol_version": "1.18", + "reason": "not_enabled_for_this_worker_registration" + }, + "sticky_execution": { + "supported": false, + "minimum_protocol_version": "1.18", + "reason": "not_enabled_for_this_worker_registration" + } + }, + "status": "active" + }, + "advertised_as_cluster_info": true + }, + "register_flat_only": { + "status": 409, + "response": { + "registered": false, + "reason": "worker_capability_manifest_mismatch", + "capability": "local_activities", + "requested_version": "1.20", + "minimum_protocol_version": "1.18", + "remediation": "Keep the structured supported flag and the flat routing capability in exact agreement." + } + }, + "register_manifest_only": { + "status": 409, + "response": { + "registered": false, + "reason": "worker_capability_manifest_mismatch", + "capability": "local_activities", + "requested_version": "1.20", + "minimum_protocol_version": "1.18", + "remediation": "Keep the structured supported flag and the flat routing capability in exact agreement." + } + }, + "refused_protocol_floor": { + "status": 409, + "response": { + "task_id": "task-1", + "workflow_task_attempt": 1, + "outcome": "rejected", + "recorded": false, + "reason": "portable_worker_affinity_protocol_floor_unavailable", + "capability": "local_activities", + "requested_version": "1.17", + "minimum_protocol_version": "1.18", + "remediation": "Use cold durable replay unless the registered worker manifest explicitly supports the required capability and exact cache identity." + } + }, + "refused_wrong_owner": { + "status": 409, + "response": { + "task_id": "task-1", + "workflow_task_attempt": 1, + "error": "Workflow task lease is owned by another worker.", + "reason": "lease_owner_mismatch", + "lease_owner": "{id}-rla-a" + } + }, + "refused_wrong_owner_invalid_report": { + "status": 409, + "response": { + "task_id": "task-1", + "workflow_task_attempt": 1, + "error": "Workflow task lease is owned by another worker.", + "reason": "lease_owner_mismatch", + "lease_owner": "{id}-rla-a" + } + }, + "invalid_transport": { + "status": 422, + "response": { + "message": "The selected commands.0.outcome is invalid. (and 2 more errors)", + "reason": "validation_failed", + "errors": { + "commands.0.outcome": [ + "The selected commands.0.outcome is invalid." + ], + "commands.0.attempts.0.attempt_number": [ + "The commands.0.attempts.0.attempt_number field is required." + ], + "commands.0.attempts.0.outcome": [ + "The selected commands.0.attempts.0.outcome is invalid." + ] + }, + "validation_errors": { + "commands.0.outcome": [ + "The selected commands.0.outcome is invalid." + ], + "commands.0.attempts.0.attempt_number": [ + "The commands.0.attempts.0.attempt_number field is required." + ], + "commands.0.attempts.0.outcome": [ + "The selected commands.0.attempts.0.outcome is invalid." + ] + } + } + }, + "invalid_worker_session": { + "status": 422, + "response": { + "message": "worker_session is only supported for schedule_activity commands.", + "reason": "validation_failed", + "errors": { + "commands.0.worker_session": [ + "worker_session is only supported for schedule_activity commands." + ] + }, + "validation_errors": { + "commands.0.worker_session": [ + "worker_session is only supported for schedule_activity commands." + ] + } + } + }, + "invalid_cancellation_policy": { + "status": 422, + "response": { + "message": "cancellation_policy is only supported for start_child_workflow, schedule_activity or prepared local Activity commands.", + "reason": "validation_failed", + "errors": { + "commands.0.cancellation_policy": [ + "cancellation_policy is only supported for start_child_workflow, schedule_activity or prepared local Activity commands." + ] + }, + "validation_errors": { + "commands.0.cancellation_policy": [ + "cancellation_policy is only supported for start_child_workflow, schedule_activity or prepared local Activity commands." + ] + } + } + }, + "invalid_timeout_envelope": { + "status": 422, + "response": { + "message": "heartbeat_timeout cannot exceed start_to_close_timeout.", + "reason": "validation_failed", + "errors": { + "commands.0.heartbeat_timeout": [ + "heartbeat_timeout cannot exceed start_to_close_timeout." + ] + }, + "validation_errors": { + "commands.0.heartbeat_timeout": [ + "heartbeat_timeout cannot exceed start_to_close_timeout." + ] + } + } + }, + "invalid_field_scope": { + "status": 422, + "response": { + "message": "Workflow task command field [schedule_to_start_timeout] is not valid on a record_local_activity command. schedule_to_start_timeout limits queue wait before an activity attempt starts and only applies to a schedule_activity command.", + "reason": "validation_failed", + "errors": { + "commands.0.schedule_to_start_timeout": [ + "Workflow task command field [schedule_to_start_timeout] is not valid on a record_local_activity command. schedule_to_start_timeout limits queue wait before an activity attempt starts and only applies to a schedule_activity command." + ] + }, + "validation_errors": { + "commands.0.schedule_to_start_timeout": [ + "Workflow task command field [schedule_to_start_timeout] is not valid on a record_local_activity command. schedule_to_start_timeout limits queue wait before an activity attempt starts and only applies to a schedule_activity command." + ] + } + } + }, + "invalid_activity_type": { + "status": 422, + "response": { + "message": "Local activity records require a non-empty activity_type.", + "reason": "validation_failed", + "errors": { + "commands.0.activity_type": [ + "Local activity records require a non-empty activity_type." + ] + }, + "validation_errors": { + "commands.0.activity_type": [ + "Local activity records require a non-empty activity_type." + ] + } + } + }, + "invalid_outcome": { + "status": 422, + "response": { + "message": "Local activity outcome must be completed, failed, timed_out, or cancelled.", + "reason": "validation_failed", + "errors": { + "commands.0.outcome": [ + "Local activity outcome must be completed, failed, timed_out, or cancelled." + ] + }, + "validation_errors": { + "commands.0.outcome": [ + "Local activity outcome must be completed, failed, timed_out, or cancelled." + ] + } + } + }, + "invalid_report": { + "status": 422, + "response": { + "message": "Completed local activities cannot be marked non-retryable. (and 5 more errors)", + "reason": "validation_failed", + "errors": { + "commands.0.non_retryable": [ + "Completed local activities cannot be marked non-retryable." + ], + "commands.0.attempts.0.extra": [ + "Field [extra] is not supported in a local activity attempt report." + ], + "commands.0.attempts.0.attempt_number": [ + "Local activity attempt_number must be the ordered one-based value 1." + ], + "commands.0.attempts.0.outcome": [ + "The final local activity attempt outcome must match the command outcome." + ], + "commands.0.attempts.0.message": [ + "Failed, timed out, and cancelled local activity attempts require a message." + ], + "commands.0.result": [ + "Completed local activities require a result payload envelope." + ] + }, + "validation_errors": { + "commands.0.non_retryable": [ + "Completed local activities cannot be marked non-retryable." + ], + "commands.0.attempts.0.extra": [ + "Field [extra] is not supported in a local activity attempt report." + ], + "commands.0.attempts.0.attempt_number": [ + "Local activity attempt_number must be the ordered one-based value 1." + ], + "commands.0.attempts.0.outcome": [ + "The final local activity attempt outcome must match the command outcome." + ], + "commands.0.attempts.0.message": [ + "Failed, timed out, and cancelled local activity attempts require a message." + ], + "commands.0.result": [ + "Completed local activities require a result payload envelope." + ] + } + } + }, + "invalid_missing_attempts": { + "status": 422, + "response": { + "message": "Local activity attempts must be a non-empty ordered list.", + "reason": "validation_failed", + "errors": { + "commands.0.attempts": [ + "Local activity attempts must be a non-empty ordered list." + ] + }, + "validation_errors": { + "commands.0.attempts": [ + "Local activity attempts must be a non-empty ordered list." + ] + } + } + }, + "invalid_attempt_budget": { + "status": 422, + "response": { + "message": "Local activity report contains 2 attempts but retry_policy.max_attempts is 1.", + "reason": "validation_failed", + "errors": { + "commands.0.attempts": [ + "Local activity report contains 2 attempts but retry_policy.max_attempts is 1." + ] + }, + "validation_errors": { + "commands.0.attempts": [ + "Local activity report contains 2 attempts but retry_policy.max_attempts is 1." + ] + } + } + }, + "invalid_retry_policy": { + "status": 422, + "response": { + "message": "Field [jitter] is not supported in a local activity retry policy report. (and 2 more errors)", + "reason": "validation_failed", + "errors": { + "commands.0.retry_policy.jitter": [ + "Field [jitter] is not supported in a local activity retry policy report." + ], + "commands.0.retry_policy.non_retryable_error_types": [ + "Local activity retry policy non_retryable_error_types must not contain duplicates." + ], + "commands.0.retry_policy.max_attempts": [ + "Local activity retry policy max_attempts must not exceed 100." + ] + }, + "validation_errors": { + "commands.0.retry_policy.jitter": [ + "Field [jitter] is not supported in a local activity retry policy report." + ], + "commands.0.retry_policy.non_retryable_error_types": [ + "Local activity retry policy non_retryable_error_types must not contain duplicates." + ], + "commands.0.retry_policy.max_attempts": [ + "Local activity retry policy max_attempts must not exceed 100." + ] + } + } + }, + "invalid_retry_attempts": { + "status": 422, + "response": { + "message": "Only failed or timed_out local activity attempts may be followed by a retry. (and 8 more errors)", + "reason": "validation_failed", + "errors": { + "commands.0.attempts.0.outcome": [ + "Only failed or timed_out local activity attempts may be followed by a retry." + ], + "commands.0.attempts.0.non_retryable": [ + "A non-retryable local activity attempt cannot be followed by a retry." + ], + "commands.0.attempts.0.retry_reason": [ + "A retried completed local activity attempt requires a compatible retry_reason." + ], + "commands.0.attempts.0.backoff_seconds": [ + "A retried local activity attempt requires backoff_seconds." + ], + "commands.0.attempts.1.non_retryable": [ + "A non-retryable local activity attempt cannot be followed by a retry." + ], + "commands.0.attempts.1.timeout_kind": [ + "Only timed out local activity attempts may declare timeout_kind." + ], + "commands.0.attempts.1.retry_reason": [ + "A retried failed local activity attempt requires a compatible retry_reason." + ], + "commands.0.attempts.2.attempt_id": [ + "Local activity attempt_id must be unique within the reported sequence." + ], + "commands.0.attempts.2": [ + "The terminal local activity attempt cannot declare retry_reason or backoff_seconds." + ] + }, + "validation_errors": { + "commands.0.attempts.0.outcome": [ + "Only failed or timed_out local activity attempts may be followed by a retry." + ], + "commands.0.attempts.0.non_retryable": [ + "A non-retryable local activity attempt cannot be followed by a retry." + ], + "commands.0.attempts.0.retry_reason": [ + "A retried completed local activity attempt requires a compatible retry_reason." + ], + "commands.0.attempts.0.backoff_seconds": [ + "A retried local activity attempt requires backoff_seconds." + ], + "commands.0.attempts.1.non_retryable": [ + "A non-retryable local activity attempt cannot be followed by a retry." + ], + "commands.0.attempts.1.timeout_kind": [ + "Only timed out local activity attempts may declare timeout_kind." + ], + "commands.0.attempts.1.retry_reason": [ + "A retried failed local activity attempt requires a compatible retry_reason." + ], + "commands.0.attempts.2.attempt_id": [ + "Local activity attempt_id must be unique within the reported sequence." + ], + "commands.0.attempts.2": [ + "The terminal local activity attempt cannot declare retry_reason or backoff_seconds." + ] + } + } + }, + "reordered_attempts_report": { + "status": 422, + "response": { + "message": "Only failed or timed_out local activity attempts may be followed by a retry. (and 8 more errors)", + "reason": "validation_failed", + "errors": { + "commands.0.attempts.0.outcome": [ + "Only failed or timed_out local activity attempts may be followed by a retry." + ], + "commands.0.attempts.0.non_retryable": [ + "A non-retryable local activity attempt cannot be followed by a retry." + ], + "commands.0.attempts.0.retry_reason": [ + "A retried completed local activity attempt requires a compatible retry_reason." + ], + "commands.0.attempts.0.backoff_seconds": [ + "A retried local activity attempt requires backoff_seconds." + ], + "commands.0.attempts.1.non_retryable": [ + "A non-retryable local activity attempt cannot be followed by a retry." + ], + "commands.0.attempts.1.timeout_kind": [ + "Only timed out local activity attempts may declare timeout_kind." + ], + "commands.0.attempts.1.retry_reason": [ + "A retried failed local activity attempt requires a compatible retry_reason." + ], + "commands.0.attempts.2.attempt_id": [ + "Local activity attempt_id must be unique within the reported sequence." + ], + "commands.0.attempts.2": [ + "The terminal local activity attempt cannot declare retry_reason or backoff_seconds." + ] + }, + "validation_errors": { + "commands.0.attempts.0.outcome": [ + "Only failed or timed_out local activity attempts may be followed by a retry." + ], + "commands.0.attempts.0.non_retryable": [ + "A non-retryable local activity attempt cannot be followed by a retry." + ], + "commands.0.attempts.0.retry_reason": [ + "A retried completed local activity attempt requires a compatible retry_reason." + ], + "commands.0.attempts.0.backoff_seconds": [ + "A retried local activity attempt requires backoff_seconds." + ], + "commands.0.attempts.1.non_retryable": [ + "A non-retryable local activity attempt cannot be followed by a retry." + ], + "commands.0.attempts.1.timeout_kind": [ + "Only timed out local activity attempts may declare timeout_kind." + ], + "commands.0.attempts.1.retry_reason": [ + "A retried failed local activity attempt requires a compatible retry_reason." + ], + "commands.0.attempts.2.attempt_id": [ + "Local activity attempt_id must be unique within the reported sequence." + ], + "commands.0.attempts.2": [ + "The terminal local activity attempt cannot declare retry_reason or backoff_seconds." + ] + } + } + }, + "invalid_timed_out": { + "status": 422, + "response": { + "message": "The terminal local activity attempt non_retryable value must match the command outcome. (and 3 more errors)", + "reason": "validation_failed", + "errors": { + "commands.0.attempts.0.non_retryable": [ + "The terminal local activity attempt non_retryable value must match the command outcome." + ], + "commands.0.attempts.0.timeout_kind": [ + "Timed out local activity attempts require start_to_close, schedule_to_close, or heartbeat timeout_kind." + ], + "commands.0.timeout_kind": [ + "Timed out local activities require a timeout_kind." + ], + "commands.0.message": [ + "Failed, timed out, and cancelled local activities require a message." + ] + }, + "validation_errors": { + "commands.0.attempts.0.non_retryable": [ + "The terminal local activity attempt non_retryable value must match the command outcome." + ], + "commands.0.attempts.0.timeout_kind": [ + "Timed out local activity attempts require start_to_close, schedule_to_close, or heartbeat timeout_kind." + ], + "commands.0.timeout_kind": [ + "Timed out local activities require a timeout_kind." + ], + "commands.0.message": [ + "Failed, timed out, and cancelled local activities require a message." + ] + } + } + }, + "invalid_heartbeats": { + "status": 422, + "response": { + "message": "Local activity heartbeat elapsed_ms values must be ordered monotonically. (and 2 more errors)", + "reason": "validation_failed", + "errors": { + "commands.0.attempts.0.heartbeats.1.elapsed_ms": [ + "Local activity heartbeat elapsed_ms values must be ordered monotonically." + ], + "commands.0.attempts.0.heartbeats.1.details": [ + "Local activity heartbeat details must be an object when provided." + ], + "commands.0.attempts.0.heartbeats.2.note": [ + "Field [note] is not supported in a local activity heartbeat report." + ] + }, + "validation_errors": { + "commands.0.attempts.0.heartbeats.1.elapsed_ms": [ + "Local activity heartbeat elapsed_ms values must be ordered monotonically." + ], + "commands.0.attempts.0.heartbeats.1.details": [ + "Local activity heartbeat details must be an object when provided." + ], + "commands.0.attempts.0.heartbeats.2.note": [ + "Field [note] is not supported in a local activity heartbeat report." + ] + } + } + }, + "reordered_heartbeats_report": { + "status": 422, + "response": { + "message": "Local activity heartbeat elapsed_ms values must be ordered monotonically. (and 3 more errors)", + "reason": "validation_failed", + "errors": { + "commands.0.attempts.0.heartbeats.1.elapsed_ms": [ + "Local activity heartbeat elapsed_ms values must be ordered monotonically." + ], + "commands.0.attempts.0.heartbeats.1.details": [ + "Local activity heartbeat details must be an object when provided." + ], + "commands.0.attempts.0.heartbeats.2.note": [ + "Field [note] is not supported in a local activity heartbeat report." + ], + "commands.0.attempts.0.heartbeats.3": [ + "Each local activity heartbeat must be an object." + ] + }, + "validation_errors": { + "commands.0.attempts.0.heartbeats.1.elapsed_ms": [ + "Local activity heartbeat elapsed_ms values must be ordered monotonically." + ], + "commands.0.attempts.0.heartbeats.1.details": [ + "Local activity heartbeat details must be an object when provided." + ], + "commands.0.attempts.0.heartbeats.2.note": [ + "Field [note] is not supported in a local activity heartbeat report." + ], + "commands.0.attempts.0.heartbeats.3": [ + "Each local activity heartbeat must be an object." + ] + } + } + }, + "invalid_second_command": { + "status": 422, + "response": { + "message": "Local activity records require a non-empty activity_type.", + "reason": "validation_failed", + "errors": { + "commands.1.activity_type": [ + "Local activity records require a non-empty activity_type." + ] + }, + "validation_errors": { + "commands.1.activity_type": [ + "Local activity records require a non-empty activity_type." + ] + } + } + }, + "completed": { + "status": 200, + "response": { + "task_id": "task-1", + "workflow_task_attempt": 1, + "outcome": "completed", + "recorded": true, + "run_id": "run-1", + "run_status": "waiting", + "created_task_ids": [], + "reason": null + } + }, + "completed_run": { + "event_types": [ + "StartAccepted", + "WorkflowStarted", + "ActivityScheduled", + "ActivityStarted", + "ActivityHeartbeatRecorded", + "ActivityRetryScheduled", + "ActivityStarted", + "ActivityCompleted", + "SignalWaitOpened" + ], + "activity_events": [ + { + "sequence": 3, + "event_type": "ActivityScheduled", + "timestamp": "{time}", + "principal": null, + "payload": { + "execution_mode": "local", + "local_activity": true, + "activity_execution_id": "execution-1", + "activity_class": "parity.v1.local", + "activity_type": "parity.v1.local", + "sequence": 1, + "workflow_task_id": "task-1", + "activity": { + "id": "execution-1", + "idempotency_key": "execution-1", + "sequence": 1, + "type": "parity.v1.local", + "class": "parity.v1.local", + "execution_mode": "local", + "local_activity": true, + "status": "pending", + "payload_codec": "avro", + "attempt_count": 0, + "retry_policy": { + "max_attempts": 2, + "backoff_seconds": [ + 1 + ] + }, + "queue": "{id}-rla-a-queue", + "created_at": "{time}", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + } + }, + "task": { + "id": "task-1", + "type": "workflow", + "status": "leased", + "available_at": "{time}", + "leased_at": "{time}", + "lease_owner": "{id}-rla-a", + "lease_expires_at": "{time}", + "attempt_count": 1, + "repair_count": 0, + "connection": "sqlite", + "queue": "{id}-rla-a-queue", + "last_dispatch_attempt_at": "{time}", + "last_dispatched_at": "{time}" + } + } + }, + { + "sequence": 4, + "event_type": "ActivityStarted", + "timestamp": "{time}", + "principal": null, + "payload": { + "execution_mode": "local", + "local_activity": true, + "activity_execution_id": "execution-1", + "activity_class": "parity.v1.local", + "activity_type": "parity.v1.local", + "sequence": 1, + "workflow_task_id": "task-1", + "activity": { + "id": "execution-1", + "idempotency_key": "execution-1", + "sequence": 1, + "type": "parity.v1.local", + "class": "parity.v1.local", + "execution_mode": "local", + "local_activity": true, + "attempt_id": "attempt-1:upper", + "status": "running", + "payload_codec": "avro", + "attempt_count": 1, + "retry_policy": { + "max_attempts": 2, + "backoff_seconds": [ + 1 + ] + }, + "queue": "{id}-rla-a-queue", + "created_at": "{time}", + "started_at": "@-2700ms", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + } + }, + "activity_attempt_id": "attempt-1:upper", + "worker_attempt_id": "w-1", + "attempt_number": 1, + "activity_attempt": { + "id": "attempt-1:upper", + "worker_attempt_id": "w-1", + "attempt_number": 1, + "status": "running", + "task_id": "task-1", + "lease_owner": "{id}-rla-a", + "started_at": "@-2700ms" + }, + "task": "(the first event's task)" + } + }, + { + "sequence": 5, + "event_type": "ActivityHeartbeatRecorded", + "timestamp": "{time}", + "principal": null, + "payload": { + "execution_mode": "local", + "local_activity": true, + "activity_execution_id": "execution-1", + "activity_class": "parity.v1.local", + "activity_type": "parity.v1.local", + "sequence": 1, + "workflow_task_id": "task-1", + "activity": { + "id": "execution-1", + "idempotency_key": "execution-1", + "sequence": 1, + "type": "parity.v1.local", + "class": "parity.v1.local", + "execution_mode": "local", + "local_activity": true, + "attempt_id": "attempt-1:upper", + "status": "running", + "payload_codec": "avro", + "attempt_count": 1, + "retry_policy": { + "max_attempts": 2, + "backoff_seconds": [ + 1 + ] + }, + "queue": "{id}-rla-a-queue", + "created_at": "{time}", + "started_at": "@-2700ms", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "last_heartbeat_at": "@-2200ms" + }, + "activity_attempt_id": "attempt-1:upper", + "worker_attempt_id": "w-1", + "attempt_number": 1, + "activity_attempt": { + "id": "attempt-1:upper", + "worker_attempt_id": "w-1", + "attempt_number": 1, + "status": "running", + "task_id": "task-1", + "lease_owner": "{id}-rla-a", + "started_at": "@-2700ms", + "last_heartbeat_at": "@-2200ms" + }, + "heartbeat_at": "@-2200ms", + "lease_expires_at": null, + "progress": { + "step": "one" + }, + "task": "(the first event's task)" + } + }, + { + "sequence": 6, + "event_type": "ActivityRetryScheduled", + "timestamp": "{time}", + "principal": null, + "payload": { + "execution_mode": "local", + "local_activity": true, + "activity_execution_id": "execution-1", + "activity_class": "parity.v1.local", + "activity_type": "parity.v1.local", + "sequence": 1, + "workflow_task_id": "task-1", + "activity": { + "id": "execution-1", + "idempotency_key": "execution-1", + "sequence": 1, + "type": "parity.v1.local", + "class": "parity.v1.local", + "execution_mode": "local", + "local_activity": true, + "attempt_id": "attempt-1:upper", + "status": "pending", + "payload_codec": "avro", + "attempt_count": 1, + "retry_policy": { + "max_attempts": 2, + "backoff_seconds": [ + 1 + ] + }, + "queue": "{id}-rla-a-queue", + "created_at": "{time}", + "started_at": "@-2700ms", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "last_heartbeat_at": "@-2200ms" + }, + "activity_attempt_id": "attempt-1:upper", + "worker_attempt_id": "w-1", + "activity_attempt": { + "id": "attempt-1:upper", + "worker_attempt_id": "w-1", + "attempt_number": 1, + "status": "failed", + "task_id": "task-1", + "lease_owner": "{id}-rla-a", + "started_at": "@-2700ms", + "last_heartbeat_at": "@-2200ms", + "closed_at": "@-1200ms" + }, + "retry_after_attempt_id": "attempt-1:upper", + "retry_after_attempt": 1, + "message": "first λ", + "exception_type": "RuntimeException", + "retry_reason": "failure", + "retry_backoff_seconds": 1, + "task": "(the first event's task)" + } + }, + { + "sequence": 7, + "event_type": "ActivityStarted", + "timestamp": "{time}", + "principal": null, + "payload": { + "execution_mode": "local", + "local_activity": true, + "activity_execution_id": "execution-1", + "activity_class": "parity.v1.local", + "activity_type": "parity.v1.local", + "sequence": 1, + "workflow_task_id": "task-1", + "activity": { + "id": "execution-1", + "idempotency_key": "execution-1", + "sequence": 1, + "type": "parity.v1.local", + "class": "parity.v1.local", + "execution_mode": "local", + "local_activity": true, + "attempt_id": "attempt-2:upper", + "status": "running", + "payload_codec": "avro", + "attempt_count": 2, + "retry_policy": { + "max_attempts": 2, + "backoff_seconds": [ + 1 + ] + }, + "queue": "{id}-rla-a-queue", + "last_heartbeat_at": "@-2200ms", + "created_at": "{time}", + "started_at": "@-2700ms", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + } + }, + "activity_attempt_id": "attempt-2:upper", + "worker_attempt_id": "w-2", + "attempt_number": 2, + "activity_attempt": { + "id": "attempt-2:upper", + "worker_attempt_id": "w-2", + "attempt_number": 2, + "status": "running", + "task_id": "task-1", + "lease_owner": "{id}-rla-a", + "started_at": "@-200ms" + }, + "task": "(the first event's task)" + } + }, + { + "sequence": 8, + "event_type": "ActivityCompleted", + "timestamp": "{time}", + "principal": null, + "payload": { + "execution_mode": "local", + "local_activity": true, + "activity_execution_id": "execution-1", + "activity_class": "parity.v1.local", + "activity_type": "parity.v1.local", + "sequence": 1, + "workflow_task_id": "task-1", + "activity": { + "id": "execution-1", + "idempotency_key": "execution-1", + "sequence": 1, + "type": "parity.v1.local", + "class": "parity.v1.local", + "execution_mode": "local", + "local_activity": true, + "attempt_id": "attempt-2:upper", + "status": "completed", + "payload_codec": "avro", + "attempt_count": 2, + "retry_policy": { + "max_attempts": 2, + "backoff_seconds": [ + 1 + ] + }, + "queue": "{id}-rla-a-queue", + "last_heartbeat_at": "@-2200ms", + "created_at": "{time}", + "started_at": "@-2700ms", + "closed_at": "@0ms", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoIZG9uZQ==" + } + }, + "activity_attempt_id": "attempt-2:upper", + "worker_attempt_id": "w-2", + "attempt_number": 2, + "activity_attempt": { + "id": "attempt-2:upper", + "worker_attempt_id": "w-2", + "attempt_number": 2, + "status": "completed", + "task_id": "task-1", + "lease_owner": "{id}-rla-a", + "started_at": "@-200ms", + "closed_at": "@0ms" + }, + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoIZG9uZQ==" + }, + "payload_codec": "avro", + "task": "(the first event's task)" + } + } + ], + "activities": [ + { + "id": "execution-1", + "idempotency_key": "execution-1", + "sequence": 1, + "type": "parity.v1.local", + "class": "parity.v1.local", + "execution_mode": "local", + "local_activity": true, + "history_authority": "typed_history", + "diagnostic_only": false, + "history_event_types": [ + "ActivityScheduled", + "ActivityStarted", + "ActivityHeartbeatRecorded", + "ActivityRetryScheduled", + "ActivityCompleted" + ], + "history_unsupported_reason": null, + "reused_from_run_id": null, + "reused_activity_execution_id": null, + "reused_recorded_at": null, + "row_status": null, + "parallel_group_kind": null, + "parallel_group_id": null, + "parallel_group_base_sequence": null, + "parallel_group_size": null, + "parallel_group_index": null, + "parallel_group_path": [], + "attempt_id": "attempt-2:upper", + "worker_attempt_id": "w-2", + "status": "completed", + "attempt_count": 2, + "retry_policy": { + "max_attempts": 2, + "backoff_seconds": [ + 1 + ] + }, + "connection": null, + "queue": "{id}-rla-a-queue", + "last_heartbeat_at": "@-2200ms", + "last_heartbeat_progress": null, + "created_at": "{time}", + "started_at": "@-2700ms", + "closed_at": "@0ms", + "attempts": [ + { + "id": "attempt-1:upper", + "worker_attempt_id": "w-1", + "attempt_number": 1, + "status": "failed", + "task_id": "task-1", + "lease_owner": "{id}-rla-a", + "lease_expires_at": null, + "started_at": "@-2700ms", + "last_heartbeat_at": "@-2200ms", + "last_heartbeat_progress": null, + "closed_at": "@-1200ms", + "can_continue": false, + "cancel_requested": false, + "stop_reason": "attempt_closed", + "failure_reason": null + }, + { + "id": "attempt-2:upper", + "worker_attempt_id": "w-2", + "attempt_number": 2, + "status": "completed", + "task_id": "task-1", + "lease_owner": "{id}-rla-a", + "lease_expires_at": null, + "started_at": "@-200ms", + "last_heartbeat_at": null, + "last_heartbeat_progress": null, + "closed_at": "@0ms", + "can_continue": false, + "cancel_requested": false, + "stop_reason": "attempt_closed", + "failure_reason": null + } + ] + } + ], + "run": { + "status": "waiting", + "status_bucket": "running", + "is_terminal": false, + "closed_reason": null, + "wait_kind": "signal", + "wait_reason": "Waiting for signal resume" + }, + "attempt_identity_order": [ + "attempt-2:upper", + "attempt-1:upper" + ] + }, + "failed": { + "status": 200, + "response": { + "task_id": "task-2", + "workflow_task_attempt": 1, + "outcome": "completed", + "recorded": true, + "run_id": "run-2", + "run_status": "completed", + "created_task_ids": [], + "reason": null + } + }, + "failed_run": { + "event_types": [ + "StartAccepted", + "WorkflowStarted", + "ActivityScheduled", + "ActivityStarted", + "ActivityHeartbeatRecorded", + "ActivityHeartbeatRecorded", + "ActivityHeartbeatRecorded", + "ActivityRetryScheduled", + "ActivityStarted", + "ActivityRetryScheduled", + "ActivityStarted", + "ActivityFailed", + "ActivityScheduled", + "ActivityStarted", + "ActivityTimedOut", + "WorkflowCompleted" + ], + "activity_events": [ + { + "sequence": 3, + "event_type": "ActivityScheduled", + "timestamp": "{time}", + "principal": null, + "payload": { + "execution_mode": "local", + "local_activity": true, + "activity_execution_id": "execution-2", + "activity_class": "parity.v1.local", + "activity_type": "parity.v1.local", + "sequence": 1, + "workflow_task_id": "task-2", + "activity": { + "id": "execution-2", + "idempotency_key": "execution-2", + "sequence": 1, + "type": "parity.v1.local", + "class": "parity.v1.local", + "execution_mode": "local", + "local_activity": true, + "status": "pending", + "payload_codec": "avro", + "attempt_count": 0, + "retry_policy": { + "max_attempts": 3, + "backoff_seconds": [ + 2, + 0 + ], + "non_retryable_error_types": [ + "App\\Errors\\Fatal" + ] + }, + "queue": "{id}-rla-b-queue", + "created_at": "{time}", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + } + }, + "task": { + "id": "task-2", + "type": "workflow", + "status": "leased", + "available_at": "{time}", + "leased_at": "{time}", + "lease_owner": "{id}-rla-b", + "lease_expires_at": "{time}", + "attempt_count": 1, + "repair_count": 0, + "connection": "sqlite", + "queue": "{id}-rla-b-queue", + "last_dispatch_attempt_at": "{time}", + "last_dispatched_at": "{time}" + } + } + }, + { + "sequence": 4, + "event_type": "ActivityStarted", + "timestamp": "{time}", + "principal": null, + "payload": { + "execution_mode": "local", + "local_activity": true, + "activity_execution_id": "execution-2", + "activity_class": "parity.v1.local", + "activity_type": "parity.v1.local", + "sequence": 1, + "workflow_task_id": "task-2", + "activity": { + "id": "execution-2", + "idempotency_key": "execution-2", + "sequence": 1, + "type": "parity.v1.local", + "class": "parity.v1.local", + "execution_mode": "local", + "local_activity": true, + "attempt_id": "attempt-3:upper", + "status": "running", + "payload_codec": "avro", + "attempt_count": 1, + "retry_policy": { + "max_attempts": 3, + "backoff_seconds": [ + 2, + 0 + ], + "non_retryable_error_types": [ + "App\\Errors\\Fatal" + ] + }, + "queue": "{id}-rla-b-queue", + "created_at": "{time}", + "started_at": "@-5040ms", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + } + }, + "activity_attempt_id": "attempt-3:upper", + "worker_attempt_id": "b-1", + "attempt_number": 1, + "activity_attempt": { + "id": "attempt-3:upper", + "worker_attempt_id": "b-1", + "attempt_number": 1, + "status": "running", + "task_id": "task-2", + "lease_owner": "{id}-rla-b", + "started_at": "@-5040ms" + }, + "task": "(the first event's task)" + } + }, + { + "sequence": 5, + "event_type": "ActivityHeartbeatRecorded", + "timestamp": "{time}", + "principal": null, + "payload": { + "execution_mode": "local", + "local_activity": true, + "activity_execution_id": "execution-2", + "activity_class": "parity.v1.local", + "activity_type": "parity.v1.local", + "sequence": 1, + "workflow_task_id": "task-2", + "activity": { + "id": "execution-2", + "idempotency_key": "execution-2", + "sequence": 1, + "type": "parity.v1.local", + "class": "parity.v1.local", + "execution_mode": "local", + "local_activity": true, + "attempt_id": "attempt-3:upper", + "status": "running", + "payload_codec": "avro", + "attempt_count": 1, + "retry_policy": { + "max_attempts": 3, + "backoff_seconds": [ + 2, + 0 + ], + "non_retryable_error_types": [ + "App\\Errors\\Fatal" + ] + }, + "queue": "{id}-rla-b-queue", + "created_at": "{time}", + "started_at": "@-5040ms", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "last_heartbeat_at": "@-4040ms" + }, + "activity_attempt_id": "attempt-3:upper", + "worker_attempt_id": "b-1", + "attempt_number": 1, + "activity_attempt": { + "id": "attempt-3:upper", + "worker_attempt_id": "b-1", + "attempt_number": 1, + "status": "running", + "task_id": "task-2", + "lease_owner": "{id}-rla-b", + "started_at": "@-5040ms", + "last_heartbeat_at": "@-4040ms" + }, + "heartbeat_at": "@-4040ms", + "lease_expires_at": null, + "progress": [], + "task": "(the first event's task)" + } + }, + { + "sequence": 6, + "event_type": "ActivityHeartbeatRecorded", + "timestamp": "{time}", + "principal": null, + "payload": { + "execution_mode": "local", + "local_activity": true, + "activity_execution_id": "execution-2", + "activity_class": "parity.v1.local", + "activity_type": "parity.v1.local", + "sequence": 1, + "workflow_task_id": "task-2", + "activity": { + "id": "execution-2", + "idempotency_key": "execution-2", + "sequence": 1, + "type": "parity.v1.local", + "class": "parity.v1.local", + "execution_mode": "local", + "local_activity": true, + "attempt_id": "attempt-3:upper", + "status": "running", + "payload_codec": "avro", + "attempt_count": 1, + "retry_policy": { + "max_attempts": 3, + "backoff_seconds": [ + 2, + 0 + ], + "non_retryable_error_types": [ + "App\\Errors\\Fatal" + ] + }, + "queue": "{id}-rla-b-queue", + "created_at": "{time}", + "started_at": "@-5040ms", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "last_heartbeat_at": "@-4040ms" + }, + "activity_attempt_id": "attempt-3:upper", + "worker_attempt_id": "b-1", + "attempt_number": 1, + "activity_attempt": { + "id": "attempt-3:upper", + "worker_attempt_id": "b-1", + "attempt_number": 1, + "status": "running", + "task_id": "task-2", + "lease_owner": "{id}-rla-b", + "started_at": "@-5040ms", + "last_heartbeat_at": "@-4040ms" + }, + "heartbeat_at": "@-4040ms", + "lease_expires_at": null, + "progress": [], + "task": "(the first event's task)" + } + }, + { + "sequence": 7, + "event_type": "ActivityHeartbeatRecorded", + "timestamp": "{time}", + "principal": null, + "payload": { + "execution_mode": "local", + "local_activity": true, + "activity_execution_id": "execution-2", + "activity_class": "parity.v1.local", + "activity_type": "parity.v1.local", + "sequence": 1, + "workflow_task_id": "task-2", + "activity": { + "id": "execution-2", + "idempotency_key": "execution-2", + "sequence": 1, + "type": "parity.v1.local", + "class": "parity.v1.local", + "execution_mode": "local", + "local_activity": true, + "attempt_id": "attempt-3:upper", + "status": "running", + "payload_codec": "avro", + "attempt_count": 1, + "retry_policy": { + "max_attempts": 3, + "backoff_seconds": [ + 2, + 0 + ], + "non_retryable_error_types": [ + "App\\Errors\\Fatal" + ] + }, + "queue": "{id}-rla-b-queue", + "created_at": "{time}", + "started_at": "@-5040ms", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "last_heartbeat_at": "@-2540ms" + }, + "activity_attempt_id": "attempt-3:upper", + "worker_attempt_id": "b-1", + "attempt_number": 1, + "activity_attempt": { + "id": "attempt-3:upper", + "worker_attempt_id": "b-1", + "attempt_number": 1, + "status": "running", + "task_id": "task-2", + "lease_owner": "{id}-rla-b", + "started_at": "@-5040ms", + "last_heartbeat_at": "@-2540ms" + }, + "heartbeat_at": "@-2540ms", + "lease_expires_at": null, + "progress": { + "p": 2 + }, + "task": "(the first event's task)" + } + }, + { + "sequence": 8, + "event_type": "ActivityRetryScheduled", + "timestamp": "{time}", + "principal": null, + "payload": { + "execution_mode": "local", + "local_activity": true, + "activity_execution_id": "execution-2", + "activity_class": "parity.v1.local", + "activity_type": "parity.v1.local", + "sequence": 1, + "workflow_task_id": "task-2", + "activity": { + "id": "execution-2", + "idempotency_key": "execution-2", + "sequence": 1, + "type": "parity.v1.local", + "class": "parity.v1.local", + "execution_mode": "local", + "local_activity": true, + "attempt_id": "attempt-3:upper", + "status": "pending", + "payload_codec": "avro", + "attempt_count": 1, + "retry_policy": { + "max_attempts": 3, + "backoff_seconds": [ + 2, + 0 + ], + "non_retryable_error_types": [ + "App\\Errors\\Fatal" + ] + }, + "queue": "{id}-rla-b-queue", + "created_at": "{time}", + "started_at": "@-5040ms", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "last_heartbeat_at": "@-2540ms" + }, + "activity_attempt_id": "attempt-3:upper", + "worker_attempt_id": "b-1", + "activity_attempt": { + "id": "attempt-3:upper", + "worker_attempt_id": "b-1", + "attempt_number": 1, + "status": "failed", + "task_id": "task-2", + "lease_owner": "{id}-rla-b", + "started_at": "@-5040ms", + "last_heartbeat_at": "@-2540ms", + "closed_at": "@-2040ms" + }, + "retry_after_attempt_id": "attempt-3:upper", + "retry_after_attempt": 1, + "message": "slow", + "exception_type": null, + "retry_reason": "timeout", + "retry_backoff_seconds": 2, + "task": "(the first event's task)" + } + }, + { + "sequence": 9, + "event_type": "ActivityStarted", + "timestamp": "{time}", + "principal": null, + "payload": { + "execution_mode": "local", + "local_activity": true, + "activity_execution_id": "execution-2", + "activity_class": "parity.v1.local", + "activity_type": "parity.v1.local", + "sequence": 1, + "workflow_task_id": "task-2", + "activity": { + "id": "execution-2", + "idempotency_key": "execution-2", + "sequence": 1, + "type": "parity.v1.local", + "class": "parity.v1.local", + "execution_mode": "local", + "local_activity": true, + "attempt_id": "attempt-4:upper", + "status": "running", + "payload_codec": "avro", + "attempt_count": 2, + "retry_policy": { + "max_attempts": 3, + "backoff_seconds": [ + 2, + 0 + ], + "non_retryable_error_types": [ + "App\\Errors\\Fatal" + ] + }, + "queue": "{id}-rla-b-queue", + "last_heartbeat_at": "@-2540ms", + "created_at": "{time}", + "started_at": "@-5040ms", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + } + }, + "activity_attempt_id": "attempt-4:upper", + "worker_attempt_id": "b-2", + "attempt_number": 2, + "activity_attempt": { + "id": "attempt-4:upper", + "worker_attempt_id": "b-2", + "attempt_number": 2, + "status": "running", + "task_id": "task-2", + "lease_owner": "{id}-rla-b", + "started_at": "@-40ms" + }, + "task": "(the first event's task)" + } + }, + { + "sequence": 10, + "event_type": "ActivityRetryScheduled", + "timestamp": "{time}", + "principal": null, + "payload": { + "execution_mode": "local", + "local_activity": true, + "activity_execution_id": "execution-2", + "activity_class": "parity.v1.local", + "activity_type": "parity.v1.local", + "sequence": 1, + "workflow_task_id": "task-2", + "activity": { + "id": "execution-2", + "idempotency_key": "execution-2", + "sequence": 1, + "type": "parity.v1.local", + "class": "parity.v1.local", + "execution_mode": "local", + "local_activity": true, + "attempt_id": "attempt-4:upper", + "status": "pending", + "payload_codec": "avro", + "attempt_count": 2, + "retry_policy": { + "max_attempts": 3, + "backoff_seconds": [ + 2, + 0 + ], + "non_retryable_error_types": [ + "App\\Errors\\Fatal" + ] + }, + "queue": "{id}-rla-b-queue", + "last_heartbeat_at": "@-2540ms", + "created_at": "{time}", + "started_at": "@-5040ms", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + } + }, + "activity_attempt_id": "attempt-4:upper", + "worker_attempt_id": "b-2", + "activity_attempt": { + "id": "attempt-4:upper", + "worker_attempt_id": "b-2", + "attempt_number": 2, + "status": "failed", + "task_id": "task-2", + "lease_owner": "{id}-rla-b", + "started_at": "@-40ms", + "closed_at": "@-40ms" + }, + "retry_after_attempt_id": "attempt-4:upper", + "retry_after_attempt": 2, + "message": "flaky", + "exception_type": "RuntimeException", + "retry_reason": "failure", + "retry_backoff_seconds": 0, + "task": "(the first event's task)" + } + }, + { + "sequence": 11, + "event_type": "ActivityStarted", + "timestamp": "{time}", + "principal": null, + "payload": { + "execution_mode": "local", + "local_activity": true, + "activity_execution_id": "execution-2", + "activity_class": "parity.v1.local", + "activity_type": "parity.v1.local", + "sequence": 1, + "workflow_task_id": "task-2", + "activity": { + "id": "execution-2", + "idempotency_key": "execution-2", + "sequence": 1, + "type": "parity.v1.local", + "class": "parity.v1.local", + "execution_mode": "local", + "local_activity": true, + "attempt_id": "attempt-5:upper", + "status": "running", + "payload_codec": "avro", + "attempt_count": 3, + "retry_policy": { + "max_attempts": 3, + "backoff_seconds": [ + 2, + 0 + ], + "non_retryable_error_types": [ + "App\\Errors\\Fatal" + ] + }, + "queue": "{id}-rla-b-queue", + "last_heartbeat_at": "@-2540ms", + "created_at": "{time}", + "started_at": "@-5040ms", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + } + }, + "activity_attempt_id": "attempt-5:upper", + "worker_attempt_id": "b-3", + "attempt_number": 3, + "activity_attempt": { + "id": "attempt-5:upper", + "worker_attempt_id": "b-3", + "attempt_number": 3, + "status": "running", + "task_id": "task-2", + "lease_owner": "{id}-rla-b", + "started_at": "@-40ms" + }, + "task": "(the first event's task)" + } + }, + { + "sequence": 12, + "event_type": "ActivityFailed", + "timestamp": "{time}", + "principal": null, + "payload": { + "execution_mode": "local", + "local_activity": true, + "activity_execution_id": "execution-2", + "activity_class": "parity.v1.local", + "activity_type": "parity.v1.local", + "sequence": 1, + "workflow_task_id": "task-2", + "activity": { + "id": "execution-2", + "idempotency_key": "execution-2", + "sequence": 1, + "type": "parity.v1.local", + "class": "parity.v1.local", + "execution_mode": "local", + "local_activity": true, + "attempt_id": "attempt-5:upper", + "status": "failed", + "payload_codec": "avro", + "attempt_count": 3, + "retry_policy": { + "max_attempts": 3, + "backoff_seconds": [ + 2, + 0 + ], + "non_retryable_error_types": [ + "App\\Errors\\Fatal" + ] + }, + "queue": "{id}-rla-b-queue", + "last_heartbeat_at": "@-2540ms", + "created_at": "{time}", + "started_at": "@-5040ms", + "closed_at": "@0ms", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "exception": "wwHioz3/VYAiNwoOZ2F2ZSB1cA==" + }, + "activity_attempt_id": "attempt-5:upper", + "worker_attempt_id": "b-3", + "attempt_number": 3, + "activity_attempt": { + "id": "attempt-5:upper", + "worker_attempt_id": "b-3", + "attempt_number": 3, + "status": "failed", + "task_id": "task-2", + "lease_owner": "{id}-rla-b", + "started_at": "@-40ms", + "closed_at": "@0ms" + }, + "failure_id": "failure-1", + "failure_category": "application", + "message": "gave up", + "exception_type": "App\\Errors\\Fatal", + "non_retryable": true, + "task": "(the first event's task)" + } + }, + { + "sequence": 13, + "event_type": "ActivityScheduled", + "timestamp": "{time}", + "principal": null, + "payload": { + "execution_mode": "local", + "local_activity": true, + "activity_execution_id": "execution-3", + "activity_class": "parity.v1.local.slow", + "activity_type": "parity.v1.local.slow", + "sequence": 2, + "workflow_task_id": "task-2", + "activity": { + "id": "execution-3", + "idempotency_key": "execution-3", + "sequence": 2, + "type": "parity.v1.local.slow", + "class": "parity.v1.local.slow", + "execution_mode": "local", + "local_activity": true, + "status": "pending", + "payload_codec": "avro", + "attempt_count": 0, + "queue": "{id}-rla-b-queue", + "created_at": "{time}", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + } + }, + "task": "(the first event's task)" + } + }, + { + "sequence": 14, + "event_type": "ActivityStarted", + "timestamp": "{time}", + "principal": null, + "payload": { + "execution_mode": "local", + "local_activity": true, + "activity_execution_id": "execution-3", + "activity_class": "parity.v1.local.slow", + "activity_type": "parity.v1.local.slow", + "sequence": 2, + "workflow_task_id": "task-2", + "activity": { + "id": "execution-3", + "idempotency_key": "execution-3", + "sequence": 2, + "type": "parity.v1.local.slow", + "class": "parity.v1.local.slow", + "execution_mode": "local", + "local_activity": true, + "attempt_id": "attempt-6:upper", + "status": "running", + "payload_codec": "avro", + "attempt_count": 1, + "queue": "{id}-rla-b-queue", + "created_at": "{time}", + "started_at": "@-30000ms", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + } + }, + "activity_attempt_id": "attempt-6:upper", + "worker_attempt_id": null, + "attempt_number": 1, + "activity_attempt": { + "id": "attempt-6:upper", + "attempt_number": 1, + "status": "running", + "task_id": "task-2", + "lease_owner": "{id}-rla-b", + "started_at": "@-30000ms" + }, + "task": "(the first event's task)" + } + }, + { + "sequence": 15, + "event_type": "ActivityTimedOut", + "timestamp": "{time}", + "principal": null, + "payload": { + "execution_mode": "local", + "local_activity": true, + "activity_execution_id": "execution-3", + "activity_class": "parity.v1.local.slow", + "activity_type": "parity.v1.local.slow", + "sequence": 2, + "workflow_task_id": "task-2", + "activity": { + "id": "execution-3", + "idempotency_key": "execution-3", + "sequence": 2, + "type": "parity.v1.local.slow", + "class": "parity.v1.local.slow", + "execution_mode": "local", + "local_activity": true, + "attempt_id": "attempt-6:upper", + "status": "failed", + "payload_codec": "avro", + "attempt_count": 1, + "queue": "{id}-rla-b-queue", + "created_at": "{time}", + "started_at": "@-30000ms", + "closed_at": "@0ms", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "exception": "wwHioz3/VYAiNwoQZGVhZGxpbmU=" + }, + "activity_attempt_id": "attempt-6:upper", + "worker_attempt_id": null, + "attempt_number": 1, + "activity_attempt": { + "id": "attempt-6:upper", + "attempt_number": 1, + "status": "failed", + "task_id": "task-2", + "lease_owner": "{id}-rla-b", + "started_at": "@-30000ms", + "closed_at": "@0ms" + }, + "failure_id": "failure-2", + "failure_category": "timeout", + "timeout_kind": "schedule_to_close", + "message": "deadline", + "exception_class": null, + "task": "(the first event's task)" + } + } + ], + "activities": [ + { + "id": "execution-2", + "idempotency_key": "execution-2", + "sequence": 1, + "type": "parity.v1.local", + "class": "parity.v1.local", + "execution_mode": "local", + "local_activity": true, + "history_authority": "typed_history", + "diagnostic_only": false, + "history_event_types": [ + "ActivityScheduled", + "ActivityStarted", + "ActivityHeartbeatRecorded", + "ActivityRetryScheduled", + "ActivityFailed" + ], + "history_unsupported_reason": null, + "reused_from_run_id": null, + "reused_activity_execution_id": null, + "reused_recorded_at": null, + "row_status": null, + "parallel_group_kind": null, + "parallel_group_id": null, + "parallel_group_base_sequence": null, + "parallel_group_size": null, + "parallel_group_index": null, + "parallel_group_path": [], + "attempt_id": "attempt-5:upper", + "worker_attempt_id": "b-3", + "status": "failed", + "attempt_count": 3, + "retry_policy": { + "max_attempts": 3, + "backoff_seconds": [ + 2, + 0 + ], + "non_retryable_error_types": [ + "App\\Errors\\Fatal" + ] + }, + "connection": null, + "queue": "{id}-rla-b-queue", + "last_heartbeat_at": "@-2540ms", + "last_heartbeat_progress": null, + "created_at": "{time}", + "started_at": "@-5040ms", + "closed_at": "@0ms", + "attempts": [ + { + "id": "attempt-3:upper", + "worker_attempt_id": "b-1", + "attempt_number": 1, + "status": "failed", + "task_id": "task-2", + "lease_owner": "{id}-rla-b", + "lease_expires_at": null, + "started_at": "@-5040ms", + "last_heartbeat_at": "@-2540ms", + "last_heartbeat_progress": null, + "closed_at": "@-2040ms", + "can_continue": false, + "cancel_requested": false, + "stop_reason": "attempt_closed", + "failure_reason": null + }, + { + "id": "attempt-4:upper", + "worker_attempt_id": "b-2", + "attempt_number": 2, + "status": "failed", + "task_id": "task-2", + "lease_owner": "{id}-rla-b", + "lease_expires_at": null, + "started_at": "@-40ms", + "last_heartbeat_at": null, + "last_heartbeat_progress": null, + "closed_at": "@-40ms", + "can_continue": false, + "cancel_requested": false, + "stop_reason": "attempt_closed", + "failure_reason": null + }, + { + "id": "attempt-5:upper", + "worker_attempt_id": "b-3", + "attempt_number": 3, + "status": "failed", + "task_id": "task-2", + "lease_owner": "{id}-rla-b", + "lease_expires_at": null, + "started_at": "@-40ms", + "last_heartbeat_at": null, + "last_heartbeat_progress": null, + "closed_at": "@0ms", + "can_continue": false, + "cancel_requested": false, + "stop_reason": "attempt_closed", + "failure_reason": null + } + ] + }, + { + "id": "execution-3", + "idempotency_key": "execution-3", + "sequence": 2, + "type": "parity.v1.local.slow", + "class": "parity.v1.local.slow", + "execution_mode": "local", + "local_activity": true, + "history_authority": "typed_history", + "diagnostic_only": false, + "history_event_types": [ + "ActivityScheduled", + "ActivityStarted", + "ActivityTimedOut" + ], + "history_unsupported_reason": null, + "reused_from_run_id": null, + "reused_activity_execution_id": null, + "reused_recorded_at": null, + "row_status": null, + "parallel_group_kind": null, + "parallel_group_id": null, + "parallel_group_base_sequence": null, + "parallel_group_size": null, + "parallel_group_index": null, + "parallel_group_path": [], + "attempt_id": "attempt-6:upper", + "worker_attempt_id": null, + "status": "failed", + "attempt_count": 1, + "retry_policy": null, + "connection": null, + "queue": "{id}-rla-b-queue", + "last_heartbeat_at": null, + "last_heartbeat_progress": null, + "created_at": "{time}", + "started_at": "@-30000ms", + "closed_at": "@0ms", + "attempts": [ + { + "id": "attempt-6:upper", + "worker_attempt_id": null, + "attempt_number": 1, + "status": "expired", + "task_id": "task-2", + "lease_owner": "{id}-rla-b", + "lease_expires_at": null, + "started_at": "@-30000ms", + "last_heartbeat_at": null, + "last_heartbeat_progress": null, + "closed_at": "@0ms", + "can_continue": false, + "cancel_requested": false, + "stop_reason": "attempt_expired", + "failure_reason": null + } + ] + } + ], + "run": { + "status": "completed", + "status_bucket": "completed", + "is_terminal": true, + "closed_reason": "completed", + "wait_kind": null, + "wait_reason": null + }, + "attempt_identity_order": [ + "attempt-5:upper", + "attempt-4:upper", + "attempt-3:upper", + "attempt-6:upper" + ] + }, + "protocol_1_18_missing_result": { + "status": 422, + "response": { + "message": "Completed local activities require a result payload envelope.", + "reason": "validation_failed", + "errors": { + "commands.0.result": [ + "Completed local activities require a result payload envelope." + ] + }, + "validation_errors": { + "commands.0.result": [ + "Completed local activities require a result payload envelope." + ] + } + } + }, + "protocol_1_18": { + "status": 200, + "response": { + "task_id": "task-3", + "workflow_task_attempt": 1, + "outcome": "completed", + "recorded": true, + "run_id": "run-3", + "run_status": "waiting", + "created_task_ids": [], + "reason": null + } + }, + "protocol_1_18_run": { + "event_types": [ + "StartAccepted", + "WorkflowStarted", + "ActivityScheduled", + "ActivityStarted", + "ActivityFailed", + "ActivityScheduled", + "ActivityStarted", + "ActivityTimedOut", + "ActivityScheduled", + "ActivityStarted", + "ActivityCancelled", + "ActivityScheduled", + "ActivityStarted", + "ActivityHeartbeatRecorded", + "ActivityCompleted", + "SignalWaitOpened" + ], + "activity_events": [ + { + "sequence": 3, + "event_type": "ActivityScheduled", + "timestamp": "{time}", + "principal": null, + "payload": { + "execution_mode": "local", + "local_activity": true, + "activity_execution_id": "execution-4", + "activity_class": "parity.v1.local", + "activity_type": "parity.v1.local", + "sequence": 1, + "workflow_task_id": "task-3", + "activity": { + "id": "execution-4", + "idempotency_key": "execution-4", + "sequence": 1, + "type": "parity.v1.local", + "class": "parity.v1.local", + "execution_mode": "local", + "local_activity": true, + "status": "pending", + "payload_codec": "avro", + "attempt_count": 0, + "queue": "{id}-rla-c-queue", + "created_at": "{time}", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + } + }, + "task": { + "id": "task-3", + "type": "workflow", + "status": "leased", + "available_at": "{time}", + "leased_at": "{time}", + "lease_owner": "{id}-rla-c", + "lease_expires_at": "{time}", + "attempt_count": 1, + "repair_count": 0, + "connection": "sqlite", + "queue": "{id}-rla-c-queue", + "last_dispatch_attempt_at": "{time}", + "last_dispatched_at": "{time}" + } + } + }, + { + "sequence": 4, + "event_type": "ActivityStarted", + "timestamp": "{time}", + "principal": null, + "payload": { + "execution_mode": "local", + "local_activity": true, + "activity_execution_id": "execution-4", + "activity_class": "parity.v1.local", + "activity_type": "parity.v1.local", + "sequence": 1, + "workflow_task_id": "task-3", + "activity": { + "id": "execution-4", + "idempotency_key": "execution-4", + "sequence": 1, + "type": "parity.v1.local", + "class": "parity.v1.local", + "execution_mode": "local", + "local_activity": true, + "attempt_id": "attempt-7:upper", + "status": "running", + "payload_codec": "avro", + "attempt_count": 1, + "queue": "{id}-rla-c-queue", + "created_at": "{time}", + "started_at": "@0ms", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + } + }, + "activity_attempt_id": "attempt-7:upper", + "worker_attempt_id": null, + "attempt_number": 1, + "activity_attempt": { + "id": "attempt-7:upper", + "attempt_number": 1, + "status": "running", + "task_id": "task-3", + "lease_owner": "{id}-rla-c", + "started_at": "@0ms" + }, + "task": "(the first event's task)" + } + }, + { + "sequence": 5, + "event_type": "ActivityFailed", + "timestamp": "{time}", + "principal": null, + "payload": { + "execution_mode": "local", + "local_activity": true, + "activity_execution_id": "execution-4", + "activity_class": "parity.v1.local", + "activity_type": "parity.v1.local", + "sequence": 1, + "workflow_task_id": "task-3", + "activity": { + "id": "execution-4", + "idempotency_key": "execution-4", + "sequence": 1, + "type": "parity.v1.local", + "class": "parity.v1.local", + "execution_mode": "local", + "local_activity": true, + "attempt_id": "attempt-7:upper", + "status": "failed", + "payload_codec": "avro", + "attempt_count": 1, + "queue": "{id}-rla-c-queue", + "created_at": "{time}", + "started_at": "@0ms", + "closed_at": "@0ms", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "exception": "wwHioz3/VYAiNwoWbm8gYXR0ZW1wdHM=" + }, + "activity_attempt_id": "attempt-7:upper", + "worker_attempt_id": null, + "attempt_number": 1, + "activity_attempt": { + "id": "attempt-7:upper", + "attempt_number": 1, + "status": "failed", + "task_id": "task-3", + "lease_owner": "{id}-rla-c", + "started_at": "@0ms", + "closed_at": "@0ms" + }, + "failure_id": "failure-3", + "failure_category": "application", + "message": "no attempts", + "exception_type": "RuntimeException", + "non_retryable": false, + "task": "(the first event's task)" + } + }, + { + "sequence": 6, + "event_type": "ActivityScheduled", + "timestamp": "{time}", + "principal": null, + "payload": { + "execution_mode": "local", + "local_activity": true, + "activity_execution_id": "execution-5", + "activity_class": "parity.v1.local", + "activity_type": "parity.v1.local", + "sequence": 2, + "workflow_task_id": "task-3", + "activity": { + "id": "execution-5", + "idempotency_key": "execution-5", + "sequence": 2, + "type": "parity.v1.local", + "class": "parity.v1.local", + "execution_mode": "local", + "local_activity": true, + "status": "pending", + "payload_codec": "avro", + "attempt_count": 0, + "queue": "{id}-rla-c-queue", + "created_at": "{time}", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + } + }, + "task": "(the first event's task)" + } + }, + { + "sequence": 7, + "event_type": "ActivityStarted", + "timestamp": "{time}", + "principal": null, + "payload": { + "execution_mode": "local", + "local_activity": true, + "activity_execution_id": "execution-5", + "activity_class": "parity.v1.local", + "activity_type": "parity.v1.local", + "sequence": 2, + "workflow_task_id": "task-3", + "activity": { + "id": "execution-5", + "idempotency_key": "execution-5", + "sequence": 2, + "type": "parity.v1.local", + "class": "parity.v1.local", + "execution_mode": "local", + "local_activity": true, + "attempt_id": "attempt-8:upper", + "status": "running", + "payload_codec": "avro", + "attempt_count": 1, + "queue": "{id}-rla-c-queue", + "created_at": "{time}", + "started_at": "@0ms", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + } + }, + "activity_attempt_id": "attempt-8:upper", + "worker_attempt_id": null, + "attempt_number": 1, + "activity_attempt": { + "id": "attempt-8:upper", + "attempt_number": 1, + "status": "running", + "task_id": "task-3", + "lease_owner": "{id}-rla-c", + "started_at": "@0ms" + }, + "task": "(the first event's task)" + } + }, + { + "sequence": 8, + "event_type": "ActivityTimedOut", + "timestamp": "{time}", + "principal": null, + "payload": { + "execution_mode": "local", + "local_activity": true, + "activity_execution_id": "execution-5", + "activity_class": "parity.v1.local", + "activity_type": "parity.v1.local", + "sequence": 2, + "workflow_task_id": "task-3", + "activity": { + "id": "execution-5", + "idempotency_key": "execution-5", + "sequence": 2, + "type": "parity.v1.local", + "class": "parity.v1.local", + "execution_mode": "local", + "local_activity": true, + "attempt_id": "attempt-8:upper", + "status": "failed", + "payload_codec": "avro", + "attempt_count": 1, + "queue": "{id}-rla-c-queue", + "created_at": "{time}", + "started_at": "@0ms", + "closed_at": "@0ms", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "exception": "wwHioz3/VYAiNwoWbm8gYXR0ZW1wdHM=" + }, + "activity_attempt_id": "attempt-8:upper", + "worker_attempt_id": null, + "attempt_number": 1, + "activity_attempt": { + "id": "attempt-8:upper", + "attempt_number": 1, + "status": "failed", + "task_id": "task-3", + "lease_owner": "{id}-rla-c", + "started_at": "@0ms", + "closed_at": "@0ms" + }, + "failure_id": "failure-4", + "failure_category": "timeout", + "timeout_kind": "heartbeat", + "message": "no attempts", + "exception_class": null, + "task": "(the first event's task)" + } + }, + { + "sequence": 9, + "event_type": "ActivityScheduled", + "timestamp": "{time}", + "principal": null, + "payload": { + "execution_mode": "local", + "local_activity": true, + "activity_execution_id": "execution-6", + "activity_class": "parity.v1.local", + "activity_type": "parity.v1.local", + "sequence": 3, + "workflow_task_id": "task-3", + "activity": { + "id": "execution-6", + "idempotency_key": "execution-6", + "sequence": 3, + "type": "parity.v1.local", + "class": "parity.v1.local", + "execution_mode": "local", + "local_activity": true, + "status": "pending", + "payload_codec": "avro", + "attempt_count": 0, + "queue": "{id}-rla-c-queue", + "created_at": "{time}", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + } + }, + "task": "(the first event's task)" + } + }, + { + "sequence": 10, + "event_type": "ActivityStarted", + "timestamp": "{time}", + "principal": null, + "payload": { + "execution_mode": "local", + "local_activity": true, + "activity_execution_id": "execution-6", + "activity_class": "parity.v1.local", + "activity_type": "parity.v1.local", + "sequence": 3, + "workflow_task_id": "task-3", + "activity": { + "id": "execution-6", + "idempotency_key": "execution-6", + "sequence": 3, + "type": "parity.v1.local", + "class": "parity.v1.local", + "execution_mode": "local", + "local_activity": true, + "attempt_id": "attempt-9:upper", + "status": "running", + "payload_codec": "avro", + "attempt_count": 1, + "queue": "{id}-rla-c-queue", + "created_at": "{time}", + "started_at": "@0ms", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + } + }, + "activity_attempt_id": "attempt-9:upper", + "worker_attempt_id": null, + "attempt_number": 1, + "activity_attempt": { + "id": "attempt-9:upper", + "attempt_number": 1, + "status": "running", + "task_id": "task-3", + "lease_owner": "{id}-rla-c", + "started_at": "@0ms" + }, + "task": "(the first event's task)" + } + }, + { + "sequence": 11, + "event_type": "ActivityCancelled", + "timestamp": "{time}", + "principal": null, + "payload": { + "execution_mode": "local", + "local_activity": true, + "activity_execution_id": "execution-6", + "activity_class": "parity.v1.local", + "activity_type": "parity.v1.local", + "sequence": 3, + "workflow_task_id": "task-3", + "activity": { + "id": "execution-6", + "idempotency_key": "execution-6", + "sequence": 3, + "type": "parity.v1.local", + "class": "parity.v1.local", + "execution_mode": "local", + "local_activity": true, + "attempt_id": "attempt-9:upper", + "status": "cancelled", + "payload_codec": "avro", + "attempt_count": 1, + "queue": "{id}-rla-c-queue", + "created_at": "{time}", + "started_at": "@0ms", + "closed_at": "@0ms", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "exception": "wwHioz3/VYAiNwoWbm8gYXR0ZW1wdHM=" + }, + "activity_attempt_id": "attempt-9:upper", + "worker_attempt_id": null, + "attempt_number": 1, + "activity_attempt": { + "id": "attempt-9:upper", + "attempt_number": 1, + "status": "cancelled", + "task_id": "task-3", + "lease_owner": "{id}-rla-c", + "started_at": "@0ms", + "closed_at": "@0ms" + }, + "cancelled_at": "@0ms", + "task": "(the first event's task)" + } + }, + { + "sequence": 12, + "event_type": "ActivityScheduled", + "timestamp": "{time}", + "principal": null, + "payload": { + "execution_mode": "local", + "local_activity": true, + "activity_execution_id": "execution-7", + "activity_class": "parity.v1.local", + "activity_type": "parity.v1.local", + "sequence": 4, + "workflow_task_id": "task-3", + "activity": { + "id": "execution-7", + "idempotency_key": "execution-7", + "sequence": 4, + "type": "parity.v1.local", + "class": "parity.v1.local", + "execution_mode": "local", + "local_activity": true, + "status": "pending", + "payload_codec": "avro", + "attempt_count": 0, + "retry_policy": { + "max_attempts": 1 + }, + "queue": "{id}-rla-c-queue", + "created_at": "{time}", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + } + }, + "task": "(the first event's task)" + } + }, + { + "sequence": 13, + "event_type": "ActivityStarted", + "timestamp": "{time}", + "principal": null, + "payload": { + "execution_mode": "local", + "local_activity": true, + "activity_execution_id": "execution-7", + "activity_class": "parity.v1.local", + "activity_type": "parity.v1.local", + "sequence": 4, + "workflow_task_id": "task-3", + "activity": { + "id": "execution-7", + "idempotency_key": "execution-7", + "sequence": 4, + "type": "parity.v1.local", + "class": "parity.v1.local", + "execution_mode": "local", + "local_activity": true, + "attempt_id": "attempt-10:upper", + "status": "running", + "payload_codec": "avro", + "attempt_count": 1, + "retry_policy": { + "max_attempts": 1 + }, + "queue": "{id}-rla-c-queue", + "created_at": "{time}", + "started_at": "@-5ms", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + } + }, + "activity_attempt_id": "attempt-10:upper", + "worker_attempt_id": null, + "attempt_number": 1, + "activity_attempt": { + "id": "attempt-10:upper", + "attempt_number": 1, + "status": "running", + "task_id": "task-3", + "lease_owner": "{id}-rla-c", + "started_at": "@-5ms" + }, + "task": "(the first event's task)" + } + }, + { + "sequence": 14, + "event_type": "ActivityHeartbeatRecorded", + "timestamp": "{time}", + "principal": null, + "payload": { + "execution_mode": "local", + "local_activity": true, + "activity_execution_id": "execution-7", + "activity_class": "parity.v1.local", + "activity_type": "parity.v1.local", + "sequence": 4, + "workflow_task_id": "task-3", + "activity": { + "id": "execution-7", + "idempotency_key": "execution-7", + "sequence": 4, + "type": "parity.v1.local", + "class": "parity.v1.local", + "execution_mode": "local", + "local_activity": true, + "attempt_id": "attempt-10:upper", + "status": "running", + "payload_codec": "avro", + "attempt_count": 1, + "retry_policy": { + "max_attempts": 1 + }, + "queue": "{id}-rla-c-queue", + "created_at": "{time}", + "started_at": "@-5ms", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "last_heartbeat_at": "@0ms" + }, + "activity_attempt_id": "attempt-10:upper", + "worker_attempt_id": null, + "attempt_number": 1, + "activity_attempt": { + "id": "attempt-10:upper", + "attempt_number": 1, + "status": "running", + "task_id": "task-3", + "lease_owner": "{id}-rla-c", + "started_at": "@-5ms", + "last_heartbeat_at": "@0ms" + }, + "heartbeat_at": "@0ms", + "lease_expires_at": null, + "task": "(the first event's task)" + } + }, + { + "sequence": 15, + "event_type": "ActivityCompleted", + "timestamp": "{time}", + "principal": null, + "payload": { + "execution_mode": "local", + "local_activity": true, + "activity_execution_id": "execution-7", + "activity_class": "parity.v1.local", + "activity_type": "parity.v1.local", + "sequence": 4, + "workflow_task_id": "task-3", + "activity": { + "id": "execution-7", + "idempotency_key": "execution-7", + "sequence": 4, + "type": "parity.v1.local", + "class": "parity.v1.local", + "execution_mode": "local", + "local_activity": true, + "attempt_id": "attempt-10:upper", + "status": "completed", + "payload_codec": "avro", + "attempt_count": 1, + "retry_policy": { + "max_attempts": 1 + }, + "queue": "{id}-rla-c-queue", + "last_heartbeat_at": "@0ms", + "created_at": "{time}", + "started_at": "@-5ms", + "closed_at": "@0ms", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoIZG9uZQ==" + } + }, + "activity_attempt_id": "attempt-10:upper", + "worker_attempt_id": null, + "attempt_number": 1, + "activity_attempt": { + "id": "attempt-10:upper", + "attempt_number": 1, + "status": "completed", + "task_id": "task-3", + "lease_owner": "{id}-rla-c", + "started_at": "@-5ms", + "last_heartbeat_at": "@0ms", + "closed_at": "@0ms" + }, + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoIZG9uZQ==" + }, + "payload_codec": "avro", + "task": "(the first event's task)" + } + } + ], + "activities": [ + { + "id": "execution-4", + "idempotency_key": "execution-4", + "sequence": 1, + "type": "parity.v1.local", + "class": "parity.v1.local", + "execution_mode": "local", + "local_activity": true, + "history_authority": "typed_history", + "diagnostic_only": false, + "history_event_types": [ + "ActivityScheduled", + "ActivityStarted", + "ActivityFailed" + ], + "history_unsupported_reason": null, + "reused_from_run_id": null, + "reused_activity_execution_id": null, + "reused_recorded_at": null, + "row_status": null, + "parallel_group_kind": null, + "parallel_group_id": null, + "parallel_group_base_sequence": null, + "parallel_group_size": null, + "parallel_group_index": null, + "parallel_group_path": [], + "attempt_id": "attempt-7:upper", + "worker_attempt_id": null, + "status": "failed", + "attempt_count": 1, + "retry_policy": null, + "connection": null, + "queue": "{id}-rla-c-queue", + "last_heartbeat_at": null, + "last_heartbeat_progress": null, + "created_at": "{time}", + "started_at": "@0ms", + "closed_at": "@0ms", + "attempts": [ + { + "id": "attempt-7:upper", + "worker_attempt_id": null, + "attempt_number": 1, + "status": "failed", + "task_id": "task-3", + "lease_owner": "{id}-rla-c", + "lease_expires_at": null, + "started_at": "@0ms", + "last_heartbeat_at": null, + "last_heartbeat_progress": null, + "closed_at": "@0ms", + "can_continue": false, + "cancel_requested": false, + "stop_reason": "attempt_closed", + "failure_reason": null + } + ] + }, + { + "id": "execution-5", + "idempotency_key": "execution-5", + "sequence": 2, + "type": "parity.v1.local", + "class": "parity.v1.local", + "execution_mode": "local", + "local_activity": true, + "history_authority": "typed_history", + "diagnostic_only": false, + "history_event_types": [ + "ActivityScheduled", + "ActivityStarted", + "ActivityTimedOut" + ], + "history_unsupported_reason": null, + "reused_from_run_id": null, + "reused_activity_execution_id": null, + "reused_recorded_at": null, + "row_status": null, + "parallel_group_kind": null, + "parallel_group_id": null, + "parallel_group_base_sequence": null, + "parallel_group_size": null, + "parallel_group_index": null, + "parallel_group_path": [], + "attempt_id": "attempt-8:upper", + "worker_attempt_id": null, + "status": "failed", + "attempt_count": 1, + "retry_policy": null, + "connection": null, + "queue": "{id}-rla-c-queue", + "last_heartbeat_at": null, + "last_heartbeat_progress": null, + "created_at": "{time}", + "started_at": "@0ms", + "closed_at": "@0ms", + "attempts": [ + { + "id": "attempt-8:upper", + "worker_attempt_id": null, + "attempt_number": 1, + "status": "expired", + "task_id": "task-3", + "lease_owner": "{id}-rla-c", + "lease_expires_at": null, + "started_at": "@0ms", + "last_heartbeat_at": null, + "last_heartbeat_progress": null, + "closed_at": "@0ms", + "can_continue": false, + "cancel_requested": false, + "stop_reason": "attempt_expired", + "failure_reason": null + } + ] + }, + { + "id": "execution-6", + "idempotency_key": "execution-6", + "sequence": 3, + "type": "parity.v1.local", + "class": "parity.v1.local", + "execution_mode": "local", + "local_activity": true, + "history_authority": "typed_history", + "diagnostic_only": false, + "history_event_types": [ + "ActivityScheduled", + "ActivityStarted", + "ActivityCancelled" + ], + "history_unsupported_reason": null, + "reused_from_run_id": null, + "reused_activity_execution_id": null, + "reused_recorded_at": null, + "row_status": null, + "parallel_group_kind": null, + "parallel_group_id": null, + "parallel_group_base_sequence": null, + "parallel_group_size": null, + "parallel_group_index": null, + "parallel_group_path": [], + "attempt_id": "attempt-9:upper", + "worker_attempt_id": null, + "status": "cancelled", + "attempt_count": 1, + "retry_policy": null, + "connection": null, + "queue": "{id}-rla-c-queue", + "last_heartbeat_at": null, + "last_heartbeat_progress": null, + "created_at": "{time}", + "started_at": "@0ms", + "closed_at": "@0ms", + "attempts": [ + { + "id": "attempt-9:upper", + "worker_attempt_id": null, + "attempt_number": 1, + "status": "cancelled", + "task_id": "task-3", + "lease_owner": "{id}-rla-c", + "lease_expires_at": null, + "started_at": "@0ms", + "last_heartbeat_at": null, + "last_heartbeat_progress": null, + "closed_at": "@0ms", + "can_continue": false, + "cancel_requested": true, + "stop_reason": "attempt_cancelled", + "failure_reason": null + } + ] + }, + { + "id": "execution-7", + "idempotency_key": "execution-7", + "sequence": 4, + "type": "parity.v1.local", + "class": "parity.v1.local", + "execution_mode": "local", + "local_activity": true, + "history_authority": "typed_history", + "diagnostic_only": false, + "history_event_types": [ + "ActivityScheduled", + "ActivityStarted", + "ActivityHeartbeatRecorded", + "ActivityCompleted" + ], + "history_unsupported_reason": null, + "reused_from_run_id": null, + "reused_activity_execution_id": null, + "reused_recorded_at": null, + "row_status": null, + "parallel_group_kind": null, + "parallel_group_id": null, + "parallel_group_base_sequence": null, + "parallel_group_size": null, + "parallel_group_index": null, + "parallel_group_path": [], + "attempt_id": "attempt-10:upper", + "worker_attempt_id": null, + "status": "completed", + "attempt_count": 1, + "retry_policy": { + "max_attempts": 1 + }, + "connection": null, + "queue": "{id}-rla-c-queue", + "last_heartbeat_at": "@0ms", + "last_heartbeat_progress": null, + "created_at": "{time}", + "started_at": "@-5ms", + "closed_at": "@0ms", + "attempts": [ + { + "id": "attempt-10:upper", + "worker_attempt_id": null, + "attempt_number": 1, + "status": "completed", + "task_id": "task-3", + "lease_owner": "{id}-rla-c", + "lease_expires_at": null, + "started_at": "@-5ms", + "last_heartbeat_at": "@0ms", + "last_heartbeat_progress": null, + "closed_at": "@0ms", + "can_continue": false, + "cancel_requested": false, + "stop_reason": "attempt_closed", + "failure_reason": null + } + ] + } + ], + "run": { + "status": "waiting", + "status_bucket": "running", + "is_terminal": false, + "closed_reason": null, + "wait_kind": "signal", + "wait_reason": "Waiting for signal resume" + }, + "attempt_identity_order": [ + "attempt-7:upper", + "attempt-8:upper", + "attempt-9:upper", + "attempt-10:upper" + ] + }, + "reordered_valid_report": { + "status": 200, + "response": { + "task_id": "task-4", + "workflow_task_attempt": 1, + "outcome": "completed", + "recorded": true, + "run_id": "run-4", + "run_status": "waiting", + "created_task_ids": [], + "reason": null + } + }, + "refused_capability": { + "status": 409, + "response": { + "task_id": "task-5", + "workflow_task_attempt": 1, + "outcome": "rejected", + "recorded": false, + "reason": "worker_capability_not_supported", + "capability": "local_activities", + "requested_version": "1.20", + "minimum_protocol_version": "1.18", + "remediation": "Use cold durable replay unless the registered worker manifest explicitly supports the required capability and exact cache identity." + } + }, + "refused_capability_below_floor": { + "status": 409, + "response": { + "task_id": "task-5", + "workflow_task_attempt": 1, + "outcome": "rejected", + "recorded": false, + "reason": "portable_worker_affinity_protocol_floor_unavailable", + "capability": "local_activities", + "requested_version": "1.17", + "minimum_protocol_version": "1.18", + "remediation": "Use cold durable replay unless the registered worker manifest explicitly supports the required capability and exact cache identity." + } + }, + "released_without_local": 200 + }, + "published_defects": { + "validated_reorders_nested_reports": { + "summary": "WorkerController::mutateWorkflowTaskCommands passes Laravel's validated() commands to WorkflowCommandNormalizer. validated() rebuilds each nested list from the rules in rule order, so when an optional attempt field (attempt_id) or heartbeat field (details) is absent from the first entry but present on a later one, the list's keys are reordered (attempts [1,2,0], heartbeats [1,0,2]) and the normalizer refuses the report as not an ordered list; a heartbeat entry that is not an object is dropped. A valid report is refused 422, and an invalid one reports the ordering error instead of its own errors. The intended answers are the normalizer's own on the commands as sent, computed with WorkflowCommandNormalizer::normalize in the published image.", + "published": { + "reordered_attempts_report": { + "status": 422, + "response": { + "message": "Local activity attempts must be a non-empty ordered list.", + "reason": "validation_failed", + "errors": { + "commands.0.attempts": [ + "Local activity attempts must be a non-empty ordered list." + ] + }, + "validation_errors": { + "commands.0.attempts": [ + "Local activity attempts must be a non-empty ordered list." + ] + } + } + }, + "reordered_heartbeats_report": { + "status": 422, + "response": { + "message": "Local activity attempt heartbeats must be an ordered list.", + "reason": "validation_failed", + "errors": { + "commands.0.attempts.0.heartbeats": [ + "Local activity attempt heartbeats must be an ordered list." + ] + }, + "validation_errors": { + "commands.0.attempts.0.heartbeats": [ + "Local activity attempt heartbeats must be an ordered list." + ] + } + } + }, + "reordered_valid_report": { + "status": 422, + "response": { + "message": "Local activity attempts must be a non-empty ordered list.", + "reason": "validation_failed", + "errors": { + "commands.0.attempts": [ + "Local activity attempts must be a non-empty ordered list." + ] + }, + "validation_errors": { + "commands.0.attempts": [ + "Local activity attempts must be a non-empty ordered list." + ] + } + } + } + } + } + } + }, + "contract": "POST /api/worker/workflow-tasks/{task}/complete with record_local_activity commands (worker protocol 1.18+), as published Server 2.11.1 serves them, in their own namespace. Cluster info and registration advertise the local_activities contract object (supported third) with record_local_activity among the workflow task commands; the prepared-local flags are present, typed and in agreement, and their values belong to the prepared-local routes. Registration refuses a flat capability and manifest entry that disagree. A completion is refused 409 below protocol 1.18 and for a worker whose registration lacks the capability or its manifest entry, after the ownership guard, which wins with a wrong owner even over an invalid report. Transport rules, controller scopes and the timeout envelope refuse first (422), then the command normalizer's own messages in its order: field scope, activity type, outcome, retry policy, attempt budget, ordered attempts and heartbeats, retry fields, timeout kinds, result and message, across the whole batch. A success writes ActivityScheduled, then per attempt ActivityStarted, its heartbeats and, before a retry, ActivityRetryScheduled, then the terminal event, with uppercase attempt identities generated last attempt first, times walked back from the close by durations and backoffs, the run's queue (queue and connection are ignored), and an ActivityFailed or ActivityTimedOut failure identity. At protocol 1.18 a missing attempt list becomes one terminal attempt without a worker identity and unknown nested fields are ignored. One published defect is bounded: transport validation reorders a nested report whose first entry lacks an optional field; the published refusals or the normalizer's own answers on the reports as sent are accepted. Embedded applications have no worker protocol." +} diff --git a/tests/Fixtures/ServerParityExpected/record-local-activity-observation.json b/tests/Fixtures/ServerParityExpected/record-local-activity-observation.json new file mode 100644 index 000000000..fdd111f9e --- /dev/null +++ b/tests/Fixtures/ServerParityExpected/record-local-activity-observation.json @@ -0,0 +1,3070 @@ +{ + "applicable": true, + "namespace": "pr1-record-local-activity-rla", + "cluster_info": { + "local_activities": { + "schema": "durable-workflow.v2.local-activity.contract", + "version": 1, + "supported": true, + "worker_capability": "local_activities", + "minimum_worker_protocol_version": "1.18", + "command_type": "record_local_activity", + "command": { + "type": "record_local_activity", + "minimum_protocol_version": "1.18", + "required_fields": [ + "activity_type", + "outcome", + "attempts" + ], + "attempt_reports": { + "field": "attempts", + "shape": "ordered_attempt_reports", + "required_fields": [ + "attempt_number", + "outcome" + ], + "maximum": 100, + "required_from_protocol_version": "1.19", + "worker_attempt_identity": { + "report_field": "attempt_id", + "persistence_field": "worker_attempt_id", + "history_field": "worker_attempt_id", + "durable_field": "activity_attempt_id" + } + }, + "heartbeat_reports": { + "field": "heartbeats", + "shape": "ordered_heartbeat_reports", + "required_fields": [ + "elapsed_ms" + ], + "maximum_per_attempt": 1000, + "maximum_per_command": 1000, + "elapsed_ms_origin": "attempt_started_at", + "persisted_last_heartbeat_at": "attempt_started_at_plus_last_elapsed_ms" + }, + "rolling_upgrade": { + "retained_protocol_version": "1.18", + "missing_attempts": "normalize_to_one_terminal_attempt_without_worker_identity_or_heartbeats", + "strict_shape_selected_by": "negotiated_worker_protocol_version", + "retained_nested_object_unknown_fields": "ignore", + "strict_nested_object_shape_from_protocol_version": "1.19" + } + }, + "api": { + "functions": [ + "Workflow\\V2\\localActivity" + ], + "workflow_facade": [ + "Workflow\\V2\\Workflow::localActivity", + "Workflow\\V2\\Workflow::executeLocalActivity" + ], + "options": "Workflow\\V2\\Support\\LocalActivityOptions" + }, + "execution": { + "mode": "local", + "same_process": true, + "ordinary_activity_task_created": false, + "history_events": [ + "ActivityScheduled", + "ActivityStarted", + "ActivityHeartbeatRecorded", + "ActivityRetryScheduled", + "ActivityCompleted", + "ActivityFailed", + "ActivityTimedOut", + "ActivityCancelled" + ], + "history_marker": { + "execution_mode": "local", + "local_activity": true + } + }, + "heartbeating": { + "lease": "workflow_task", + "automatic_points": [ + "before_attempt_start", + "after_activity_heartbeat" + ], + "long_running_requirement": "activity_code_calls_heartbeat_before_the_workflow_task_lease_can_expire" + }, + "cancellation": { + "preemption": "cooperative", + "observed_at": [ + "heartbeat", + "timeout_enforcement", + "attempt_completion" + ], + "terminal_event": "ActivityCancelled" + }, + "shutdown": { + "graceful": "worker_stops_claiming_new_workflow_tasks_and_allows_in_process_attempt_to_finish_or_heartbeat", + "crash": "workflow_task_lease_expires_and_task_repair_reclaims_the_workflow_task", + "cold_replay": "committed_terminal_event_is_replayed_or_started_attempt_without_terminal_event_schedules_local_retry" + }, + "routing": { + "admission": "activity_class_must_resolve_in_the_workflow_worker_process", + "queue_bypassed": true, + "rejected_options": [ + "connection", + "queue", + "worker_session", + "schedule_to_start_timeout" + ] + }, + "retry": { + "new_attempt_when": [ + "a local attempt starts", + "a retry workflow task replays after backoff", + "cold replay finds a previously started local attempt without a terminal event" + ], + "backoff": "durable_workflow_task_available_at", + "cold_replay_reason": "cold_replay" + }, + "timeouts": { + "start_to_close_timeout": "one_local_attempt", + "schedule_to_close_timeout": "entire_local_execution_including_retries", + "heartbeat_timeout": "gap_between_recorded_local_activity_heartbeats", + "enforcement": "cooperative_at_heartbeat_attempt_start_and_attempt_completion_boundaries" + }, + "visibility": { + "activity_execution_marker": "activity_options.execution_mode", + "history_marker": "payload.execution_mode", + "metrics_marker": "activities.local_*", + "history_export": "same_activity_event_payloads_with_execution_mode_local" + } + }, + "record_local_activity_command": true, + "prepared_flags": { + "prepared_local_activities": "bool", + "prepared_local_activity_groups": "bool", + "prepared_local_activity_cancellation_policies": "ordered subset", + "agree": true + }, + "cluster_capabilities_local_keys": [] + }, + "register_local": { + "status": 201, + "response": { + "registered": true, + "capabilities": [ + "local_activities" + ], + "capability_manifest": { + "local_activities": { + "supported": true, + "minimum_protocol_version": "1.18", + "implementation": "record_local_activity" + }, + "worker_sessions": { + "supported": false, + "minimum_protocol_version": "1.18", + "reason": "not_enabled_for_this_worker_registration" + }, + "sticky_execution": { + "supported": false, + "minimum_protocol_version": "1.18", + "reason": "not_enabled_for_this_worker_registration" + } + }, + "status": "active" + }, + "advertised_as_cluster_info": true + }, + "register_flat_only": { + "status": 409, + "response": { + "registered": false, + "reason": "worker_capability_manifest_mismatch", + "capability": "local_activities", + "requested_version": "1.20", + "minimum_protocol_version": "1.18", + "remediation": "Keep the structured supported flag and the flat routing capability in exact agreement." + } + }, + "register_manifest_only": { + "status": 409, + "response": { + "registered": false, + "reason": "worker_capability_manifest_mismatch", + "capability": "local_activities", + "requested_version": "1.20", + "minimum_protocol_version": "1.18", + "remediation": "Keep the structured supported flag and the flat routing capability in exact agreement." + } + }, + "refused_protocol_floor": { + "status": 409, + "response": { + "task_id": "task-1", + "workflow_task_attempt": 1, + "outcome": "rejected", + "recorded": false, + "reason": "portable_worker_affinity_protocol_floor_unavailable", + "capability": "local_activities", + "requested_version": "1.17", + "minimum_protocol_version": "1.18", + "remediation": "Use cold durable replay unless the registered worker manifest explicitly supports the required capability and exact cache identity." + } + }, + "refused_wrong_owner": { + "status": 409, + "response": { + "task_id": "task-1", + "workflow_task_attempt": 1, + "error": "Workflow task lease is owned by another worker.", + "reason": "lease_owner_mismatch", + "lease_owner": "pr1-record-local-activity-rla-a" + } + }, + "refused_wrong_owner_invalid_report": { + "status": 409, + "response": { + "task_id": "task-1", + "workflow_task_attempt": 1, + "error": "Workflow task lease is owned by another worker.", + "reason": "lease_owner_mismatch", + "lease_owner": "pr1-record-local-activity-rla-a" + } + }, + "invalid_transport": { + "status": 422, + "response": { + "message": "The selected commands.0.outcome is invalid. (and 2 more errors)", + "reason": "validation_failed", + "errors": { + "commands.0.outcome": [ + "The selected commands.0.outcome is invalid." + ], + "commands.0.attempts.0.attempt_number": [ + "The commands.0.attempts.0.attempt_number field is required." + ], + "commands.0.attempts.0.outcome": [ + "The selected commands.0.attempts.0.outcome is invalid." + ] + }, + "validation_errors": { + "commands.0.outcome": [ + "The selected commands.0.outcome is invalid." + ], + "commands.0.attempts.0.attempt_number": [ + "The commands.0.attempts.0.attempt_number field is required." + ], + "commands.0.attempts.0.outcome": [ + "The selected commands.0.attempts.0.outcome is invalid." + ] + } + } + }, + "invalid_worker_session": { + "status": 422, + "response": { + "message": "worker_session is only supported for schedule_activity commands.", + "reason": "validation_failed", + "errors": { + "commands.0.worker_session": [ + "worker_session is only supported for schedule_activity commands." + ] + }, + "validation_errors": { + "commands.0.worker_session": [ + "worker_session is only supported for schedule_activity commands." + ] + } + } + }, + "invalid_cancellation_policy": { + "status": 422, + "response": { + "message": "cancellation_policy is only supported for start_child_workflow, schedule_activity or prepared local Activity commands.", + "reason": "validation_failed", + "errors": { + "commands.0.cancellation_policy": [ + "cancellation_policy is only supported for start_child_workflow, schedule_activity or prepared local Activity commands." + ] + }, + "validation_errors": { + "commands.0.cancellation_policy": [ + "cancellation_policy is only supported for start_child_workflow, schedule_activity or prepared local Activity commands." + ] + } + } + }, + "invalid_timeout_envelope": { + "status": 422, + "response": { + "message": "heartbeat_timeout cannot exceed start_to_close_timeout.", + "reason": "validation_failed", + "errors": { + "commands.0.heartbeat_timeout": [ + "heartbeat_timeout cannot exceed start_to_close_timeout." + ] + }, + "validation_errors": { + "commands.0.heartbeat_timeout": [ + "heartbeat_timeout cannot exceed start_to_close_timeout." + ] + } + } + }, + "invalid_field_scope": { + "status": 422, + "response": { + "message": "Workflow task command field [schedule_to_start_timeout] is not valid on a record_local_activity command. schedule_to_start_timeout limits queue wait before an activity attempt starts and only applies to a schedule_activity command.", + "reason": "validation_failed", + "errors": { + "commands.0.schedule_to_start_timeout": [ + "Workflow task command field [schedule_to_start_timeout] is not valid on a record_local_activity command. schedule_to_start_timeout limits queue wait before an activity attempt starts and only applies to a schedule_activity command." + ] + }, + "validation_errors": { + "commands.0.schedule_to_start_timeout": [ + "Workflow task command field [schedule_to_start_timeout] is not valid on a record_local_activity command. schedule_to_start_timeout limits queue wait before an activity attempt starts and only applies to a schedule_activity command." + ] + } + } + }, + "invalid_activity_type": { + "status": 422, + "response": { + "message": "Local activity records require a non-empty activity_type.", + "reason": "validation_failed", + "errors": { + "commands.0.activity_type": [ + "Local activity records require a non-empty activity_type." + ] + }, + "validation_errors": { + "commands.0.activity_type": [ + "Local activity records require a non-empty activity_type." + ] + } + } + }, + "invalid_outcome": { + "status": 422, + "response": { + "message": "Local activity outcome must be completed, failed, timed_out, or cancelled.", + "reason": "validation_failed", + "errors": { + "commands.0.outcome": [ + "Local activity outcome must be completed, failed, timed_out, or cancelled." + ] + }, + "validation_errors": { + "commands.0.outcome": [ + "Local activity outcome must be completed, failed, timed_out, or cancelled." + ] + } + } + }, + "invalid_report": { + "status": 422, + "response": { + "message": "Completed local activities cannot be marked non-retryable. (and 5 more errors)", + "reason": "validation_failed", + "errors": { + "commands.0.non_retryable": [ + "Completed local activities cannot be marked non-retryable." + ], + "commands.0.attempts.0.extra": [ + "Field [extra] is not supported in a local activity attempt report." + ], + "commands.0.attempts.0.attempt_number": [ + "Local activity attempt_number must be the ordered one-based value 1." + ], + "commands.0.attempts.0.outcome": [ + "The final local activity attempt outcome must match the command outcome." + ], + "commands.0.attempts.0.message": [ + "Failed, timed out, and cancelled local activity attempts require a message." + ], + "commands.0.result": [ + "Completed local activities require a result payload envelope." + ] + }, + "validation_errors": { + "commands.0.non_retryable": [ + "Completed local activities cannot be marked non-retryable." + ], + "commands.0.attempts.0.extra": [ + "Field [extra] is not supported in a local activity attempt report." + ], + "commands.0.attempts.0.attempt_number": [ + "Local activity attempt_number must be the ordered one-based value 1." + ], + "commands.0.attempts.0.outcome": [ + "The final local activity attempt outcome must match the command outcome." + ], + "commands.0.attempts.0.message": [ + "Failed, timed out, and cancelled local activity attempts require a message." + ], + "commands.0.result": [ + "Completed local activities require a result payload envelope." + ] + } + } + }, + "invalid_missing_attempts": { + "status": 422, + "response": { + "message": "Local activity attempts must be a non-empty ordered list.", + "reason": "validation_failed", + "errors": { + "commands.0.attempts": [ + "Local activity attempts must be a non-empty ordered list." + ] + }, + "validation_errors": { + "commands.0.attempts": [ + "Local activity attempts must be a non-empty ordered list." + ] + } + } + }, + "invalid_attempt_budget": { + "status": 422, + "response": { + "message": "Local activity report contains 2 attempts but retry_policy.max_attempts is 1.", + "reason": "validation_failed", + "errors": { + "commands.0.attempts": [ + "Local activity report contains 2 attempts but retry_policy.max_attempts is 1." + ] + }, + "validation_errors": { + "commands.0.attempts": [ + "Local activity report contains 2 attempts but retry_policy.max_attempts is 1." + ] + } + } + }, + "invalid_retry_policy": { + "status": 422, + "response": { + "message": "Field [jitter] is not supported in a local activity retry policy report. (and 2 more errors)", + "reason": "validation_failed", + "errors": { + "commands.0.retry_policy.jitter": [ + "Field [jitter] is not supported in a local activity retry policy report." + ], + "commands.0.retry_policy.non_retryable_error_types": [ + "Local activity retry policy non_retryable_error_types must not contain duplicates." + ], + "commands.0.retry_policy.max_attempts": [ + "Local activity retry policy max_attempts must not exceed 100." + ] + }, + "validation_errors": { + "commands.0.retry_policy.jitter": [ + "Field [jitter] is not supported in a local activity retry policy report." + ], + "commands.0.retry_policy.non_retryable_error_types": [ + "Local activity retry policy non_retryable_error_types must not contain duplicates." + ], + "commands.0.retry_policy.max_attempts": [ + "Local activity retry policy max_attempts must not exceed 100." + ] + } + } + }, + "invalid_retry_attempts": { + "status": 422, + "response": { + "message": "Only failed or timed_out local activity attempts may be followed by a retry. (and 8 more errors)", + "reason": "validation_failed", + "errors": { + "commands.0.attempts.0.outcome": [ + "Only failed or timed_out local activity attempts may be followed by a retry." + ], + "commands.0.attempts.0.non_retryable": [ + "A non-retryable local activity attempt cannot be followed by a retry." + ], + "commands.0.attempts.0.retry_reason": [ + "A retried completed local activity attempt requires a compatible retry_reason." + ], + "commands.0.attempts.0.backoff_seconds": [ + "A retried local activity attempt requires backoff_seconds." + ], + "commands.0.attempts.1.non_retryable": [ + "A non-retryable local activity attempt cannot be followed by a retry." + ], + "commands.0.attempts.1.timeout_kind": [ + "Only timed out local activity attempts may declare timeout_kind." + ], + "commands.0.attempts.1.retry_reason": [ + "A retried failed local activity attempt requires a compatible retry_reason." + ], + "commands.0.attempts.2.attempt_id": [ + "Local activity attempt_id must be unique within the reported sequence." + ], + "commands.0.attempts.2": [ + "The terminal local activity attempt cannot declare retry_reason or backoff_seconds." + ] + }, + "validation_errors": { + "commands.0.attempts.0.outcome": [ + "Only failed or timed_out local activity attempts may be followed by a retry." + ], + "commands.0.attempts.0.non_retryable": [ + "A non-retryable local activity attempt cannot be followed by a retry." + ], + "commands.0.attempts.0.retry_reason": [ + "A retried completed local activity attempt requires a compatible retry_reason." + ], + "commands.0.attempts.0.backoff_seconds": [ + "A retried local activity attempt requires backoff_seconds." + ], + "commands.0.attempts.1.non_retryable": [ + "A non-retryable local activity attempt cannot be followed by a retry." + ], + "commands.0.attempts.1.timeout_kind": [ + "Only timed out local activity attempts may declare timeout_kind." + ], + "commands.0.attempts.1.retry_reason": [ + "A retried failed local activity attempt requires a compatible retry_reason." + ], + "commands.0.attempts.2.attempt_id": [ + "Local activity attempt_id must be unique within the reported sequence." + ], + "commands.0.attempts.2": [ + "The terminal local activity attempt cannot declare retry_reason or backoff_seconds." + ] + } + } + }, + "reordered_attempts_report": { + "status": 422, + "response": { + "message": "Local activity attempts must be a non-empty ordered list.", + "reason": "validation_failed", + "errors": { + "commands.0.attempts": [ + "Local activity attempts must be a non-empty ordered list." + ] + }, + "validation_errors": { + "commands.0.attempts": [ + "Local activity attempts must be a non-empty ordered list." + ] + } + } + }, + "invalid_timed_out": { + "status": 422, + "response": { + "message": "The terminal local activity attempt non_retryable value must match the command outcome. (and 3 more errors)", + "reason": "validation_failed", + "errors": { + "commands.0.attempts.0.non_retryable": [ + "The terminal local activity attempt non_retryable value must match the command outcome." + ], + "commands.0.attempts.0.timeout_kind": [ + "Timed out local activity attempts require start_to_close, schedule_to_close, or heartbeat timeout_kind." + ], + "commands.0.timeout_kind": [ + "Timed out local activities require a timeout_kind." + ], + "commands.0.message": [ + "Failed, timed out, and cancelled local activities require a message." + ] + }, + "validation_errors": { + "commands.0.attempts.0.non_retryable": [ + "The terminal local activity attempt non_retryable value must match the command outcome." + ], + "commands.0.attempts.0.timeout_kind": [ + "Timed out local activity attempts require start_to_close, schedule_to_close, or heartbeat timeout_kind." + ], + "commands.0.timeout_kind": [ + "Timed out local activities require a timeout_kind." + ], + "commands.0.message": [ + "Failed, timed out, and cancelled local activities require a message." + ] + } + } + }, + "invalid_heartbeats": { + "status": 422, + "response": { + "message": "Local activity heartbeat elapsed_ms values must be ordered monotonically. (and 2 more errors)", + "reason": "validation_failed", + "errors": { + "commands.0.attempts.0.heartbeats.1.elapsed_ms": [ + "Local activity heartbeat elapsed_ms values must be ordered monotonically." + ], + "commands.0.attempts.0.heartbeats.1.details": [ + "Local activity heartbeat details must be an object when provided." + ], + "commands.0.attempts.0.heartbeats.2.note": [ + "Field [note] is not supported in a local activity heartbeat report." + ] + }, + "validation_errors": { + "commands.0.attempts.0.heartbeats.1.elapsed_ms": [ + "Local activity heartbeat elapsed_ms values must be ordered monotonically." + ], + "commands.0.attempts.0.heartbeats.1.details": [ + "Local activity heartbeat details must be an object when provided." + ], + "commands.0.attempts.0.heartbeats.2.note": [ + "Field [note] is not supported in a local activity heartbeat report." + ] + } + } + }, + "reordered_heartbeats_report": { + "status": 422, + "response": { + "message": "Local activity attempt heartbeats must be an ordered list.", + "reason": "validation_failed", + "errors": { + "commands.0.attempts.0.heartbeats": [ + "Local activity attempt heartbeats must be an ordered list." + ] + }, + "validation_errors": { + "commands.0.attempts.0.heartbeats": [ + "Local activity attempt heartbeats must be an ordered list." + ] + } + } + }, + "invalid_second_command": { + "status": 422, + "response": { + "message": "Local activity records require a non-empty activity_type.", + "reason": "validation_failed", + "errors": { + "commands.1.activity_type": [ + "Local activity records require a non-empty activity_type." + ] + }, + "validation_errors": { + "commands.1.activity_type": [ + "Local activity records require a non-empty activity_type." + ] + } + } + }, + "completed": { + "status": 200, + "response": { + "task_id": "task-1", + "workflow_task_attempt": 1, + "outcome": "completed", + "recorded": true, + "run_id": "run-1", + "run_status": "waiting", + "created_task_ids": [], + "reason": null + } + }, + "completed_run": { + "event_types": [ + "StartAccepted", + "WorkflowStarted", + "ActivityScheduled", + "ActivityStarted", + "ActivityHeartbeatRecorded", + "ActivityRetryScheduled", + "ActivityStarted", + "ActivityCompleted", + "SignalWaitOpened" + ], + "activity_events": [ + { + "sequence": 3, + "event_type": "ActivityScheduled", + "timestamp": "{time}", + "principal": null, + "payload": { + "execution_mode": "local", + "local_activity": true, + "activity_execution_id": "execution-1", + "activity_class": "parity.v1.local", + "activity_type": "parity.v1.local", + "sequence": 1, + "workflow_task_id": "task-1", + "activity": { + "id": "execution-1", + "idempotency_key": "execution-1", + "sequence": 1, + "type": "parity.v1.local", + "class": "parity.v1.local", + "execution_mode": "local", + "local_activity": true, + "status": "pending", + "payload_codec": "avro", + "attempt_count": 0, + "retry_policy": { + "max_attempts": 2, + "backoff_seconds": [ + 1 + ] + }, + "queue": "pr1-record-local-activity-rla-a-queue", + "created_at": "{time}", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + } + }, + "task": { + "id": "task-1", + "type": "workflow", + "status": "leased", + "available_at": "{time}", + "leased_at": "{time}", + "lease_owner": "pr1-record-local-activity-rla-a", + "lease_expires_at": "{time}", + "attempt_count": 1, + "repair_count": 0, + "connection": "sqlite", + "queue": "pr1-record-local-activity-rla-a-queue", + "last_dispatch_attempt_at": "{time}", + "last_dispatched_at": "{time}" + } + } + }, + { + "sequence": 4, + "event_type": "ActivityStarted", + "timestamp": "{time}", + "principal": null, + "payload": { + "execution_mode": "local", + "local_activity": true, + "activity_execution_id": "execution-1", + "activity_class": "parity.v1.local", + "activity_type": "parity.v1.local", + "sequence": 1, + "workflow_task_id": "task-1", + "activity": { + "id": "execution-1", + "idempotency_key": "execution-1", + "sequence": 1, + "type": "parity.v1.local", + "class": "parity.v1.local", + "execution_mode": "local", + "local_activity": true, + "attempt_id": "attempt-1:upper", + "status": "running", + "payload_codec": "avro", + "attempt_count": 1, + "retry_policy": { + "max_attempts": 2, + "backoff_seconds": [ + 1 + ] + }, + "queue": "pr1-record-local-activity-rla-a-queue", + "created_at": "{time}", + "started_at": "@-2700ms", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + } + }, + "activity_attempt_id": "attempt-1:upper", + "worker_attempt_id": "w-1", + "attempt_number": 1, + "activity_attempt": { + "id": "attempt-1:upper", + "worker_attempt_id": "w-1", + "attempt_number": 1, + "status": "running", + "task_id": "task-1", + "lease_owner": "pr1-record-local-activity-rla-a", + "started_at": "@-2700ms" + }, + "task": "(the first event's task)" + } + }, + { + "sequence": 5, + "event_type": "ActivityHeartbeatRecorded", + "timestamp": "{time}", + "principal": null, + "payload": { + "execution_mode": "local", + "local_activity": true, + "activity_execution_id": "execution-1", + "activity_class": "parity.v1.local", + "activity_type": "parity.v1.local", + "sequence": 1, + "workflow_task_id": "task-1", + "activity": { + "id": "execution-1", + "idempotency_key": "execution-1", + "sequence": 1, + "type": "parity.v1.local", + "class": "parity.v1.local", + "execution_mode": "local", + "local_activity": true, + "attempt_id": "attempt-1:upper", + "status": "running", + "payload_codec": "avro", + "attempt_count": 1, + "retry_policy": { + "max_attempts": 2, + "backoff_seconds": [ + 1 + ] + }, + "queue": "pr1-record-local-activity-rla-a-queue", + "created_at": "{time}", + "started_at": "@-2700ms", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "last_heartbeat_at": "@-2200ms" + }, + "activity_attempt_id": "attempt-1:upper", + "worker_attempt_id": "w-1", + "attempt_number": 1, + "activity_attempt": { + "id": "attempt-1:upper", + "worker_attempt_id": "w-1", + "attempt_number": 1, + "status": "running", + "task_id": "task-1", + "lease_owner": "pr1-record-local-activity-rla-a", + "started_at": "@-2700ms", + "last_heartbeat_at": "@-2200ms" + }, + "heartbeat_at": "@-2200ms", + "lease_expires_at": null, + "progress": { + "step": "one" + }, + "task": "(the first event's task)" + } + }, + { + "sequence": 6, + "event_type": "ActivityRetryScheduled", + "timestamp": "{time}", + "principal": null, + "payload": { + "execution_mode": "local", + "local_activity": true, + "activity_execution_id": "execution-1", + "activity_class": "parity.v1.local", + "activity_type": "parity.v1.local", + "sequence": 1, + "workflow_task_id": "task-1", + "activity": { + "id": "execution-1", + "idempotency_key": "execution-1", + "sequence": 1, + "type": "parity.v1.local", + "class": "parity.v1.local", + "execution_mode": "local", + "local_activity": true, + "attempt_id": "attempt-1:upper", + "status": "pending", + "payload_codec": "avro", + "attempt_count": 1, + "retry_policy": { + "max_attempts": 2, + "backoff_seconds": [ + 1 + ] + }, + "queue": "pr1-record-local-activity-rla-a-queue", + "created_at": "{time}", + "started_at": "@-2700ms", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "last_heartbeat_at": "@-2200ms" + }, + "activity_attempt_id": "attempt-1:upper", + "worker_attempt_id": "w-1", + "activity_attempt": { + "id": "attempt-1:upper", + "worker_attempt_id": "w-1", + "attempt_number": 1, + "status": "failed", + "task_id": "task-1", + "lease_owner": "pr1-record-local-activity-rla-a", + "started_at": "@-2700ms", + "last_heartbeat_at": "@-2200ms", + "closed_at": "@-1200ms" + }, + "retry_after_attempt_id": "attempt-1:upper", + "retry_after_attempt": 1, + "message": "first λ", + "exception_type": "RuntimeException", + "retry_reason": "failure", + "retry_backoff_seconds": 1, + "task": "(the first event's task)" + } + }, + { + "sequence": 7, + "event_type": "ActivityStarted", + "timestamp": "{time}", + "principal": null, + "payload": { + "execution_mode": "local", + "local_activity": true, + "activity_execution_id": "execution-1", + "activity_class": "parity.v1.local", + "activity_type": "parity.v1.local", + "sequence": 1, + "workflow_task_id": "task-1", + "activity": { + "id": "execution-1", + "idempotency_key": "execution-1", + "sequence": 1, + "type": "parity.v1.local", + "class": "parity.v1.local", + "execution_mode": "local", + "local_activity": true, + "attempt_id": "attempt-2:upper", + "status": "running", + "payload_codec": "avro", + "attempt_count": 2, + "retry_policy": { + "max_attempts": 2, + "backoff_seconds": [ + 1 + ] + }, + "queue": "pr1-record-local-activity-rla-a-queue", + "last_heartbeat_at": "@-2200ms", + "created_at": "{time}", + "started_at": "@-2700ms", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + } + }, + "activity_attempt_id": "attempt-2:upper", + "worker_attempt_id": "w-2", + "attempt_number": 2, + "activity_attempt": { + "id": "attempt-2:upper", + "worker_attempt_id": "w-2", + "attempt_number": 2, + "status": "running", + "task_id": "task-1", + "lease_owner": "pr1-record-local-activity-rla-a", + "started_at": "@-200ms" + }, + "task": "(the first event's task)" + } + }, + { + "sequence": 8, + "event_type": "ActivityCompleted", + "timestamp": "{time}", + "principal": null, + "payload": { + "execution_mode": "local", + "local_activity": true, + "activity_execution_id": "execution-1", + "activity_class": "parity.v1.local", + "activity_type": "parity.v1.local", + "sequence": 1, + "workflow_task_id": "task-1", + "activity": { + "id": "execution-1", + "idempotency_key": "execution-1", + "sequence": 1, + "type": "parity.v1.local", + "class": "parity.v1.local", + "execution_mode": "local", + "local_activity": true, + "attempt_id": "attempt-2:upper", + "status": "completed", + "payload_codec": "avro", + "attempt_count": 2, + "retry_policy": { + "max_attempts": 2, + "backoff_seconds": [ + 1 + ] + }, + "queue": "pr1-record-local-activity-rla-a-queue", + "last_heartbeat_at": "@-2200ms", + "created_at": "{time}", + "started_at": "@-2700ms", + "closed_at": "@0ms", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoIZG9uZQ==" + } + }, + "activity_attempt_id": "attempt-2:upper", + "worker_attempt_id": "w-2", + "attempt_number": 2, + "activity_attempt": { + "id": "attempt-2:upper", + "worker_attempt_id": "w-2", + "attempt_number": 2, + "status": "completed", + "task_id": "task-1", + "lease_owner": "pr1-record-local-activity-rla-a", + "started_at": "@-200ms", + "closed_at": "@0ms" + }, + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoIZG9uZQ==" + }, + "payload_codec": "avro", + "task": "(the first event's task)" + } + } + ], + "activities": [ + { + "id": "execution-1", + "idempotency_key": "execution-1", + "sequence": 1, + "type": "parity.v1.local", + "class": "parity.v1.local", + "execution_mode": "local", + "local_activity": true, + "history_authority": "typed_history", + "diagnostic_only": false, + "history_event_types": [ + "ActivityScheduled", + "ActivityStarted", + "ActivityHeartbeatRecorded", + "ActivityRetryScheduled", + "ActivityCompleted" + ], + "history_unsupported_reason": null, + "reused_from_run_id": null, + "reused_activity_execution_id": null, + "reused_recorded_at": null, + "row_status": null, + "parallel_group_kind": null, + "parallel_group_id": null, + "parallel_group_base_sequence": null, + "parallel_group_size": null, + "parallel_group_index": null, + "parallel_group_path": [], + "attempt_id": "attempt-2:upper", + "worker_attempt_id": "w-2", + "status": "completed", + "attempt_count": 2, + "retry_policy": { + "max_attempts": 2, + "backoff_seconds": [ + 1 + ] + }, + "connection": null, + "queue": "pr1-record-local-activity-rla-a-queue", + "last_heartbeat_at": "@-2200ms", + "last_heartbeat_progress": null, + "created_at": "{time}", + "started_at": "@-2700ms", + "closed_at": "@0ms", + "attempts": [ + { + "id": "attempt-1:upper", + "worker_attempt_id": "w-1", + "attempt_number": 1, + "status": "failed", + "task_id": "task-1", + "lease_owner": "pr1-record-local-activity-rla-a", + "lease_expires_at": null, + "started_at": "@-2700ms", + "last_heartbeat_at": "@-2200ms", + "last_heartbeat_progress": null, + "closed_at": "@-1200ms", + "can_continue": false, + "cancel_requested": false, + "stop_reason": "attempt_closed", + "failure_reason": null + }, + { + "id": "attempt-2:upper", + "worker_attempt_id": "w-2", + "attempt_number": 2, + "status": "completed", + "task_id": "task-1", + "lease_owner": "pr1-record-local-activity-rla-a", + "lease_expires_at": null, + "started_at": "@-200ms", + "last_heartbeat_at": null, + "last_heartbeat_progress": null, + "closed_at": "@0ms", + "can_continue": false, + "cancel_requested": false, + "stop_reason": "attempt_closed", + "failure_reason": null + } + ] + } + ], + "run": { + "status": "waiting", + "status_bucket": "running", + "is_terminal": false, + "closed_reason": null, + "wait_kind": "signal", + "wait_reason": "Waiting for signal resume" + }, + "attempt_identity_order": [ + "attempt-2:upper", + "attempt-1:upper" + ] + }, + "failed": { + "status": 200, + "response": { + "task_id": "task-2", + "workflow_task_attempt": 1, + "outcome": "completed", + "recorded": true, + "run_id": "run-2", + "run_status": "completed", + "created_task_ids": [], + "reason": null + } + }, + "failed_run": { + "event_types": [ + "StartAccepted", + "WorkflowStarted", + "ActivityScheduled", + "ActivityStarted", + "ActivityHeartbeatRecorded", + "ActivityHeartbeatRecorded", + "ActivityHeartbeatRecorded", + "ActivityRetryScheduled", + "ActivityStarted", + "ActivityRetryScheduled", + "ActivityStarted", + "ActivityFailed", + "ActivityScheduled", + "ActivityStarted", + "ActivityTimedOut", + "WorkflowCompleted" + ], + "activity_events": [ + { + "sequence": 3, + "event_type": "ActivityScheduled", + "timestamp": "{time}", + "principal": null, + "payload": { + "execution_mode": "local", + "local_activity": true, + "activity_execution_id": "execution-2", + "activity_class": "parity.v1.local", + "activity_type": "parity.v1.local", + "sequence": 1, + "workflow_task_id": "task-2", + "activity": { + "id": "execution-2", + "idempotency_key": "execution-2", + "sequence": 1, + "type": "parity.v1.local", + "class": "parity.v1.local", + "execution_mode": "local", + "local_activity": true, + "status": "pending", + "payload_codec": "avro", + "attempt_count": 0, + "retry_policy": { + "max_attempts": 3, + "backoff_seconds": [ + 2, + 0 + ], + "non_retryable_error_types": [ + "App\\Errors\\Fatal" + ] + }, + "queue": "pr1-record-local-activity-rla-b-queue", + "created_at": "{time}", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + } + }, + "task": { + "id": "task-2", + "type": "workflow", + "status": "leased", + "available_at": "{time}", + "leased_at": "{time}", + "lease_owner": "pr1-record-local-activity-rla-b", + "lease_expires_at": "{time}", + "attempt_count": 1, + "repair_count": 0, + "connection": "sqlite", + "queue": "pr1-record-local-activity-rla-b-queue", + "last_dispatch_attempt_at": "{time}", + "last_dispatched_at": "{time}" + } + } + }, + { + "sequence": 4, + "event_type": "ActivityStarted", + "timestamp": "{time}", + "principal": null, + "payload": { + "execution_mode": "local", + "local_activity": true, + "activity_execution_id": "execution-2", + "activity_class": "parity.v1.local", + "activity_type": "parity.v1.local", + "sequence": 1, + "workflow_task_id": "task-2", + "activity": { + "id": "execution-2", + "idempotency_key": "execution-2", + "sequence": 1, + "type": "parity.v1.local", + "class": "parity.v1.local", + "execution_mode": "local", + "local_activity": true, + "attempt_id": "attempt-3:upper", + "status": "running", + "payload_codec": "avro", + "attempt_count": 1, + "retry_policy": { + "max_attempts": 3, + "backoff_seconds": [ + 2, + 0 + ], + "non_retryable_error_types": [ + "App\\Errors\\Fatal" + ] + }, + "queue": "pr1-record-local-activity-rla-b-queue", + "created_at": "{time}", + "started_at": "@-5040ms", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + } + }, + "activity_attempt_id": "attempt-3:upper", + "worker_attempt_id": "b-1", + "attempt_number": 1, + "activity_attempt": { + "id": "attempt-3:upper", + "worker_attempt_id": "b-1", + "attempt_number": 1, + "status": "running", + "task_id": "task-2", + "lease_owner": "pr1-record-local-activity-rla-b", + "started_at": "@-5040ms" + }, + "task": "(the first event's task)" + } + }, + { + "sequence": 5, + "event_type": "ActivityHeartbeatRecorded", + "timestamp": "{time}", + "principal": null, + "payload": { + "execution_mode": "local", + "local_activity": true, + "activity_execution_id": "execution-2", + "activity_class": "parity.v1.local", + "activity_type": "parity.v1.local", + "sequence": 1, + "workflow_task_id": "task-2", + "activity": { + "id": "execution-2", + "idempotency_key": "execution-2", + "sequence": 1, + "type": "parity.v1.local", + "class": "parity.v1.local", + "execution_mode": "local", + "local_activity": true, + "attempt_id": "attempt-3:upper", + "status": "running", + "payload_codec": "avro", + "attempt_count": 1, + "retry_policy": { + "max_attempts": 3, + "backoff_seconds": [ + 2, + 0 + ], + "non_retryable_error_types": [ + "App\\Errors\\Fatal" + ] + }, + "queue": "pr1-record-local-activity-rla-b-queue", + "created_at": "{time}", + "started_at": "@-5040ms", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "last_heartbeat_at": "@-4040ms" + }, + "activity_attempt_id": "attempt-3:upper", + "worker_attempt_id": "b-1", + "attempt_number": 1, + "activity_attempt": { + "id": "attempt-3:upper", + "worker_attempt_id": "b-1", + "attempt_number": 1, + "status": "running", + "task_id": "task-2", + "lease_owner": "pr1-record-local-activity-rla-b", + "started_at": "@-5040ms", + "last_heartbeat_at": "@-4040ms" + }, + "heartbeat_at": "@-4040ms", + "lease_expires_at": null, + "progress": [], + "task": "(the first event's task)" + } + }, + { + "sequence": 6, + "event_type": "ActivityHeartbeatRecorded", + "timestamp": "{time}", + "principal": null, + "payload": { + "execution_mode": "local", + "local_activity": true, + "activity_execution_id": "execution-2", + "activity_class": "parity.v1.local", + "activity_type": "parity.v1.local", + "sequence": 1, + "workflow_task_id": "task-2", + "activity": { + "id": "execution-2", + "idempotency_key": "execution-2", + "sequence": 1, + "type": "parity.v1.local", + "class": "parity.v1.local", + "execution_mode": "local", + "local_activity": true, + "attempt_id": "attempt-3:upper", + "status": "running", + "payload_codec": "avro", + "attempt_count": 1, + "retry_policy": { + "max_attempts": 3, + "backoff_seconds": [ + 2, + 0 + ], + "non_retryable_error_types": [ + "App\\Errors\\Fatal" + ] + }, + "queue": "pr1-record-local-activity-rla-b-queue", + "created_at": "{time}", + "started_at": "@-5040ms", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "last_heartbeat_at": "@-4040ms" + }, + "activity_attempt_id": "attempt-3:upper", + "worker_attempt_id": "b-1", + "attempt_number": 1, + "activity_attempt": { + "id": "attempt-3:upper", + "worker_attempt_id": "b-1", + "attempt_number": 1, + "status": "running", + "task_id": "task-2", + "lease_owner": "pr1-record-local-activity-rla-b", + "started_at": "@-5040ms", + "last_heartbeat_at": "@-4040ms" + }, + "heartbeat_at": "@-4040ms", + "lease_expires_at": null, + "progress": [], + "task": "(the first event's task)" + } + }, + { + "sequence": 7, + "event_type": "ActivityHeartbeatRecorded", + "timestamp": "{time}", + "principal": null, + "payload": { + "execution_mode": "local", + "local_activity": true, + "activity_execution_id": "execution-2", + "activity_class": "parity.v1.local", + "activity_type": "parity.v1.local", + "sequence": 1, + "workflow_task_id": "task-2", + "activity": { + "id": "execution-2", + "idempotency_key": "execution-2", + "sequence": 1, + "type": "parity.v1.local", + "class": "parity.v1.local", + "execution_mode": "local", + "local_activity": true, + "attempt_id": "attempt-3:upper", + "status": "running", + "payload_codec": "avro", + "attempt_count": 1, + "retry_policy": { + "max_attempts": 3, + "backoff_seconds": [ + 2, + 0 + ], + "non_retryable_error_types": [ + "App\\Errors\\Fatal" + ] + }, + "queue": "pr1-record-local-activity-rla-b-queue", + "created_at": "{time}", + "started_at": "@-5040ms", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "last_heartbeat_at": "@-2540ms" + }, + "activity_attempt_id": "attempt-3:upper", + "worker_attempt_id": "b-1", + "attempt_number": 1, + "activity_attempt": { + "id": "attempt-3:upper", + "worker_attempt_id": "b-1", + "attempt_number": 1, + "status": "running", + "task_id": "task-2", + "lease_owner": "pr1-record-local-activity-rla-b", + "started_at": "@-5040ms", + "last_heartbeat_at": "@-2540ms" + }, + "heartbeat_at": "@-2540ms", + "lease_expires_at": null, + "progress": { + "p": 2 + }, + "task": "(the first event's task)" + } + }, + { + "sequence": 8, + "event_type": "ActivityRetryScheduled", + "timestamp": "{time}", + "principal": null, + "payload": { + "execution_mode": "local", + "local_activity": true, + "activity_execution_id": "execution-2", + "activity_class": "parity.v1.local", + "activity_type": "parity.v1.local", + "sequence": 1, + "workflow_task_id": "task-2", + "activity": { + "id": "execution-2", + "idempotency_key": "execution-2", + "sequence": 1, + "type": "parity.v1.local", + "class": "parity.v1.local", + "execution_mode": "local", + "local_activity": true, + "attempt_id": "attempt-3:upper", + "status": "pending", + "payload_codec": "avro", + "attempt_count": 1, + "retry_policy": { + "max_attempts": 3, + "backoff_seconds": [ + 2, + 0 + ], + "non_retryable_error_types": [ + "App\\Errors\\Fatal" + ] + }, + "queue": "pr1-record-local-activity-rla-b-queue", + "created_at": "{time}", + "started_at": "@-5040ms", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "last_heartbeat_at": "@-2540ms" + }, + "activity_attempt_id": "attempt-3:upper", + "worker_attempt_id": "b-1", + "activity_attempt": { + "id": "attempt-3:upper", + "worker_attempt_id": "b-1", + "attempt_number": 1, + "status": "failed", + "task_id": "task-2", + "lease_owner": "pr1-record-local-activity-rla-b", + "started_at": "@-5040ms", + "last_heartbeat_at": "@-2540ms", + "closed_at": "@-2040ms" + }, + "retry_after_attempt_id": "attempt-3:upper", + "retry_after_attempt": 1, + "message": "slow", + "exception_type": null, + "retry_reason": "timeout", + "retry_backoff_seconds": 2, + "task": "(the first event's task)" + } + }, + { + "sequence": 9, + "event_type": "ActivityStarted", + "timestamp": "{time}", + "principal": null, + "payload": { + "execution_mode": "local", + "local_activity": true, + "activity_execution_id": "execution-2", + "activity_class": "parity.v1.local", + "activity_type": "parity.v1.local", + "sequence": 1, + "workflow_task_id": "task-2", + "activity": { + "id": "execution-2", + "idempotency_key": "execution-2", + "sequence": 1, + "type": "parity.v1.local", + "class": "parity.v1.local", + "execution_mode": "local", + "local_activity": true, + "attempt_id": "attempt-4:upper", + "status": "running", + "payload_codec": "avro", + "attempt_count": 2, + "retry_policy": { + "max_attempts": 3, + "backoff_seconds": [ + 2, + 0 + ], + "non_retryable_error_types": [ + "App\\Errors\\Fatal" + ] + }, + "queue": "pr1-record-local-activity-rla-b-queue", + "last_heartbeat_at": "@-2540ms", + "created_at": "{time}", + "started_at": "@-5040ms", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + } + }, + "activity_attempt_id": "attempt-4:upper", + "worker_attempt_id": "b-2", + "attempt_number": 2, + "activity_attempt": { + "id": "attempt-4:upper", + "worker_attempt_id": "b-2", + "attempt_number": 2, + "status": "running", + "task_id": "task-2", + "lease_owner": "pr1-record-local-activity-rla-b", + "started_at": "@-40ms" + }, + "task": "(the first event's task)" + } + }, + { + "sequence": 10, + "event_type": "ActivityRetryScheduled", + "timestamp": "{time}", + "principal": null, + "payload": { + "execution_mode": "local", + "local_activity": true, + "activity_execution_id": "execution-2", + "activity_class": "parity.v1.local", + "activity_type": "parity.v1.local", + "sequence": 1, + "workflow_task_id": "task-2", + "activity": { + "id": "execution-2", + "idempotency_key": "execution-2", + "sequence": 1, + "type": "parity.v1.local", + "class": "parity.v1.local", + "execution_mode": "local", + "local_activity": true, + "attempt_id": "attempt-4:upper", + "status": "pending", + "payload_codec": "avro", + "attempt_count": 2, + "retry_policy": { + "max_attempts": 3, + "backoff_seconds": [ + 2, + 0 + ], + "non_retryable_error_types": [ + "App\\Errors\\Fatal" + ] + }, + "queue": "pr1-record-local-activity-rla-b-queue", + "last_heartbeat_at": "@-2540ms", + "created_at": "{time}", + "started_at": "@-5040ms", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + } + }, + "activity_attempt_id": "attempt-4:upper", + "worker_attempt_id": "b-2", + "activity_attempt": { + "id": "attempt-4:upper", + "worker_attempt_id": "b-2", + "attempt_number": 2, + "status": "failed", + "task_id": "task-2", + "lease_owner": "pr1-record-local-activity-rla-b", + "started_at": "@-40ms", + "closed_at": "@-40ms" + }, + "retry_after_attempt_id": "attempt-4:upper", + "retry_after_attempt": 2, + "message": "flaky", + "exception_type": "RuntimeException", + "retry_reason": "failure", + "retry_backoff_seconds": 0, + "task": "(the first event's task)" + } + }, + { + "sequence": 11, + "event_type": "ActivityStarted", + "timestamp": "{time}", + "principal": null, + "payload": { + "execution_mode": "local", + "local_activity": true, + "activity_execution_id": "execution-2", + "activity_class": "parity.v1.local", + "activity_type": "parity.v1.local", + "sequence": 1, + "workflow_task_id": "task-2", + "activity": { + "id": "execution-2", + "idempotency_key": "execution-2", + "sequence": 1, + "type": "parity.v1.local", + "class": "parity.v1.local", + "execution_mode": "local", + "local_activity": true, + "attempt_id": "attempt-5:upper", + "status": "running", + "payload_codec": "avro", + "attempt_count": 3, + "retry_policy": { + "max_attempts": 3, + "backoff_seconds": [ + 2, + 0 + ], + "non_retryable_error_types": [ + "App\\Errors\\Fatal" + ] + }, + "queue": "pr1-record-local-activity-rla-b-queue", + "last_heartbeat_at": "@-2540ms", + "created_at": "{time}", + "started_at": "@-5040ms", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + } + }, + "activity_attempt_id": "attempt-5:upper", + "worker_attempt_id": "b-3", + "attempt_number": 3, + "activity_attempt": { + "id": "attempt-5:upper", + "worker_attempt_id": "b-3", + "attempt_number": 3, + "status": "running", + "task_id": "task-2", + "lease_owner": "pr1-record-local-activity-rla-b", + "started_at": "@-40ms" + }, + "task": "(the first event's task)" + } + }, + { + "sequence": 12, + "event_type": "ActivityFailed", + "timestamp": "{time}", + "principal": null, + "payload": { + "execution_mode": "local", + "local_activity": true, + "activity_execution_id": "execution-2", + "activity_class": "parity.v1.local", + "activity_type": "parity.v1.local", + "sequence": 1, + "workflow_task_id": "task-2", + "activity": { + "id": "execution-2", + "idempotency_key": "execution-2", + "sequence": 1, + "type": "parity.v1.local", + "class": "parity.v1.local", + "execution_mode": "local", + "local_activity": true, + "attempt_id": "attempt-5:upper", + "status": "failed", + "payload_codec": "avro", + "attempt_count": 3, + "retry_policy": { + "max_attempts": 3, + "backoff_seconds": [ + 2, + 0 + ], + "non_retryable_error_types": [ + "App\\Errors\\Fatal" + ] + }, + "queue": "pr1-record-local-activity-rla-b-queue", + "last_heartbeat_at": "@-2540ms", + "created_at": "{time}", + "started_at": "@-5040ms", + "closed_at": "@0ms", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "exception": "wwHioz3/VYAiNwoOZ2F2ZSB1cA==" + }, + "activity_attempt_id": "attempt-5:upper", + "worker_attempt_id": "b-3", + "attempt_number": 3, + "activity_attempt": { + "id": "attempt-5:upper", + "worker_attempt_id": "b-3", + "attempt_number": 3, + "status": "failed", + "task_id": "task-2", + "lease_owner": "pr1-record-local-activity-rla-b", + "started_at": "@-40ms", + "closed_at": "@0ms" + }, + "failure_id": "failure-1", + "failure_category": "application", + "message": "gave up", + "exception_type": "App\\Errors\\Fatal", + "non_retryable": true, + "task": "(the first event's task)" + } + }, + { + "sequence": 13, + "event_type": "ActivityScheduled", + "timestamp": "{time}", + "principal": null, + "payload": { + "execution_mode": "local", + "local_activity": true, + "activity_execution_id": "execution-3", + "activity_class": "parity.v1.local.slow", + "activity_type": "parity.v1.local.slow", + "sequence": 2, + "workflow_task_id": "task-2", + "activity": { + "id": "execution-3", + "idempotency_key": "execution-3", + "sequence": 2, + "type": "parity.v1.local.slow", + "class": "parity.v1.local.slow", + "execution_mode": "local", + "local_activity": true, + "status": "pending", + "payload_codec": "avro", + "attempt_count": 0, + "queue": "pr1-record-local-activity-rla-b-queue", + "created_at": "{time}", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + } + }, + "task": "(the first event's task)" + } + }, + { + "sequence": 14, + "event_type": "ActivityStarted", + "timestamp": "{time}", + "principal": null, + "payload": { + "execution_mode": "local", + "local_activity": true, + "activity_execution_id": "execution-3", + "activity_class": "parity.v1.local.slow", + "activity_type": "parity.v1.local.slow", + "sequence": 2, + "workflow_task_id": "task-2", + "activity": { + "id": "execution-3", + "idempotency_key": "execution-3", + "sequence": 2, + "type": "parity.v1.local.slow", + "class": "parity.v1.local.slow", + "execution_mode": "local", + "local_activity": true, + "attempt_id": "attempt-6:upper", + "status": "running", + "payload_codec": "avro", + "attempt_count": 1, + "queue": "pr1-record-local-activity-rla-b-queue", + "created_at": "{time}", + "started_at": "@-30000ms", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + } + }, + "activity_attempt_id": "attempt-6:upper", + "worker_attempt_id": null, + "attempt_number": 1, + "activity_attempt": { + "id": "attempt-6:upper", + "attempt_number": 1, + "status": "running", + "task_id": "task-2", + "lease_owner": "pr1-record-local-activity-rla-b", + "started_at": "@-30000ms" + }, + "task": "(the first event's task)" + } + }, + { + "sequence": 15, + "event_type": "ActivityTimedOut", + "timestamp": "{time}", + "principal": null, + "payload": { + "execution_mode": "local", + "local_activity": true, + "activity_execution_id": "execution-3", + "activity_class": "parity.v1.local.slow", + "activity_type": "parity.v1.local.slow", + "sequence": 2, + "workflow_task_id": "task-2", + "activity": { + "id": "execution-3", + "idempotency_key": "execution-3", + "sequence": 2, + "type": "parity.v1.local.slow", + "class": "parity.v1.local.slow", + "execution_mode": "local", + "local_activity": true, + "attempt_id": "attempt-6:upper", + "status": "failed", + "payload_codec": "avro", + "attempt_count": 1, + "queue": "pr1-record-local-activity-rla-b-queue", + "created_at": "{time}", + "started_at": "@-30000ms", + "closed_at": "@0ms", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "exception": "wwHioz3/VYAiNwoQZGVhZGxpbmU=" + }, + "activity_attempt_id": "attempt-6:upper", + "worker_attempt_id": null, + "attempt_number": 1, + "activity_attempt": { + "id": "attempt-6:upper", + "attempt_number": 1, + "status": "failed", + "task_id": "task-2", + "lease_owner": "pr1-record-local-activity-rla-b", + "started_at": "@-30000ms", + "closed_at": "@0ms" + }, + "failure_id": "failure-2", + "failure_category": "timeout", + "timeout_kind": "schedule_to_close", + "message": "deadline", + "exception_class": null, + "task": "(the first event's task)" + } + } + ], + "activities": [ + { + "id": "execution-2", + "idempotency_key": "execution-2", + "sequence": 1, + "type": "parity.v1.local", + "class": "parity.v1.local", + "execution_mode": "local", + "local_activity": true, + "history_authority": "typed_history", + "diagnostic_only": false, + "history_event_types": [ + "ActivityScheduled", + "ActivityStarted", + "ActivityHeartbeatRecorded", + "ActivityRetryScheduled", + "ActivityFailed" + ], + "history_unsupported_reason": null, + "reused_from_run_id": null, + "reused_activity_execution_id": null, + "reused_recorded_at": null, + "row_status": null, + "parallel_group_kind": null, + "parallel_group_id": null, + "parallel_group_base_sequence": null, + "parallel_group_size": null, + "parallel_group_index": null, + "parallel_group_path": [], + "attempt_id": "attempt-5:upper", + "worker_attempt_id": "b-3", + "status": "failed", + "attempt_count": 3, + "retry_policy": { + "max_attempts": 3, + "backoff_seconds": [ + 2, + 0 + ], + "non_retryable_error_types": [ + "App\\Errors\\Fatal" + ] + }, + "connection": null, + "queue": "pr1-record-local-activity-rla-b-queue", + "last_heartbeat_at": "@-2540ms", + "last_heartbeat_progress": null, + "created_at": "{time}", + "started_at": "@-5040ms", + "closed_at": "@0ms", + "attempts": [ + { + "id": "attempt-3:upper", + "worker_attempt_id": "b-1", + "attempt_number": 1, + "status": "failed", + "task_id": "task-2", + "lease_owner": "pr1-record-local-activity-rla-b", + "lease_expires_at": null, + "started_at": "@-5040ms", + "last_heartbeat_at": "@-2540ms", + "last_heartbeat_progress": null, + "closed_at": "@-2040ms", + "can_continue": false, + "cancel_requested": false, + "stop_reason": "attempt_closed", + "failure_reason": null + }, + { + "id": "attempt-4:upper", + "worker_attempt_id": "b-2", + "attempt_number": 2, + "status": "failed", + "task_id": "task-2", + "lease_owner": "pr1-record-local-activity-rla-b", + "lease_expires_at": null, + "started_at": "@-40ms", + "last_heartbeat_at": null, + "last_heartbeat_progress": null, + "closed_at": "@-40ms", + "can_continue": false, + "cancel_requested": false, + "stop_reason": "attempt_closed", + "failure_reason": null + }, + { + "id": "attempt-5:upper", + "worker_attempt_id": "b-3", + "attempt_number": 3, + "status": "failed", + "task_id": "task-2", + "lease_owner": "pr1-record-local-activity-rla-b", + "lease_expires_at": null, + "started_at": "@-40ms", + "last_heartbeat_at": null, + "last_heartbeat_progress": null, + "closed_at": "@0ms", + "can_continue": false, + "cancel_requested": false, + "stop_reason": "attempt_closed", + "failure_reason": null + } + ] + }, + { + "id": "execution-3", + "idempotency_key": "execution-3", + "sequence": 2, + "type": "parity.v1.local.slow", + "class": "parity.v1.local.slow", + "execution_mode": "local", + "local_activity": true, + "history_authority": "typed_history", + "diagnostic_only": false, + "history_event_types": [ + "ActivityScheduled", + "ActivityStarted", + "ActivityTimedOut" + ], + "history_unsupported_reason": null, + "reused_from_run_id": null, + "reused_activity_execution_id": null, + "reused_recorded_at": null, + "row_status": null, + "parallel_group_kind": null, + "parallel_group_id": null, + "parallel_group_base_sequence": null, + "parallel_group_size": null, + "parallel_group_index": null, + "parallel_group_path": [], + "attempt_id": "attempt-6:upper", + "worker_attempt_id": null, + "status": "failed", + "attempt_count": 1, + "retry_policy": null, + "connection": null, + "queue": "pr1-record-local-activity-rla-b-queue", + "last_heartbeat_at": null, + "last_heartbeat_progress": null, + "created_at": "{time}", + "started_at": "@-30000ms", + "closed_at": "@0ms", + "attempts": [ + { + "id": "attempt-6:upper", + "worker_attempt_id": null, + "attempt_number": 1, + "status": "expired", + "task_id": "task-2", + "lease_owner": "pr1-record-local-activity-rla-b", + "lease_expires_at": null, + "started_at": "@-30000ms", + "last_heartbeat_at": null, + "last_heartbeat_progress": null, + "closed_at": "@0ms", + "can_continue": false, + "cancel_requested": false, + "stop_reason": "attempt_expired", + "failure_reason": null + } + ] + } + ], + "run": { + "status": "completed", + "status_bucket": "completed", + "is_terminal": true, + "closed_reason": "completed", + "wait_kind": null, + "wait_reason": null + }, + "attempt_identity_order": [ + "attempt-5:upper", + "attempt-4:upper", + "attempt-3:upper", + "attempt-6:upper" + ] + }, + "protocol_1_18_missing_result": { + "status": 422, + "response": { + "message": "Completed local activities require a result payload envelope.", + "reason": "validation_failed", + "errors": { + "commands.0.result": [ + "Completed local activities require a result payload envelope." + ] + }, + "validation_errors": { + "commands.0.result": [ + "Completed local activities require a result payload envelope." + ] + } + } + }, + "protocol_1_18": { + "status": 200, + "response": { + "task_id": "task-3", + "workflow_task_attempt": 1, + "outcome": "completed", + "recorded": true, + "run_id": "run-3", + "run_status": "waiting", + "created_task_ids": [], + "reason": null + } + }, + "protocol_1_18_run": { + "event_types": [ + "StartAccepted", + "WorkflowStarted", + "ActivityScheduled", + "ActivityStarted", + "ActivityFailed", + "ActivityScheduled", + "ActivityStarted", + "ActivityTimedOut", + "ActivityScheduled", + "ActivityStarted", + "ActivityCancelled", + "ActivityScheduled", + "ActivityStarted", + "ActivityHeartbeatRecorded", + "ActivityCompleted", + "SignalWaitOpened" + ], + "activity_events": [ + { + "sequence": 3, + "event_type": "ActivityScheduled", + "timestamp": "{time}", + "principal": null, + "payload": { + "execution_mode": "local", + "local_activity": true, + "activity_execution_id": "execution-4", + "activity_class": "parity.v1.local", + "activity_type": "parity.v1.local", + "sequence": 1, + "workflow_task_id": "task-3", + "activity": { + "id": "execution-4", + "idempotency_key": "execution-4", + "sequence": 1, + "type": "parity.v1.local", + "class": "parity.v1.local", + "execution_mode": "local", + "local_activity": true, + "status": "pending", + "payload_codec": "avro", + "attempt_count": 0, + "queue": "pr1-record-local-activity-rla-c-queue", + "created_at": "{time}", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + } + }, + "task": { + "id": "task-3", + "type": "workflow", + "status": "leased", + "available_at": "{time}", + "leased_at": "{time}", + "lease_owner": "pr1-record-local-activity-rla-c", + "lease_expires_at": "{time}", + "attempt_count": 1, + "repair_count": 0, + "connection": "sqlite", + "queue": "pr1-record-local-activity-rla-c-queue", + "last_dispatch_attempt_at": "{time}", + "last_dispatched_at": "{time}" + } + } + }, + { + "sequence": 4, + "event_type": "ActivityStarted", + "timestamp": "{time}", + "principal": null, + "payload": { + "execution_mode": "local", + "local_activity": true, + "activity_execution_id": "execution-4", + "activity_class": "parity.v1.local", + "activity_type": "parity.v1.local", + "sequence": 1, + "workflow_task_id": "task-3", + "activity": { + "id": "execution-4", + "idempotency_key": "execution-4", + "sequence": 1, + "type": "parity.v1.local", + "class": "parity.v1.local", + "execution_mode": "local", + "local_activity": true, + "attempt_id": "attempt-7:upper", + "status": "running", + "payload_codec": "avro", + "attempt_count": 1, + "queue": "pr1-record-local-activity-rla-c-queue", + "created_at": "{time}", + "started_at": "@0ms", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + } + }, + "activity_attempt_id": "attempt-7:upper", + "worker_attempt_id": null, + "attempt_number": 1, + "activity_attempt": { + "id": "attempt-7:upper", + "attempt_number": 1, + "status": "running", + "task_id": "task-3", + "lease_owner": "pr1-record-local-activity-rla-c", + "started_at": "@0ms" + }, + "task": "(the first event's task)" + } + }, + { + "sequence": 5, + "event_type": "ActivityFailed", + "timestamp": "{time}", + "principal": null, + "payload": { + "execution_mode": "local", + "local_activity": true, + "activity_execution_id": "execution-4", + "activity_class": "parity.v1.local", + "activity_type": "parity.v1.local", + "sequence": 1, + "workflow_task_id": "task-3", + "activity": { + "id": "execution-4", + "idempotency_key": "execution-4", + "sequence": 1, + "type": "parity.v1.local", + "class": "parity.v1.local", + "execution_mode": "local", + "local_activity": true, + "attempt_id": "attempt-7:upper", + "status": "failed", + "payload_codec": "avro", + "attempt_count": 1, + "queue": "pr1-record-local-activity-rla-c-queue", + "created_at": "{time}", + "started_at": "@0ms", + "closed_at": "@0ms", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "exception": "wwHioz3/VYAiNwoWbm8gYXR0ZW1wdHM=" + }, + "activity_attempt_id": "attempt-7:upper", + "worker_attempt_id": null, + "attempt_number": 1, + "activity_attempt": { + "id": "attempt-7:upper", + "attempt_number": 1, + "status": "failed", + "task_id": "task-3", + "lease_owner": "pr1-record-local-activity-rla-c", + "started_at": "@0ms", + "closed_at": "@0ms" + }, + "failure_id": "failure-3", + "failure_category": "application", + "message": "no attempts", + "exception_type": "RuntimeException", + "non_retryable": false, + "task": "(the first event's task)" + } + }, + { + "sequence": 6, + "event_type": "ActivityScheduled", + "timestamp": "{time}", + "principal": null, + "payload": { + "execution_mode": "local", + "local_activity": true, + "activity_execution_id": "execution-5", + "activity_class": "parity.v1.local", + "activity_type": "parity.v1.local", + "sequence": 2, + "workflow_task_id": "task-3", + "activity": { + "id": "execution-5", + "idempotency_key": "execution-5", + "sequence": 2, + "type": "parity.v1.local", + "class": "parity.v1.local", + "execution_mode": "local", + "local_activity": true, + "status": "pending", + "payload_codec": "avro", + "attempt_count": 0, + "queue": "pr1-record-local-activity-rla-c-queue", + "created_at": "{time}", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + } + }, + "task": "(the first event's task)" + } + }, + { + "sequence": 7, + "event_type": "ActivityStarted", + "timestamp": "{time}", + "principal": null, + "payload": { + "execution_mode": "local", + "local_activity": true, + "activity_execution_id": "execution-5", + "activity_class": "parity.v1.local", + "activity_type": "parity.v1.local", + "sequence": 2, + "workflow_task_id": "task-3", + "activity": { + "id": "execution-5", + "idempotency_key": "execution-5", + "sequence": 2, + "type": "parity.v1.local", + "class": "parity.v1.local", + "execution_mode": "local", + "local_activity": true, + "attempt_id": "attempt-8:upper", + "status": "running", + "payload_codec": "avro", + "attempt_count": 1, + "queue": "pr1-record-local-activity-rla-c-queue", + "created_at": "{time}", + "started_at": "@0ms", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + } + }, + "activity_attempt_id": "attempt-8:upper", + "worker_attempt_id": null, + "attempt_number": 1, + "activity_attempt": { + "id": "attempt-8:upper", + "attempt_number": 1, + "status": "running", + "task_id": "task-3", + "lease_owner": "pr1-record-local-activity-rla-c", + "started_at": "@0ms" + }, + "task": "(the first event's task)" + } + }, + { + "sequence": 8, + "event_type": "ActivityTimedOut", + "timestamp": "{time}", + "principal": null, + "payload": { + "execution_mode": "local", + "local_activity": true, + "activity_execution_id": "execution-5", + "activity_class": "parity.v1.local", + "activity_type": "parity.v1.local", + "sequence": 2, + "workflow_task_id": "task-3", + "activity": { + "id": "execution-5", + "idempotency_key": "execution-5", + "sequence": 2, + "type": "parity.v1.local", + "class": "parity.v1.local", + "execution_mode": "local", + "local_activity": true, + "attempt_id": "attempt-8:upper", + "status": "failed", + "payload_codec": "avro", + "attempt_count": 1, + "queue": "pr1-record-local-activity-rla-c-queue", + "created_at": "{time}", + "started_at": "@0ms", + "closed_at": "@0ms", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "exception": "wwHioz3/VYAiNwoWbm8gYXR0ZW1wdHM=" + }, + "activity_attempt_id": "attempt-8:upper", + "worker_attempt_id": null, + "attempt_number": 1, + "activity_attempt": { + "id": "attempt-8:upper", + "attempt_number": 1, + "status": "failed", + "task_id": "task-3", + "lease_owner": "pr1-record-local-activity-rla-c", + "started_at": "@0ms", + "closed_at": "@0ms" + }, + "failure_id": "failure-4", + "failure_category": "timeout", + "timeout_kind": "heartbeat", + "message": "no attempts", + "exception_class": null, + "task": "(the first event's task)" + } + }, + { + "sequence": 9, + "event_type": "ActivityScheduled", + "timestamp": "{time}", + "principal": null, + "payload": { + "execution_mode": "local", + "local_activity": true, + "activity_execution_id": "execution-6", + "activity_class": "parity.v1.local", + "activity_type": "parity.v1.local", + "sequence": 3, + "workflow_task_id": "task-3", + "activity": { + "id": "execution-6", + "idempotency_key": "execution-6", + "sequence": 3, + "type": "parity.v1.local", + "class": "parity.v1.local", + "execution_mode": "local", + "local_activity": true, + "status": "pending", + "payload_codec": "avro", + "attempt_count": 0, + "queue": "pr1-record-local-activity-rla-c-queue", + "created_at": "{time}", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + } + }, + "task": "(the first event's task)" + } + }, + { + "sequence": 10, + "event_type": "ActivityStarted", + "timestamp": "{time}", + "principal": null, + "payload": { + "execution_mode": "local", + "local_activity": true, + "activity_execution_id": "execution-6", + "activity_class": "parity.v1.local", + "activity_type": "parity.v1.local", + "sequence": 3, + "workflow_task_id": "task-3", + "activity": { + "id": "execution-6", + "idempotency_key": "execution-6", + "sequence": 3, + "type": "parity.v1.local", + "class": "parity.v1.local", + "execution_mode": "local", + "local_activity": true, + "attempt_id": "attempt-9:upper", + "status": "running", + "payload_codec": "avro", + "attempt_count": 1, + "queue": "pr1-record-local-activity-rla-c-queue", + "created_at": "{time}", + "started_at": "@0ms", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + } + }, + "activity_attempt_id": "attempt-9:upper", + "worker_attempt_id": null, + "attempt_number": 1, + "activity_attempt": { + "id": "attempt-9:upper", + "attempt_number": 1, + "status": "running", + "task_id": "task-3", + "lease_owner": "pr1-record-local-activity-rla-c", + "started_at": "@0ms" + }, + "task": "(the first event's task)" + } + }, + { + "sequence": 11, + "event_type": "ActivityCancelled", + "timestamp": "{time}", + "principal": null, + "payload": { + "execution_mode": "local", + "local_activity": true, + "activity_execution_id": "execution-6", + "activity_class": "parity.v1.local", + "activity_type": "parity.v1.local", + "sequence": 3, + "workflow_task_id": "task-3", + "activity": { + "id": "execution-6", + "idempotency_key": "execution-6", + "sequence": 3, + "type": "parity.v1.local", + "class": "parity.v1.local", + "execution_mode": "local", + "local_activity": true, + "attempt_id": "attempt-9:upper", + "status": "cancelled", + "payload_codec": "avro", + "attempt_count": 1, + "queue": "pr1-record-local-activity-rla-c-queue", + "created_at": "{time}", + "started_at": "@0ms", + "closed_at": "@0ms", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "exception": "wwHioz3/VYAiNwoWbm8gYXR0ZW1wdHM=" + }, + "activity_attempt_id": "attempt-9:upper", + "worker_attempt_id": null, + "attempt_number": 1, + "activity_attempt": { + "id": "attempt-9:upper", + "attempt_number": 1, + "status": "cancelled", + "task_id": "task-3", + "lease_owner": "pr1-record-local-activity-rla-c", + "started_at": "@0ms", + "closed_at": "@0ms" + }, + "cancelled_at": "@0ms", + "task": "(the first event's task)" + } + }, + { + "sequence": 12, + "event_type": "ActivityScheduled", + "timestamp": "{time}", + "principal": null, + "payload": { + "execution_mode": "local", + "local_activity": true, + "activity_execution_id": "execution-7", + "activity_class": "parity.v1.local", + "activity_type": "parity.v1.local", + "sequence": 4, + "workflow_task_id": "task-3", + "activity": { + "id": "execution-7", + "idempotency_key": "execution-7", + "sequence": 4, + "type": "parity.v1.local", + "class": "parity.v1.local", + "execution_mode": "local", + "local_activity": true, + "status": "pending", + "payload_codec": "avro", + "attempt_count": 0, + "retry_policy": { + "max_attempts": 1 + }, + "queue": "pr1-record-local-activity-rla-c-queue", + "created_at": "{time}", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + } + }, + "task": "(the first event's task)" + } + }, + { + "sequence": 13, + "event_type": "ActivityStarted", + "timestamp": "{time}", + "principal": null, + "payload": { + "execution_mode": "local", + "local_activity": true, + "activity_execution_id": "execution-7", + "activity_class": "parity.v1.local", + "activity_type": "parity.v1.local", + "sequence": 4, + "workflow_task_id": "task-3", + "activity": { + "id": "execution-7", + "idempotency_key": "execution-7", + "sequence": 4, + "type": "parity.v1.local", + "class": "parity.v1.local", + "execution_mode": "local", + "local_activity": true, + "attempt_id": "attempt-10:upper", + "status": "running", + "payload_codec": "avro", + "attempt_count": 1, + "retry_policy": { + "max_attempts": 1 + }, + "queue": "pr1-record-local-activity-rla-c-queue", + "created_at": "{time}", + "started_at": "@-5ms", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + } + }, + "activity_attempt_id": "attempt-10:upper", + "worker_attempt_id": null, + "attempt_number": 1, + "activity_attempt": { + "id": "attempt-10:upper", + "attempt_number": 1, + "status": "running", + "task_id": "task-3", + "lease_owner": "pr1-record-local-activity-rla-c", + "started_at": "@-5ms" + }, + "task": "(the first event's task)" + } + }, + { + "sequence": 14, + "event_type": "ActivityHeartbeatRecorded", + "timestamp": "{time}", + "principal": null, + "payload": { + "execution_mode": "local", + "local_activity": true, + "activity_execution_id": "execution-7", + "activity_class": "parity.v1.local", + "activity_type": "parity.v1.local", + "sequence": 4, + "workflow_task_id": "task-3", + "activity": { + "id": "execution-7", + "idempotency_key": "execution-7", + "sequence": 4, + "type": "parity.v1.local", + "class": "parity.v1.local", + "execution_mode": "local", + "local_activity": true, + "attempt_id": "attempt-10:upper", + "status": "running", + "payload_codec": "avro", + "attempt_count": 1, + "retry_policy": { + "max_attempts": 1 + }, + "queue": "pr1-record-local-activity-rla-c-queue", + "created_at": "{time}", + "started_at": "@-5ms", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "last_heartbeat_at": "@0ms" + }, + "activity_attempt_id": "attempt-10:upper", + "worker_attempt_id": null, + "attempt_number": 1, + "activity_attempt": { + "id": "attempt-10:upper", + "attempt_number": 1, + "status": "running", + "task_id": "task-3", + "lease_owner": "pr1-record-local-activity-rla-c", + "started_at": "@-5ms", + "last_heartbeat_at": "@0ms" + }, + "heartbeat_at": "@0ms", + "lease_expires_at": null, + "task": "(the first event's task)" + } + }, + { + "sequence": 15, + "event_type": "ActivityCompleted", + "timestamp": "{time}", + "principal": null, + "payload": { + "execution_mode": "local", + "local_activity": true, + "activity_execution_id": "execution-7", + "activity_class": "parity.v1.local", + "activity_type": "parity.v1.local", + "sequence": 4, + "workflow_task_id": "task-3", + "activity": { + "id": "execution-7", + "idempotency_key": "execution-7", + "sequence": 4, + "type": "parity.v1.local", + "class": "parity.v1.local", + "execution_mode": "local", + "local_activity": true, + "attempt_id": "attempt-10:upper", + "status": "completed", + "payload_codec": "avro", + "attempt_count": 1, + "retry_policy": { + "max_attempts": 1 + }, + "queue": "pr1-record-local-activity-rla-c-queue", + "last_heartbeat_at": "@0ms", + "created_at": "{time}", + "started_at": "@-5ms", + "closed_at": "@0ms", + "arguments": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwwA" + }, + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoIZG9uZQ==" + } + }, + "activity_attempt_id": "attempt-10:upper", + "worker_attempt_id": null, + "attempt_number": 1, + "activity_attempt": { + "id": "attempt-10:upper", + "attempt_number": 1, + "status": "completed", + "task_id": "task-3", + "lease_owner": "pr1-record-local-activity-rla-c", + "started_at": "@-5ms", + "last_heartbeat_at": "@0ms", + "closed_at": "@0ms" + }, + "result": { + "codec": "avro", + "blob": "wwHioz3/VYAiNwoIZG9uZQ==" + }, + "payload_codec": "avro", + "task": "(the first event's task)" + } + } + ], + "activities": [ + { + "id": "execution-4", + "idempotency_key": "execution-4", + "sequence": 1, + "type": "parity.v1.local", + "class": "parity.v1.local", + "execution_mode": "local", + "local_activity": true, + "history_authority": "typed_history", + "diagnostic_only": false, + "history_event_types": [ + "ActivityScheduled", + "ActivityStarted", + "ActivityFailed" + ], + "history_unsupported_reason": null, + "reused_from_run_id": null, + "reused_activity_execution_id": null, + "reused_recorded_at": null, + "row_status": null, + "parallel_group_kind": null, + "parallel_group_id": null, + "parallel_group_base_sequence": null, + "parallel_group_size": null, + "parallel_group_index": null, + "parallel_group_path": [], + "attempt_id": "attempt-7:upper", + "worker_attempt_id": null, + "status": "failed", + "attempt_count": 1, + "retry_policy": null, + "connection": null, + "queue": "pr1-record-local-activity-rla-c-queue", + "last_heartbeat_at": null, + "last_heartbeat_progress": null, + "created_at": "{time}", + "started_at": "@0ms", + "closed_at": "@0ms", + "attempts": [ + { + "id": "attempt-7:upper", + "worker_attempt_id": null, + "attempt_number": 1, + "status": "failed", + "task_id": "task-3", + "lease_owner": "pr1-record-local-activity-rla-c", + "lease_expires_at": null, + "started_at": "@0ms", + "last_heartbeat_at": null, + "last_heartbeat_progress": null, + "closed_at": "@0ms", + "can_continue": false, + "cancel_requested": false, + "stop_reason": "attempt_closed", + "failure_reason": null + } + ] + }, + { + "id": "execution-5", + "idempotency_key": "execution-5", + "sequence": 2, + "type": "parity.v1.local", + "class": "parity.v1.local", + "execution_mode": "local", + "local_activity": true, + "history_authority": "typed_history", + "diagnostic_only": false, + "history_event_types": [ + "ActivityScheduled", + "ActivityStarted", + "ActivityTimedOut" + ], + "history_unsupported_reason": null, + "reused_from_run_id": null, + "reused_activity_execution_id": null, + "reused_recorded_at": null, + "row_status": null, + "parallel_group_kind": null, + "parallel_group_id": null, + "parallel_group_base_sequence": null, + "parallel_group_size": null, + "parallel_group_index": null, + "parallel_group_path": [], + "attempt_id": "attempt-8:upper", + "worker_attempt_id": null, + "status": "failed", + "attempt_count": 1, + "retry_policy": null, + "connection": null, + "queue": "pr1-record-local-activity-rla-c-queue", + "last_heartbeat_at": null, + "last_heartbeat_progress": null, + "created_at": "{time}", + "started_at": "@0ms", + "closed_at": "@0ms", + "attempts": [ + { + "id": "attempt-8:upper", + "worker_attempt_id": null, + "attempt_number": 1, + "status": "expired", + "task_id": "task-3", + "lease_owner": "pr1-record-local-activity-rla-c", + "lease_expires_at": null, + "started_at": "@0ms", + "last_heartbeat_at": null, + "last_heartbeat_progress": null, + "closed_at": "@0ms", + "can_continue": false, + "cancel_requested": false, + "stop_reason": "attempt_expired", + "failure_reason": null + } + ] + }, + { + "id": "execution-6", + "idempotency_key": "execution-6", + "sequence": 3, + "type": "parity.v1.local", + "class": "parity.v1.local", + "execution_mode": "local", + "local_activity": true, + "history_authority": "typed_history", + "diagnostic_only": false, + "history_event_types": [ + "ActivityScheduled", + "ActivityStarted", + "ActivityCancelled" + ], + "history_unsupported_reason": null, + "reused_from_run_id": null, + "reused_activity_execution_id": null, + "reused_recorded_at": null, + "row_status": null, + "parallel_group_kind": null, + "parallel_group_id": null, + "parallel_group_base_sequence": null, + "parallel_group_size": null, + "parallel_group_index": null, + "parallel_group_path": [], + "attempt_id": "attempt-9:upper", + "worker_attempt_id": null, + "status": "cancelled", + "attempt_count": 1, + "retry_policy": null, + "connection": null, + "queue": "pr1-record-local-activity-rla-c-queue", + "last_heartbeat_at": null, + "last_heartbeat_progress": null, + "created_at": "{time}", + "started_at": "@0ms", + "closed_at": "@0ms", + "attempts": [ + { + "id": "attempt-9:upper", + "worker_attempt_id": null, + "attempt_number": 1, + "status": "cancelled", + "task_id": "task-3", + "lease_owner": "pr1-record-local-activity-rla-c", + "lease_expires_at": null, + "started_at": "@0ms", + "last_heartbeat_at": null, + "last_heartbeat_progress": null, + "closed_at": "@0ms", + "can_continue": false, + "cancel_requested": true, + "stop_reason": "attempt_cancelled", + "failure_reason": null + } + ] + }, + { + "id": "execution-7", + "idempotency_key": "execution-7", + "sequence": 4, + "type": "parity.v1.local", + "class": "parity.v1.local", + "execution_mode": "local", + "local_activity": true, + "history_authority": "typed_history", + "diagnostic_only": false, + "history_event_types": [ + "ActivityScheduled", + "ActivityStarted", + "ActivityHeartbeatRecorded", + "ActivityCompleted" + ], + "history_unsupported_reason": null, + "reused_from_run_id": null, + "reused_activity_execution_id": null, + "reused_recorded_at": null, + "row_status": null, + "parallel_group_kind": null, + "parallel_group_id": null, + "parallel_group_base_sequence": null, + "parallel_group_size": null, + "parallel_group_index": null, + "parallel_group_path": [], + "attempt_id": "attempt-10:upper", + "worker_attempt_id": null, + "status": "completed", + "attempt_count": 1, + "retry_policy": { + "max_attempts": 1 + }, + "connection": null, + "queue": "pr1-record-local-activity-rla-c-queue", + "last_heartbeat_at": "@0ms", + "last_heartbeat_progress": null, + "created_at": "{time}", + "started_at": "@-5ms", + "closed_at": "@0ms", + "attempts": [ + { + "id": "attempt-10:upper", + "worker_attempt_id": null, + "attempt_number": 1, + "status": "completed", + "task_id": "task-3", + "lease_owner": "pr1-record-local-activity-rla-c", + "lease_expires_at": null, + "started_at": "@-5ms", + "last_heartbeat_at": "@0ms", + "last_heartbeat_progress": null, + "closed_at": "@0ms", + "can_continue": false, + "cancel_requested": false, + "stop_reason": "attempt_closed", + "failure_reason": null + } + ] + } + ], + "run": { + "status": "waiting", + "status_bucket": "running", + "is_terminal": false, + "closed_reason": null, + "wait_kind": "signal", + "wait_reason": "Waiting for signal resume" + }, + "attempt_identity_order": [ + "attempt-7:upper", + "attempt-8:upper", + "attempt-9:upper", + "attempt-10:upper" + ] + }, + "reordered_valid_report": { + "status": 422, + "response": { + "message": "Local activity attempts must be a non-empty ordered list.", + "reason": "validation_failed", + "errors": { + "commands.0.attempts": [ + "Local activity attempts must be a non-empty ordered list." + ] + }, + "validation_errors": { + "commands.0.attempts": [ + "Local activity attempts must be a non-empty ordered list." + ] + } + } + }, + "refused_capability": { + "status": 409, + "response": { + "task_id": "task-5", + "workflow_task_attempt": 1, + "outcome": "rejected", + "recorded": false, + "reason": "worker_capability_not_supported", + "capability": "local_activities", + "requested_version": "1.20", + "minimum_protocol_version": "1.18", + "remediation": "Use cold durable replay unless the registered worker manifest explicitly supports the required capability and exact cache identity." + } + }, + "refused_capability_below_floor": { + "status": 409, + "response": { + "task_id": "task-5", + "workflow_task_attempt": 1, + "outcome": "rejected", + "recorded": false, + "reason": "portable_worker_affinity_protocol_floor_unavailable", + "capability": "local_activities", + "requested_version": "1.17", + "minimum_protocol_version": "1.18", + "remediation": "Use cold durable replay unless the registered worker manifest explicitly supports the required capability and exact cache identity." + } + }, + "released_without_local": 200 +} diff --git a/tests/Unit/ServerParityRecordLocalActivityContractTest.mjs b/tests/Unit/ServerParityRecordLocalActivityContractTest.mjs new file mode 100644 index 000000000..611fc2309 --- /dev/null +++ b/tests/Unit/ServerParityRecordLocalActivityContractTest.mjs @@ -0,0 +1,99 @@ +import assert from 'node:assert/strict'; +import {readFileSync} from 'node:fs'; +import test from 'node:test'; +import {checkRecordLocalActivity} from '../../scripts/conformance/server-parity/record-local-activity-contract.mjs'; + +const fixture = JSON.parse(readFileSync(new URL('../Fixtures/ServerParity/record-local-activity.json', import.meta.url))); +// The observation published Server 2.11.1 returned for this fixture. +const recorded = JSON.parse(readFileSync( + new URL('../Fixtures/ServerParityExpected/record-local-activity-observation.json', import.meta.url))); +const id = 'pr1-record-local-activity'; +const observation = (change = () => {}, base = recorded) => { + const state = structuredClone(base); + change(state); + return {mode: 'http', record_local_activity: state}; +}; +const check = state => checkRecordLocalActivity(fixture, state, id); +// The intended answers: the normalizer sees each report as sent. +const intended = JSON.parse(JSON.stringify(fixture.record_local_activity.expected).replaceAll('{id}', id)); +const events = (state, run) => state[run].activity_events; + +test('the published 2.11.1 recording satisfies the contract', () => { + assert.deepStrictEqual(check(observation()), {scope: 'record_local_activity'}); +}); + +test('the normalizer\'s own answers on the reports as sent are accepted too', () => { + assert.deepStrictEqual(check(observation(() => {}, intended)), {scope: 'record_local_activity'}); + assert.equal(intended.reordered_valid_report.status, 200); + assert.deepStrictEqual(intended.reordered_attempts_report.response.errors, intended.invalid_retry_attempts.response.errors); +}); + +test('embedded applications are explicitly inapplicable', () => { + assert.deepStrictEqual(check({mode: 'embedded', record_local_activity: + {applicable: false, reason: 'embedded_has_no_worker_protocol'}}), {scope: 'record_local_activity'}); +}); + +for (const [label, change, message] of [ + // Native advertised a bare flag and refused the command before this fixture. + ['a bare local_activities flag', state => { + state.cluster_info.local_activities = false; + }, /record local activity cluster_info/], + ['an unsupported command', state => { + state.completed = {status: 422, response: {reason: 'unsupported_workflow_command', message: 'unsupported_workflow_command'}}; + }, /record local activity completed/], + ['prepared flags that disagree', state => { + state.cluster_info.prepared_flags.agree = false; + }, /record local activity cluster_info/], + ['the capability guard before ownership', state => { + state.refused_wrong_owner = structuredClone(state.refused_capability); + }, /record local activity refused_wrong_owner/], + ['normalizer errors before ownership', state => { + state.refused_wrong_owner_invalid_report = structuredClone(intended.invalid_report); + }, /record local activity refused_wrong_owner_invalid_report/], + ['only part of the published reordering defect', state => { + state.reordered_valid_report = structuredClone(intended.reordered_valid_report); + }, /record local activity reordered_attempts_report/], + ['a reordered report that is accepted without its own errors', state => { + Object.assign(state, structuredClone({reordered_valid_report: intended.reordered_valid_report, + reordered_heartbeats_report: intended.reordered_heartbeats_report})); + }, /record local activity reordered_attempts_report/], + ['normalizer errors in another order', state => { + const errors = Object.entries(state.invalid_report.response.errors).reverse(); + state.invalid_report.response.message = `${errors[0][1][0]} (and 5 more errors)`; + }, /record local activity invalid_report/], + ['lowercase attempt identities', state => { + const text = JSON.stringify(state.completed_run).replaceAll(/(attempt-\d+):upper/g, '$1:lower'); + state.completed_run = JSON.parse(text); + }, /record local activity completed_run/], + ['attempt identities generated first attempt first', state => { + state.completed_run.attempt_identity_order.reverse(); + }, /record local activity completed_run/], + ['times walked forward from the first start', state => { + events(state, 'completed_run')[1].payload.activity_attempt.started_at = '@0ms'; + }, /record local activity completed_run/], + ['the command\'s queue instead of the run\'s', state => { + events(state, 'completed_run')[0].payload.activity.queue = 'ignored-queue'; + }, /record local activity completed_run/], + ['empty heartbeat details kept as an object', state => { + const heartbeat = events(state, 'failed_run').find(event => event.event_type === 'ActivityHeartbeatRecorded' + && Array.isArray(event.payload.progress)); + heartbeat.payload.progress = {}; + }, /record local activity failed_run/], + ['a synthesized 1.18 attempt with a worker identity', state => { + events(state, 'protocol_1_18_run')[1].payload.worker_attempt_id = 'w-1'; + }, /record local activity protocol_1_18_run/], + ['a timed out activity recorded as failed', state => { + const timedOut = events(state, 'failed_run').find(event => event.event_type === 'ActivityTimedOut'); + timedOut.event_type = 'ActivityFailed'; + }, /record local activity failed_run/], + ['a missing step', state => { + delete state.invalid_heartbeats; + }, /record local activity invalid_heartbeats/], + ['an extra step', state => { + state.extra = {status: 200}; + }, /record local activity steps/], +]) { + test(`the contract refuses ${label}`, () => { + assert.throws(() => check(observation(change)), message); + }); +} From 514498a6fa30f8a4344140663604641bcbb4de46 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Sat, 24 Oct 2026 23:09:43 +0000 Subject: [PATCH 2/9] Rust #325: record local activities like Server 2.11.1 Native now serves the record_local_activity workflow task command (worker protocol 1.18 and newer) as published Server 2.11.1 does with Workflow 2.10.2: - It advertises the local activity contract in server_capabilities and lists the command among the supported workflow task commands. Prepared local activities stay unadvertised, with no groups and no cancellation policies. - The portable worker affinity guard runs first, then the command normalizer's field, outcome, attempt and payload rules with PHP's messages. - The bridge records the activity's execution, attempts, history and failure under one authored sequence. - The run's activity view and attempt status follow Workflow 2.10+: each attempt carries failure_reason, and an expired attempt keeps a recorded lease loss as its stop reason. One published defect is not reproduced: PHP's validated() rebuilds a nested report in rule order, so a report whose first attempt lacks an optional field is refused as unordered. The record-local-activity fixture bounds that difference. The 2.5.22 activity metadata corpus predates the attempt failure_reason. Its test now requires that field to be null (no corpus case records a lease loss) and compares the rest as recorded. A payload unit test now expects PHP's final message for a completed record, whose missing-result check replaces the resolver's message under the same key. Native passes the record-local-activity fixture recorded from published 2.11.1, along with the capability-sensitive fixtures (cluster-info, admission, worker listing, namespaces, worker replacement). Co-Authored-By: Claude Opus 5.5 --- rust/src/runtime/activity_metadata.rs | 34 +- rust/src/runtime/activity_summary.rs | 47 +- rust/src/runtime/completion_rules.rs | 21 +- rust/src/runtime/local_activities.rs | 1216 ++++++++++++++++++++++ rust/src/runtime/mod.rs | 1 + rust/src/runtime/store.rs | 29 +- rust/src/runtime/wait_projection.rs | 5 + rust/tests/execution.rs | 2 + rust/tests/execution/local_activities.rs | 176 ++++ 9 files changed, 1501 insertions(+), 30 deletions(-) create mode 100644 rust/src/runtime/local_activities.rs create mode 100644 rust/tests/execution/local_activities.rs diff --git a/rust/src/runtime/activity_metadata.rs b/rust/src/runtime/activity_metadata.rs index 5b9a53291..f3c64abe9 100644 --- a/rust/src/runtime/activity_metadata.rs +++ b/rust/src/runtime/activity_metadata.rs @@ -498,12 +498,15 @@ fn present_attempt(attempt: &Value, execution: Option<&Value>) -> Value { }); let cancelled = status == Some("cancelled") || execution.is_some_and(|row| row["status"] == "cancelled"); + // Workflow 2.10's attempt view keeps a recorded lease loss as the + // expired attempt's failure reason. + let failure = string(&attempt["failure_reason"]); let reason = if status == Some("cancelled") { Some("attempt_cancelled") + } else if status == Some("expired") { + Some(failure.unwrap_or(if cancelled { "activity_cancelled" } else { "attempt_expired" })) } else if cancelled { Some("activity_cancelled") - } else if status == Some("expired") { - Some("attempt_expired") } else if matches!(status, Some("completed" | "failed")) { Some("attempt_closed") } else if execution @@ -518,7 +521,7 @@ fn present_attempt(attempt: &Value, execution: Option<&Value>) -> Value { let mut result = json!({"id":attempt["id"],"worker_attempt_id":attempt["worker_attempt_id"], "attempt_number":attempt["attempt_number"],"status":attempt["status"],"task_id":attempt["workflow_task_id"], "lease_owner":attempt["lease_owner"],"last_heartbeat_progress":progress(&attempt["last_heartbeat_progress"]), - "can_continue":can_continue,"cancel_requested":cancelled,"stop_reason":reason}); + "can_continue":can_continue,"cancel_requested":cancelled,"stop_reason":reason,"failure_reason":failure}); for field in [ "started_at", "last_heartbeat_at", @@ -559,7 +562,7 @@ fn synthetic_attempt(state: &Value, row: &Value) -> Option { }; let mut result = json!({"id":id,"attempt_number":count,"status":status,"task_id":null, "lease_owner":null,"lease_expires_at":null,"can_continue":can_continue, - "cancel_requested":cancelled,"stop_reason":reason, + "cancel_requested":cancelled,"stop_reason":reason,"failure_reason":null, "last_heartbeat_progress":progress(&state["last_heartbeat_progress"])}); for field in ["started_at", "last_heartbeat_at", "closed_at"] { result[field] = first(&[&state[field], &row[field]]); @@ -806,15 +809,22 @@ mod tests { assert_eq!(cases.len(), 32); for case in cases { let input = &case["input"]; - assert_eq!( - project( - input["events"].as_array().unwrap(), - input["executions"].as_array().unwrap() - ), - case["expected"], - "{}", - case["name"] + let mut projected = project( + input["events"].as_array().unwrap(), + input["executions"].as_array().unwrap(), ); + // Published Workflow 2.11.1's RunActivityView adds each attempt's + // `failure_reason` (a recorded lease loss, otherwise null). This + // corpus comes from 2.5.22, before that field, and none of its + // cases records a lease loss, so the field must be null here and + // the rest is compared as recorded. + for activity in projected.as_array_mut().into_iter().flatten() { + for attempt in activity["attempts"].as_array_mut().into_iter().flatten() { + let reason = attempt.as_object_mut().unwrap().remove("failure_reason"); + assert_eq!(reason, Some(Value::Null), "{}", case["name"]); + } + } + assert_eq!(projected, case["expected"], "{}", case["name"]); } } } diff --git a/rust/src/runtime/activity_summary.rs b/rust/src/runtime/activity_summary.rs index 40db1fb8e..e88e5f026 100644 --- a/rust/src/runtime/activity_summary.rs +++ b/rust/src/runtime/activity_summary.rs @@ -13,6 +13,8 @@ use std::collections::BTreeMap; #[derive(Default)] pub(super) struct ActivitySummaryHistory { open: BTreeMap, + /// Recorded local activities not yet closed by their terminal event. + local: BTreeMap, } struct ExpectedActivity { @@ -50,15 +52,12 @@ impl ActivitySummaryHistory { { return false; } - // Local execution still requires its separately qualified history. - if snapshot["id"] != id - || payload["sequence"] != sequence - || payload["activity_type"] != kind - || snapshot["local_activity"] == true - || snapshot["execution_mode"] == "local" - { + if snapshot["id"] != id || payload["sequence"] != sequence || payload["activity_type"] != kind { return false; } + if snapshot["local_activity"] == true || snapshot["execution_mode"] == "local" { + return self.apply_local(event, id, sequence, kind, attempts, payload); + } let status = match event { "ActivityScheduled" | "ActivityRetryScheduled" => "pending", "ActivityStarted" | "ActivityHeartbeatRecorded" => "running", @@ -127,6 +126,38 @@ impl ActivitySummaryHistory { } true } + + /// A `record_local_activity` writes its whole history in the completion + /// that reports it: scheduled, then each attempt, then the terminal event, + /// which closes it without a wait. A local activity left open (a prepared + /// one) is not qualified here. + fn apply_local(&mut self, event: &str, id: &str, sequence: i64, kind: &str, attempts: i64, payload: &Value) -> bool { + let status = match event { + "ActivityScheduled" | "ActivityRetryScheduled" => "pending", + "ActivityStarted" | "ActivityHeartbeatRecorded" => "running", + "ActivityCompleted" => "completed", + "ActivityFailed" | "ActivityTimedOut" => "failed", + "ActivityCancelled" => "cancelled", + _ => return false, + }; + if payload["execution_mode"] != "local" || payload["local_activity"] != true || payload["activity"]["status"] != status { + return false; + } + if event == "ActivityScheduled" { + if attempts != 0 || self.open.contains_key(id) || self.local.contains_key(id) { + return false; + } + self.local.insert(id.to_owned(), (sequence, kind.to_owned())); + return true; + } + if self.local.get(id) != Some(&(sequence, kind.to_owned())) { + return false; + } + if matches!(status, "completed" | "failed" | "cancelled") { + self.local.remove(id); + } + true + } } impl ExpectedActivity { @@ -184,7 +215,7 @@ where ) -> Result { let rows = Self::query("SELECT w.*,a.id AS activity_id,a.sequence AS activity_sequence,a.activity_type,a.status AS activity_status,a.attempt_count,a.current_attempt_id,a.parallel_group_path AS activity_group_path,q.id AS matched_task_id,q.task_type AS matched_task_type,q.payload AS matched_task_payload FROM workflow_run_waits w LEFT JOIN activity_executions a ON a.id=w.resume_source_id AND a.workflow_run_id=w.workflow_run_id LEFT JOIN workflow_tasks q ON q.id=w.task_id AND q.workflow_run_id=w.workflow_run_id WHERE w.workflow_run_id=$1 AND w.kind='activity' AND w.status='open' ORDER BY w.sequence,w.opened_at,w.resume_source_id LIMIT 101") .bind(run_id).fetch_all(&mut **tx).await?; - if rows.len() != history.open.len() || rows.len() > 100 { + if rows.len() != history.open.len() || rows.len() > 100 || !history.local.is_empty() { return Ok(false); } let mut selected_activity_task = false; diff --git a/rust/src/runtime/completion_rules.rs b/rust/src/runtime/completion_rules.rs index dc887b57b..fc5b38113 100644 --- a/rust/src/runtime/completion_rules.rs +++ b/rust/src/runtime/completion_rules.rs @@ -540,14 +540,12 @@ const FIELD_SCOPES: &[(&str, &[&str], &str)] = &[ ("entries", &["upsert_memo"], "entries contains the non-indexed memo patch and only applies to an upsert_memo command."), ]; -/// The command normalizer's per-command checks native shares with PHP: -/// `assertCommandFieldScope`, then the timer, version marker and wait rules. -/// PHP collects these across the batch; native stops at the first command. -pub(super) fn normalizer_checks(index: usize, command: &Value) -> Result<()> { +/// `assertCommandFieldScope`: each field this command type does not take. +pub(super) fn field_scope_errors(index: usize, command: &Value) -> Vec<(String, Vec)> { let Some(kind) = command.get("type").and_then(Value::as_str) else { - return Ok(()); + return Vec::new(); }; - let mut found: Vec<(String, Vec)> = Vec::new(); + let mut found = Vec::new(); for (field, allowed, guidance) in FIELD_SCOPES { // A marker's wire shape is frozen; PHP drops its payload codec. if !has_value(command, field) || allowed.contains(&kind) || kind == "record_version_marker" && *field == "payload_codec" { @@ -556,6 +554,17 @@ pub(super) fn normalizer_checks(index: usize, command: &Value) -> Result<()> { found.push((format!("commands.{index}.{field}"), vec![format!("Workflow task command field [{field}] is not valid on a {kind} command. {guidance}")])); } + found +} + +/// The command normalizer's per-command checks native shares with PHP: +/// `assertCommandFieldScope`, then the timer, version marker and wait rules. +/// PHP collects these across the batch; native stops at the first command. +pub(super) fn normalizer_checks(index: usize, command: &Value) -> Result<()> { + let Some(kind) = command.get("type").and_then(Value::as_str) else { + return Ok(()); + }; + let mut found = field_scope_errors(index, command); let whole = |field: &str| command.get(field).and_then(Value::as_i64); match kind { "start_timer" if !whole("delay_seconds").is_some_and(|delay| delay >= 0) => found.push(( diff --git a/rust/src/runtime/local_activities.rs b/rust/src/runtime/local_activities.rs new file mode 100644 index 000000000..078aa0e0b --- /dev/null +++ b/rust/src/runtime/local_activities.rs @@ -0,0 +1,1216 @@ +//! `record_local_activity`, the workflow task command for a local activity +//! the worker already ran (worker protocol 1.18 and newer), as published +//! Server 2.11.1 serves it with Workflow 2.10.2: the advertised contract +//! (`LocalActivityContract::manifest`), the portable worker affinity guard +//! (`WorkerController::guardPortableWorkerAffinityCompletion`), the command +//! normalizer's rules (`WorkflowCommandNormalizer::normalize`) and the +//! bridge's recording (`DefaultWorkflowTaskBridge::applyRecordLocalActivity`). +//! +//! The normalizer sees each command as the worker sent it. Published PHP +//! hands it Laravel's `validated()` commands instead, which rebuild a nested +//! list in rule order: a report whose first attempt lacks `attempt_id`, or +//! whose first heartbeat lacks `details`, while a later entry has it reaches +//! the normalizer reordered and is refused as unordered. Native does not +//! reproduce that defect; the record-local-activity fixture bounds it. +use super::laravel_validation::failure_messages; +use super::store::{id, now}; +use super::transaction::Transaction; +use super::{Result, RuntimeError, refuse}; +use axum::http::StatusCode; +use chrono::{DateTime, Duration, SecondsFormat, Utc}; +use serde_json::{Map, Value, json}; +use std::collections::{BTreeMap, HashSet}; +use std::sync::Arc; +use tokio::sync::Semaphore; + +pub(super) const COMMAND: &str = "record_local_activity"; +/// `WorkerProtocolVersion::PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION`. +pub(super) const MINIMUM_PROTOCOL_VERSION: &str = "1.18"; +/// `LOCAL_ACTIVITY_ATTEMPT_REPORTS_MINIMUM_PROTOCOL_VERSION`. +const ATTEMPT_REPORTS_PROTOCOL_VERSION: &str = "1.19"; +const MAX_ATTEMPTS: i64 = 100; +const MAX_HEARTBEATS_PER_ATTEMPT: usize = 1000; +const MAX_HEARTBEATS: usize = 1000; +const OUTCOMES: [&str; 4] = ["completed", "failed", "timed_out", "cancelled"]; +const TIMEOUT_KINDS: [&str; 3] = ["start_to_close", "schedule_to_close", "heartbeat"]; +const RETRY_POLICY_FIELDS: [&str; 3] = ["max_attempts", "backoff_seconds", "non_retryable_error_types"]; +const ATTEMPT_FIELDS: [&str; 11] = ["attempt_id", "attempt_number", "outcome", "duration_ms", "message", + "exception_type", "non_retryable", "timeout_kind", "retry_reason", "backoff_seconds", "heartbeats"]; +const HEARTBEAT_FIELDS: [&str; 2] = ["elapsed_ms", "details"]; +/// `PARALLEL_METADATA_FIELDS` and the group path, which belong only to the +/// scheduling commands. +const PARALLEL_FIELDS: [&str; 12] = ["parallel_group_id", "parallel_group_kind", "parallel_group_mode", + "parallel_group_base_sequence", "parallel_group_size", "parallel_group_index", "selection_member_key", + "selection_member_index", "selection_member_base_sequence", "selection_member_size", "selection_member_kind", + "parallel_group_path"]; +const TIMEOUT_EXCEPTION: &str = "Workflow\\V2\\Exceptions\\ActivityTimeoutException"; + +/// `server_capabilities.local_activities`: `LocalActivityContract::manifest` +/// with `supported` from the configured protocol, which native fixes at 1.20. +pub(super) fn manifest() -> Value { + json!({ + "schema": "durable-workflow.v2.local-activity.contract", + "version": 1, + "supported": true, + "worker_capability": "local_activities", + "minimum_worker_protocol_version": MINIMUM_PROTOCOL_VERSION, + "command_type": COMMAND, + "command": { + "type": COMMAND, + "minimum_protocol_version": MINIMUM_PROTOCOL_VERSION, + "required_fields": ["activity_type", "outcome", "attempts"], + "attempt_reports": { + "field": "attempts", + "shape": "ordered_attempt_reports", + "required_fields": ["attempt_number", "outcome"], + "maximum": MAX_ATTEMPTS, + "required_from_protocol_version": ATTEMPT_REPORTS_PROTOCOL_VERSION, + "worker_attempt_identity": {"report_field": "attempt_id", "persistence_field": "worker_attempt_id", + "history_field": "worker_attempt_id", "durable_field": "activity_attempt_id"} + }, + "heartbeat_reports": { + "field": "heartbeats", + "shape": "ordered_heartbeat_reports", + "required_fields": ["elapsed_ms"], + "maximum_per_attempt": MAX_HEARTBEATS_PER_ATTEMPT, + "maximum_per_command": MAX_HEARTBEATS, + "elapsed_ms_origin": "attempt_started_at", + "persisted_last_heartbeat_at": "attempt_started_at_plus_last_elapsed_ms" + }, + "rolling_upgrade": { + "retained_protocol_version": MINIMUM_PROTOCOL_VERSION, + "missing_attempts": "normalize_to_one_terminal_attempt_without_worker_identity_or_heartbeats", + "strict_shape_selected_by": "negotiated_worker_protocol_version", + "retained_nested_object_unknown_fields": "ignore", + "strict_nested_object_shape_from_protocol_version": ATTEMPT_REPORTS_PROTOCOL_VERSION + } + }, + "api": { + "functions": ["Workflow\\V2\\localActivity"], + "workflow_facade": ["Workflow\\V2\\Workflow::localActivity", "Workflow\\V2\\Workflow::executeLocalActivity"], + "options": "Workflow\\V2\\Support\\LocalActivityOptions" + }, + "execution": { + "mode": "local", + "same_process": true, + "ordinary_activity_task_created": false, + "history_events": ["ActivityScheduled", "ActivityStarted", "ActivityHeartbeatRecorded", "ActivityRetryScheduled", + "ActivityCompleted", "ActivityFailed", "ActivityTimedOut", "ActivityCancelled"], + "history_marker": {"execution_mode": "local", "local_activity": true} + }, + "heartbeating": { + "lease": "workflow_task", + "automatic_points": ["before_attempt_start", "after_activity_heartbeat"], + "long_running_requirement": "activity_code_calls_heartbeat_before_the_workflow_task_lease_can_expire" + }, + "cancellation": { + "preemption": "cooperative", + "observed_at": ["heartbeat", "timeout_enforcement", "attempt_completion"], + "terminal_event": "ActivityCancelled" + }, + "shutdown": { + "graceful": "worker_stops_claiming_new_workflow_tasks_and_allows_in_process_attempt_to_finish_or_heartbeat", + "crash": "workflow_task_lease_expires_and_task_repair_reclaims_the_workflow_task", + "cold_replay": "committed_terminal_event_is_replayed_or_started_attempt_without_terminal_event_schedules_local_retry" + }, + "routing": { + "admission": "activity_class_must_resolve_in_the_workflow_worker_process", + "queue_bypassed": true, + "rejected_options": ["connection", "queue", "worker_session", "schedule_to_start_timeout"] + }, + "retry": { + "new_attempt_when": ["a local attempt starts", "a retry workflow task replays after backoff", + "cold replay finds a previously started local attempt without a terminal event"], + "backoff": "durable_workflow_task_available_at", + "cold_replay_reason": "cold_replay" + }, + "timeouts": { + "start_to_close_timeout": "one_local_attempt", + "schedule_to_close_timeout": "entire_local_execution_including_retries", + "heartbeat_timeout": "gap_between_recorded_local_activity_heartbeats", + "enforcement": "cooperative_at_heartbeat_attempt_start_and_attempt_completion_boundaries" + }, + "visibility": { + "activity_execution_marker": "activity_options.execution_mode", + "history_marker": "payload.execution_mode", + "metrics_marker": "activities.local_*", + "history_export": "same_activity_event_payloads_with_execution_mode_local" + } + }) +} + +/// `WorkerProtocolVersion::supportsFeatureVersion`: the same major version +/// and at least the minimum minor version. +fn supports(candidate: &str, minimum: &str) -> bool { + let parts = |version: &str| { + let (major, minor) = version.split_once('.')?; + let digits = |part: &str| (!part.is_empty() && part.bytes().all(|byte| byte.is_ascii_digit())).then(|| part.parse::().ok()).flatten(); + Some((digits(major)?, digits(minor)?)) + }; + matches!((parts(candidate), parts(minimum)), (Some((major, minor)), Some((least_major, least_minor))) + if major == least_major && minor >= least_minor) +} + +/// `version_compare($requested, $minimum, '<')` for the guard's numeric +/// `major.minor` versions; anything else is below the floor. +fn below(requested: Option<&str>, minimum: &str) -> bool { + let parse = |version: &str| -> Option> { version.split('.').map(|part| part.parse().ok()).collect() }; + match (requested.and_then(parse), parse(minimum)) { + (Some(requested), Some(minimum)) => requested < minimum, + _ => true, + } +} + +/// PHP's `trim`. +fn trim(text: &str) -> &str { + text.trim_matches([' ', '\t', '\n', '\r', '\0', '\x0B']) +} + +/// A JSON object key PHP's decoder turns into an integer array key. +fn integer_key(key: &str) -> bool { + let digits = key.strip_prefix('-').unwrap_or(key); + !digits.is_empty() && digits.bytes().all(|byte| byte.is_ascii_digit()) + && (digits == "0" || !digits.starts_with('0')) && key != "-0" && key.parse::().is_ok() +} + +/// PHP's `is_array($value) && array_is_list($value)` for a decoded JSON +/// value: an array, or an empty object. +fn list(value: &Value) -> bool { + match value { + Value::Array(_) => true, + Value::Object(fields) => fields.is_empty(), + _ => false, + } +} + +/// PHP's `is_array($value) && ! array_is_list($value)`. +fn object(value: &Value) -> Option<&Map> { + value.as_object().filter(|fields| !fields.is_empty()) +} + +/// PHP's `array_values`: an object's values or an array's elements. +fn values(value: &Value) -> Vec<&Value> { + match value { + Value::Array(items) => items.iter().collect(), + Value::Object(fields) => fields.values().collect(), + _ => Vec::new(), + } +} + +/// A decoded PHP array encoded again: an empty object becomes `[]`. +fn php_json(value: &Value) -> Value { + match value { + Value::Object(fields) if fields.is_empty() => json!([]), + Value::Object(fields) => Value::Object(fields.iter().map(|(key, value)| (key.clone(), php_json(value))).collect()), + Value::Array(items) => Value::Array(items.iter().map(php_json).collect()), + other => other.clone(), + } +} + +/// The normalizer's error map: a later message for a field replaces the +/// earlier one in place, as PHP's `$errors[$field] = [...]` does. +#[derive(Default)] +struct Errors(Vec<(String, Vec)>); + +impl Errors { + fn set(&mut self, field: impl Into, message: impl Into) { + let (field, messages) = (field.into(), vec![message.into()]); + match self.0.iter_mut().find(|(seen, _)| *seen == field) { + Some((_, current)) => *current = messages, + None => self.0.push((field, messages)), + } + } +} + +fn present(command: &Value, field: &str) -> bool { + command.get(field).is_some_and(|value| !value.is_null()) +} + +/// `optionalCommandString`. +fn command_string(command: &Value, field: &str, index: usize, errors: &mut Errors) -> Option { + let value = command.get(field).filter(|value| !value.is_null())?; + match value.as_str().map(trim).filter(|text| !text.is_empty()) { + Some(text) => Some(text.to_owned()), + None => { + errors.set(format!("commands.{index}.{field}"), + format!("Workflow task command field [{field}] must be a non-empty string when provided.")); + None + } + } +} + +/// `optionalNestedString`. +fn nested_string(value: &Value, field: &str, path: &str, errors: &mut Errors) -> Option { + let value = value.get(field).filter(|value| !value.is_null())?; + match value.as_str().map(trim).filter(|text| !text.is_empty()) { + Some(text) => Some(text.to_owned()), + None => { + errors.set(format!("{path}.{field}"), format!("Local activity attempt field [{field}] must be a non-empty string.")); + None + } + } +} + +/// `optionalPositiveInt`. +fn positive(command: &Value, field: &str, index: usize, errors: &mut Errors) -> Option { + let value = command.get(field).filter(|value| !value.is_null())?; + match value.as_i64().filter(|seconds| *seconds >= 1) { + Some(seconds) => Some(seconds), + None => { + errors.set(format!("commands.{index}.{field}"), + format!("Workflow task command field [{field}] must be a positive integer when provided.")); + None + } + } +} + +/// `optionalRetryPolicy` for a local activity. +fn retry_policy(command: &Value, index: usize, errors: &mut Errors) -> Option> { + let raw = command.get("retry_policy").filter(|value| !value.is_null())?; + let path = format!("commands.{index}.retry_policy"); + let Some(raw) = raw.as_object() else { + if !raw.is_array() { + errors.set(path, "Workflow task command field [retry_policy] must be an object when provided."); + } + return None; + }; + let mut policy = Map::new(); + if let Some(value) = raw.get("max_attempts") { + match value.as_i64().filter(|attempts| *attempts >= 1) { + Some(attempts) => { + policy.insert("max_attempts".into(), json!(attempts)); + } + None => errors.set(format!("{path}.max_attempts"), + "Local activity retry policy max_attempts must be a positive integer when provided."), + } + } + if let Some(value) = raw.get("backoff_seconds") { + if value.is_array() || value.is_object() { + let mut backoff = Vec::new(); + for (position, entry) in values(value).into_iter().enumerate() { + match entry.as_i64().filter(|seconds| *seconds >= 0) { + Some(seconds) => backoff.push(json!(seconds)), + None => errors.set(format!("{path}.backoff_seconds.{position}"), + "Local activity retry policy backoff_seconds entries must be non-negative integers."), + } + } + policy.insert("backoff_seconds".into(), Value::Array(backoff)); + } else { + errors.set(format!("{path}.backoff_seconds"), + "Local activity retry policy backoff_seconds must be a list of non-negative integers."); + } + } + if let Some(value) = raw.get("non_retryable_error_types") { + if value.is_array() || value.is_object() { + let mut types: Vec = Vec::new(); + for (position, entry) in values(value).into_iter().enumerate() { + match entry.as_str().map(trim).filter(|text| !text.is_empty()) { + Some(text) => { + if !types.iter().any(|seen| seen == text) { + types.push(text.to_owned()); + } + } + None => errors.set(format!("{path}.non_retryable_error_types.{position}"), + "Local activity retry policy non_retryable_error_types entries must be non-empty strings."), + } + } + policy.insert("non_retryable_error_types".into(), json!(types)); + } else { + errors.set(format!("{path}.non_retryable_error_types"), + "Local activity retry policy non_retryable_error_types must be a list of strings."); + } + } + (!policy.is_empty()).then_some(policy) +} + +/// `rejectUnknownNestedFields`: a decoded integer key is never refused. +fn unknown_fields(value: &Map, allowed: &[&str], path: &str, subject: &str, errors: &mut Errors) { + for field in value.keys().filter(|field| !integer_key(field) && !allowed.contains(&field.as_str())) { + errors.set(format!("{path}.{field}"), format!("Field [{field}] is not supported in a {subject} report.")); + } +} + +/// `validateLocalActivityRetryPolicyShape`. +fn retry_policy_shape(command: &Value, strict: bool, index: usize, errors: &mut Errors) { + let Some(raw) = command.get("retry_policy").and_then(Value::as_object) else { + return; + }; + let path = format!("commands.{index}.retry_policy"); + if strict { + unknown_fields(raw, &RETRY_POLICY_FIELDS, &path, "local activity retry policy", errors); + } + for field in ["backoff_seconds", "non_retryable_error_types"] { + if raw.get(field).and_then(object).is_some() { + errors.set(format!("{path}.{field}"), format!("Local activity retry policy {field} must be an ordered list.")); + } + } + if let Some(types) = raw.get("non_retryable_error_types").filter(|types| list(types)) { + let types = values(types); + let mut unique: Vec<&Value> = Vec::new(); + for entry in &types { + if !unique.contains(entry) { + unique.push(entry); + } + } + if unique.len() != types.len() { + errors.set(format!("{path}.non_retryable_error_types"), + "Local activity retry policy non_retryable_error_types must not contain duplicates."); + } + } +} + +/// `normalizeLocalActivityHeartbeats`. +fn heartbeats(value: Option<&Value>, attempt_path: &str, strict: bool, errors: &mut Errors) -> Vec { + let Some(value) = value.filter(|value| !value.is_null()) else { + return Vec::new(); + }; + if !list(value) { + errors.set(format!("{attempt_path}.heartbeats"), "Local activity attempt heartbeats must be an ordered list."); + return Vec::new(); + } + let entries = values(value); + if entries.len() > MAX_HEARTBEATS_PER_ATTEMPT { + errors.set(format!("{attempt_path}.heartbeats"), + format!("A local activity attempt may contain at most {MAX_HEARTBEATS_PER_ATTEMPT} heartbeats.")); + } + let mut normalized = Vec::new(); + let mut last = -1; + for (position, heartbeat) in entries.into_iter().enumerate() { + let path = format!("{attempt_path}.heartbeats.{position}"); + let Some(fields) = object(heartbeat) else { + errors.set(path, "Each local activity heartbeat must be an object."); + continue; + }; + if strict { + unknown_fields(fields, &HEARTBEAT_FIELDS, &path, "local activity heartbeat", errors); + } + let Some(elapsed) = fields.get("elapsed_ms").and_then(Value::as_i64).filter(|elapsed| *elapsed >= 0) else { + errors.set(format!("{path}.elapsed_ms"), "Local activity heartbeat elapsed_ms must be a non-negative integer."); + continue; + }; + if elapsed < last { + errors.set(format!("{path}.elapsed_ms"), "Local activity heartbeat elapsed_ms values must be ordered monotonically."); + } + last = elapsed; + let mut entry = json!({"elapsed_ms": elapsed}); + if let Some(details) = fields.get("details").filter(|details| !details.is_null()) { + if object(details).is_some() || list(details) && values(details).is_empty() { + entry["details"] = php_json(details); + } else { + errors.set(format!("{path}.details"), "Local activity heartbeat details must be an object when provided."); + } + } + normalized.push(entry); + } + normalized +} + +/// `normalizeLocalActivityAttempts`. +fn attempts(command: &Value, outcome: &str, terminal_non_retryable: Option, policy: Option<&Map>, + protocol: &str, index: usize, errors: &mut Errors) -> Option> { + let synthesized; + let mut raw = command.get("attempts"); + // Protocol 1.18 workers may omit the list: one terminal attempt. + if raw.is_none() && supports(protocol, MINIMUM_PROTOCOL_VERSION) && !supports(protocol, ATTEMPT_REPORTS_PROTOCOL_VERSION) { + let mut attempt = json!({"attempt_number": 1, "outcome": outcome}); + if let Some(message) = command.get("message").and_then(Value::as_str) { + attempt["message"] = json!(trim(message)); + } + if let Some(exception) = nested_string(command, "exception_type", &format!("commands.{index}"), errors) { + attempt["exception_type"] = json!(exception); + } + if let Some(non_retryable) = terminal_non_retryable { + attempt["non_retryable"] = json!(non_retryable); + } + if let Some(kind) = command.get("timeout_kind").and_then(Value::as_str) { + attempt["timeout_kind"] = json!(trim(kind)); + } + attempt["heartbeats"] = json!([]); + synthesized = json!([attempt]); + raw = Some(&synthesized); + } + let Some(raw) = raw.filter(|raw| list(raw) && !values(raw).is_empty()) else { + errors.set(format!("commands.{index}.attempts"), "Local activity attempts must be a non-empty ordered list."); + return None; + }; + let raw = values(raw); + let count = raw.len() as i64; + let strict = supports(protocol, ATTEMPT_REPORTS_PROTOCOL_VERSION); + let max_attempts = policy.and_then(|policy| policy.get("max_attempts")).and_then(Value::as_i64).unwrap_or(1); + retry_policy_shape(command, strict, index, errors); + if max_attempts > MAX_ATTEMPTS { + errors.set(format!("commands.{index}.retry_policy.max_attempts"), + format!("Local activity retry policy max_attempts must not exceed {MAX_ATTEMPTS}.")); + } + if count > MAX_ATTEMPTS { + errors.set(format!("commands.{index}.attempts"), format!("Local activity reports may contain at most {MAX_ATTEMPTS} attempts.")); + } else if count > max_attempts { + errors.set(format!("commands.{index}.attempts"), + format!("Local activity report contains {count} attempts but retry_policy.max_attempts is {max_attempts}.")); + } + let schedule: Vec = policy.and_then(|policy| policy.get("backoff_seconds")).and_then(Value::as_array) + .map(|entries| entries.iter().filter_map(Value::as_i64).collect()).unwrap_or_default(); + if schedule.len() as i64 > (max_attempts - 1).max(0) { + errors.set(format!("commands.{index}.retry_policy.backoff_seconds"), + "Local activity retry backoff entries must not exceed the number of possible retries."); + } + let mut normalized = Vec::new(); + let mut identities = HashSet::new(); + let mut heartbeat_count = 0; + let last = raw.len() - 1; + for (position, attempt) in raw.into_iter().enumerate() { + let path = format!("commands.{index}.attempts.{position}"); + let Some(fields) = object(attempt) else { + errors.set(path, "Each local activity attempt must be an object."); + continue; + }; + if strict { + unknown_fields(fields, &ATTEMPT_FIELDS, &path, "local activity attempt", errors); + } + let expected = position as i64 + 1; + let number = fields.get("attempt_number").and_then(Value::as_i64); + if number != Some(expected) { + errors.set(format!("{path}.attempt_number"), + format!("Local activity attempt_number must be the ordered one-based value {expected}.")); + } + let Some(attempt_outcome) = fields.get("outcome").and_then(Value::as_str).filter(|value| OUTCOMES.contains(value)) else { + errors.set(format!("{path}.outcome"), "Local activity attempt outcome must be completed, failed, timed_out, or cancelled."); + continue; + }; + let terminal = position == last; + if terminal && attempt_outcome != outcome { + errors.set(format!("{path}.outcome"), "The final local activity attempt outcome must match the command outcome."); + } + if !terminal && !matches!(attempt_outcome, "failed" | "timed_out") { + errors.set(format!("{path}.outcome"), "Only failed or timed_out local activity attempts may be followed by a retry."); + } + let identity = nested_string(attempt, "attempt_id", &path, errors); + match &identity { + Some(identity) if identity.len() > 255 => errors.set(format!("{path}.attempt_id"), "Local activity attempt_id must not exceed 255 bytes."), + Some(identity) if identities.contains(identity) => errors.set(format!("{path}.attempt_id"), + "Local activity attempt_id must be unique within the reported sequence."), + Some(identity) => { + identities.insert(identity.clone()); + } + None => {} + } + let duration = match fields.get("duration_ms") { + None => None, + Some(value) => match value.as_i64().filter(|duration| *duration >= 0) { + Some(duration) => Some(duration), + None => { + errors.set(format!("{path}.duration_ms"), "Local activity attempt duration_ms must be a non-negative integer."); + None + } + }, + }; + let message = nested_string(attempt, "message", &path, errors); + let exception_type = nested_string(attempt, "exception_type", &path, errors); + if attempt_outcome != "completed" && message.is_none() { + errors.set(format!("{path}.message"), "Failed, timed out, and cancelled local activity attempts require a message."); + } + let non_retryable = match fields.get("non_retryable") { + None => None, + Some(Value::Bool(value)) => { + if !terminal && *value { + errors.set(format!("{path}.non_retryable"), "A non-retryable local activity attempt cannot be followed by a retry."); + } + Some(*value) + } + Some(_) => { + errors.set(format!("{path}.non_retryable"), "Local activity attempt non_retryable must be boolean when provided."); + None + } + }; + if terminal && non_retryable.is_some_and(|value| value != terminal_non_retryable.unwrap_or(false)) { + errors.set(format!("{path}.non_retryable"), + "The terminal local activity attempt non_retryable value must match the command outcome."); + } + let timeout_kind = nested_string(attempt, "timeout_kind", &path, errors); + if attempt_outcome == "timed_out" && !timeout_kind.as_deref().is_some_and(|kind| TIMEOUT_KINDS.contains(&kind)) { + errors.set(format!("{path}.timeout_kind"), + "Timed out local activity attempts require start_to_close, schedule_to_close, or heartbeat timeout_kind."); + } + if attempt_outcome != "timed_out" && timeout_kind.is_some() { + errors.set(format!("{path}.timeout_kind"), "Only timed out local activity attempts may declare timeout_kind."); + } + let retry_reason = nested_string(attempt, "retry_reason", &path, errors); + let backoff = match fields.get("backoff_seconds") { + None => None, + Some(value) => match value.as_i64().filter(|seconds| *seconds >= 0) { + Some(seconds) => Some(seconds), + None => { + errors.set(format!("{path}.backoff_seconds"), "Local activity attempt backoff_seconds must be a non-negative integer."); + None + } + }, + }; + if !terminal { + let allowed: [&str; 2] = if attempt_outcome == "timed_out" { ["timeout", "cold_replay"] } else { ["failure", "cold_replay"] }; + if !retry_reason.as_deref().is_some_and(|reason| allowed.contains(&reason)) { + errors.set(format!("{path}.retry_reason"), + format!("A retried {attempt_outcome} local activity attempt requires a compatible retry_reason.")); + } + match backoff { + None => errors.set(format!("{path}.backoff_seconds"), "A retried local activity attempt requires backoff_seconds."), + Some(seconds) if schedule.get(position).is_some_and(|scheduled| *scheduled != seconds) => errors.set( + format!("{path}.backoff_seconds"), "Local activity attempt backoff_seconds must match retry_policy.backoff_seconds."), + Some(_) => {} + } + } else if retry_reason.is_some() || backoff.is_some() { + errors.set(path.clone(), "The terminal local activity attempt cannot declare retry_reason or backoff_seconds."); + } + let beats = heartbeats(fields.get("heartbeats"), &path, strict, errors); + heartbeat_count += beats.len(); + let mut entry = Map::new(); + let mut put = |key: &str, value: Option| { + if let Some(value) = value { + entry.insert(key.to_owned(), value); + } + }; + put("attempt_id", identity.map(Value::from)); + put("attempt_number", Some(json!(number.unwrap_or(expected)))); + put("outcome", Some(json!(attempt_outcome))); + put("duration_ms", duration.map(Value::from)); + put("message", message.map(Value::from)); + put("exception_type", exception_type.map(Value::from)); + put("non_retryable", non_retryable.map(Value::from)); + put("timeout_kind", timeout_kind.map(Value::from)); + put("retry_reason", retry_reason.map(Value::from)); + put("backoff_seconds", backoff.map(Value::from)); + put("heartbeats", Some(Value::Array(beats))); + normalized.push(Value::Object(entry)); + } + if heartbeat_count > MAX_HEARTBEATS { + errors.set(format!("commands.{index}.attempts"), + format!("Local activity reports may contain at most {MAX_HEARTBEATS} total heartbeats.")); + } + Some(normalized) +} + +/// A payload field after `PayloadEnvelopeResolver`: its blob and codec. +/// External references are resolved before the normalizer in PHP; native +/// does not resolve them for local activities yet and refuses them. +fn payload(command: &Value, field: &str, index: usize, errors: &mut Errors) -> Result<(Option, Option)> { + let not_payload = |errors: &mut Errors| errors.set(format!("commands.{index}.{field}"), + format!("Workflow task command field [{field}] must be a string or a payload envelope when provided.")); + match command.get(field) { + None | Some(Value::Null) => Ok((None, None)), + // Arguments that are an empty string carry no payload. + Some(Value::String(blob)) => Ok(((field == "result" || !blob.is_empty()).then(|| blob.clone()), None)), + Some(value @ (Value::Object(_) | Value::Array(_))) => { + let envelope = value.as_object().filter(|fields| { + let mut keys: Vec<&str> = fields.keys().map(String::as_str).collect(); + keys.sort_unstable(); + fields.contains_key("codec") && (keys == ["blob", "codec"] || keys == ["codec", "external_storage"]) + }); + let Some(envelope) = envelope else { + // A plain value is no payload for arguments and refused as a result. + if field == "result" { + not_payload(errors); + } + return Ok((None, None)); + }; + let path = format!("commands.{index}.{field}"); + if !envelope["codec"].as_str().is_some_and(|codec| !codec.is_empty()) { + errors.set(format!("{path}.codec"), "The payload envelope codec must be a non-empty string."); + return Ok((None, None)); + } + if let Some(message) = super::input_envelope::codec_refusal(&envelope["codec"]) { + errors.set(format!("{path}.codec"), message); + return Ok((None, None)); + } + if envelope.contains_key("external_storage") { + return Err(refuse(StatusCode::UNPROCESSABLE_ENTITY, "local_activity_external_payload_unsupported")); + } + let Some(blob) = envelope["blob"].as_str() else { + errors.set(format!("{path}.blob"), "The payload envelope blob must be a string."); + return Ok((None, None)); + }; + Ok(((field == "result" || !blob.is_empty()).then(|| blob.to_owned()), Some("avro".to_owned()))) + } + Some(_) => { + not_payload(errors); + Ok((None, None)) + } + } +} + +/// `payloadCodecForResolvedPayload`. +fn payload_codec(command: &Value, resolved: Option<&str>, field: &str, index: usize, errors: &mut Errors) -> Option { + let explicit = match command.get("payload_codec").filter(|codec| !codec.is_null()) { + None => None, + Some(codec) => match codec.as_str().map(trim).filter(|codec| !codec.is_empty()) { + None => { + errors.set(format!("commands.{index}.payload_codec"), + "Workflow task command field [payload_codec] must be a non-empty string when provided."); + None + } + Some(codec) => match super::input_envelope::codec_refusal(&json!(codec)) { + Some(message) => { + errors.set(format!("commands.{index}.payload_codec"), message); + None + } + None => Some("avro".to_owned()), + }, + }, + }; + if let (Some(explicit), Some(resolved)) = (&explicit, resolved) + && explicit != resolved + { + errors.set(format!("commands.{index}.payload_codec"), + format!("Workflow task command field [payload_codec] must match the {field} envelope codec.")); + } + explicit.or_else(|| resolved.map(str::to_owned)) +} + +/// One record after the normalizer, as the bridge applies it. +#[derive(Debug)] +pub(super) struct Record { + activity_type: String, + arguments: Option, + result: Option, + payload_codec: Option, + outcome: String, + message: Option, + exception_type: Option, + non_retryable: Option, + timeout_kind: Option, + attempts: Vec, + retry_policy: Option>, + timeouts: [Option; 3], +} + +impl Record { + /// The message the execution's exception carries, when it has one. + pub(super) fn exception_message(&self) -> Option { + (self.outcome != "completed").then(|| self.message.clone().unwrap_or_else(|| "Local activity failed.".to_owned())) + } +} + +/// `WorkflowCommandNormalizer::normalize` for each record in the batch, at +/// the request's protocol version (1.20 without one), with its field scopes +/// and messages, collected across the batch. +pub(super) fn normalize(commands: &[Value], protocol: Option<&str>) -> Result> { + let protocol = protocol.unwrap_or(super::WORKER_PROTOCOL_VERSION); + let mut errors = Errors::default(); + let mut records = BTreeMap::new(); + for (index, command) in commands.iter().enumerate().filter(|(_, command)| command["type"] == COMMAND) { + for (field, messages) in super::completion_rules::field_scope_errors(index, command) { + for message in messages { + errors.set(field.clone(), message); + } + } + for field in PARALLEL_FIELDS.iter().filter(|field| present(command, field)) { + errors.set(format!("commands.{index}.{field}"), + format!("{field} is only supported for activity, timer, and child-workflow scheduling commands.")); + } + // `optionalOperationScopeMetadata`. + if let Some(scope) = command.get("cancellation_scope_id") { + if !scope.as_str().is_some_and(|scope| !trim(scope).is_empty() && scope.len() <= 255) { + errors.set(format!("commands.{index}.cancellation_scope_id"), "Expected a nonempty canonical scope identity."); + } else if !supports(protocol, "1.20") { + errors.set(format!("commands.{index}.cancellation_scope_id"), "Operation scope membership requires candidate protocol 1.20."); + } + } + let Some(activity_type) = command["activity_type"].as_str().map(trim).filter(|text| !text.is_empty()) else { + errors.set(format!("commands.{index}.activity_type"), "Local activity records require a non-empty activity_type."); + continue; + }; + let outcome = command_string(command, "outcome", index, &mut errors); + let Some(outcome) = outcome.filter(|outcome| OUTCOMES.contains(&outcome.as_str())) else { + errors.set(format!("commands.{index}.outcome"), "Local activity outcome must be completed, failed, timed_out, or cancelled."); + continue; + }; + let policy = retry_policy(command, index, &mut errors); + let start = positive(command, "start_to_close_timeout", index, &mut errors); + let close = positive(command, "schedule_to_close_timeout", index, &mut errors); + let heartbeat = positive(command, "heartbeat_timeout", index, &mut errors); + // `assertActivityTimeoutOrdering`. + if let (Some(start), Some(close)) = (start, close) + && close < start + { + errors.set(format!("commands.{index}.schedule_to_close_timeout"), format!("schedule_to_close_timeout ({close}) must be greater than or equal to start_to_close_timeout ({start}). schedule_to_close covers the whole activity execution including retries; one attempt cannot exceed the total budget.")); + } + if let (Some(start), Some(heartbeat)) = (start, heartbeat) + && heartbeat > start + { + errors.set(format!("commands.{index}.heartbeat_timeout"), format!("heartbeat_timeout ({heartbeat}) must be less than or equal to start_to_close_timeout ({start}). heartbeat_timeout polices liveness within one attempt and cannot exceed the per-attempt budget.")); + } + let non_retryable = match command.get("non_retryable") { + None => None, + Some(Value::Bool(value)) => { + if outcome == "completed" { + errors.set(format!("commands.{index}.non_retryable"), "Completed local activities cannot be marked non-retryable."); + } + Some(*value) + } + Some(_) => { + errors.set(format!("commands.{index}.non_retryable"), "Local activity non_retryable must be boolean when provided."); + None + } + }; + let attempts = attempts(command, &outcome, non_retryable, policy.as_ref(), protocol, index, &mut errors); + let timeout_kind = command_string(command, "timeout_kind", index, &mut errors); + if timeout_kind.as_deref().is_some_and(|kind| !TIMEOUT_KINDS.contains(&kind)) { + errors.set(format!("commands.{index}.timeout_kind"), + "Local activity timeout_kind must be start_to_close, schedule_to_close, or heartbeat."); + } + if outcome == "timed_out" && timeout_kind.is_none() { + errors.set(format!("commands.{index}.timeout_kind"), "Timed out local activities require a timeout_kind."); + } + if outcome != "timed_out" && timeout_kind.is_some() { + errors.set(format!("commands.{index}.timeout_kind"), "Only timed out local activities may declare timeout_kind."); + } + let (arguments, arguments_codec) = payload(command, "arguments", index, &mut errors)?; + let (result, result_codec) = payload(command, "result", index, &mut errors)?; + let codec = if arguments.is_some() { + payload_codec(command, arguments_codec.as_deref(), "arguments", index, &mut errors) + } else { + payload_codec(command, result_codec.as_deref(), "result", index, &mut errors) + }; + if outcome == "completed" && result.is_none() { + errors.set(format!("commands.{index}.result"), "Completed local activities require a result payload envelope."); + } + let message = command["message"].as_str().map(|message| trim(message).to_owned()); + if outcome != "completed" && message.as_deref().is_none_or(str::is_empty) { + errors.set(format!("commands.{index}.message"), "Failed, timed out, and cancelled local activities require a message."); + } + let exception_type = command_string(command, "exception_type", index, &mut errors); + if let Some(attempts) = attempts { + records.insert(index, Record { + activity_type: activity_type.to_owned(), + arguments, + result, + payload_codec: codec, + outcome, + message, + exception_type, + non_retryable, + timeout_kind, + attempts, + retry_policy: policy, + timeouts: [start, close, heartbeat], + }); + } + } + match failure_messages(errors.0) { + Some(error) => Err(error), + None => Ok(records), + } +} + +/// `toJSON()`: microseconds and `Z`. +fn time(at: DateTime) -> Value { + json!(at.to_rfc3339_opts(SecondsFormat::Micros, true)) +} + +fn milliseconds(value: Option<&Value>) -> i64 { + value.and_then(Value::as_i64).unwrap_or(0) +} + +/// One reported attempt with the times the bridge gives it. +struct Attempt<'a> { + report: &'a Value, + id: String, + started_at: DateTime, + last_heartbeat_at: Option>, + closed_at: DateTime, +} + +impl Attempt<'_> { + fn number(&self) -> i64 { + self.report["attempt_number"].as_i64().unwrap_or(1) + } + fn worker_id(&self) -> Value { + self.report.get("attempt_id").cloned().unwrap_or(Value::Null) + } + fn outcome(&self) -> &str { + self.report["outcome"].as_str().unwrap_or("failed") + } + fn status(&self) -> &'static str { + match self.outcome() { + "completed" => "completed", + "cancelled" => "cancelled", + _ => "failed", + } + } + fn heartbeats(&self) -> &[Value] { + self.report["heartbeats"].as_array().map_or(&[], Vec::as_slice) + } +} + +/// Inserts the fields that have values, in order. +fn filtered(fields: &[(&str, Value)]) -> Value { + Value::Object(fields.iter().filter(|(_, value)| !value.is_null()).map(|(key, value)| ((*key).to_owned(), value.clone())).collect()) +} + +/// Encodes each record's exception message with the official codec. +pub(super) async fn exceptions(records: &BTreeMap, codec: Arc) -> Result> { + let messages: Vec<(usize, String)> = records.iter().filter_map(|(index, record)| record.exception_message().map(|message| (*index, message))).collect(); + if messages.is_empty() { + return Ok(BTreeMap::new()); + } + let unavailable = || refuse(StatusCode::SERVICE_UNAVAILABLE, "local_activity_codec_unavailable"); + let permit = codec.acquire_owned().await.map_err(|_| unavailable())?; + tokio::task::spawn_blocking(move || { + let _permit = permit; + let codec = crate::codec::ValueCodec::new().map_err(|_| unavailable())?; + messages.into_iter() + .map(|(index, message)| codec.encode(&crate::codec::Value::String(message)).map(|blob| (index, blob)).map_err(|_| unavailable())) + .collect() + }) + .await + .map_err(|_| unavailable())? +} + +/// `portableWorkerAffinityRejection`. +fn affinity_rejection(task_id: &str, attempt: &Value, reason: &str, requested: Option<&str>) -> RuntimeError { + RuntimeError::Protocol { + status: StatusCode::CONFLICT, + response: json!({"task_id": task_id, "workflow_task_attempt": attempt, "outcome": "rejected", "recorded": false, + "reason": reason, "capability": "local_activities", "requested_version": requested, + "minimum_protocol_version": MINIMUM_PROTOCOL_VERSION, + "remediation": "Use cold durable replay unless the registered worker manifest explicitly supports the required capability and exact cache identity.", + "protocol_version": super::WORKER_PROTOCOL_VERSION, "server_capabilities": super::store::capabilities()}), + } +} + +store_methods! { + /// `guardPortableWorkerAffinityCompletion` for a batch with a local + /// activity record, after the ownership guard: protocol 1.18 or newer, + /// and a lease owner whose registration lists `local_activities` with a + /// supported manifest entry. + pub(super) async fn guard_local_activities(tx: &mut Transaction<'_, DB>, namespace: &str, task: &DB::Row, body: &Value, + protocol: Option<&str>) -> Result<()> { + let task_id = DB::string(task, "id")?; + let attempt = body["workflow_task_attempt"].as_i64().map_or(Value::Null, Value::from); + if below(protocol, MINIMUM_PROTOCOL_VERSION) { + return Err(affinity_rejection(&task_id, &attempt, "portable_worker_affinity_protocol_floor_unavailable", protocol)); + } + let worker = Self::query("SELECT capabilities,capability_manifest FROM workflow_worker_registrations WHERE namespace=$1 AND worker_id=$2") + .bind(namespace).bind(super::text(body, "lease_owner")?).fetch_optional(&mut **tx).await?; + let (capabilities, manifest) = match &worker { + Some(row) => (DB::optional_document_row(row, "capabilities")?, DB::optional_document_row(row, "capability_manifest")?), + None => (None, None), + }; + let listed = capabilities.as_ref().and_then(Value::as_array) + .is_some_and(|capabilities| capabilities.iter().any(|capability| capability.as_str() == Some("local_activities"))); + let supported = manifest.as_ref().and_then(|manifest| manifest.get("local_activities")) + .is_some_and(|entry| (entry.is_object() || entry.is_array()) && entry["supported"] == Value::Bool(true)); + if !listed || !supported { + return Err(affinity_rejection(&task_id, &attempt, "worker_capability_not_supported", protocol)); + } + Ok(()) + } + + /// `applyRecordLocalActivity`: the execution, its attempts with times + /// walked back from now, and the history PHP writes for them. + #[allow(clippy::too_many_arguments)] + pub(super) async fn record_local_activity(tx: &mut Transaction<'_, DB>, run: &DB::Row, task: &DB::Row, record: &Record, + exception: Option<&str>, sequence: i64) -> Result<()> { + let run_id = DB::string(run, "id")?; + let task_id = DB::string(task, "id")?; + let lease_owner = DB::optional_string(task, "lease_owner")?; + let codec = record.payload_codec.clone() + .or(DB::optional_string(run, "payload_codec")?.filter(|codec| !codec.is_empty())) + .unwrap_or_else(|| "avro".to_owned()); + let at = now(); + // Reports carry durations, not times: walk back from the close so + // each attempt ends before the next one's declared backoff. Identities + // are made last attempt first, as PHP makes them, and are uppercase. + let mut attempts: Vec> = Vec::with_capacity(record.attempts.len()); + let mut closed_at = at; + let mut latest_heartbeat: Option> = None; + for (position, report) in record.attempts.iter().enumerate().rev() { + let beats = report["heartbeats"].as_array().map_or(&[][..], Vec::as_slice); + let last_elapsed = milliseconds(beats.last().and_then(|beat| beat.get("elapsed_ms"))); + let duration = milliseconds(report.get("duration_ms")); + let started_at = closed_at - Duration::milliseconds(last_elapsed.max(duration)); + let last_heartbeat_at = (!beats.is_empty()).then(|| started_at + Duration::milliseconds(last_elapsed)); + if let Some(beat) = last_heartbeat_at && latest_heartbeat.is_none_or(|latest| beat > latest) { + latest_heartbeat = Some(beat); + } + attempts.push(Attempt { report, id: id().to_uppercase(), started_at, last_heartbeat_at, closed_at }); + if position > 0 { + let backoff = milliseconds(record.attempts[position - 1].get("backoff_seconds")); + closed_at = started_at - Duration::seconds(backoff); + } + } + attempts.reverse(); + let last = attempts.last().expect("a normalized record has attempts"); + let status = match record.outcome.as_str() { + "completed" => "completed", + "cancelled" => "cancelled", + _ => "failed", + }; + let mut options = json!({"execution_mode": "local", "queue_bypassed": true, "routing": "same_process_workflow_task"}); + for (field, value) in ["start_to_close_timeout", "schedule_to_close_timeout", "heartbeat_timeout"].into_iter().zip(record.timeouts) { + if let Some(seconds) = value { + options[field] = json!(seconds); + } + } + let policy = record.retry_policy.clone().map(Value::Object); + let execution_id = id(); + let connection = DB::optional_string(run, "connection")?; + let queue = DB::optional_string(run, "queue")?; + let started_at = attempts[0].started_at; + Self::query("INSERT INTO activity_executions(id,workflow_run_id,sequence,activity_class,activity_type,status,payload_codec,arguments,result,exception,connection,queue,attempt_count,current_attempt_id,retry_policy,activity_options,started_at,closed_at,last_heartbeat_at,created_at,updated_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21)") + .bind(&execution_id).bind(&run_id).bind(sequence).bind(&record.activity_type).bind(&record.activity_type).bind(status) + .bind(&codec).bind(record.arguments.as_deref()).bind(record.result.as_deref()).bind(exception) + .bind(connection.as_deref()).bind(queue.as_deref()).bind(attempts.len() as i64).bind(&last.id) + .bind(policy.as_ref().map(DB::document)).bind(DB::document(&options)).bind(DB::bind_time(started_at)) + .bind(DB::bind_time(at)).bind(DB::optional_time(latest_heartbeat)).bind(DB::bind_time(at)).bind(DB::bind_time(at)) + .execute(&mut **tx).await?; + // `ActivitySnapshot::fromExecution`. + let wire = |blob: &Option| blob.as_deref().map_or(Value::Null, |blob| json!({"codec": codec, "blob": blob})); + let terminal_activity = filtered(&[ + ("id", json!(execution_id)), ("idempotency_key", json!(execution_id)), ("sequence", json!(sequence)), + ("type", json!(record.activity_type)), ("class", json!(record.activity_type)), ("execution_mode", json!("local")), + ("local_activity", json!(true)), ("attempt_id", json!(last.id)), ("status", json!(status)), + ("payload_codec", json!(codec)), ("attempt_count", json!(attempts.len())), ("retry_policy", policy.clone().unwrap_or(Value::Null)), + ("connection", json!(connection)), ("queue", json!(queue)), ("last_heartbeat_at", latest_heartbeat.map_or(Value::Null, time)), + ("created_at", time(at)), ("started_at", time(started_at)), ("closed_at", time(at)), + ("arguments", wire(&record.arguments)), ("result", wire(&record.result)), ("exception", json!(exception)), + ]); + let without = |snapshot: &Value, fields: &[&str]| { + let mut snapshot = snapshot.clone(); + for field in fields { + snapshot.as_object_mut().expect("snapshot").remove(*field); + } + snapshot + }; + let mut scheduled_activity = without(&terminal_activity, &["attempt_id", "started_at", "last_heartbeat_at", "closed_at", "result", "exception"]); + scheduled_activity["status"] = json!("pending"); + scheduled_activity["attempt_count"] = json!(0); + let snapshot = Self::task_snapshot(task)?; + let base = json!({"execution_mode": "local", "local_activity": true, "activity_execution_id": execution_id, + "activity_class": record.activity_type, "activity_type": record.activity_type, "sequence": sequence, + "workflow_task_id": task_id, "task": snapshot}); + let event = |activity: Value, fields: Value| { + let mut payload = base.clone(); + payload["activity"] = activity; + for (key, value) in fields.as_object().expect("event fields") { + payload[key] = value.clone(); + } + payload + }; + Self::append(tx, &run_id, "ActivityScheduled", event(scheduled_activity, json!({})), Some(&task_id), None).await?; + let mut emitted: Option> = None; + for (position, attempt) in attempts.iter().enumerate() { + let worker_attempt_id = attempt.report.get("attempt_id").and_then(Value::as_str); + Self::query("INSERT INTO activity_attempts(id,workflow_run_id,activity_execution_id,workflow_task_id,worker_attempt_id,attempt_number,status,lease_owner,started_at,last_heartbeat_at,lease_expires_at,closed_at,created_at,updated_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,NULL,$11,$12,$13)") + .bind(&attempt.id).bind(&run_id).bind(&execution_id).bind(&task_id).bind(worker_attempt_id) + .bind(attempt.number()).bind(attempt.status()).bind(lease_owner.as_deref()).bind(DB::bind_time(attempt.started_at)) + .bind(DB::optional_time(attempt.last_heartbeat_at)).bind(DB::bind_time(attempt.closed_at)) + .bind(DB::bind_time(at)).bind(DB::bind_time(at)).execute(&mut **tx).await?; + let terminal_attempt = filtered(&[ + ("id", json!(attempt.id)), ("worker_attempt_id", attempt.worker_id()), ("attempt_number", json!(attempt.number())), + ("status", json!(attempt.status())), ("task_id", json!(task_id)), ("lease_owner", json!(lease_owner)), + ("started_at", time(attempt.started_at)), ("last_heartbeat_at", attempt.last_heartbeat_at.map_or(Value::Null, time)), + ("closed_at", time(attempt.closed_at)), + ]); + let mut running_attempt = without(&terminal_attempt, &["last_heartbeat_at", "closed_at"]); + running_attempt["status"] = json!("running"); + let mut running_activity = without(&terminal_activity, &["closed_at", "result", "exception", "last_heartbeat_at"]); + running_activity["attempt_id"] = json!(attempt.id); + running_activity["attempt_count"] = json!(attempt.number()); + running_activity["status"] = json!("running"); + if let Some(emitted) = emitted { + running_activity["last_heartbeat_at"] = time(emitted); + } + let started = json!({"activity_attempt_id": attempt.id, "worker_attempt_id": attempt.worker_id(), + "attempt_number": attempt.number(), "activity_attempt": running_attempt}); + Self::append(tx, &run_id, "ActivityStarted", event(running_activity.clone(), started.clone()), Some(&task_id), None).await?; + for heartbeat in attempt.heartbeats() { + let heartbeat_at = attempt.started_at + Duration::milliseconds(milliseconds(heartbeat.get("elapsed_ms"))); + if emitted.is_none_or(|latest| heartbeat_at > latest) { + emitted = Some(heartbeat_at); + } + let mut activity = running_activity.clone(); + activity["last_heartbeat_at"] = time(emitted.expect("a heartbeat was emitted")); + let mut fields = started.clone(); + fields["activity_attempt"]["last_heartbeat_at"] = time(heartbeat_at); + fields["heartbeat_at"] = time(heartbeat_at); + fields["lease_expires_at"] = Value::Null; + if let Some(details) = heartbeat.get("details") { + fields["progress"] = details.clone(); + } + Self::append(tx, &run_id, "ActivityHeartbeatRecorded", event(activity, fields), Some(&task_id), None).await?; + } + if position + 1 < attempts.len() { + let mut activity = running_activity.clone(); + activity["status"] = json!("pending"); + match emitted { + Some(emitted) => activity["last_heartbeat_at"] = time(emitted), + None => { + activity.as_object_mut().expect("snapshot").remove("last_heartbeat_at"); + } + } + let report = attempt.report; + let retry = json!({"activity_attempt_id": attempt.id, "worker_attempt_id": attempt.worker_id(), + "activity_attempt": terminal_attempt, "retry_after_attempt_id": attempt.id, "retry_after_attempt": attempt.number(), + "message": report.get("message").cloned().unwrap_or_else(|| json!("Local activity attempt failed.")), + "exception_type": report.get("exception_type").cloned().unwrap_or(Value::Null), + "retry_reason": report.get("retry_reason").cloned().unwrap_or_else(|| json!("failure")), + "retry_backoff_seconds": report.get("backoff_seconds").cloned().unwrap_or_else(|| json!(0))}); + Self::append(tx, &run_id, "ActivityRetryScheduled", event(activity, retry), Some(&task_id), None).await?; + } + } + let mut terminal = json!({"activity_attempt_id": last.id, "worker_attempt_id": last.worker_id(), "attempt_number": last.number(), + "activity_attempt": filtered(&[ + ("id", json!(last.id)), ("worker_attempt_id", last.worker_id()), ("attempt_number", json!(last.number())), + ("status", json!(last.status())), ("task_id", json!(task_id)), ("lease_owner", json!(lease_owner)), + ("started_at", time(last.started_at)), ("last_heartbeat_at", last.last_heartbeat_at.map_or(Value::Null, time)), + ("closed_at", time(last.closed_at)), + ])}); + let event_type = match record.outcome.as_str() { + "completed" => "ActivityCompleted", + "cancelled" => "ActivityCancelled", + "timed_out" => "ActivityTimedOut", + _ => "ActivityFailed", + }; + match record.outcome.as_str() { + "completed" => { + terminal["result"] = wire(&record.result); + terminal["payload_codec"] = json!(codec); + } + "cancelled" => terminal["cancelled_at"] = time(at), + outcome => { + let timed_out = outcome == "timed_out"; + let failure_id = id(); + let message = record.message.clone().unwrap_or_default(); + let class = record.exception_type.clone() + .unwrap_or_else(|| if timed_out { TIMEOUT_EXCEPTION } else { "RuntimeException" }.to_owned()); + let non_retryable = record.non_retryable.unwrap_or(false); + Self::query("INSERT INTO workflow_failures(id,workflow_run_id,source_kind,source_id,propagation_kind,failure_category,non_retryable,handled,exception_class,message,file,line,trace_preview,created_at,updated_at) VALUES ($1,$2,'activity_execution',$3,$4,$5,CASE WHEN $6='true' THEN TRUE ELSE FALSE END,FALSE,$7,$8,'',0,'',$9,$10)") + .bind(&failure_id).bind(&run_id).bind(&execution_id).bind(if timed_out { "timeout" } else { "activity" }) + .bind(if timed_out { "timeout" } else { "activity" }).bind(if non_retryable { "true" } else { "false" }) + .bind(&class).bind(&message).bind(DB::bind_time(at)).bind(DB::bind_time(at)).execute(&mut **tx).await?; + terminal["failure_id"] = json!(failure_id); + if timed_out { + terminal["failure_category"] = json!("timeout"); + terminal["timeout_kind"] = json!(record.timeout_kind.as_deref().unwrap_or("start_to_close")); + terminal["message"] = json!(record.message.as_deref().unwrap_or("Local activity timed out.")); + terminal["exception_class"] = json!(record.exception_type); + } else { + terminal["failure_category"] = json!("application"); + terminal["message"] = json!(record.message.as_deref().unwrap_or("Local activity failed.")); + terminal["exception_type"] = json!(record.exception_type); + terminal["non_retryable"] = json!(non_retryable); + } + } + } + Self::append(tx, &run_id, event_type, event(terminal_activity, terminal), Some(&task_id), None).await?; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn refusal(commands: Value, protocol: &str) -> Value { + match normalize(commands.as_array().unwrap(), Some(protocol)) { + Err(RuntimeError::Protocol { response, .. }) => response, + other => panic!("{other:?}"), + } + } + + fn record(fields: Value) -> Value { + let mut command = json!({"type": COMMAND, "activity_type": "parity.v1.local", "arguments": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}, + "result": {"codec": "avro", "blob": "wwHioz3/VYAiNwoIZG9uZQ=="}, "outcome": "completed", + "attempts": [{"attempt_id": "w-1", "attempt_number": 1, "outcome": "completed", "duration_ms": 25}]}); + for (key, value) in fields.as_object().unwrap() { + command[key] = value.clone(); + } + command + } + + #[test] + fn protocol_versions_compare_like_php() { + assert!(supports("1.18", "1.18") && supports("1.20", "1.19") && !supports("1.17", "1.18") && !supports("2.20", "1.18")); + assert!(!supports("1.x", "1.18") && !supports("1", "1.18")); + assert!(below(Some("1.17"), "1.18") && below(Some("1.9"), "1.18") && !below(Some("1.18"), "1.18") && below(None, "1.18")); + } + + #[test] + fn a_later_message_for_a_field_replaces_it_in_place() { + // The published multi-error report: six fields, PHP's order. + let response = refusal(json!([record(json!({"non_retryable": false, "result": null, + "attempts": [{"attempt_number": 2, "outcome": "failed", "extra": 1}]}))]), "1.20"); + assert_eq!(response["message"], "Completed local activities cannot be marked non-retryable. (and 5 more errors)"); + let fields: Vec<&String> = response["errors"].as_object().unwrap().keys().collect(); + assert_eq!(fields.len(), 6); + assert_eq!(response["errors"]["commands.0.attempts.0.outcome"], + json!(["The final local activity attempt outcome must match the command outcome."])); + // The timed out report: the timeout kind's second message wins. + let response = refusal(json!([record(json!({"outcome": "timed_out", "result": null, "message": "late", + "timeout_kind": "bogus", "attempts": [{"attempt_number": 1, "outcome": "timed_out", "message": "x", "timeout_kind": "heartbeat"}]}))]), "1.20"); + assert_eq!(response["errors"]["commands.0.timeout_kind"], + json!(["Local activity timeout_kind must be start_to_close, schedule_to_close, or heartbeat."])); + } + + #[test] + fn protocol_1_18_synthesizes_one_terminal_attempt_and_ignores_unknown_fields() { + let mut command = record(json!({"outcome": "failed", "result": null, "message": " no attempts ", "exception_type": "E"})); + command.as_object_mut().unwrap().remove("attempts"); + let records = normalize(&[command.clone()], Some("1.18")).unwrap(); + assert_eq!(records[&0].attempts, vec![json!({"attempt_number": 1, "outcome": "failed", "message": "no attempts", + "exception_type": "E", "heartbeats": []})]); + assert_eq!(refusal(json!([command]), "1.19")["message"], "Local activity attempts must be a non-empty ordered list."); + let lenient = record(json!({"retry_policy": {"max_attempts": 1, "jitter": 1}, + "attempts": [{"attempt_number": 1, "outcome": "completed", "shard": "x", "heartbeats": [{"elapsed_ms": 5, "note": "x"}]}]})); + let records = normalize(&[lenient.clone()], Some("1.18")).unwrap(); + assert_eq!(records[&0].attempts[0]["heartbeats"], json!([{"elapsed_ms": 5}])); + assert_eq!(records[&0].retry_policy, Some(json!({"max_attempts": 1}).as_object().unwrap().clone())); + let strict = refusal(json!([lenient]), "1.20"); + assert_eq!(strict["message"], "Field [jitter] is not supported in a local activity retry policy report. (and 2 more errors)"); + } + + #[test] + fn reports_are_read_as_sent() { + // Published PHP reorders this report before its normalizer sees it. + let response = refusal(json!([record(json!({"retry_policy": {"max_attempts": 2, "backoff_seconds": [0]}, + "attempts": [{"attempt_number": 1, "outcome": "failed", "message": "one", "retry_reason": "failure", "backoff_seconds": 0}, + {"attempt_id": "e-2", "attempt_number": 2, "outcome": "completed", "heartbeats": [{"elapsed_ms": 9}, {"elapsed_ms": 5, "details": {}}, "beat"]}]}))]), "1.20"); + assert_eq!(response["errors"], json!({ + "commands.0.attempts.1.heartbeats.1.elapsed_ms": ["Local activity heartbeat elapsed_ms values must be ordered monotonically."], + "commands.0.attempts.1.heartbeats.2": ["Each local activity heartbeat must be an object."]})); + // Empty heartbeat details are PHP's empty array. + let records = normalize(&[record(json!({"attempts": [{"attempt_number": 1, "outcome": "completed", + "heartbeats": [{"elapsed_ms": 1, "details": {}}, {"elapsed_ms": 2, "details": {"a": {}}}]}]}))], Some("1.20")).unwrap(); + assert_eq!(records[&0].attempts[0]["heartbeats"], json!([{"elapsed_ms": 1, "details": []}, {"elapsed_ms": 2, "details": {"a": []}}])); + } + + #[test] + fn errors_are_collected_across_the_batch_and_scopes_come_first() { + let response = refusal(json!([record(json!({"schedule_to_start_timeout": 5, "activity_type": null})), + record(json!({"parallel_group_id": "g"})), {"type": "start_timer", "delay_seconds": 1}]), "1.20"); + let fields: Vec<&String> = response["errors"].as_object().unwrap().keys().collect(); + assert_eq!(fields.len(), 3, "{response}"); + assert!(response["message"].as_str().unwrap().starts_with("Workflow task command field [schedule_to_start_timeout]")); + assert_eq!(response["errors"]["commands.1.parallel_group_id"], + json!(["parallel_group_id is only supported for activity, timer, and child-workflow scheduling commands."])); + } + + #[test] + fn payloads_follow_the_resolver() { + let records = normalize(&[record(json!({"arguments": "wwHioz3/VYAiNwwA", "payload_codec": "avro"}))], None).unwrap(); + assert_eq!(records[&0].arguments.as_deref(), Some("wwHioz3/VYAiNwwA")); + assert_eq!(records[&0].payload_codec.as_deref(), Some("avro")); + // Plain arguments are no payload; a plain result is refused. For a + // completed record PHP then replaces the resolver's message with the + // missing-result one under the same key. + let records = normalize(&[record(json!({"arguments": {"a": 1}}))], None).unwrap(); + assert_eq!(records[&0].arguments, None); + assert_eq!(refusal(json!([record(json!({"result": [1]}))]), "1.20")["errors"]["commands.0.result"], + json!(["Completed local activities require a result payload envelope."])); + assert_eq!(refusal(json!([record(json!({"result": [1], "outcome": "failed", "message": "no", + "attempts": [{"attempt_number": 1, "outcome": "failed", "message": "no"}]}))]), "1.20")["errors"]["commands.0.result"], + json!(["Workflow task command field [result] must be a string or a payload envelope when provided."])); + let records = normalize(&[record(json!({"outcome": "cancelled", "result": null, "message": "stop", + "attempts": [{"attempt_number": 1, "outcome": "cancelled", "message": "stop"}]}))], None).unwrap(); + assert_eq!(records[&0].exception_message().as_deref(), Some("stop")); + } +} diff --git a/rust/src/runtime/mod.rs b/rust/src/runtime/mod.rs index a38f42362..aca0099c9 100644 --- a/rust/src/runtime/mod.rs +++ b/rust/src/runtime/mod.rs @@ -61,6 +61,7 @@ mod input_envelope; mod history_presentation; mod http; mod laravel_validation; +mod local_activities; mod memo_json; mod memo_limits; mod standalone_activities; diff --git a/rust/src/runtime/store.rs b/rust/src/runtime/store.rs index edeba7358..4cd10d81a 100644 --- a/rust/src/runtime/store.rs +++ b/rust/src/runtime/store.rs @@ -2177,10 +2177,14 @@ where // options cannot be silently accepted as unsupported durable effects. for (index, command) in commands.iter().enumerate() { // The command normalizer's field scopes and value rules, with - // PHP's messages, before native's own refusals. - super::completion_rules::normalizer_checks(index, command)?; + // PHP's messages, before native's own refusals. A local activity + // record meets the normalizer after the ownership guards, as in PHP. + if command["type"] != super::local_activities::COMMAND { + super::completion_rules::normalizer_checks(index, command)?; + } let command_type = text(command, "type")?; match command_type { + "record_local_activity" => {} "fail_workflow" => { super::workflow_failures::validate(command)?; } @@ -2353,6 +2357,15 @@ where }); } } + // `guardPortableWorkerAffinityCompletion`, then the normalizer's + // local activity rules inside the transaction. + let mut local = std::collections::BTreeMap::new(); + let mut exceptions = std::collections::BTreeMap::new(); + if commands.iter().any(|command| command["type"] == super::local_activities::COMMAND) { + Self::guard_local_activities(&mut tx, self.namespace.as_ref(), &task, &body, protocol).await?; + local = super::local_activities::normalize(commands, protocol)?; + exceptions = super::local_activities::exceptions(&local, transition_codec.clone()).await?; + } // `parseCommands`: at most one terminal command, anywhere in the // batch. It applies after every other command. if commands.iter().filter(|command| is_terminal(command)).count() > 1 { @@ -2430,6 +2443,11 @@ where } else if command["type"] == "start_child_workflow" { sequence += 1; created.push(Self::start_child(&mut tx, &run, task_id, command, sequence).await?); + } else if command["type"] == super::local_activities::COMMAND { + // `applyRecordLocalActivity`: one authored sequence. + sequence += 1; + let record = local.get(&index).expect("a normalized local activity record"); + Self::record_local_activity(&mut tx, &run, &task, record, exceptions.get(&index).map(String::as_str), sequence).await?; } else if command["type"] == "record_side_effect" { // `applyRecordSideEffect`: one authored sequence and its result. sequence += 1; @@ -4052,10 +4070,13 @@ pub(super) fn reject_fields(body: &Value, allowed: &[&str]) -> Result<()> { } pub(crate) fn capabilities() -> Value { - let mut value = json!({"supported_workflow_task_commands": ["schedule_activity", "start_timer", "start_child_workflow", "open_condition_wait", "open_signal_wait", "record_version_marker", "record_side_effect", "complete_workflow", "fail_workflow", "continue_as_new", "complete_update", "fail_update"], + // Prepared local activities are not served yet: like a PHP server whose + // bridge lacks the prepared role, no groups and no cancellation policies. + let mut value = json!({"supported_workflow_task_commands": ["schedule_activity", "start_timer", "start_child_workflow", "open_condition_wait", "open_signal_wait", "record_version_marker", "record_side_effect", "record_local_activity", "complete_workflow", "fail_workflow", "continue_as_new", "complete_update", "fail_update"], "worker_deregistration_fencing":{"schema":"durable-workflow.v2.worker-deregistration.v1","supported":true,"receipt_retention_seconds":600,"endpoint":"/worker/registrations/{workerId}/deregister"}, "workflow_memo_updates": false, "cooperative_cancellation": true, "prepared_local_activities": false, - "worker_sessions": false, "sticky_execution": false, "local_activities": false, "message_streams": false, + "prepared_local_activity_groups": false, "prepared_local_activity_cancellation_policies": [], + "worker_sessions": false, "sticky_execution": false, "local_activities": super::local_activities::manifest(), "message_streams": false, "workflow_updates": true, "query_tasks": true, "query_task_authority":true, "query_task_poll_request_idempotency": true, "activity_lease_renewal": true}); value["runtime_external_payload_transport"] = super::payload_discovery::transport_manifest(); diff --git a/rust/src/runtime/wait_projection.rs b/rust/src/runtime/wait_projection.rs index 6467aa436..6adb64b95 100644 --- a/rust/src/runtime/wait_projection.rs +++ b/rust/src/runtime/wait_projection.rs @@ -174,6 +174,11 @@ where ) { return Ok(()); } + // A recorded local activity opens no wait: it ran inside the + // workflow task that reports it. + if payload["local_activity"] == true { + return Ok(()); + } let activity_id = super::text(payload, "activity_execution_id")?; let wait_id = format!("activity:{activity_id}"); if event_type == "ActivityRetryScheduled" { diff --git a/rust/tests/execution.rs b/rust/tests/execution.rs index 96f2db7e4..5841b5e7f 100644 --- a/rust/tests/execution.rs +++ b/rust/tests/execution.rs @@ -103,6 +103,8 @@ mod child_policies; mod definition_routing; #[path = "execution/worker_admission.rs"] mod worker_admission; +#[path = "execution/local_activities.rs"] +mod local_activities; /// `WorkflowDefinitionRouting::ENTRY_VALIDATION_CAPABILITY`. const ENTRY_VALIDATION: &str = "workflow_entry_arguments_validation"; #[path = "execution/parent_close_policies.rs"] diff --git a/rust/tests/execution/local_activities.rs b/rust/tests/execution/local_activities.rs new file mode 100644 index 000000000..41490976b --- /dev/null +++ b/rust/tests/execution/local_activities.rs @@ -0,0 +1,176 @@ +use super::*; + +// `record_local_activity` as published Server 2.11.1 records it (the +// record-local-activity parity fixture holds the PHP recordings). + +const EMPTY: &str = "wwHioz3/VYAiNwwA"; +const DONE: &str = "wwHioz3/VYAiNwoIZG9uZQ=="; + +async fn local_worker(app: &Router, worker: &str, capabilities: Value) { + let (status, body) = request_protocol(app, "POST", "/api/worker/register", json!({"worker_id": worker, + "task_queue": "test", "runtime": "python", "supported_workflow_types": ["echo"], "supported_activity_types": [], + "capabilities": capabilities}), "1.20").await; + assert_eq!(status, StatusCode::CREATED, "{body}"); +} + +async fn complete(app: &Router, task: &Value, commands: Value, protocol: &str) -> (StatusCode, Value) { + let path = format!("/api/worker/workflow-tasks/{}/complete", task["task_id"].as_str().unwrap()); + request_protocol(app, "POST", &path, completion(task, commands), protocol).await +} + +fn record(fields: Value) -> Value { + let mut command = json!({"type": "record_local_activity", "activity_type": "local.echo", + "arguments": {"codec": "avro", "blob": EMPTY}, "result": {"codec": "avro", "blob": DONE}, "outcome": "completed", + "attempts": [{"attempt_id": "w-1", "attempt_number": 1, "outcome": "completed", "duration_ms": 25}]}); + for (key, value) in fields.as_object().unwrap() { + command[key] = value.clone(); + } + command +} + +fn wait() -> Value { + json!({"type": "open_signal_wait", "signal_name": "resume"}) +} + +fn instant(value: &Value) -> chrono::DateTime { + chrono::DateTime::parse_from_rfc3339(value.as_str().unwrap_or_else(|| panic!("a time: {value}"))).unwrap() +} + +#[tokio::test] +async fn advertised_contract_and_capability_gates() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + let (status, cluster) = request(&app, "GET", "/api/cluster/info", Value::Null).await; + assert_eq!(status, StatusCode::OK, "{cluster}"); + let capabilities = &cluster["worker_protocol"]["server_capabilities"]; + assert_eq!(capabilities["local_activities"]["supported"], true); + assert_eq!(capabilities["local_activities"]["command"]["attempt_reports"]["required_from_protocol_version"], "1.19"); + assert!(capabilities["supported_workflow_task_commands"].as_array().unwrap().contains(&json!("record_local_activity"))); + // Prepared local activities are not served: no groups and no policies. + assert_eq!((&capabilities["prepared_local_activities"], &capabilities["prepared_local_activity_groups"], + &capabilities["prepared_local_activity_cancellation_policies"]), (&json!(false), &json!(false), &json!([]))); + + local_worker(&app, "plain", json!([])).await; + start(&app, "local-gates").await; + let task = poll(&app, "plain", "workflow").await; + // Ownership first, then the protocol floor, then the capability. + let mut foreign = task.clone(); + foreign["lease_owner"] = json!("someone-else"); + let invalid = record(json!({"non_retryable": false})); + let (status, body) = complete(&app, &foreign, json!([invalid]), "1.17").await; + assert_eq!((status, &body["reason"]), (StatusCode::CONFLICT, &json!("lease_owner_mismatch")), "{body}"); + let (status, body) = complete(&app, &task, json!([record(json!({})), wait()]), "1.17").await; + assert_eq!(status, StatusCode::CONFLICT, "{body}"); + assert_eq!(body["reason"], "portable_worker_affinity_protocol_floor_unavailable"); + assert_eq!((&body["capability"], &body["requested_version"], &body["minimum_protocol_version"]), + (&json!("local_activities"), &json!("1.17"), &json!("1.18"))); + let (status, body) = complete(&app, &task, json!([record(json!({})), wait()]), "1.20").await; + assert_eq!((status, &body["reason"]), (StatusCode::CONFLICT, &json!("worker_capability_not_supported")), "{body}"); + assert_eq!((&body["outcome"], &body["recorded"]), (&json!("rejected"), &json!(false))); + // The claim survives every refusal. + let (status, body) = complete(&app, &task, json!([wait()]), "1.20").await; + assert_eq!(status, StatusCode::OK, "{body}"); +} + +#[tokio::test] +async fn normalizer_refusals_follow_ownership_and_name_php_fields() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + local_worker(&app, "local", json!(["local_activities"])).await; + let run = start(&app, "local-refusals").await; + let task = poll(&app, "local", "workflow").await; + let report = record(json!({"non_retryable": false, "result": null, "schedule_to_start_timeout": 5, + "attempts": [{"attempt_number": 2, "outcome": "failed", "extra": 1}]})); + let (status, body) = complete(&app, &task, json!([report]), "1.20").await; + assert_eq!(status, StatusCode::UNPROCESSABLE_ENTITY, "{body}"); + assert_eq!(body["message"], "Workflow task command field [schedule_to_start_timeout] is not valid on a record_local_activity command. schedule_to_start_timeout limits queue wait before an activity attempt starts and only applies to a schedule_activity command. (and 6 more errors)"); + assert_eq!(body["errors"]["commands.0.attempts.0.extra"], json!(["Field [extra] is not supported in a local activity attempt report."])); + let mut missing = record(json!({})); + missing.as_object_mut().unwrap().remove("attempts"); + let (status, body) = complete(&app, &task, json!([missing.clone(), wait()]), "1.19").await; + assert_eq!((status, &body["message"]), (StatusCode::UNPROCESSABLE_ENTITY, &json!("Local activity attempts must be a non-empty ordered list."))); + // Protocol 1.18 makes the one terminal attempt itself. + let (status, body) = complete(&app, &task, json!([missing, wait()]), "1.18").await; + assert_eq!(status, StatusCode::OK, "{body}"); + let events = run_history(&app, &run["workflow_id"], &run["run_id"]).await; + let started = events.as_array().unwrap().iter().find(|event| event["event_type"] == "ActivityStarted").unwrap(); + assert_eq!(started["payload"]["worker_attempt_id"], Value::Null); + assert!(started["payload"]["activity_attempt"].get("worker_attempt_id").is_none(), "{started}"); +} + +#[tokio::test] +async fn a_recorded_local_activity_writes_its_attempts_history_and_failure() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + local_worker(&app, "local", json!(["local_activities"])).await; + let run = start(&app, "local-recorded").await; + let task = poll(&app, "local", "workflow").await; + let retried = record(json!({"arguments": EMPTY, "payload_codec": "avro", "queue": "ignored", "connection": "ignored", + "retry_policy": {"max_attempts": 2, "backoff_seconds": [1]}, "start_to_close_timeout": 300, + "attempts": [ + {"attempt_id": "w-1", "attempt_number": 1, "outcome": "failed", "duration_ms": 1500, "message": " first ", + "exception_type": "RuntimeException", "retry_reason": "failure", "backoff_seconds": 1, + "heartbeats": [{"elapsed_ms": 500, "details": {}}]}, + {"attempt_id": "w-2", "attempt_number": 2, "outcome": "completed", "duration_ms": 200}]})); + let failed = record(json!({"activity_type": "local.fatal", "outcome": "failed", "result": null, "message": "gave up", + "exception_type": "App\\Fatal", "non_retryable": true, + "attempts": [{"attempt_number": 1, "outcome": "failed", "message": "gave up", "non_retryable": true}]})); + let timed_out = record(json!({"activity_type": "local.slow", "outcome": "timed_out", "result": null, "message": "late", + "timeout_kind": "heartbeat", "attempts": [{"attempt_number": 1, "outcome": "timed_out", "message": "late", + "timeout_kind": "heartbeat", "duration_ms": 10}]})); + let (status, body) = complete(&app, &task, json!([retried, failed, timed_out, wait()]), "1.20").await; + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!((&body["recorded"], &body["run_status"], &body["created_task_ids"]), (&json!(true), &json!("waiting"), &json!([]))); + + let events = run_history(&app, &run["workflow_id"], &run["run_id"]).await; + let events = events.as_array().unwrap(); + let types: Vec<_> = events.iter().map(|event| event["event_type"].as_str().unwrap()).collect(); + assert_eq!(types[2..], ["ActivityScheduled", "ActivityStarted", "ActivityHeartbeatRecorded", "ActivityRetryScheduled", + "ActivityStarted", "ActivityCompleted", "ActivityScheduled", "ActivityStarted", "ActivityFailed", "ActivityScheduled", + "ActivityStarted", "ActivityTimedOut", "SignalWaitOpened"]); + let payload = |index: usize| &events[index]["payload"]; + for index in 2..14 { + assert_eq!((&payload(index)["execution_mode"], &payload(index)["local_activity"]), (&json!("local"), &json!(true))); + assert_eq!(payload(index)["workflow_task_id"], task["task_id"]); + } + // The run's queue, not the command's; the sequences the commands consume. + assert_eq!(payload(2)["activity"]["queue"], "test"); + assert_eq!((&payload(2)["sequence"], &payload(8)["sequence"], &payload(11)["sequence"]), (&json!(1), &json!(2), &json!(3))); + // Attempt identities are uppercase and made last attempt first. + let first = payload(3)["activity_attempt_id"].as_str().unwrap(); + let second = payload(6)["activity_attempt_id"].as_str().unwrap(); + assert!(first.len() == 26 && first == first.to_uppercase() && first > second, "{first} {second}"); + // Times walk back from the close by durations and the backoff. + let close = instant(&payload(7)["activity"]["closed_at"]); + let offset = |value: &Value| (instant(value) - close).num_milliseconds(); + assert_eq!(offset(&payload(6)["activity_attempt"]["started_at"]), -200); + assert_eq!(offset(&payload(5)["activity_attempt"]["closed_at"]), -1200); + assert_eq!(offset(&payload(3)["activity_attempt"]["started_at"]), -2700); + assert_eq!(offset(&payload(4)["heartbeat_at"]), -2200); + assert_eq!(payload(4)["progress"], json!([]), "empty details are PHP's empty array"); + assert_eq!((&payload(5)["message"], &payload(5)["retry_backoff_seconds"]), (&json!("first"), &json!(1))); + assert_eq!(payload(7)["result"], json!({"codec": "avro", "blob": DONE})); + assert_eq!(payload(7)["activity"]["retry_policy"], json!({"max_attempts": 2, "backoff_seconds": [1]})); + assert!(payload(10)["failure_id"].is_string(), "{}", payload(10)); + assert_eq!((&payload(10)["failure_category"], &payload(10)["exception_type"], &payload(10)["non_retryable"]), + (&json!("application"), &json!("App\\Fatal"), &json!(true))); + assert!(payload(10)["activity"]["exception"].is_string(), "the serialized message: {}", payload(10)); + assert_eq!((&payload(13)["failure_category"], &payload(13)["timeout_kind"], &payload(13)["exception_class"]), + (&json!("timeout"), &json!("heartbeat"), &Value::Null)); + + let (status, view) = request(&app, "GET", &format!("/api/workflows/local-recorded/runs/{}/activities", + run["run_id"].as_str().unwrap()), Value::Null).await; + assert_eq!(status, StatusCode::OK, "{view}"); + let activities = view["activities"].as_array().unwrap(); + assert_eq!(activities.len(), 3); + assert_eq!((&activities[0]["status"], &activities[0]["local_activity"], &activities[0]["attempt_count"]), + (&json!("completed"), &json!(true), &json!(2))); + assert_eq!(activities[0]["attempts"][0]["worker_attempt_id"], "w-1"); + assert_eq!(activities[0]["attempts"][0]["failure_reason"], Value::Null); + assert_eq!(activities[2]["status"], "failed"); + // Closed local activities leave the run waiting on its signal. + let (status, described) = request(&app, "GET", &format!("/api/workflows/local-recorded/runs/{}", + run["run_id"].as_str().unwrap()), Value::Null).await; + assert_eq!(status, StatusCode::OK, "{described}"); + assert_eq!((&described["status"], &described["wait_kind"]), (&json!("waiting"), &json!("signal")), "{described}"); +} From d6b96bd0b5953017b0cf840af4b9172a89814807 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Sun, 25 Oct 2026 19:43:42 +0000 Subject: [PATCH 3/9] Server parity: compare the task snapshot's connection per backend The record-local-activity fixture was recorded on SQLite, so each local activity event's workflow task snapshot names the sqlite connection. On the PostgreSQL and MySQL corpus lanes that snapshot names pgsql or mysql, the backend that recorded it, and the contract compared the name exactly. It now requires one known storage connection per observation and compares it as {connection}. Every other step still compares exactly. Co-Authored-By: Claude Opus 5.5 --- .../record-local-activity-contract.mjs | 39 ++++++++++++++++--- ...rParityRecordLocalActivityContractTest.mjs | 17 ++++++++ 2 files changed, 50 insertions(+), 6 deletions(-) diff --git a/scripts/conformance/server-parity/record-local-activity-contract.mjs b/scripts/conformance/server-parity/record-local-activity-contract.mjs index dca18c5ba..9b372a1f9 100644 --- a/scripts/conformance/server-parity/record-local-activity-contract.mjs +++ b/scripts/conformance/server-parity/record-local-activity-contract.mjs @@ -14,6 +14,32 @@ import assert from 'node:assert/strict'; // unordered (fixture `published_defects`). At those steps the published // refusals, all of them, or the normalizer's own answers on the reports as // sent are accepted; nothing else is relaxed. +// +// A workflow task snapshot names the storage connection that recorded it: +// sqlite where the fixture was recorded, pgsql or mysql on the other +// backends. Each observation names one of them throughout, compared as +// `{connection}`. +const CONNECTIONS = new Set(['sqlite', 'pgsql', 'mysql']); + +export function normalizeConnections(state, label) { + const names = new Set(); + const walk = value => { + if (Array.isArray(value)) return value.map(walk); + if (value === null || typeof value !== 'object') return value; + return Object.fromEntries(Object.entries(value).map(([key, item]) => { + if (key === 'connection' && typeof item === 'string') { + names.add(item); + return [key, '{connection}']; + } + return [key, walk(item)]; + })); + }; + const normalized = walk(state); + assert.ok(names.size <= 1, `${label} names one storage connection: ${[...names].join(', ')}`); + for (const name of names) assert.ok(CONNECTIONS.has(name), `${label} storage connection ${name}`); + return normalized; +} + export function checkRecordLocalActivity(fixture, observation, workflowId) { const state = observation.record_local_activity; const projection = {scope: 'record_local_activity'}; @@ -24,8 +50,9 @@ export function checkRecordLocalActivity(fixture, observation, workflowId) { return projection; } const definition = fixture.record_local_activity; - const actual = JSON.parse(JSON.stringify(state).replaceAll(workflowId, '{id}')); - const defect = definition.published_defects.validated_reorders_nested_reports.published; + const actual = normalizeConnections(JSON.parse(JSON.stringify(state).replaceAll(workflowId, '{id}')), 'actual'); + const expected = normalizeConnections(definition.expected, 'expected'); + const defect = normalizeConnections(definition.published_defects.validated_reorders_nested_reports.published, 'published'); const steps = Object.keys(defect); let published = true; try { @@ -34,11 +61,11 @@ export function checkRecordLocalActivity(fixture, observation, workflowId) { published = false; } if (published) { - for (const step of steps) actual[step] = definition.expected[step]; + for (const step of steps) actual[step] = expected[step]; } - for (const [step, expected] of Object.entries(definition.expected)) { - assert.deepStrictEqual(actual[step], expected, `record local activity ${step}`); + for (const [step, value] of Object.entries(expected)) { + assert.deepStrictEqual(actual[step], value, `record local activity ${step}`); } - assert.deepStrictEqual(Object.keys(actual).sort(), Object.keys(definition.expected).sort(), 'record local activity steps'); + assert.deepStrictEqual(Object.keys(actual).sort(), Object.keys(expected).sort(), 'record local activity steps'); return projection; } diff --git a/tests/Unit/ServerParityRecordLocalActivityContractTest.mjs b/tests/Unit/ServerParityRecordLocalActivityContractTest.mjs index 611fc2309..8e49ab2a4 100644 --- a/tests/Unit/ServerParityRecordLocalActivityContractTest.mjs +++ b/tests/Unit/ServerParityRecordLocalActivityContractTest.mjs @@ -97,3 +97,20 @@ for (const [label, change, message] of [ assert.throws(() => check(observation(change)), message); }); } + +test('a task snapshot names the backend\'s own storage connection, one per observation', () => { + for (const name of ['pgsql', 'mysql']) { + const renamed = observation(state => { + for (const run of ['completed_run', 'failed_run', 'protocol_1_18_run']) events(state, run)[0].payload.task.connection = name; + }); + assert.deepStrictEqual(check(renamed), {scope: 'record_local_activity'}); + } + const mixed = observation(state => { + events(state, 'failed_run')[0].payload.task.connection = 'pgsql'; + }); + assert.throws(() => check(mixed), /actual names one storage connection: sqlite, pgsql/); + const unknown = observation(state => { + for (const run of ['completed_run', 'failed_run', 'protocol_1_18_run']) events(state, run)[0].payload.task.connection = 'redis'; + }); + assert.throws(() => check(unknown), /actual storage connection redis/); +}); From 0ed8de6c3d537f3ec9967960f14488b3246e4832 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Sun, 25 Oct 2026 22:34:17 +0000 Subject: [PATCH 4/9] Rust #325: bound and meter workflow task completions like Workflow #930 Native applies phase A of Workflow #930 (Workflow PR #934) to record_local_activity and to every workflow task completion: - record_local_activity caps duration_ms and heartbeat elapsed_ms at 7 days and backoff_seconds at 1 day, with the package's field errors. The walk back from completion is summed with checked arithmetic and refused with local_activity_timestamp_out_of_range before it reaches the earliest timestamp every supported column stores. - After a run's deadline close, which is never metered, the local activities' exact events and rows (other commands at one event) are checked against DW_V2_LIMIT_HISTORY_TRANSACTION_SIZE and _ROWS, and a completion past them is refused with completion_budget_exceeded. - The rest of the completion runs under a meter. Each history event charges one event, its HistoryBudget bytes and two rows, and each local activity row is charged before it is written. A refusal answers history_transaction_limit_exceeded and rolls back, leaving the lease. A tripped meter also refuses the commit. Co-Authored-By: Claude Opus 5.5 --- docs/rust-server-port.md | 25 ++++ rust/src/runtime/history_meter.rs | 144 +++++++++++++++++++ rust/src/runtime/local_activities.rs | 163 +++++++++++++++++++++- rust/src/runtime/memo_limits.rs | 15 ++ rust/src/runtime/mod.rs | 1 + rust/src/runtime/store.rs | 26 ++++ rust/src/runtime/transaction.rs | 24 ++++ rust/tests/execution.rs | 2 + rust/tests/execution/completion_budget.rs | 129 +++++++++++++++++ rust/tests/execution/structural_limits.rs | 1 + 10 files changed, 525 insertions(+), 5 deletions(-) create mode 100644 rust/src/runtime/history_meter.rs create mode 100644 rust/tests/execution/completion_budget.rs diff --git a/docs/rust-server-port.md b/docs/rust-server-port.md index d45b88ed6..d9caeaeab 100644 --- a/docs/rust-server-port.md +++ b/docs/rust-server-port.md @@ -226,6 +226,31 @@ contract fixture before its correction. batch size, and still caps parallel groups at 100 members and a run at 64 updates over its lifetime. #612 tracks the rest ([checkpoint](rust-server-port-checkpoints/2026-10-23-structural-limits.md)). +- **Completion budget, ahead of published PHP.** Native applies Phase A of + [Workflow #930](https://github.com/durable-workflow/workflow/issues/930) + ([Workflow #934](https://github.com/durable-workflow/workflow/pull/934)), + which published 2.11.1 does not have yet: + - `record_local_activity` caps `duration_ms` and heartbeat `elapsed_ms` at + 7 days and `backoff_seconds` at 1 day, each with PHP's field error. The walk + back from completion is overflow-checked, and it is refused with + `local_activity_timestamp_out_of_range` before it reaches + 1970-01-01T00:00:01Z. + - After a run's deadline close, which is never metered, a completion whose + local activities' exact events and rows exceed + `DW_V2_LIMIT_HISTORY_TRANSACTION_SIZE` or `_ROWS` (other commands count one + event) is refused with `completion_budget_exceeded`. + - The rest of the completion runs under a meter. Each history event charges + one event, its bytes as `HistoryBudget` measures them (against + `_BYTES`) and two rows, and each local activity row is charged before it + is written. A refusal answers `history_transaction_limit_exceeded`, writes + nothing and leaves the task's lease as it was. A tripped meter also refuses + the commit. + - Native never stores an external payload object while applying a + completion: workers upload objects first, and a completion only claims + their references. Native also dispatches no lifecycle events. So #934's + staging and event deferral have no native counterpart. + - The structural-limit snapshot in operator metrics stays as published + 2.11.1 reports it until the PHP Server takes the Workflow release. - **Environment settings.** Of PHP's 166 contract settings, native honors 43 and ignores 57 that change nothing it serves. It refuses to start on a non-default value of the other 66 diff --git a/rust/src/runtime/history_meter.rs b/rust/src/runtime/history_meter.rs new file mode 100644 index 000000000..7d8c3e5f6 --- /dev/null +++ b/rust/src/runtime/history_meter.rs @@ -0,0 +1,144 @@ +//! `HistoryTransactionMeter` (Workflow #930): the whole-transaction budget +//! of one workflow task completion. Each history event is charged where it +//! is written ([`Store::record`]): one event, its bytes as `HistoryBudget` +//! measures a stored event, and two rows (the event and the run position it +//! advances). Local activity rows are charged before each write. Past a +//! hard structural limit the completion is refused with PHP's response and +//! its transaction rolls back, so nothing it wrote survives and the task +//! keeps its lease. A run's deadline close happens before the meter starts. +use super::memo_limits::StructuralLimits; +use super::RuntimeError; +use axum::http::StatusCode; +use serde_json::{Value, json}; + +/// The early whole-completion check's count for each command other than +/// `record_local_activity`: one event, a lower bound +/// (`WorkflowCommandNormalizer::MIN_COMMAND_HISTORY_EVENTS`). +pub(super) const MIN_COMMAND_HISTORY_EVENTS: usize = 1; + +#[derive(Debug)] +pub(super) struct Meter { + limits: [Option; 3], + used: [usize; 3], + /// The fields of PHP's refused completion response. + refusal: Value, + tripped: bool, +} + +impl Meter { + pub(super) fn new(limits: &StructuralLimits, task_id: &str, workflow_task_attempt: &Value) -> Self { + Self { + limits: [limits.history_events, limits.history_bytes, limits.history_rows], + used: [0; 3], + refusal: json!({"task_id": task_id, "workflow_task_attempt": workflow_task_attempt, "outcome": "completed", + "recorded": false, "run_id": null, "run_status": null, "created_task_ids": [], + "reason": "history_transaction_limit_exceeded"}), + tripped: false, + } + } + + /// One history event about to be stored. + pub(super) fn charge_event(&mut self, event_type: &str, payload: &Value) -> Result<(), RuntimeError> { + self.charge(0, 1)?; + self.charge(1, event_type.len().saturating_add(php_json_len(payload)))?; + self.charge(2, 2) + } + + /// Rows the next statement will write, charged before it runs. + pub(super) fn charge_rows(&mut self, rows: usize) -> Result<(), RuntimeError> { + self.charge(2, rows) + } + + fn charge(&mut self, dimension: usize, amount: usize) -> Result<(), RuntimeError> { + if self.tripped { + return Err(self.refused()); + } + // An overflow can only exceed every limit. + let next = self.used[dimension].checked_add(amount).unwrap_or(usize::MAX); + if self.limits[dimension].is_some_and(|limit| limit > 0 && next > limit) { + self.tripped = true; + return Err(self.refused()); + } + self.used[dimension] = next; + Ok(()) + } + + /// The refusal again, if any charge has been refused. + pub(super) fn tripped(&self) -> Option { + self.tripped.then(|| self.refused()) + } + + fn refused(&self) -> RuntimeError { + RuntimeError::Protocol { status: StatusCode::CONFLICT, response: self.refusal.clone() } + } + + /// The completion's refused response with another reason. + pub(super) fn refused_with(&self, reason: &str) -> RuntimeError { + let mut response = self.refusal.clone(); + response["reason"] = json!(reason); + RuntimeError::Protocol { status: StatusCode::CONFLICT, response } + } +} + +/// The length of `json_encode($payload, JSON_UNESCAPED_SLASHES | +/// JSON_UNESCAPED_UNICODE)`: serde's output, plus the escapes PHP still +/// writes for U+2028 and U+2029 under JSON_UNESCAPED_UNICODE. +pub(super) fn php_json_len(payload: &Value) -> usize { + let encoded = serde_json::to_string(payload).unwrap_or_default(); + let separators = encoded.matches(['\u{2028}', '\u{2029}']).count(); + encoded.len().saturating_add(separators.saturating_mul(3)) +} + +/// The exact history events and rows a normalized `record_local_activity` +/// record writes (`localActivityHistoryEvents`, `localActivityRows`): one +/// scheduled event; per attempt a started event, its heartbeats and, before +/// the next attempt, a retry event; one terminal event. Each event is two +/// rows; the execution, each attempt and a failure add theirs. +pub(super) fn local_activity_budget(attempts: &[Value], failed: bool) -> (usize, usize) { + let mut events = 2usize.saturating_add(attempts.len().saturating_sub(1)); + for attempt in attempts { + let heartbeats = attempt["heartbeats"].as_array().map_or(0, Vec::len); + events = events.saturating_add(1).saturating_add(heartbeats); + } + let rows = events.saturating_mul(2).saturating_add(1).saturating_add(attempts.len()).saturating_add(usize::from(failed)); + (events, rows) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn limits(events: usize, bytes: usize, rows: usize) -> StructuralLimits { + StructuralLimits { history_events: Some(events), history_bytes: Some(bytes), history_rows: Some(rows), ..StructuralLimits::default() } + } + + #[test] + fn events_bytes_and_rows_follow_php_s_measure_and_limits() { + assert_eq!(php_json_len(&json!({"a": "b/é"})), r#"{"a":"b/é"}"#.len()); + assert_eq!(php_json_len(&json!({"a": "\u{2028}"})), r#"{"a":"\u2028"}"#.len()); + let mut meter = Meter::new(&limits(2, 1_000, 5), "task", &json!(1)); + meter.charge_event("E", &json!({})).unwrap(); + meter.charge_rows(1).unwrap(); + // The second event would be the fourth and fifth rows: allowed. + meter.charge_event("E", &json!({})).unwrap(); + let Err(RuntimeError::Protocol { status, response }) = meter.charge_rows(1) else { panic!("one row over") }; + assert_eq!(status, StatusCode::CONFLICT); + assert_eq!(response["reason"], "history_transaction_limit_exceeded"); + assert_eq!((response["task_id"].clone(), response["recorded"].clone()), (json!("task"), json!(false))); + // A tripped meter refuses everything after, and its commit. + assert!(meter.charge_rows(0).is_err()); + assert!(matches!(meter.tripped(), Some(RuntimeError::Protocol { status: StatusCode::CONFLICT, .. }))); + // A disabled limit (0) refuses nothing. + let mut open = Meter::new(&limits(0, 0, 0), "task", &json!(1)); + for _ in 0..10 { open.charge_event("E", &json!({"x": "y"})).unwrap(); } + assert!(open.tripped().is_none()); + } + + #[test] + fn local_activity_budget_matches_the_package_formula() { + let attempts = vec![json!({"heartbeats": [{}, {}]}), json!({"heartbeats": []})]; + // 2 + 1 retry + (1 + 2) + (1 + 0) = 7 events; 14 + 1 + 2 + 1 rows. + assert_eq!(local_activity_budget(&attempts, true), (7, 18)); + assert_eq!(local_activity_budget(&[json!({"heartbeats": []})], false), (3, 8)); + } +} diff --git a/rust/src/runtime/local_activities.rs b/rust/src/runtime/local_activities.rs index 078aa0e0b..1c00ac578 100644 --- a/rust/src/runtime/local_activities.rs +++ b/rust/src/runtime/local_activities.rs @@ -31,6 +31,11 @@ const ATTEMPT_REPORTS_PROTOCOL_VERSION: &str = "1.19"; const MAX_ATTEMPTS: i64 = 100; const MAX_HEARTBEATS_PER_ATTEMPT: usize = 1000; const MAX_HEARTBEATS: usize = 1000; +/// `WorkflowCommandNormalizer::MAX_LOCAL_ACTIVITY_DURATION_MS` (Workflow +/// #930): the longest attempt `duration_ms` or heartbeat `elapsed_ms`. +pub(super) const MAX_DURATION_MS: i64 = 604_800_000; +/// `MAX_LOCAL_ACTIVITY_BACKOFF_SECONDS`: the longest attempt backoff. +pub(super) const MAX_BACKOFF_SECONDS: i64 = 86_400; const OUTCOMES: [&str; 4] = ["completed", "failed", "timed_out", "cancelled"]; const TIMEOUT_KINDS: [&str; 3] = ["start_to_close", "schedule_to_close", "heartbeat"]; const RETRY_POLICY_FIELDS: [&str; 3] = ["max_attempts", "backoff_seconds", "non_retryable_error_types"]; @@ -388,6 +393,10 @@ fn heartbeats(value: Option<&Value>, attempt_path: &str, strict: bool, errors: & errors.set(format!("{path}.elapsed_ms"), "Local activity heartbeat elapsed_ms must be a non-negative integer."); continue; }; + if elapsed > MAX_DURATION_MS { + errors.set(format!("{path}.elapsed_ms"), format!("Local activity heartbeat elapsed_ms must not exceed {MAX_DURATION_MS}.")); + continue; + } if elapsed < last { errors.set(format!("{path}.elapsed_ms"), "Local activity heartbeat elapsed_ms values must be ordered monotonically."); } @@ -497,6 +506,10 @@ fn attempts(command: &Value, outcome: &str, terminal_non_retryable: Option let duration = match fields.get("duration_ms") { None => None, Some(value) => match value.as_i64().filter(|duration| *duration >= 0) { + Some(duration) if duration > MAX_DURATION_MS => { + errors.set(format!("{path}.duration_ms"), format!("Local activity attempt duration_ms must not exceed {MAX_DURATION_MS}.")); + None + } Some(duration) => Some(duration), None => { errors.set(format!("{path}.duration_ms"), "Local activity attempt duration_ms must be a non-negative integer."); @@ -538,6 +551,11 @@ fn attempts(command: &Value, outcome: &str, terminal_non_retryable: Option let backoff = match fields.get("backoff_seconds") { None => None, Some(value) => match value.as_i64().filter(|seconds| *seconds >= 0) { + Some(seconds) if seconds > MAX_BACKOFF_SECONDS => { + errors.set(format!("{path}.backoff_seconds"), + format!("Local activity attempt backoff_seconds must not exceed {MAX_BACKOFF_SECONDS}.")); + None + } Some(seconds) => Some(seconds), None => { errors.set(format!("{path}.backoff_seconds"), "Local activity attempt backoff_seconds must be a non-negative integer."); @@ -551,8 +569,12 @@ fn attempts(command: &Value, outcome: &str, terminal_non_retryable: Option errors.set(format!("{path}.retry_reason"), format!("A retried {attempt_outcome} local activity attempt requires a compatible retry_reason.")); } + // A given value keeps its own refusal (Workflow #930). match backoff { - None => errors.set(format!("{path}.backoff_seconds"), "A retried local activity attempt requires backoff_seconds."), + None if !fields.contains_key("backoff_seconds") => { + errors.set(format!("{path}.backoff_seconds"), "A retried local activity attempt requires backoff_seconds."); + } + None => {} Some(seconds) if schedule.get(position).is_some_and(|scheduled| *scheduled != seconds) => errors.set( format!("{path}.backoff_seconds"), "Local activity attempt backoff_seconds must match retry_policy.backoff_seconds."), Some(_) => {} @@ -682,6 +704,11 @@ pub(super) struct Record { } impl Record { + /// The exact history events and rows this record writes (Workflow #930). + pub(super) fn budget(&self) -> (usize, usize) { + super::history_meter::local_activity_budget(&self.attempts, matches!(self.outcome.as_str(), "failed" | "timed_out")) + } + /// The message the execution's exception carries, when it has one. pub(super) fn exception_message(&self) -> Option { (self.outcome != "completed").then(|| self.message.clone().unwrap_or_else(|| "Local activity failed.".to_owned())) @@ -809,6 +836,47 @@ fn milliseconds(value: Option<&Value>) -> i64 { value.and_then(Value::as_i64).unwrap_or(0) } +/// The earliest timestamp every supported column type stores (MySQL +/// TIMESTAMP's 1970-01-01T00:00:01Z), in Unix milliseconds. +const EARLIEST_STORED_TIMESTAMP_MS: i64 = 1000; + +/// How far a report walks back from completion: each attempt's span (its +/// duration or last heartbeat, whichever is later) and, before the next +/// attempt, its backoff. None if the sum overflows. +fn backdated_ms(attempts: &[Value]) -> Option { + let mut total: i64 = 0; + for (position, report) in attempts.iter().enumerate() { + let beats = report["heartbeats"].as_array().map_or(&[][..], Vec::as_slice); + let span = milliseconds(beats.last().and_then(|beat| beat.get("elapsed_ms"))) + .max(milliseconds(report.get("duration_ms"))); + let backoff = if position + 1 < attempts.len() { + milliseconds(report.get("backoff_seconds")).checked_mul(1000)? + } else { + 0 + }; + total = total.checked_add(span)?.checked_add(backoff)?; + } + Some(total) +} + +/// Whether the walk's earliest timestamp, `backdated` before `at_ms`, is +/// one every supported column type stores. +fn storable(at_ms: i64, backdated: i64) -> bool { + at_ms.checked_sub(backdated).is_some_and(|earliest| earliest >= EARLIEST_STORED_TIMESTAMP_MS) +} + +/// Checked timestamp arithmetic: the bounds above make these infallible, and +/// a failure is refused rather than unwound. +fn checked_sub(at: DateTime, delta: Option) -> Result> { + delta.and_then(|delta| at.checked_sub_signed(delta)) + .ok_or_else(|| refuse(StatusCode::CONFLICT, "local_activity_timestamp_out_of_range")) +} + +fn checked_add(at: DateTime, delta: Option) -> Result> { + delta.and_then(|delta| at.checked_add_signed(delta)) + .ok_or_else(|| refuse(StatusCode::CONFLICT, "local_activity_timestamp_out_of_range")) +} + /// One reported attempt with the times the bridge gives it. struct Attempt<'a> { report: &'a Value, @@ -904,6 +972,13 @@ store_methods! { Ok(()) } + + /// `local_activity_timestamp_out_of_range`, in the completion's response. + fn timestamp_out_of_range(tx: &Transaction<'_, DB>) -> RuntimeError { + const REASON: &str = "local_activity_timestamp_out_of_range"; + tx.meter.as_ref().map_or_else(|| refuse(StatusCode::CONFLICT, REASON), |meter| meter.refused_with(REASON)) + } + /// `applyRecordLocalActivity`: the execution, its attempts with times /// walked back from now, and the history PHP writes for them. #[allow(clippy::too_many_arguments)] @@ -916,6 +991,12 @@ store_methods! { .or(DB::optional_string(run, "payload_codec")?.filter(|codec| !codec.is_empty())) .unwrap_or_else(|| "avro".to_owned()); let at = now(); + // Workflow #930: the normalizer bounds every timing field; the walk's + // total is also checked, and its earliest timestamp must be one every + // supported column type stores, before anything is written. + if !backdated_ms(&record.attempts).is_some_and(|backdated| storable(at.timestamp_millis(), backdated)) { + return Err(Self::timestamp_out_of_range(tx)); + } // Reports carry durations, not times: walk back from the close so // each attempt ends before the next one's declared backoff. Identities // are made last attempt first, as PHP makes them, and are uppercase. @@ -926,15 +1007,19 @@ store_methods! { let beats = report["heartbeats"].as_array().map_or(&[][..], Vec::as_slice); let last_elapsed = milliseconds(beats.last().and_then(|beat| beat.get("elapsed_ms"))); let duration = milliseconds(report.get("duration_ms")); - let started_at = closed_at - Duration::milliseconds(last_elapsed.max(duration)); - let last_heartbeat_at = (!beats.is_empty()).then(|| started_at + Duration::milliseconds(last_elapsed)); + let started_at = checked_sub(closed_at, Duration::try_milliseconds(last_elapsed.max(duration)))?; + let last_heartbeat_at = if beats.is_empty() { + None + } else { + Some(checked_add(started_at, Duration::try_milliseconds(last_elapsed))?) + }; if let Some(beat) = last_heartbeat_at && latest_heartbeat.is_none_or(|latest| beat > latest) { latest_heartbeat = Some(beat); } attempts.push(Attempt { report, id: id().to_uppercase(), started_at, last_heartbeat_at, closed_at }); if position > 0 { let backoff = milliseconds(record.attempts[position - 1].get("backoff_seconds")); - closed_at = started_at - Duration::seconds(backoff); + closed_at = checked_sub(started_at, Duration::try_seconds(backoff))?; } } attempts.reverse(); @@ -955,6 +1040,7 @@ store_methods! { let connection = DB::optional_string(run, "connection")?; let queue = DB::optional_string(run, "queue")?; let started_at = attempts[0].started_at; + tx.charge_rows(1)?; Self::query("INSERT INTO activity_executions(id,workflow_run_id,sequence,activity_class,activity_type,status,payload_codec,arguments,result,exception,connection,queue,attempt_count,current_attempt_id,retry_policy,activity_options,started_at,closed_at,last_heartbeat_at,created_at,updated_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21)") .bind(&execution_id).bind(&run_id).bind(sequence).bind(&record.activity_type).bind(&record.activity_type).bind(status) .bind(&codec).bind(record.arguments.as_deref()).bind(record.result.as_deref()).bind(exception) @@ -999,6 +1085,7 @@ store_methods! { let mut emitted: Option> = None; for (position, attempt) in attempts.iter().enumerate() { let worker_attempt_id = attempt.report.get("attempt_id").and_then(Value::as_str); + tx.charge_rows(1)?; Self::query("INSERT INTO activity_attempts(id,workflow_run_id,activity_execution_id,workflow_task_id,worker_attempt_id,attempt_number,status,lease_owner,started_at,last_heartbeat_at,lease_expires_at,closed_at,created_at,updated_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,NULL,$11,$12,$13)") .bind(&attempt.id).bind(&run_id).bind(&execution_id).bind(&task_id).bind(worker_attempt_id) .bind(attempt.number()).bind(attempt.status()).bind(lease_owner.as_deref()).bind(DB::bind_time(attempt.started_at)) @@ -1023,7 +1110,7 @@ store_methods! { "attempt_number": attempt.number(), "activity_attempt": running_attempt}); Self::append(tx, &run_id, "ActivityStarted", event(running_activity.clone(), started.clone()), Some(&task_id), None).await?; for heartbeat in attempt.heartbeats() { - let heartbeat_at = attempt.started_at + Duration::milliseconds(milliseconds(heartbeat.get("elapsed_ms"))); + let heartbeat_at = checked_add(attempt.started_at, Duration::try_milliseconds(milliseconds(heartbeat.get("elapsed_ms"))))?; if emitted.is_none_or(|latest| heartbeat_at > latest) { emitted = Some(heartbeat_at); } @@ -1083,6 +1170,7 @@ store_methods! { let class = record.exception_type.clone() .unwrap_or_else(|| if timed_out { TIMEOUT_EXCEPTION } else { "RuntimeException" }.to_owned()); let non_retryable = record.non_retryable.unwrap_or(false); + tx.charge_rows(1)?; Self::query("INSERT INTO workflow_failures(id,workflow_run_id,source_kind,source_id,propagation_kind,failure_category,non_retryable,handled,exception_class,message,file,line,trace_preview,created_at,updated_at) VALUES ($1,$2,'activity_execution',$3,$4,$5,CASE WHEN $6='true' THEN TRUE ELSE FALSE END,FALSE,$7,$8,'',0,'',$9,$10)") .bind(&failure_id).bind(&run_id).bind(&execution_id).bind(if timed_out { "timeout" } else { "activity" }) .bind(if timed_out { "timeout" } else { "activity" }).bind(if non_retryable { "true" } else { "false" }) @@ -1127,6 +1215,71 @@ mod tests { command } + #[test] + fn each_timing_field_admits_its_maximum_and_refuses_one_more() { + let cases = [ + ("commands.0.attempts.0.duration_ms", "Local activity attempt duration_ms must not exceed 604800000."), + ("commands.0.attempts.0.heartbeats.0.elapsed_ms", "Local activity heartbeat elapsed_ms must not exceed 604800000."), + ("commands.0.attempts.0.backoff_seconds", "Local activity attempt backoff_seconds must not exceed 86400."), + ]; + let report = |duration: i64, elapsed: i64, backoff: i64| record(json!({"retry_policy": {"max_attempts": 2}, + "attempts": [ + {"attempt_number": 1, "outcome": "failed", "message": "retry", "retry_reason": "failure", + "duration_ms": duration, "backoff_seconds": backoff, "heartbeats": [{"elapsed_ms": elapsed}]}, + {"attempt_number": 2, "outcome": "completed", "duration_ms": 25}]})); + assert!(normalize(&[report(MAX_DURATION_MS, MAX_DURATION_MS, MAX_BACKOFF_SECONDS)], Some("1.20")).is_ok()); + for over in [1, i64::MAX] { + for (index, (field, message)) in cases.iter().enumerate() { + let bump = |value: i64, own: usize| if index == own { value.saturating_add(over) } else { value }; + let response = refusal(json!([report(bump(MAX_DURATION_MS, 0), bump(MAX_DURATION_MS, 1), + bump(MAX_BACKOFF_SECONDS, 2))]), "1.20"); + let fields: Vec<&String> = response["errors"].as_object().unwrap().keys().collect(); + assert_eq!(fields, [field], "{response}"); + assert_eq!(response["errors"][field], json!([message]), "{response}"); + } + } + } + + #[test] + fn a_retried_attempt_needs_backoff_seconds_only_when_absent() { + let response = refusal(json!([record(json!({"retry_policy": {"max_attempts": 2}, "attempts": [ + {"attempt_number": 1, "outcome": "failed", "message": "retry", "retry_reason": "failure"}, + {"attempt_number": 2, "outcome": "completed"}]}))]), "1.20"); + assert_eq!(response["errors"]["commands.0.attempts.0.backoff_seconds"], + json!(["A retried local activity attempt requires backoff_seconds."])); + let response = refusal(json!([record(json!({"retry_policy": {"max_attempts": 2}, "attempts": [ + {"attempt_number": 1, "outcome": "failed", "message": "retry", "retry_reason": "failure", "backoff_seconds": 86_401}, + {"attempt_number": 2, "outcome": "completed"}]}))]), "1.20"); + assert_eq!(response["errors"]["commands.0.attempts.0.backoff_seconds"], + json!(["Local activity attempt backoff_seconds must not exceed 86400."])); + } + + #[test] + fn the_walk_is_checked_and_must_stay_storable() { + let attempt = |duration: i64, elapsed: Option, backoff: i64| { + let mut attempt = json!({"duration_ms": duration, "backoff_seconds": backoff}); + if let Some(elapsed) = elapsed { + attempt["heartbeats"] = json!([{"elapsed_ms": elapsed}]); + } + attempt + }; + // A span is the later of the duration and the last heartbeat; the + // last attempt's backoff does not count. + assert_eq!(backdated_ms(&[attempt(1_000, Some(5_000), 2), attempt(3_000, None, 9)]), Some(5_000 + 2_000 + 3_000)); + // The longest report: 100 seven-day attempts and 99 one-day backoffs. + let longest: Vec = (0..100).map(|_| attempt(MAX_DURATION_MS, Some(MAX_DURATION_MS), MAX_BACKOFF_SECONDS)).collect(); + assert_eq!(backdated_ms(&longest), Some(100 * MAX_DURATION_MS + 99 * MAX_BACKOFF_SECONDS * 1_000)); + // Arithmetic past i64 fails closed. + assert_eq!(backdated_ms(&[attempt(i64::MAX, None, 0), attempt(1, None, 0)]), None); + assert_eq!(backdated_ms(&[attempt(0, None, i64::MAX), attempt(0, None, 0)]), None); + // The earliest storable instant is 1970-01-01T00:00:01Z. + let one_day = 86_400_000; + assert!(storable(one_day, one_day - EARLIEST_STORED_TIMESTAMP_MS)); + assert!(!storable(one_day, one_day - EARLIEST_STORED_TIMESTAMP_MS + 1)); + assert!(!storable(one_day, 2 * one_day)); + assert!(!storable(i64::MIN + 1, 2)); + } + #[test] fn protocol_versions_compare_like_php() { assert!(supports("1.18", "1.18") && supports("1.20", "1.19") && !supports("1.17", "1.18") && !supports("2.20", "1.18")); diff --git a/rust/src/runtime/memo_limits.rs b/rust/src/runtime/memo_limits.rs index fbf931e67..afe1bfcf3 100644 --- a/rust/src/runtime/memo_limits.rs +++ b/rust/src/runtime/memo_limits.rs @@ -90,6 +90,15 @@ pub struct StructuralLimits { pub pending_updates: Option, /// Bytes of one serialized payload blob (DW_V2_LIMIT_PAYLOAD_SIZE_BYTES). pub payload_bytes: Option, + /// History events one workflow task completion may write + /// (DW_V2_LIMIT_HISTORY_TRANSACTION_SIZE; Workflow #930). + pub history_events: Option, + /// History bytes one completion may write, measured as `HistoryBudget` + /// measures an event (DW_V2_LIMIT_HISTORY_TRANSACTION_BYTES). + pub history_bytes: Option, + /// Durable rows one completion may write + /// (DW_V2_LIMIT_HISTORY_TRANSACTION_ROWS). + pub history_rows: Option, } impl Default for StructuralLimits { @@ -98,6 +107,9 @@ impl Default for StructuralLimits { pending_signals: Some(5000), pending_updates: Some(500), payload_bytes: Some(2_097_152), + history_events: Some(5000), + history_bytes: Some(16_777_216), + history_rows: Some(20_000), } } } @@ -110,6 +122,9 @@ impl StructuralLimits { pending_signals: configured("PENDING_SIGNALS", defaults.pending_signals.unwrap(), invalid)?, pending_updates: configured("PENDING_UPDATES", defaults.pending_updates.unwrap(), invalid)?, payload_bytes: configured("PAYLOAD_SIZE_BYTES", defaults.payload_bytes.unwrap(), invalid)?, + history_events: configured("HISTORY_TRANSACTION_SIZE", defaults.history_events.unwrap(), invalid)?, + history_bytes: configured("HISTORY_TRANSACTION_BYTES", defaults.history_bytes.unwrap(), invalid)?, + history_rows: configured("HISTORY_TRANSACTION_ROWS", defaults.history_rows.unwrap(), invalid)?, }) } } diff --git a/rust/src/runtime/mod.rs b/rust/src/runtime/mod.rs index 0c10e3f9d..35787c2e8 100644 --- a/rust/src/runtime/mod.rs +++ b/rust/src/runtime/mod.rs @@ -65,6 +65,7 @@ mod laravel_validation; mod local_activities; mod memo_json; mod memo_limits; +mod history_meter; mod standalone_activities; mod archive; mod command_payloads; diff --git a/rust/src/runtime/store.rs b/rust/src/runtime/store.rs index 89f018b65..7ee692074 100644 --- a/rust/src/runtime/store.rs +++ b/rust/src/runtime/store.rs @@ -2697,6 +2697,30 @@ where self.wake.notify_waiters(); return Err(super::RuntimeError::Protocol { status: StatusCode::CONFLICT, response }); } + // Workflow #930: the deadline close above is never metered. Before + // the worker's first write, its local activities' exact events and + // rows (other commands at one event) must fit the limits; then the + // rest of the completion runs under the history meter. + let (mut events, mut rows) = (0usize, 0usize); + for index in 0..commands.len() { + let (command_events, command_rows) = match local.get(&index) { + Some(record) => record.budget(), + None => (super::history_meter::MIN_COMMAND_HISTORY_EVENTS, 2 * super::history_meter::MIN_COMMAND_HISTORY_EVENTS), + }; + events = events.saturating_add(command_events); + rows = rows.saturating_add(command_rows); + } + if limits.history_events.is_some_and(|limit| limit > 0 && events > limit) + || limits.history_rows.is_some_and(|limit| limit > 0 && rows > limit) + { + return Err(super::RuntimeError::Protocol { + status: StatusCode::CONFLICT, + response: json!({"task_id": task_id, "workflow_task_attempt": body["workflow_task_attempt"], "outcome": "completed", + "recorded": false, "run_id": run_id, "run_status": null, "created_task_ids": [], + "reason": "completion_budget_exceeded"}), + }); + } + tx.meter = Some(super::history_meter::Meter::new(&limits, task_id, &body["workflow_task_attempt"])); let task_payload = DB::document_row(&task, "payload")?; if Self::cancellation_request(&run)?.is_some() && commands.iter().any(|command| { @@ -3818,6 +3842,8 @@ where { payload["command"] = snapshot; } + // A metered completion (Workflow #930) charges the event as stored. + tx.charge_event(event_type, &payload)?; Self::query("INSERT INTO workflow_history_events(id,workflow_run_id,sequence,event_type,payload,workflow_task_id,workflow_command_id,recorded_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8)") .bind(id()).bind(run_id).bind(sequence).bind(event_type).bind(DB::document(&payload)).bind(task_id).bind(command_id).bind(DB::bind_time(recorded_at)).execute(&mut **tx).await?; Self::project_activity_wait_event(tx, run_id, event_type, &payload, recorded_at).await?; diff --git a/rust/src/runtime/transaction.rs b/rust/src/runtime/transaction.rs index a6b58831d..eae71126d 100644 --- a/rust/src/runtime/transaction.rs +++ b/rust/src/runtime/transaction.rs @@ -10,6 +10,8 @@ pub(super) struct Transaction<'a, DB: Backend> { inner: sqlx::Transaction<'a, DB>, summaries: BTreeSet, pub(super) history_budget: super::history_budget::Config, + /// A workflow task completion's history meter (Workflow #930). + pub(super) meter: Option, } impl<'a, DB: Backend> Transaction<'a, DB> { @@ -21,6 +23,7 @@ impl<'a, DB: Backend> Transaction<'a, DB> { inner, summaries: BTreeSet::new(), history_budget, + meter: None, } } @@ -28,6 +31,22 @@ impl<'a, DB: Backend> Transaction<'a, DB> { self.summaries.insert(run_id.to_owned()); } + /// Rows the next statement will write, charged to a metered completion. + pub(super) fn charge_rows(&mut self, rows: usize) -> Result<()> { + match self.meter.as_mut() { + Some(meter) => meter.charge_rows(rows), + None => Ok(()), + } + } + + /// One history event about to be stored, charged to a metered completion. + pub(super) fn charge_event(&mut self, event_type: &str, payload: &serde_json::Value) -> Result<()> { + match self.meter.as_mut() { + Some(meter) => meter.charge_event(event_type, payload), + None => Ok(()), + } + } + pub(super) async fn rollback(self) -> Result<()> { self.inner.rollback().await?; Ok(()) @@ -60,6 +79,11 @@ where usize: ColumnIndex, { pub(super) async fn commit(mut self) -> Result<()> { + // A completion whose meter refused cannot commit, even if a caller + // swallowed the refusal; dropping the transaction rolls it back. + if let Some(refusal) = self.meter.as_ref().and_then(super::history_meter::Meter::tripped) { + return Err(refusal); + } for run_id in std::mem::take(&mut self.summaries) { Store::::order_open_waits(&mut self, &run_id).await?; Store::::project_run_summary(&mut self, &run_id).await?; diff --git a/rust/tests/execution.rs b/rust/tests/execution.rs index 649103670..fffa2eb53 100644 --- a/rust/tests/execution.rs +++ b/rust/tests/execution.rs @@ -109,6 +109,8 @@ mod definition_routing; mod worker_admission; #[path = "execution/local_activities.rs"] mod local_activities; +#[path = "execution/completion_budget.rs"] +mod completion_budget; /// `WorkflowDefinitionRouting::ENTRY_VALIDATION_CAPABILITY`. const ENTRY_VALIDATION: &str = "workflow_entry_arguments_validation"; #[path = "execution/parent_close_policies.rs"] diff --git a/rust/tests/execution/completion_budget.rs b/rust/tests/execution/completion_budget.rs new file mode 100644 index 000000000..ff7a7de19 --- /dev/null +++ b/rust/tests/execution/completion_budget.rs @@ -0,0 +1,129 @@ +use super::*; +use durable_workflow_server::runtime::StructuralLimits; + +// The completion budget of Workflow #930 (phase A, Workflow PR #934): an +// early whole-completion check of the local activities' exact events and +// rows, then a meter that charges every history event (its bytes as +// `HistoryBudget` measures them, and two rows) and every local activity row. +// A refusal writes nothing and leaves the task's lease as it was. + +async fn budgeted(database: &TestDatabase, limits: StructuralLimits) -> Router { + router(database.open().await.unwrap().with_structural_limits(limits)) +} + +async fn local_worker(app: &Router) { + let (status, body) = request_protocol(app, "POST", "/api/worker/register", json!({"worker_id": "local", + "task_queue": "test", "runtime": "python", "supported_workflow_types": ["echo"], "supported_activity_types": [], + "capabilities": ["local_activities"]}), "1.20").await; + assert_eq!(status, StatusCode::CREATED, "{body}"); +} + +async fn complete(app: &Router, task: &Value, commands: Value) -> (StatusCode, Value) { + let path = format!("/api/worker/workflow-tasks/{}/complete", task["task_id"].as_str().unwrap()); + request_protocol(app, "POST", &path, completion(task, commands), "1.20").await +} + +/// One completed local activity attempt with `heartbeats` heartbeats. +fn local(heartbeats: i64) -> Value { + json!({"type": "record_local_activity", "activity_type": "local.echo", + "arguments": {"codec": "avro", "blob": "wwHioz3/VYAiNwwA"}, "result": {"codec": "avro", "blob": "wwHioz3/VYAiNwoIZG9uZQ=="}, + "outcome": "completed", "attempts": [{"attempt_id": "w-1", "attempt_number": 1, "outcome": "completed", + "duration_ms": 25, "heartbeats": (1..=heartbeats) + .map(|elapsed| json!({"elapsed_ms": elapsed, "details": {"progress": "x".repeat(200)}})) + .collect::>()}]}) +} + +fn wait() -> Value { + json!({"type": "open_signal_wait", "signal_name": "resume"}) +} + +#[tokio::test] +async fn an_over_budget_completion_writes_nothing_and_keeps_the_lease() { + let database = TestDatabase::new().await; + let app = budgeted(&database, StructuralLimits { history_events: Some(4), ..StructuralLimits::default() }).await; + local_worker(&app).await; + let run = start(&app, "budget-early").await; + let task = poll(&app, "local", "workflow").await; + let before = run_history(&app, &run["workflow_id"], &run["run_id"]).await; + + // Scheduled, started, three heartbeats and completed, then the wait: 7. + let (status, body) = complete(&app, &task, json!([local(3), wait()])).await; + + assert_eq!(status, StatusCode::CONFLICT, "{body}"); + assert_eq!(body["reason"], "completion_budget_exceeded", "{body}"); + assert_eq!((&body["recorded"], &body["run_id"], &body["outcome"]), (&json!(false), &run["run_id"], &json!("completed"))); + assert_eq!(run_history(&app, &run["workflow_id"], &run["run_id"]).await, before); + + // Exactly the budget: scheduled, started, completed and the wait. + let (status, body) = complete(&app, &task, json!([local(0), wait()])).await; + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!(body["recorded"], true, "{body}"); +} + +#[tokio::test] +async fn the_meter_refuses_past_the_byte_limit_and_keeps_the_lease() { + let database = TestDatabase::new().await; + let app = budgeted(&database, StructuralLimits { history_bytes: Some(4_000), ..StructuralLimits::default() }).await; + local_worker(&app).await; + let run = start(&app, "budget-bytes").await; + let task = poll(&app, "local", "workflow").await; + let before = run_history(&app, &run["workflow_id"], &run["run_id"]).await; + + let (status, body) = complete(&app, &task, json!([local(5), wait()])).await; + + assert_eq!(status, StatusCode::CONFLICT, "{body}"); + assert_eq!(body["reason"], "history_transaction_limit_exceeded", "{body}"); + assert_eq!((&body["recorded"], &body["run_id"], &body["created_task_ids"]), (&json!(false), &Value::Null, &json!([]))); + assert_eq!(run_history(&app, &run["workflow_id"], &run["run_id"]).await, before); + assert_eq!(task_retry::rows(&database, "SELECT COUNT(*) FROM activity_executions").await, 0); + assert_eq!(task_retry::rows(&database, "SELECT COUNT(*) FROM activity_attempts").await, 0); + + // The same lease completes a smaller task. + let (status, body) = complete(&app, &task, json!([wait()])).await; + assert_eq!(status, StatusCode::OK, "{body}"); +} + +#[tokio::test] +async fn the_meter_counts_a_local_activity_s_rows() { + let database = TestDatabase::new().await; + // Scheduled, started and completed are six rows, with the execution + // and the attempt eight; the wait's event is two more. + let app = budgeted(&database, StructuralLimits { history_rows: Some(9), ..StructuralLimits::default() }).await; + local_worker(&app).await; + let run = start(&app, "budget-rows").await; + let task = poll(&app, "local", "workflow").await; + + let (status, body) = complete(&app, &task, json!([local(0), wait()])).await; + assert_eq!(status, StatusCode::CONFLICT, "{body}"); + assert_eq!(body["reason"], "completion_budget_exceeded", "{body}"); + + let roomy = budgeted(&database, StructuralLimits { history_rows: Some(10), ..StructuralLimits::default() }).await; + // The meter charges the same ten rows the early check counted. + let (status, body) = complete(&roomy, &task, json!([local(0), wait()])).await; + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!(task_retry::rows(&database, "SELECT COUNT(*) FROM activity_attempts").await, 1); + let events = run_history(&roomy, &run["workflow_id"], &run["run_id"]).await; + assert!(events.as_array().unwrap().iter().any(|event| event["event_type"] == "SignalWaitOpened"), "{events}"); +} + +#[tokio::test] +async fn a_run_past_its_deadline_closes_before_the_budget_is_checked() { + let database = TestDatabase::new().await; + let app = budgeted(&database, StructuralLimits { history_events: Some(1), ..StructuralLimits::default() }).await; + local_worker(&app).await; + let (status, started) = request(&app, "POST", "/api/workflows", json!({"workflow_id": "budget-deadline", + "workflow_type": "echo", "task_queue": "test", "input": envelope(Payload::Array(vec![])), "run_timeout_seconds": 600})).await; + assert_eq!(status, StatusCode::CREATED, "{started}"); + let task = poll(&app, "local", "workflow").await; + database.execute("UPDATE workflow_runs SET run_deadline_at='2000-01-01 00:00:00'").await; + + // Far over the budget, but the run's deadline close comes first and is + // the server's own work: the run times out instead of staying open. + let (status, body) = complete(&app, &task, json!([local(3), wait()])).await; + + assert_eq!(status, StatusCode::CONFLICT, "{body}"); + assert_eq!(body["reason"], "run_timed_out", "{body}"); + let (status, described) = request(&app, "GET", "/api/workflows/budget-deadline", Value::Null).await; + assert_eq!(status, StatusCode::OK, "{described}"); + assert_eq!((&described["status"], &described["closed_reason"]), (&json!("failed"), &json!("timed_out"))); +} diff --git a/rust/tests/execution/structural_limits.rs b/rust/tests/execution/structural_limits.rs index 42534b31e..40687eb59 100644 --- a/rust/tests/execution/structural_limits.rs +++ b/rust/tests/execution/structural_limits.rs @@ -11,6 +11,7 @@ async fn limited(database: &TestDatabase, signals: usize, updates: usize, payloa pending_signals: Some(signals), pending_updates: Some(updates), payload_bytes: Some(payload), + ..StructuralLimits::default() }; let runtime = database.open().await.unwrap().with_structural_limits(limits); (runtime.clone(), router(runtime)) From 86178bf052de74c1c7cb9b5ca9cb45ebce616816 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Sun, 25 Oct 2026 23:09:56 +0000 Subject: [PATCH 5/9] Rust #325: the completion budget's exact-boundary matrix (WIP) Co-Authored-By: Claude Opus 5.5 --- rust/tests/execution/completion_budget.rs | 147 +++++++++++++++++++++- 1 file changed, 146 insertions(+), 1 deletion(-) diff --git a/rust/tests/execution/completion_budget.rs b/rust/tests/execution/completion_budget.rs index ff7a7de19..abb54515d 100644 --- a/rust/tests/execution/completion_budget.rs +++ b/rust/tests/execution/completion_budget.rs @@ -15,7 +15,8 @@ async fn local_worker(app: &Router) { let (status, body) = request_protocol(app, "POST", "/api/worker/register", json!({"worker_id": "local", "task_queue": "test", "runtime": "python", "supported_workflow_types": ["echo"], "supported_activity_types": [], "capabilities": ["local_activities"]}), "1.20").await; - assert_eq!(status, StatusCode::CREATED, "{body}"); + // A second runtime on the same database registers the worker again. + assert!(status == StatusCode::CREATED || status == StatusCode::OK, "{status}: {body}"); } async fn complete(app: &Router, task: &Value, commands: Value) -> (StatusCode, Value) { @@ -127,3 +128,147 @@ async fn a_run_past_its_deadline_closes_before_the_budget_is_checked() { assert_eq!(status, StatusCode::OK, "{described}"); assert_eq!((&described["status"], &described["closed_reason"]), (&json!("failed"), &json!("timed_out"))); } + +// The acceptance matrix: each limit admits its exact value and refuses one +// more, on every backend, and a refusal changes no durable row and leaves +// the lease, which then completes a smaller task. + +/// A completed local activity with a large valid argument and `heartbeats` +/// heartbeats with details. +fn large(heartbeats: i64) -> Value { + let mut command = local(heartbeats); + command["arguments"] = envelope(Payload::Array(vec![Payload::String("argument ".repeat(30_000))])); + command +} + +fn leak(sql: String) -> &'static str { + Box::leak(sql.into_boxed_str()) +} + +/// The rows of a two-column text query, written for each backend. +async fn texts(database: &TestDatabase, sqlite: &str, postgres: &str, mysql: &str) -> Vec<(String, String)> { + match database { + TestDatabase::Sqlite(dir) => { + let pool = sqlx::SqlitePool::connect(&format!("sqlite://{}", dir.path().join("runtime.sqlite").display())).await.unwrap(); + let rows = sqlx::query_as(sqlx::AssertSqlSafe(sqlite.to_owned())).fetch_all(&pool).await.unwrap(); + pool.close().await; + rows + } + TestDatabase::Postgres { options, .. } => { + let pool = sqlx::postgres::PgPoolOptions::new().connect_with(options.as_ref().clone()).await.unwrap(); + let rows = sqlx::query_as(sqlx::AssertSqlSafe(postgres.to_owned())).fetch_all(&pool).await.unwrap(); + pool.close().await; + rows + } + TestDatabase::MySql { options, .. } => { + let pool = sqlx::mysql::MySqlPoolOptions::new().connect_with(options.as_ref().clone()).await.unwrap(); + let rows = sqlx::query_as(sqlx::AssertSqlSafe(mysql.to_owned())).fetch_all(&pool).await.unwrap(); + pool.close().await; + rows + } + } +} + +/// Every table's row count, and the state of every task and run. +async fn durable_state(database: &TestDatabase) -> (Vec<(String, i64)>, Vec<(String, String)>, Vec<(String, String)>) { + let tables = texts(database, + "SELECT name, name FROM sqlite_master WHERE type='table' ORDER BY name", + "SELECT table_name::text, table_name::text FROM information_schema.tables WHERE table_schema='public' ORDER BY 1", + "SELECT CAST(table_name AS CHAR), CAST(table_name AS CHAR) FROM information_schema.tables WHERE table_schema=DATABASE() ORDER BY 1").await; + let mut counts = Vec::new(); + for (table, _) in tables { + counts.push((table.clone(), task_retry::rows(database, leak(format!("SELECT COUNT(*) FROM {table}"))).await)); + } + let tasks = texts(database, + "SELECT id, status || '|' || COALESCE(lease_owner, '') || '|' || COALESCE(lease_expires_at, '') || '|' || attempt_count FROM workflow_tasks ORDER BY id", + "SELECT id, status || '|' || COALESCE(lease_owner, '') || '|' || COALESCE(lease_expires_at::text, '') || '|' || attempt_count::text FROM workflow_tasks ORDER BY id", + "SELECT id, CONCAT(status, '|', COALESCE(lease_owner, ''), '|', COALESCE(CAST(lease_expires_at AS CHAR), ''), '|', attempt_count) FROM workflow_tasks ORDER BY id").await; + let runs = texts(database, + "SELECT id, status || '|' || last_history_sequence || '|' || COALESCE(updated_at, '') FROM workflow_runs ORDER BY id", + "SELECT id, status || '|' || last_history_sequence::text || '|' || COALESCE(updated_at::text, '') FROM workflow_runs ORDER BY id", + "SELECT id, CONCAT(status, '|', last_history_sequence, '|', COALESCE(CAST(updated_at AS CHAR), '')) FROM workflow_runs ORDER BY id").await; + (counts, tasks, runs) +} + +/// The stored history of a run, in order: each event's type and payload. +async fn stored_history(database: &TestDatabase, run_id: &str) -> Vec<(String, String)> { + texts(database, + &format!("SELECT event_type, payload FROM workflow_history_events WHERE workflow_run_id='{run_id}' ORDER BY sequence"), + &format!("SELECT event_type, payload::text FROM workflow_history_events WHERE workflow_run_id='{run_id}' ORDER BY sequence"), + &format!("SELECT event_type, CAST(payload AS CHAR) FROM workflow_history_events WHERE workflow_run_id='{run_id}' ORDER BY sequence")).await +} + +/// The bytes the meter charges for stored events: each event's type and +/// the compact JSON of its payload (none of these hold U+2028 or U+2029). +fn charged_bytes(events: &[(String, String)]) -> usize { + events.iter().map(|(event_type, payload)| { + let payload: Value = serde_json::from_str(payload).unwrap(); + event_type.len() + serde_json::to_string(&payload).unwrap().len() + }).sum() +} + +/// The exact limit admits `commands`; one less refuses them with `reason` +/// before anything is written, and the same lease then completes a wait. +async fn admits_exactly(database: &TestDatabase, tag: &str, exact: StructuralLimits, short: StructuralLimits, + commands: Value, reason: &str) { + let app = budgeted(database, exact).await; + local_worker(&app).await; + start(&app, &format!("{tag}-exact")).await; + let task = poll(&app, "local", "workflow").await; + let (status, body) = complete(&app, &task, commands.clone()).await; + assert_eq!(status, StatusCode::OK, "{tag} at its exact limit: {body}"); + + let app = budgeted(database, short).await; + start(&app, &format!("{tag}-short")).await; + let task = poll(&app, "local", "workflow").await; + let before = durable_state(database).await; + let (status, body) = complete(&app, &task, commands).await; + assert_eq!(status, StatusCode::CONFLICT, "{tag} one over: {body}"); + assert_eq!(body["reason"], reason, "{tag}: {body}"); + assert_eq!(durable_state(database).await, before, "{tag}: a refusal changed durable state"); + let (status, body) = complete(&app, &task, json!([wait()])).await; + assert_eq!(status, StatusCode::OK, "{tag}: the lease after a refusal: {body}"); +} + +fn only(events: Option, bytes: Option, rows: Option) -> StructuralLimits { + StructuralLimits { history_events: events, history_bytes: bytes, history_rows: rows, ..StructuralLimits::default() } +} + +#[tokio::test] +async fn the_event_limit_admits_its_exact_value_and_refuses_one_more() { + let database = TestDatabase::new().await; + // Scheduled, started, three heartbeats and completed, and the wait: 7. + admits_exactly(&database, "events", only(Some(7), None, None), only(Some(6), None, None), + json!([local(3), wait()]), "completion_budget_exceeded").await; +} + +#[tokio::test] +async fn the_row_limit_admits_its_exact_value_and_refuses_one_more() { + let database = TestDatabase::new().await; + // Three events (six rows), the execution and the attempt, and the wait's + // event (two rows): ten. + admits_exactly(&database, "rows", only(None, None, Some(10)), only(None, None, Some(9)), + json!([local(0), wait()]), "completion_budget_exceeded").await; +} + +#[tokio::test] +async fn the_byte_limit_admits_its_exact_value_and_refuses_one_more() { + let database = TestDatabase::new().await; + let commands = json!([large(20), wait()]); + // Measure the completion as stored, the way the meter charges it. + let probe = budgeted(&database, only(None, None, None)).await; + local_worker(&probe).await; + let run = start(&probe, "bytes-probe").await; + let run_id = run["run_id"].as_str().unwrap().to_owned(); + let task = poll(&probe, "local", "workflow").await; + let before = stored_history(&database, &run_id).await; + let (status, body) = complete(&probe, &task, commands.clone()).await; + assert_eq!(status, StatusCode::OK, "{body}"); + let after = stored_history(&database, &run_id).await; + let bytes = charged_bytes(&after[before.len()..]); + // The large argument and heartbeats make this a multi-megabyte budget. + assert!(bytes > 1_000_000, "{bytes}"); + + admits_exactly(&database, "bytes", only(None, Some(bytes), None), only(None, Some(bytes - 1), None), + commands, "history_transaction_limit_exceeded").await; +} From 0c64015a1300624ac36d7ce6df96d1fa1a8fd50c Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Sun, 25 Oct 2026 23:34:35 +0000 Subject: [PATCH 6/9] Rust #325: publish and honor the row limit with the other two Co-Authored-By: Claude Opus 5.5 --- rust/src/config_audit.rs | 9 ++++++--- rust/src/runtime/operator_metrics.rs | 1 + rust/tests/execution/structural_limits.rs | 21 +++++++++++---------- 3 files changed, 18 insertions(+), 13 deletions(-) diff --git a/rust/src/config_audit.rs b/rust/src/config_audit.rs index bb6bd30d2..cedfa3881 100644 --- a/rust/src/config_audit.rs +++ b/rust/src/config_audit.rs @@ -548,6 +548,7 @@ const UNLISTED: &[Setting] = &[ /// their legacy names warn as the contract's do. pub const NATIVE_HONORED: &[Setting] = &[ Setting { name: "DW_V2_LIMIT_HISTORY_TRANSACTION_BYTES", legacy: Some("WORKFLOW_V2_LIMIT_HISTORY_TRANSACTION_BYTES"), disposition: Honored }, + Setting { name: "DW_V2_LIMIT_HISTORY_TRANSACTION_ROWS", legacy: Some("WORKFLOW_V2_LIMIT_HISTORY_TRANSACTION_ROWS"), disposition: Honored }, ]; /// Every `vars` entry of `config/dw-contract.php`, with its contract legacy name. @@ -1086,7 +1087,7 @@ mod tests { fn the_history_transaction_limits_are_honored_settings() { let honored = |name: &str| SETTINGS.iter().chain(NATIVE_HONORED).find(|setting| setting.name == name) .is_some_and(|setting| matches!(setting.disposition, Honored)); - for name in ["DW_V2_LIMIT_HISTORY_TRANSACTION_SIZE", "DW_V2_LIMIT_HISTORY_TRANSACTION_BYTES"] { + for name in ["DW_V2_LIMIT_HISTORY_TRANSACTION_SIZE", "DW_V2_LIMIT_HISTORY_TRANSACTION_BYTES", "DW_V2_LIMIT_HISTORY_TRANSACTION_ROWS"] { assert!(honored(name), "{name}"); } // NATIVE_HONORED names are outside PHP's contract and listed once. @@ -1095,15 +1096,17 @@ mod tests { assert!(!SETTINGS.iter().any(|listed| listed.name == setting.name)); } // Strict mode starts with each primary name set. - let mut primary = environment(&[("DW_V2_LIMIT_HISTORY_TRANSACTION_SIZE", "123"), ("DW_V2_LIMIT_HISTORY_TRANSACTION_BYTES", "4567")]); + let mut primary = environment(&[("DW_V2_LIMIT_HISTORY_TRANSACTION_SIZE", "123"), ("DW_V2_LIMIT_HISTORY_TRANSACTION_BYTES", "4567"), + ("DW_V2_LIMIT_HISTORY_TRANSACTION_ROWS", "89")]); primary.insert("DW_ENV_AUDIT_STRICT".into(), "1".into()); let report = audit(&primary); assert_eq!((report.warnings.as_slice(), report.refusal()), (&[] as &[String], None)); // A legacy name warns as deprecated, naming its primary, and nothing else. let report = audit(&environment(&[("WORKFLOW_V2_LIMIT_HISTORY_TRANSACTION_SIZE", "123"), - ("WORKFLOW_V2_LIMIT_HISTORY_TRANSACTION_BYTES", "4567")])); + ("WORKFLOW_V2_LIMIT_HISTORY_TRANSACTION_BYTES", "4567"), ("WORKFLOW_V2_LIMIT_HISTORY_TRANSACTION_ROWS", "89")])); assert_eq!(report.warnings, [ "Legacy env var WORKFLOW_V2_LIMIT_HISTORY_TRANSACTION_BYTES is still honored but deprecated — rename to DW_V2_LIMIT_HISTORY_TRANSACTION_BYTES.", + "Legacy env var WORKFLOW_V2_LIMIT_HISTORY_TRANSACTION_ROWS is still honored but deprecated — rename to DW_V2_LIMIT_HISTORY_TRANSACTION_ROWS.", "Legacy env var WORKFLOW_V2_LIMIT_HISTORY_TRANSACTION_SIZE is still honored but deprecated — rename to DW_V2_LIMIT_HISTORY_TRANSACTION_SIZE.", ]); assert_eq!(report.refusal(), None); diff --git a/rust/src/runtime/operator_metrics.rs b/rust/src/runtime/operator_metrics.rs index 9fe1c20b3..8f5ff06ff 100644 --- a/rust/src/runtime/operator_metrics.rs +++ b/rust/src/runtime/operator_metrics.rs @@ -1493,6 +1493,7 @@ fn structural_limits(connection: &str, limits: &super::StructuralLimits, memo: O "search_attribute_size_bytes": 40960, "history_transaction_size": shown(limits.history_events), "history_transaction_bytes": shown(limits.history_bytes), + "history_transaction_rows": shown(limits.history_rows), "warning_threshold_percent": 80, }); let sqlite = connection == "sqlite"; diff --git a/rust/tests/execution/structural_limits.rs b/rust/tests/execution/structural_limits.rs index 2a6d49b4d..41ceaa452 100644 --- a/rust/tests/execution/structural_limits.rs +++ b/rust/tests/execution/structural_limits.rs @@ -225,7 +225,7 @@ async fn an_oversized_activity_result_fails_the_activity_like_php() { /// The history transaction ceilings as the operator metrics, the health /// snapshot and the nested backend capabilities report them. -async fn published_history_limits(app: &Router) -> Vec<(String, Value, Value)> { +async fn published_history_limits(app: &Router) -> Vec<(String, Value, Value, Value)> { let (status, metrics) = request(app, "GET", "/api/system/operator-metrics", Value::Null).await; assert_eq!(status, StatusCode::OK, "{metrics}"); let (status, health) = request(app, "GET", "/api/system/health", Value::Null).await; @@ -240,26 +240,27 @@ async fn published_history_limits(app: &Router) -> Vec<(String, Value, Value)> { ("health effective", &health["health"]["structural_limits"]["effective"]), ]; surfaces.into_iter().map(|(surface, limits)| { - (surface.to_owned(), limits["history_transaction_size"].clone(), limits["history_transaction_bytes"].clone()) + (surface.to_owned(), limits["history_transaction_size"].clone(), limits["history_transaction_bytes"].clone(), + limits["history_transaction_rows"].clone()) }).collect() } #[tokio::test] async fn every_surface_reports_the_active_history_transaction_limits() { let database = TestDatabase::new().await; - for (events, bytes, shown) in [ - (Some(4321), Some(987_654), (json!(4321), json!(987_654))), - (Some(0), None, (json!(0), json!(0))), + for (events, bytes, rows, shown) in [ + (Some(4321), Some(987_654), Some(13_579), (json!(4321), json!(987_654), json!(13_579))), + (Some(0), None, Some(0), (json!(0), json!(0), json!(0))), ] { - let limits = StructuralLimits { history_events: events, history_bytes: bytes, ..StructuralLimits::default() }; + let limits = StructuralLimits { history_events: events, history_bytes: bytes, history_rows: rows, ..StructuralLimits::default() }; let app = router(database.open().await.unwrap().with_structural_limits(limits)); - for (surface, size, published_bytes) in published_history_limits(&app).await { - assert_eq!((&size, &published_bytes), (&shown.0, &shown.1), "{surface}"); + for (surface, size, published_bytes, published_rows) in published_history_limits(&app).await { + assert_eq!((&size, &published_bytes, &published_rows), (&shown.0, &shown.1, &shown.2), "{surface}"); } } // The defaults, from no configuration. let app = router(database.open().await.unwrap()); - for (surface, size, bytes) in published_history_limits(&app).await { - assert_eq!((size, bytes), (json!(5000), json!(16_777_216)), "{surface}"); + for (surface, size, bytes, rows) in published_history_limits(&app).await { + assert_eq!((size, bytes, rows), (json!(5000), json!(16_777_216), json!(20_000)), "{surface}"); } } From 275a474a94e8e778c28cab49df481d41dc3c5ef4 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Sun, 25 Oct 2026 23:35:20 +0000 Subject: [PATCH 7/9] Rust #325: restore the row limit after merging #768; share the table snapshot helpers Co-Authored-By: Claude Opus 5.5 --- rust/src/runtime/memo_limits.rs | 10 ++++-- rust/tests/execution/completion_budget.rs | 43 +++-------------------- 2 files changed, 12 insertions(+), 41 deletions(-) diff --git a/rust/src/runtime/memo_limits.rs b/rust/src/runtime/memo_limits.rs index dd5efdd89..595b91f33 100644 --- a/rust/src/runtime/memo_limits.rs +++ b/rust/src/runtime/memo_limits.rs @@ -91,12 +91,16 @@ pub struct StructuralLimits { /// Bytes of one serialized payload blob (DW_V2_LIMIT_PAYLOAD_SIZE_BYTES). pub payload_bytes: Option, /// History events one transaction may write - /// (DW_V2_LIMIT_HISTORY_TRANSACTION_SIZE; Workflow #930): here, a - /// Waterline v1 projection import. + /// (DW_V2_LIMIT_HISTORY_TRANSACTION_SIZE; Workflow #930): a workflow task + /// completion or a Waterline v1 projection import. pub history_events: Option, /// History bytes one transaction may write, measured as PHP measures an /// event (DW_V2_LIMIT_HISTORY_TRANSACTION_BYTES). pub history_bytes: Option, + /// Metered rows one workflow task completion may write: each history + /// event and the run position it advances, and each local activity row + /// (DW_V2_LIMIT_HISTORY_TRANSACTION_ROWS). + pub history_rows: Option, } impl Default for StructuralLimits { @@ -107,6 +111,7 @@ impl Default for StructuralLimits { payload_bytes: Some(2_097_152), history_events: Some(5000), history_bytes: Some(16_777_216), + history_rows: Some(20_000), } } } @@ -121,6 +126,7 @@ impl StructuralLimits { payload_bytes: configured("PAYLOAD_SIZE_BYTES", defaults.payload_bytes.unwrap(), invalid)?, history_events: configured("HISTORY_TRANSACTION_SIZE", defaults.history_events.unwrap(), invalid)?, history_bytes: configured("HISTORY_TRANSACTION_BYTES", defaults.history_bytes.unwrap(), invalid)?, + history_rows: configured("HISTORY_TRANSACTION_ROWS", defaults.history_rows.unwrap(), invalid)?, }) } } diff --git a/rust/tests/execution/completion_budget.rs b/rust/tests/execution/completion_budget.rs index abb54515d..4ed851f36 100644 --- a/rust/tests/execution/completion_budget.rs +++ b/rust/tests/execution/completion_budget.rs @@ -141,49 +141,14 @@ fn large(heartbeats: i64) -> Value { command } -fn leak(sql: String) -> &'static str { - Box::leak(sql.into_boxed_str()) -} - -/// The rows of a two-column text query, written for each backend. -async fn texts(database: &TestDatabase, sqlite: &str, postgres: &str, mysql: &str) -> Vec<(String, String)> { - match database { - TestDatabase::Sqlite(dir) => { - let pool = sqlx::SqlitePool::connect(&format!("sqlite://{}", dir.path().join("runtime.sqlite").display())).await.unwrap(); - let rows = sqlx::query_as(sqlx::AssertSqlSafe(sqlite.to_owned())).fetch_all(&pool).await.unwrap(); - pool.close().await; - rows - } - TestDatabase::Postgres { options, .. } => { - let pool = sqlx::postgres::PgPoolOptions::new().connect_with(options.as_ref().clone()).await.unwrap(); - let rows = sqlx::query_as(sqlx::AssertSqlSafe(postgres.to_owned())).fetch_all(&pool).await.unwrap(); - pool.close().await; - rows - } - TestDatabase::MySql { options, .. } => { - let pool = sqlx::mysql::MySqlPoolOptions::new().connect_with(options.as_ref().clone()).await.unwrap(); - let rows = sqlx::query_as(sqlx::AssertSqlSafe(mysql.to_owned())).fetch_all(&pool).await.unwrap(); - pool.close().await; - rows - } - } -} - /// Every table's row count, and the state of every task and run. async fn durable_state(database: &TestDatabase) -> (Vec<(String, i64)>, Vec<(String, String)>, Vec<(String, String)>) { - let tables = texts(database, - "SELECT name, name FROM sqlite_master WHERE type='table' ORDER BY name", - "SELECT table_name::text, table_name::text FROM information_schema.tables WHERE table_schema='public' ORDER BY 1", - "SELECT CAST(table_name AS CHAR), CAST(table_name AS CHAR) FROM information_schema.tables WHERE table_schema=DATABASE() ORDER BY 1").await; - let mut counts = Vec::new(); - for (table, _) in tables { - counts.push((table.clone(), task_retry::rows(database, leak(format!("SELECT COUNT(*) FROM {table}"))).await)); - } - let tasks = texts(database, + let counts = table_counts(database).await; + let tasks = text_rows(database, "SELECT id, status || '|' || COALESCE(lease_owner, '') || '|' || COALESCE(lease_expires_at, '') || '|' || attempt_count FROM workflow_tasks ORDER BY id", "SELECT id, status || '|' || COALESCE(lease_owner, '') || '|' || COALESCE(lease_expires_at::text, '') || '|' || attempt_count::text FROM workflow_tasks ORDER BY id", "SELECT id, CONCAT(status, '|', COALESCE(lease_owner, ''), '|', COALESCE(CAST(lease_expires_at AS CHAR), ''), '|', attempt_count) FROM workflow_tasks ORDER BY id").await; - let runs = texts(database, + let runs = text_rows(database, "SELECT id, status || '|' || last_history_sequence || '|' || COALESCE(updated_at, '') FROM workflow_runs ORDER BY id", "SELECT id, status || '|' || last_history_sequence::text || '|' || COALESCE(updated_at::text, '') FROM workflow_runs ORDER BY id", "SELECT id, CONCAT(status, '|', last_history_sequence, '|', COALESCE(CAST(updated_at AS CHAR), '')) FROM workflow_runs ORDER BY id").await; @@ -192,7 +157,7 @@ async fn durable_state(database: &TestDatabase) -> (Vec<(String, i64)>, Vec<(Str /// The stored history of a run, in order: each event's type and payload. async fn stored_history(database: &TestDatabase, run_id: &str) -> Vec<(String, String)> { - texts(database, + text_rows(database, &format!("SELECT event_type, payload FROM workflow_history_events WHERE workflow_run_id='{run_id}' ORDER BY sequence"), &format!("SELECT event_type, payload::text FROM workflow_history_events WHERE workflow_run_id='{run_id}' ORDER BY sequence"), &format!("SELECT event_type, CAST(payload AS CHAR) FROM workflow_history_events WHERE workflow_run_id='{run_id}' ORDER BY sequence")).await From d4fe36e4037d9cb67414015bb2ab33e47bfc1e85 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Sun, 25 Oct 2026 23:47:53 +0000 Subject: [PATCH 8/9] Rust #325: page summary history by the run-and-sequence index on MySQL The byte matrix's large completion (a ~270 KB argument and 20 heartbeats) failed on MySQL 8.0 with "Out of sort memory": the run summary projection pages history with SELECT ... CAST(payload AS CHAR) ... ORDER BY sequence LIMIT 16, and MySQL may answer it with a filesort that carries a whole row; one large payload exceeds the default 256 KiB sort buffer and fails the completion with 503 storage_unavailable. The query now forces the workflow_run_id/sequence index, as the history page query already does. Co-Authored-By: Claude Opus 5.5 --- rust/src/runtime/backend.rs | 5 ++++- rust/tests/execution/completion_budget.rs | 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/rust/src/runtime/backend.rs b/rust/src/runtime/backend.rs index 78cfbae9c..28e8fb869 100644 --- a/rust/src/runtime/backend.rs +++ b/rust/src/runtime/backend.rs @@ -444,7 +444,10 @@ impl Backend for MySql { const ACTIVITY_METADATA_HISTORY: &str = "SELECT h.sequence,h.event_type,CASE WHEN page.projection_bytes<=8388608 THEN h.payload ELSE NULL END AS payload,h.workflow_task_id,h.recorded_at,page.projection_count FROM (SELECT id,sequence,COUNT(*) OVER () AS projection_count,SUM(payload_bytes) OVER (ORDER BY sequence) AS projection_bytes FROM (SELECT id,sequence,octet_length(CAST(payload AS CHAR CHARACTER SET utf8mb4)) AS payload_bytes FROM workflow_history_events FORCE INDEX (workflow_history_events_workflow_run_id_sequence_unique) WHERE workflow_run_id=$1 AND event_type IN ('ActivityScheduled','ActivityStarted','ActivityHeartbeatRecorded','ActivityRetryScheduled','ActivityCompleted','ActivityFailed','ActivityTimedOut','ActivityCancelled')) AS sizes ORDER BY sequence LIMIT 10001) AS page JOIN workflow_history_events h ON h.id=page.id ORDER BY page.sequence"; const PHP_CONNECTION: &str = "mysql"; const OPEN_WAIT_ORDER: &str = "WITH clocks AS (SELECT id,wait_id,opened_at,TIMESTAMPDIFF(MICROSECOND,'1970-01-01 00:00:00',opened_at) AS micros FROM workflow_run_waits WHERE workflow_run_id=? AND status='open') SELECT id FROM clocks ORDER BY CASE WHEN opened_at IS NULL THEN 1 ELSE 0 END,ROUND((CASE WHEN micros<0 THEN 2*FLOOR(micros/1000000)*1000000-micros ELSE micros END)/1000),wait_id LIMIT 100 OFFSET ?"; - const SUMMARY_HISTORY_SQL: &str = "SELECT id,sequence,event_type,CAST(payload AS CHAR) AS payload_json FROM workflow_history_events WHERE workflow_run_id=$1 AND sequence>$2 ORDER BY sequence LIMIT 16"; + // The run-and-sequence index keeps MySQL from sorting whole rows: a + // filesort of one large payload exceeds the default sort buffer and + // fails the transaction ("Out of sort memory"). + const SUMMARY_HISTORY_SQL: &str = "SELECT id,sequence,event_type,CAST(payload AS CHAR) AS payload_json FROM workflow_history_events FORCE INDEX (workflow_history_events_workflow_run_id_sequence_unique) WHERE workflow_run_id=$1 AND sequence>$2 ORDER BY sequence LIMIT 16"; const OBSERVATION_FAILURE_EVENT: &str = "SELECT id,sequence,event_type,recorded_at FROM workflow_history_events WHERE workflow_run_id=? AND CAST(JSON_UNQUOTE(JSON_EXTRACT(payload,'$.failure_id')) AS BINARY)=CAST(? AS BINARY) AND event_type IN ('ActivityFailed','ActivityTimedOut','ChildRunFailed','ChildRunCancelled','ChildRunTerminated','WorkflowFailed','WorkflowTimedOut','WorkflowCancelled','WorkflowTerminated','UpdateCompleted') ORDER BY sequence DESC LIMIT 1"; const OBSERVATION_ATTRIBUTES: &str = "SELECT `key` AS attribute_key,type,value_string,value_keyword,value_int,value_float,CASE WHEN value_bool IS NULL THEN NULL WHEN value_bool THEN 1 ELSE 0 END AS value_bool,value_datetime,value_keyword_list FROM workflow_search_attributes WHERE workflow_run_id=$1 AND workflow_instance_id=$2 AND `key` IN ($3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,$22) ORDER BY `key` LIMIT 20"; fn optional_float(row: &Self::Row, field: &str) -> Result> { diff --git a/rust/tests/execution/completion_budget.rs b/rust/tests/execution/completion_budget.rs index 4ed851f36..6fee77811 100644 --- a/rust/tests/execution/completion_budget.rs +++ b/rust/tests/execution/completion_budget.rs @@ -160,7 +160,8 @@ async fn stored_history(database: &TestDatabase, run_id: &str) -> Vec<(String, S text_rows(database, &format!("SELECT event_type, payload FROM workflow_history_events WHERE workflow_run_id='{run_id}' ORDER BY sequence"), &format!("SELECT event_type, payload::text FROM workflow_history_events WHERE workflow_run_id='{run_id}' ORDER BY sequence"), - &format!("SELECT event_type, CAST(payload AS CHAR) FROM workflow_history_events WHERE workflow_run_id='{run_id}' ORDER BY sequence")).await + &format!("SELECT event_type, CAST(payload AS CHAR) FROM workflow_history_events \ + FORCE INDEX (workflow_history_events_workflow_run_id_sequence_unique) WHERE workflow_run_id='{run_id}' ORDER BY sequence")).await } /// The bytes the meter charges for stored events: each event's type and From 5f4fd3fca4ca26b7b4ed632ceae17242ec4db8a1 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Mon, 26 Oct 2026 00:33:35 +0000 Subject: [PATCH 9/9] Rust #325: charge a terminal event's worker attribution A completed or failed run's terminal event is recorded, charged, and then rewritten to name the worker that completed the task. That growth was never charged, so a completion admitted at the exact byte limit could store more history than the limit allows. The attribution's growth is now charged before the update, measured on the same stored payload before and after, so nothing is charged twice. Controls for WorkflowCompleted and WorkflowFailed admit the stored terminal history at its exact size and refuse it one byte short; both fail without the charge. Co-Authored-By: Claude Opus 5.5 --- rust/src/runtime/history_meter.rs | 6 ++++ rust/src/runtime/store.rs | 4 +++ rust/src/runtime/transaction.rs | 8 +++++ rust/tests/execution/completion_budget.rs | 38 +++++++++++++++++++++++ 4 files changed, 56 insertions(+) diff --git a/rust/src/runtime/history_meter.rs b/rust/src/runtime/history_meter.rs index 7d8c3e5f6..93355685d 100644 --- a/rust/src/runtime/history_meter.rs +++ b/rust/src/runtime/history_meter.rs @@ -49,6 +49,12 @@ impl Meter { self.charge(2, rows) } + /// History bytes an already charged event grows by before it is + /// rewritten, charged before the update. + pub(super) fn charge_bytes(&mut self, bytes: usize) -> Result<(), RuntimeError> { + self.charge(1, bytes) + } + fn charge(&mut self, dimension: usize, amount: usize) -> Result<(), RuntimeError> { if self.tripped { return Err(self.refused()); diff --git a/rust/src/runtime/store.rs b/rust/src/runtime/store.rs index 2bd7292b1..595a00881 100644 --- a/rust/src/runtime/store.rs +++ b/rust/src/runtime/store.rs @@ -3984,7 +3984,11 @@ where return Ok(()); }; let mut payload = DB::document_row(&event, "payload")?; + let recorded = super::history_meter::php_json_len(&payload); super::command_context::worker_terminal(&mut payload, &DB::string(&event, "event_type")?, worker); + // The event was charged as recorded; the attribution's growth is + // charged before the update (Workflow #930). + tx.charge_bytes(super::history_meter::php_json_len(&payload).saturating_sub(recorded))?; Self::query("UPDATE workflow_history_events SET payload=$1 WHERE id=$2") .bind(DB::document(&payload)) .bind(DB::string(&event, "id")?) diff --git a/rust/src/runtime/transaction.rs b/rust/src/runtime/transaction.rs index eae71126d..c38433823 100644 --- a/rust/src/runtime/transaction.rs +++ b/rust/src/runtime/transaction.rs @@ -39,6 +39,14 @@ impl<'a, DB: Backend> Transaction<'a, DB> { } } + /// History bytes a stored event grows by, charged to a metered completion. + pub(super) fn charge_bytes(&mut self, bytes: usize) -> Result<()> { + match self.meter.as_mut() { + Some(meter) => meter.charge_bytes(bytes), + None => Ok(()), + } + } + /// One history event about to be stored, charged to a metered completion. pub(super) fn charge_event(&mut self, event_type: &str, payload: &serde_json::Value) -> Result<()> { match self.meter.as_mut() { diff --git a/rust/tests/execution/completion_budget.rs b/rust/tests/execution/completion_budget.rs index 6fee77811..be9cc591f 100644 --- a/rust/tests/execution/completion_budget.rs +++ b/rust/tests/execution/completion_budget.rs @@ -238,3 +238,41 @@ async fn the_byte_limit_admits_its_exact_value_and_refuses_one_more() { admits_exactly(&database, "bytes", only(None, Some(bytes), None), only(None, Some(bytes - 1), None), commands, "history_transaction_limit_exceeded").await; } + +/// A terminal event is rewritten after it is recorded, to name the worker +/// that completed or failed the run; that growth is charged too, so the +/// stored terminal history is admitted at its exact size and refused one +/// byte short. +async fn terminal_admits_exactly(database: &TestDatabase, tag: &str, terminal: Value) { + let commands = json!([large(5), terminal]); + let probe = budgeted(database, only(None, None, None)).await; + local_worker(&probe).await; + let run = start(&probe, &format!("{tag}-probe")).await; + let run_id = run["run_id"].as_str().unwrap().to_owned(); + let task = poll(&probe, "local", "workflow").await; + let before = stored_history(database, &run_id).await; + let (status, body) = complete(&probe, &task, commands.clone()).await; + assert_eq!(status, StatusCode::OK, "{body}"); + let after = stored_history(database, &run_id).await; + let terminal = after.last().unwrap(); + // The stored terminal event names the worker. + let payload: Value = serde_json::from_str(&terminal.1).unwrap(); + assert_eq!(payload["command"]["source"], "worker_protocol", "{payload}"); + assert!(payload["command"]["principal_id"].is_string(), "{payload}"); + let bytes = charged_bytes(&after[before.len()..]); + + admits_exactly(database, tag, only(None, Some(bytes), None), only(None, Some(bytes - 1), None), + commands, "history_transaction_limit_exceeded").await; +} + +#[tokio::test] +async fn a_completed_run_s_attributed_terminal_event_is_charged() { + let database = TestDatabase::new().await; + terminal_admits_exactly(&database, "done", json!({"type": "complete_workflow", "result": envelope(Payload::Long(1))})).await; +} + +#[tokio::test] +async fn a_failed_run_s_attributed_terminal_event_is_charged() { + let database = TestDatabase::new().await; + terminal_admits_exactly(&database, "fail", json!({"type": "fail_workflow", "message": "boom"})).await; +}