From a11d164b6304555441890a4f31524b3cabeb5996 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Sat, 24 Oct 2026 12:03:41 +0000 Subject: [PATCH 01/13] Server parity: worker sessions lifecycle fixture, recorded from published 2.11.1 Fixture first for #736. In its own namespace, with three workers on two queues, the probe records the five session routes: - the protocol gates (no header, 1.7, 1.21, 1.8 reaching validation), an unknown namespace on both planes, and the control-plane header; - validation of create, heartbeat and close; - registration, queue, requirements and create_if_missing refusals; - create with a padded ID, duplicate requirements and a string lease; a float lease; reuse resetting omitted options without extending the TTL deadline; - another worker's create, heartbeat and close; heartbeat of a missing session and a lease renewed from now; the per-worker session limit; - lease expiry applied on read; reacquire by another worker with no options; close by the old owner, by query worker_id, again, by another worker; heartbeat and create of a closed session, and requirements refused before the closed lookup; - the stored and the requested reacquire refusals; TTL expiry refusing heartbeat and create while the owner can still close; - a deregistered holder orphaned on read, refused on create, and the session reacquired by another worker; - a missing session's visibility, and the namespace list and metrics. Session times are whole seconds, so the probe records them as spans and presence flags; the list is compared by session ID. Three published recordings are identical, and published 2.11.1 and embedded pass. One published defect is bounded: renewHeldSession writes failure_reason => null through array_filter(... !== null), which drops it, so a reacquired session keeps its old failure_reason until a heartbeat or close. The contract accepts the published value or the intended null at exactly those paths. The unit test runs the recording, the corrected variant, and ten refused mutations, including native's 501. --- scripts/conformance/server-parity.mjs | 2 +- .../conformance/server-parity/contract.mjs | 3 + scripts/conformance/server-parity/probe.php | 6 + .../worker-sessions-contract.mjs | 38 + .../server-parity/worker-sessions-probe.php | 214 ++ .../ServerParity/worker-sessions.json | 1735 +++++++++++++++++ .../worker-sessions-observation.json | 1699 ++++++++++++++++ ...ServerParityWorkerSessionsContractTest.mjs | 72 + 8 files changed, 3768 insertions(+), 1 deletion(-) create mode 100644 scripts/conformance/server-parity/worker-sessions-contract.mjs create mode 100644 scripts/conformance/server-parity/worker-sessions-probe.php create mode 100644 tests/Fixtures/ServerParity/worker-sessions.json create mode 100644 tests/Fixtures/ServerParityExpected/worker-sessions-observation.json create mode 100644 tests/Unit/ServerParityWorkerSessionsContractTest.mjs diff --git a/scripts/conformance/server-parity.mjs b/scripts/conformance/server-parity.mjs index be069ce5a..d8537cb65 100644 --- a/scripts/conformance/server-parity.mjs +++ b/scripts/conformance/server-parity.mjs @@ -103,7 +103,7 @@ try { for (const flag of ['url', 'application-root']) { if (values[flag]) arguments_.push(`--${flag}`, values[flag]); } - const probe = captureProbe('php', arguments_, {directory: dirname(resolve(values.output)), timeout: fixture.memo_replay ? fixture.memo_replay.overall_budget_seconds * 1000 + 5000 : fixture.corrected_markers ? fixture.corrected_markers.overall_budget_seconds * 1000 + 5000 : fixture.namespace_signal_pair || fixture.selected_run_signal_pair ? 75000 : fixture.update_validation ? 90000 : fixture.worker_poll_fence ? fixture.worker_poll_fence.overall_budget_seconds * 1000 + 5000 : 45000, + const probe = captureProbe('php', arguments_, {directory: dirname(resolve(values.output)), timeout: fixture.memo_replay ? fixture.memo_replay.overall_budget_seconds * 1000 + 5000 : fixture.corrected_markers ? fixture.corrected_markers.overall_budget_seconds * 1000 + 5000 : fixture.namespace_signal_pair || fixture.selected_run_signal_pair ? 75000 : fixture.update_validation ? 90000 : fixture.worker_poll_fence ? fixture.worker_poll_fence.overall_budget_seconds * 1000 + 5000 : fixture.worker_sessions ? fixture.worker_sessions.overall_budget_seconds * 1000 + 5000 : 45000, env: {...process.env, DW_PARITY_FAILURE_EVIDENCE_DIR: dirname(resolve(values.output))}, preserveStdout: join(sourceDirectory, name)}); const item = {fixture_id: fixture.id, fixture, outcome: 'runner-blocked'}; diff --git a/scripts/conformance/server-parity/contract.mjs b/scripts/conformance/server-parity/contract.mjs index 676040e17..b57439345 100644 --- a/scripts/conformance/server-parity/contract.mjs +++ b/scripts/conformance/server-parity/contract.mjs @@ -15,6 +15,7 @@ import {checkStorageTest} from './storage-test-contract.mjs'; import {checkActivityLeaseRenewal} from './activity-lease-renewal-contract.mjs'; import {checkWorkflowPollRequest} from './workflow-poll-request-contract.mjs'; import {checkWorkerPollFence} from './worker-poll-fence-contract.mjs'; +import {checkWorkerSessions} from './worker-sessions-contract.mjs'; import {checkServiceCatalog} from './service-catalog-contract.mjs'; import {checkSystemMetrics} from './system-metrics-contract.mjs'; import {checkOperatorDashboardTypes} from './operator-dashboard-types-contract.mjs'; @@ -156,6 +157,7 @@ export function checkObservation(fixture, observation, workflowId, artifacts) { const activityLeaseRenewal = fixture.activity_lease_renewal ? checkActivityLeaseRenewal(fixture, observation, workflowId) : null; const workflowPollRequest = fixture.workflow_poll_request ? checkWorkflowPollRequest(fixture, observation, workflowId) : null; const workerPollFence = fixture.worker_poll_fence ? checkWorkerPollFence(fixture, observation, workflowId) : null; + const workerSessions = fixture.worker_sessions ? checkWorkerSessions(fixture, observation, workflowId) : null; const serviceCatalog = fixture.service_catalog ? checkServiceCatalog(fixture, observation, workflowId) : null; const systemMetrics = fixture.system_metrics ? checkSystemMetrics(fixture, observation, workflowId) : null; const operatorDashboardTypes = fixture.operator_dashboard_types ? checkOperatorDashboardTypes(fixture, observation, workflowId) : null; @@ -714,6 +716,7 @@ export function checkObservation(fixture, observation, workflowId, artifacts) { ...(activityLeaseRenewal ? {activity_lease_renewal: activityLeaseRenewal} : {}), ...(workflowPollRequest ? {workflow_poll_request: workflowPollRequest} : {}), ...(workerPollFence ? {worker_poll_fence: workerPollFence} : {}), + ...(workerSessions ? {worker_sessions: workerSessions} : {}), ...(serviceCatalog ? {service_catalog: serviceCatalog} : {}), ...(systemMetrics ? {system_metrics: systemMetrics} : {}), ...(operatorDashboardTypes ? {operator_dashboard_types: operatorDashboardTypes} : {}), diff --git a/scripts/conformance/server-parity/probe.php b/scripts/conformance/server-parity/probe.php index ac9007ca5..519d30bf9 100644 --- a/scripts/conformance/server-parity/probe.php +++ b/scripts/conformance/server-parity/probe.php @@ -433,6 +433,7 @@ function httpObservation(array $fixture, string $workflowId, string $namespace, $activityLeaseRenewal = isset($fixture['activity_lease_renewal']) ? finishHttpActivityLeaseRenewal($client, $fixture, $workflowId, $url) : null; $workflowPollRequest = isset($fixture['workflow_poll_request']) ? finishHttpWorkflowPollRequest($client, $fixture, $workflowId, $url) : null; $workerPollFence = isset($fixture['worker_poll_fence']) ? finishHttpWorkerPollFence($client, $fixture, $workflowId, $url) : null; + $workerSessions = isset($fixture['worker_sessions']) ? finishHttpWorkerSessions($client, $fixture, $workflowId, $url) : null; $serviceCatalog = isset($fixture['service_catalog']) ? finishHttpServiceCatalog($client, $fixture, $workflowId, $url) : null; $systemMetrics = isset($fixture['system_metrics']) ? finishHttpSystemMetrics($client, $fixture, $workflowId, $url) : null; $operatorDashboardTypes = isset($fixture['operator_dashboard_types']) ? finishHttpOperatorDashboardTypes($client, $fixture, $workflowId, $url) : null; @@ -544,6 +545,7 @@ function httpObservation(array $fixture, string $workflowId, string $namespace, 'activity_lease_renewal' => $activityLeaseRenewal, 'workflow_poll_request' => $workflowPollRequest, 'worker_poll_fence' => $workerPollFence, + 'worker_sessions' => $workerSessions, 'service_catalog' => $serviceCatalog, 'system_metrics' => $systemMetrics, 'operator_dashboard_types' => $operatorDashboardTypes, @@ -859,6 +861,7 @@ function embeddedObservation(array $fixture, string $workflowId, string $namespa require __DIR__.'/activity-lease-renewal-probe.php'; require __DIR__.'/workflow-poll-request-probe.php'; require __DIR__.'/worker-poll-fence-probe.php'; + require __DIR__.'/worker-sessions-probe.php'; require __DIR__.'/service-catalog-probe.php'; require __DIR__.'/system-metrics-probe.php'; require __DIR__.'/operator-dashboard-types-probe.php'; @@ -1035,6 +1038,9 @@ function embeddedObservation(array $fixture, string $workflowId, string $namespa if ($mode === 'embedded' && isset($fixture['worker_poll_fence'])) { $observation['worker_poll_fence'] = ['applicable' => false, 'reason' => 'embedded_has_no_worker_protocol']; } + if ($mode === 'embedded' && isset($fixture['worker_sessions'])) { + $observation['worker_sessions'] = ['applicable' => false, 'reason' => 'embedded_has_no_worker_protocol']; + } if ($mode === 'embedded' && isset($fixture['service_catalog'])) { $observation['service_catalog'] = ['applicable' => false, 'reason' => 'embedded_has_no_http_service_catalog']; } diff --git a/scripts/conformance/server-parity/worker-sessions-contract.mjs b/scripts/conformance/server-parity/worker-sessions-contract.mjs new file mode 100644 index 000000000..2a0c858b4 --- /dev/null +++ b/scripts/conformance/server-parity/worker-sessions-contract.mjs @@ -0,0 +1,38 @@ +import assert from 'node:assert/strict'; + +// Reviewed worker sessions (WorkerSessionController, WorkerSessionRegistry), +// recorded against published Server 2.11.1. The fixture's `expected` holds +// the whole normalized observation, with the probe's workflow prefix as +// `{id}`; session times are spans and presence flags, since PHP keeps them in +// whole seconds, and the list is compared by session ID because ties within +// a second order by ID. Every step compares exactly. +// +// Published defect boundary: a reacquired session keeps its previous +// failure_reason, because renewHeldSession filters its own `failure_reason => +// null` out (fixture `published_defects`). At those paths the published value +// or the intended null is accepted; nothing else is relaxed. +export function checkWorkerSessions(fixture, observation, workflowId) { + const state = observation.worker_sessions; + const projection = {scope: 'worker_sessions'}; + assert.ok(state && typeof state === 'object', 'actual worker sessions observation'); + if (observation.mode === 'embedded') { + assert.deepStrictEqual(state, {applicable: false, reason: 'embedded_has_no_worker_protocol'}, + 'embedded worker sessions explicitly inapplicable'); + return projection; + } + const definition = fixture.worker_sessions; + const actual = JSON.parse(JSON.stringify(state).replaceAll(workflowId, '{id}')); + const defect = definition.published_defects.reacquired_failure_reason; + const tolerate = (session, published, label) => { + assert.ok(session && typeof session === 'object', `${label}: a session`); + if (session.failure_reason === published) session.failure_reason = null; + }; + tolerate(actual.reacquired?.response?.session, defect.published[0], 'reacquired'); + tolerate(actual.orphan_reacquired?.response?.session, defect.published[1], 'orphan reacquired'); + tolerate((actual.list_sessions ?? []).find(session => session.session_id === '{id}-wss-held'), defect.published[2], 'listed held session'); + for (const [step, expected] of Object.entries(definition.expected)) { + assert.deepStrictEqual(actual[step], expected, `worker sessions ${step}`); + } + assert.deepStrictEqual(Object.keys(actual).sort(), Object.keys(definition.expected).sort(), 'worker sessions steps'); + return projection; +} diff --git a/scripts/conformance/server-parity/worker-sessions-probe.php b/scripts/conformance/server-parity/worker-sessions-probe.php new file mode 100644 index 000000000..5b093c7be --- /dev/null +++ b/scripts/conformance/server-parity/worker-sessions-probe.php @@ -0,0 +1,214 @@ +createNamespace($namespace, 'worker sessions λ', 30); + $token = getenv('DW_PARITY_TOKEN') ?: null; + $stack = HandlerStack::create(); + $stack->push(LockPressurePollRetries::middleware(), 'lock_pressure_poll_retry'); + $http = new WorkerSessionHttp(['base_uri' => rtrim($url, '/'), 'http_errors' => false, + 'connect_timeout' => 1, 'timeout' => 30, 'handler' => $stack]); + // A request on the worker plane (protocol header, null for none) or the + // control plane, in this namespace unless another is named. + $raw = static function (string $method, string $path, ?array $body = null, ?string $protocol = '1.20', + bool $control = false, ?string $as = null, ?string $json = null) use ($http, $token, $namespace): array { + $headers = array_filter(['Authorization' => $token === null ? null : 'Bearer '.$token, + 'Accept' => 'application/json', 'X-Namespace' => $as ?? $namespace, + 'X-Durable-Workflow-Protocol-Version' => $control ? null : $protocol, + 'X-Durable-Workflow-Control-Plane-Version' => $control ? '2' : null, + 'Content-Type' => $json === null ? null : 'application/json']); + $payload = $json !== null ? ['body' => $json] : ($body === null ? [] : ['json' => $body]); + $response = $http->request($method, $path, ['headers' => $headers] + $payload); + $decoded = json_decode((string) $response->getBody(), true); + if (is_array($decoded)) { + unset($decoded['server_capabilities'], $decoded['control_plane']); + } + + return ['status' => $response->getStatusCode(), 'response' => $decoded]; + }; + $at = static fn (?string $value): ?int => $value === null ? null : (new DateTimeImmutable($value))->getTimestamp(); + // A session snapshot with its times as spans and presence flags. + $snapshot = static function (array $session) use ($at): array { + $times = ['lease_expires_at', 'ttl_expires_at', 'closed_at', 'last_heartbeat_at', 'created_at', 'updated_at']; + $plain = array_diff_key($session, array_flip($times)); + if (is_array($plain['holder'] ?? null)) { + $plain['holder']['last_heartbeat_at'] = $plain['holder']['last_heartbeat_at'] === null ? null : 'present'; + } + $heartbeat = $at($session['last_heartbeat_at'] ?? null); + + return $plain + [ + 'lease_span' => isset($session['lease_expires_at']) && $heartbeat !== null ? $at($session['lease_expires_at']) - $heartbeat : null, + 'ttl_span' => isset($session['ttl_expires_at'], $session['created_at']) ? $at($session['ttl_expires_at']) - $at($session['created_at']) : null, + 'closed' => ($session['closed_at'] ?? null) !== null, + 'times' => array_map(static fn (string $key): string => ($session[$key] ?? null) === null ? 'null' : 'present', array_combine($times, $times)), + ]; + }; + $answer = static function (array $recorded) use ($snapshot): array { + $body = $recorded['response']; + if (is_array($body)) { + if (is_array($body['session'] ?? null)) { + $body['session'] = $snapshot($body['session']); + } + if (is_array($body['sessions'] ?? null)) { + $body['sessions'] = array_map($snapshot, $body['sessions']); + } + } + + return ['status' => $recorded['status'], 'response' => $body]; + }; + $ok = static function (array $recorded, int $status, string $operation): array { + if ($recorded['status'] !== $status) { + throw new RuntimeException('Worker sessions '.$operation.': '.json_encode($recorded)); + } + + return $recorded['response']; + }; + $id = static fn (string $name): string => $workflowId.'-'.$name; + $register = static function (string $worker, string $queue, array $capabilities = [], ?int $sessions = null) use ($raw, $ok, $definition): void { + $ok($raw('POST', '/api/worker/register', ['worker_id' => $worker, 'task_queue' => $queue, 'runtime' => 'php', + 'sdk_version' => 'raw-http', 'supported_workflow_types' => [$definition['workflow_type']], + 'supported_activity_types' => [$definition['activity_type']], 'capabilities' => $capabilities, + 'capability_manifest' => array_fill_keys(['local_activities', 'worker_sessions', 'sticky_execution'], + ['supported' => false, 'minimum_protocol_version' => '1.18', 'reason' => 'raw_http'])] + + ($sessions === null ? [] : ['max_concurrent_worker_sessions' => $sessions])), 201, 'registration '.$worker); + }; + $beat = static function (string ...$workers) use ($raw): void { + foreach ($workers as $worker) { + $raw('POST', '/api/worker/heartbeat', ['worker_id' => $worker]); + } + }; + $create = static fn (array $body, ?string $protocol = '1.20'): array => $raw('POST', '/api/worker/sessions', $body, $protocol); + $heartbeat = static fn (string $session, array $body): array => $raw('POST', '/api/worker/sessions/'.rawurlencode($session).'/heartbeat', $body); + $close = static fn (string $session, array $body = [], string $query = ''): array => $raw('DELETE', + '/api/worker/sessions/'.rawurlencode($session).$query, $body === [] ? null : $body); + $show = static fn (string $session): array => $raw('GET', '/api/worker-sessions/'.rawurlencode($session), null, null, true); + + [$w1, $w2, $w3] = [$id('wss-w1'), $id('wss-w2'), $id('wss-w3')]; + [$q1, $q2] = [$id('wss-q1'), $id('wss-q2')]; + $register($w1, $q1, ['gpu'], 2); + $register($w2, $q1); + $register($w3, $q2); + [$s1, $s2, $s3] = [$id('wss-s1'), $id('wss-s2'), $id('wss-s3')]; + $state = ['applicable' => true, 'namespace' => $namespace, 'workers' => [$w1, $w2, $w3], 'queues' => [$q1, $q2]]; + + // 1. Gates. + $state['gate_no_protocol'] = $answer($create(['worker_id' => $w1, 'session_id' => $s1], null)); + $state['gate_protocol_1_7'] = $answer($create(['worker_id' => $w1, 'session_id' => $s1], '1.7')); + $state['gate_protocol_1_21'] = $answer($create(['worker_id' => $w1, 'session_id' => $s1], '1.21')); + $state['gate_protocol_1_8_empty'] = $answer($create([], '1.8')); + $state['gate_unknown_namespace'] = $answer($raw('POST', '/api/worker/sessions', ['worker_id' => $w1, 'session_id' => $s1], '1.20', false, 'no-such-namespace-wss')); + $state['gate_list_without_control_plane'] = $answer($raw('GET', '/api/worker-sessions', null, '1.20')); + $state['gate_list_unknown_namespace'] = $answer($raw('GET', '/api/worker-sessions', null, null, true, 'no-such-namespace-wss')); + + // 2. Validation. + $state['invalid_empty'] = $answer($create([])); + $state['invalid_types'] = $answer($create(['worker_id' => 5, 'session_id' => ' ', 'requirements' => 'x', 'lease_seconds' => 0, + 'ttl_seconds' => 'a', 'max_concurrent_activities' => 1.5, 'create_if_missing' => 'yes', 'allow_reacquire_after_failure' => 2, + 'queue' => [], 'connection' => 7])); + $state['invalid_heartbeat'] = $answer($heartbeat($s1, ['lease_seconds' => 0])); + $state['invalid_close'] = $answer($close($s1)); + + // 3. Registration and routing. + $state['unregistered'] = $answer($create(['worker_id' => $id('wss-nobody'), 'session_id' => $s1])); + $state['queue_mismatch'] = $answer($create(['worker_id' => $w1, 'session_id' => $s1, 'queue' => $q2])); + $state['requirements_missing'] = $answer($create(['worker_id' => $w1, 'session_id' => $s1, 'requirements' => ['gpu', 'tpu', ' tpu ', 'fpga']])); + $state['missing_without_create'] = $answer($create(['worker_id' => $w1, 'session_id' => $s1, 'create_if_missing' => false])); + + // 4-6. Create, a float lease, reuse. + $state['created'] = $answer($create(['worker_id' => $w1, 'session_id' => ' '.$s1.' ', 'requirements' => ['gpu', ' gpu '], + 'lease_seconds' => '60', 'ttl_seconds' => 600, 'connection' => 'c1'])); + // A float lease passes validation as an integer; sent as JSON 7.0, since + // json_encode writes 7.0 as 7. + $state['float_lease'] = $answer($raw('POST', '/api/worker/sessions', [], '1.20', false, null, + json_encode(['worker_id' => $w1, 'session_id' => $s2, 'lease_seconds' => 7.0], JSON_PRESERVE_ZERO_FRACTION))); + $state['reused'] = $answer($create(['worker_id' => $w1, 'session_id' => $s1, 'ttl_seconds' => 9999, 'lease_seconds' => 30, + 'max_concurrent_activities' => 3, 'create_if_missing' => false])); + + // 7. Another worker. + $state['foreign_create'] = $answer($create(['worker_id' => $w2, 'session_id' => $s1])); + $state['foreign_heartbeat'] = $answer($heartbeat($s1, ['worker_id' => $w2])); + $state['foreign_close'] = $answer($close($s1, ['worker_id' => $w2])); + + // 8. Heartbeat. + $state['heartbeat_missing'] = $answer($heartbeat($id('wss-none'), ['worker_id' => $w1])); + $state['heartbeat'] = $answer($heartbeat($s1, ['worker_id' => $w1, 'lease_seconds' => 5])); + + // 9. The limit: S1 and S2 are held, so a third is refused. + $state['limit'] = $answer($create(['worker_id' => $w1, 'session_id' => $s3])); + + // 10. Lease expiry, on read. S1's lease is 5 seconds from the heartbeat. + $beat($w1, $w2, $w3); + $short = $id('wss-short'); + $state['short_created'] = $answer($create(['worker_id' => $w2, 'session_id' => $short, 'lease_seconds' => 2])); + usleep(3200000); + $state['short_expired'] = $answer($show($short)); + + // 11. Reacquire by another worker, with no options. + $state['reacquired'] = $answer($create(['worker_id' => $w3, 'session_id' => $short])); + $state['reacquired_heartbeat'] = $answer($heartbeat($short, ['worker_id' => $w3])); + + // 12. Close. + $state['close_by_old_owner'] = $answer($close($short, ['worker_id' => $w2])); + $state['closed'] = $answer($close($short, [], '?worker_id='.rawurlencode($w3))); + $state['closed_again'] = $answer($close($short, ['worker_id' => $w3])); + $state['closed_by_other'] = $answer($close($short, ['worker_id' => $w2])); + $state['closed_heartbeat'] = $answer($heartbeat($short, ['worker_id' => $w3])); + $state['closed_create'] = $answer($create(['worker_id' => $w3, 'session_id' => $short])); + $state['closed_create_unmet'] = $answer($create(['worker_id' => $w3, 'session_id' => $short, 'requirements' => ['gpu']])); + + // 13. Reacquire refused, by the stored option and by the request's. + $beat($w1, $w2, $w3); + [$terminal, $optional] = [$id('wss-terminal'), $id('wss-optional')]; + $create(['worker_id' => $w2, 'session_id' => $terminal, 'lease_seconds' => 2, 'allow_reacquire_after_failure' => false]); + $create(['worker_id' => $w2, 'session_id' => $optional, 'lease_seconds' => 2]); + usleep(3200000); + $state['reacquire_terminal'] = $answer($create(['worker_id' => $w3, 'session_id' => $terminal])); + $state['reacquire_disallowed'] = $answer($create(['worker_id' => $w3, 'session_id' => $optional, 'allow_reacquire_after_failure' => false])); + + // 14. TTL expiry is terminal; the owner can still close. + $beat($w1, $w2, $w3); + $ttl = $id('wss-ttl'); + $create(['worker_id' => $w2, 'session_id' => $ttl, 'ttl_seconds' => 2, 'lease_seconds' => 60]); + usleep(3200000); + $state['ttl_heartbeat'] = $answer($heartbeat($ttl, ['worker_id' => $w2])); + $state['ttl_create'] = $answer($create(['worker_id' => $w2, 'session_id' => $ttl])); + $state['ttl_close'] = $answer($close($ttl, ['worker_id' => $w2])); + + // 15. A holder deregistered: orphaned on read, then reacquired. + $beat($w1, $w2, $w3); + $held = $id('wss-held'); + $state['held_created'] = $answer($create(['worker_id' => $w3, 'session_id' => $held])); + $state['holder_deregistered'] = $raw('DELETE', '/api/worker/registrations/'.rawurlencode($w3))['status']; + $state['held_orphaned'] = $answer($show($held)); + $state['deregistered_create'] = $answer($create(['worker_id' => $w3, 'session_id' => $held])); + $state['orphan_reacquired'] = $answer($create(['worker_id' => $w2, 'session_id' => $held])); + $state['orphan_queue_mismatch'] = $answer($create(['worker_id' => $w2, 'session_id' => $held, 'queue' => $q2])); + + // Visibility: one session, and the namespace's list and metrics. + $state['show_missing'] = $answer($show($id('wss-none'))); + $list = $answer($raw('GET', '/api/worker-sessions', null, null, true)); + $state['list_metrics'] = $list['response']['metrics'] ?? null; + $state['list_namespace'] = $list['response']['namespace'] ?? null; + // Order ties within a second break by session ID; record membership. + $sessions = $list['response']['sessions'] ?? []; + usort($sessions, static fn (array $a, array $b): int => strcmp($a['session_id'], $b['session_id'])); + $state['list_sessions'] = $sessions; + + return $state; +} diff --git a/tests/Fixtures/ServerParity/worker-sessions.json b/tests/Fixtures/ServerParity/worker-sessions.json new file mode 100644 index 000000000..f55d91c45 --- /dev/null +++ b/tests/Fixtures/ServerParity/worker-sessions.json @@ -0,0 +1,1735 @@ +{ + "schema": "durable-workflow.server-parity-fixture/v1", + "id": "worker-sessions", + "review_state": "reviewed", + "workflow_type": "parity.v1.echo", + "input": {"message": "worker sessions λ", "count": 9007199254740993, "optional": null}, + "typed_value": {"type": "map", "value": { + "message": {"type": "string", "value": "worker sessions λ"}, + "count": {"type": "int64", "value": "9007199254740993"}, + "optional": {"type": "null", "value": null} + }}, + "activity": false, + "expected_events": ["StartAccepted", "WorkflowStarted", "WorkflowCompleted"], + "worker_sessions": { + "workflow_type": "parity.v1.sessions", + "activity_type": "parity.v1.session_activity", + "empty_blob": "wwHioz3/VYAiNwwA", + "overall_budget_seconds": 120, + "published_defects": { + "reacquired_failure_reason": { + "paths": [ + "reacquired.response.session.failure_reason", + "orphan_reacquired.response.session.failure_reason", + "list_sessions[{id}-wss-held].failure_reason" + ], + "published": [ + "lease_expired", + "worker_heartbeat_stale", + "worker_heartbeat_stale" + ], + "reason": "WorkerSessionRegistry::renewHeldSession writes failure_reason => null and closed_at => null through array_filter(... !== null), which drops both; a reacquired session stays active with the old failure_reason until a heartbeat or close clears it. Native clears it, as the code intends." + } + }, + "expected": { + "applicable": true, + "namespace": "{id}-wss", + "workers": [ + "{id}-wss-w1", + "{id}-wss-w2", + "{id}-wss-w3" + ], + "queues": [ + "{id}-wss-q1", + "{id}-wss-q2" + ], + "gate_no_protocol": { + "status": 400, + "response": { + "error": "Missing worker protocol version header.", + "reason": "missing_protocol_version", + "supported_version": "1.20", + "requested_version": null, + "remediation": "Send the X-Durable-Workflow-Protocol-Version: 1.20 header on worker protocol requests.", + "protocol_version": "1.20" + } + }, + "gate_protocol_1_7": { + "status": 409, + "response": { + "error": "Worker sessions require worker protocol 1.8 or newer.", + "reason": "worker_sessions_unavailable", + "supported_version": "1.20", + "requested_version": "1.7", + "minimum_protocol_version": "1.8", + "remediation": "Send the X-Durable-Workflow-Protocol-Version header with worker protocol 1.8 or newer and use a worker SDK that implements worker-session semantics.", + "protocol_version": "1.20" + } + }, + "gate_protocol_1_21": { + "status": 400, + "response": { + "error": "Unsupported worker protocol version.", + "reason": "unsupported_protocol_version", + "supported_version": "1.20", + "requested_version": "1.21", + "remediation": "Worker requested protocol version 1.21; this server supports 1.20. Workers may target any 1.x version with x ≤ 20. Upgrade the worker to a release that targets a compatible version, or connect to a server that matches.", + "protocol_version": "1.20" + } + }, + "gate_protocol_1_8_empty": { + "status": 422, + "response": { + "message": "The worker id field is required. (and 1 more error)", + "reason": "validation_failed", + "errors": { + "worker_id": [ + "The worker id field is required." + ], + "session_id": [ + "The session id field is required." + ] + }, + "validation_errors": { + "worker_id": [ + "The worker id field is required." + ], + "session_id": [ + "The session id field is required." + ] + }, + "protocol_version": "1.20" + } + }, + "gate_unknown_namespace": { + "status": 404, + "response": { + "message": "Namespace 'no-such-namespace-wss' does not exist.", + "reason": "namespace_not_found", + "namespace": "no-such-namespace-wss", + "remediation": "Register the namespace via POST /api/namespaces, or send an X-Namespace header naming an existing namespace.", + "protocol_version": "1.20" + } + }, + "gate_list_without_control_plane": { + "status": 400, + "response": { + "message": "Missing control-plane version header.", + "reason": "missing_control_plane_version", + "supported_version": "2", + "requested_version": null, + "remediation": "Send the X-Durable-Workflow-Control-Plane-Version: 2 header on control-plane requests." + } + }, + "gate_list_unknown_namespace": { + "status": 404, + "response": { + "message": "Namespace 'no-such-namespace-wss' does not exist.", + "reason": "namespace_not_found", + "namespace": "no-such-namespace-wss", + "remediation": "Register the namespace via POST /api/namespaces, or send an X-Namespace header naming an existing namespace." + } + }, + "invalid_empty": { + "status": 422, + "response": { + "message": "The worker id field is required. (and 1 more error)", + "reason": "validation_failed", + "errors": { + "worker_id": [ + "The worker id field is required." + ], + "session_id": [ + "The session id field is required." + ] + }, + "validation_errors": { + "worker_id": [ + "The worker id field is required." + ], + "session_id": [ + "The session id field is required." + ] + }, + "protocol_version": "1.20" + } + }, + "invalid_types": { + "status": 422, + "response": { + "message": "The worker id field must be a string. (and 9 more errors)", + "reason": "validation_failed", + "errors": { + "worker_id": [ + "The worker id field must be a string." + ], + "session_id": [ + "The session id field is required." + ], + "connection": [ + "The connection field must be a string." + ], + "queue": [ + "The queue field must be a string." + ], + "requirements": [ + "The requirements field must be an array." + ], + "lease_seconds": [ + "The lease seconds field must be at least 1." + ], + "ttl_seconds": [ + "The ttl seconds field must be an integer." + ], + "max_concurrent_activities": [ + "The max concurrent activities field must be an integer." + ], + "create_if_missing": [ + "The create if missing field must be true or false." + ], + "allow_reacquire_after_failure": [ + "The allow reacquire after failure field must be true or false." + ] + }, + "validation_errors": { + "worker_id": [ + "The worker id field must be a string." + ], + "session_id": [ + "The session id field is required." + ], + "connection": [ + "The connection field must be a string." + ], + "queue": [ + "The queue field must be a string." + ], + "requirements": [ + "The requirements field must be an array." + ], + "lease_seconds": [ + "The lease seconds field must be at least 1." + ], + "ttl_seconds": [ + "The ttl seconds field must be an integer." + ], + "max_concurrent_activities": [ + "The max concurrent activities field must be an integer." + ], + "create_if_missing": [ + "The create if missing field must be true or false." + ], + "allow_reacquire_after_failure": [ + "The allow reacquire after failure field must be true or false." + ] + }, + "protocol_version": "1.20" + } + }, + "invalid_heartbeat": { + "status": 422, + "response": { + "message": "The worker id field is required. (and 1 more error)", + "reason": "validation_failed", + "errors": { + "worker_id": [ + "The worker id field is required." + ], + "lease_seconds": [ + "The lease seconds field must be at least 1." + ] + }, + "validation_errors": { + "worker_id": [ + "The worker id field is required." + ], + "lease_seconds": [ + "The lease seconds field must be at least 1." + ] + }, + "protocol_version": "1.20" + } + }, + "invalid_close": { + "status": 422, + "response": { + "message": "The worker id field is required.", + "reason": "validation_failed", + "errors": { + "worker_id": [ + "The worker id field is required." + ] + }, + "validation_errors": { + "worker_id": [ + "The worker id field is required." + ] + }, + "protocol_version": "1.20" + } + }, + "unregistered": { + "status": 412, + "response": { + "error": "Worker must be registered before creating a worker session.", + "reason": "worker_not_registered", + "worker_id": "{id}-wss-nobody", + "protocol_version": "1.20" + } + }, + "queue_mismatch": { + "status": 409, + "response": { + "error": "Worker [{id}-wss-w1] is registered for task queue [{id}-wss-q1], not session queue [{id}-wss-q2].", + "reason": "session_queue_mismatch", + "worker_id": "{id}-wss-w1", + "registered_task_queue": "{id}-wss-q1", + "requested_session_queue": "{id}-wss-q2", + "protocol_version": "1.20" + } + }, + "requirements_missing": { + "status": 409, + "response": { + "missing_requirements": [ + "tpu", + "fpga" + ], + "admitted": false, + "error": "Worker does not advertise every worker-session requirement.", + "reason": "session_requirements_not_met", + "status": 409, + "protocol_version": "1.20" + } + }, + "missing_without_create": { + "status": 409, + "response": { + "admitted": false, + "error": "Worker session does not exist and creation is disabled.", + "reason": "session_missing", + "status": 409, + "protocol_version": "1.20" + } + }, + "created": { + "status": 201, + "response": { + "admitted": true, + "outcome": "created", + "session": { + "session_id": "{id}-wss-s1", + "namespace": "{id}-wss", + "connection": "c1", + "queue": "{id}-wss-q1", + "requirements": [ + "gpu" + ], + "status": "active", + "lease_owner": "{id}-wss-w1", + "holder": { + "worker_id": "{id}-wss-w1", + "task_queue": "{id}-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 60, + "ttl_seconds": 600, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 60, + "ttl_span": 600, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "activity_task_id": null, + "status": 201, + "reason": null, + "protocol_version": "1.20" + } + }, + "float_lease": { + "status": 201, + "response": { + "admitted": true, + "outcome": "created", + "session": { + "session_id": "{id}-wss-s2", + "namespace": "{id}-wss", + "connection": null, + "queue": "{id}-wss-q1", + "requirements": [], + "status": "active", + "lease_owner": "{id}-wss-w1", + "holder": { + "worker_id": "{id}-wss-w1", + "task_queue": "{id}-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 120, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 120, + "ttl_span": 1800, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "activity_task_id": null, + "status": 201, + "reason": null, + "protocol_version": "1.20" + } + }, + "reused": { + "status": 200, + "response": { + "admitted": true, + "outcome": "reused", + "session": { + "session_id": "{id}-wss-s1", + "namespace": "{id}-wss", + "connection": "c1", + "queue": "{id}-wss-q1", + "requirements": [], + "status": "active", + "lease_owner": "{id}-wss-w1", + "holder": { + "worker_id": "{id}-wss-w1", + "task_queue": "{id}-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 30, + "ttl_seconds": 9999, + "max_concurrent_activities": 3, + "create_if_missing": false, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 30, + "ttl_span": 600, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "activity_task_id": null, + "status": 200, + "reason": null, + "protocol_version": "1.20" + } + }, + "foreign_create": { + "status": 409, + "response": { + "session": { + "session_id": "{id}-wss-s1", + "namespace": "{id}-wss", + "connection": "c1", + "queue": "{id}-wss-q1", + "requirements": [], + "status": "active", + "lease_owner": "{id}-wss-w1", + "holder": { + "worker_id": "{id}-wss-w1", + "task_queue": "{id}-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 30, + "ttl_seconds": 9999, + "max_concurrent_activities": 3, + "create_if_missing": false, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 30, + "ttl_span": 600, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "admitted": false, + "error": "Worker session is currently leased by another worker.", + "reason": "session_owned_by_another_worker", + "status": 409, + "protocol_version": "1.20" + } + }, + "foreign_heartbeat": { + "status": 409, + "response": { + "session": { + "session_id": "{id}-wss-s1", + "namespace": "{id}-wss", + "connection": "c1", + "queue": "{id}-wss-q1", + "requirements": [], + "status": "active", + "lease_owner": "{id}-wss-w1", + "holder": { + "worker_id": "{id}-wss-w1", + "task_queue": "{id}-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 30, + "ttl_seconds": 9999, + "max_concurrent_activities": 3, + "create_if_missing": false, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 30, + "ttl_span": 600, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "admitted": false, + "error": "Worker session lease is owned by another worker.", + "reason": "session_owner_mismatch", + "status": 409, + "protocol_version": "1.20" + } + }, + "foreign_close": { + "status": 409, + "response": { + "session": { + "session_id": "{id}-wss-s1", + "namespace": "{id}-wss", + "connection": "c1", + "queue": "{id}-wss-q1", + "requirements": [], + "status": "active", + "lease_owner": "{id}-wss-w1", + "holder": { + "worker_id": "{id}-wss-w1", + "task_queue": "{id}-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 30, + "ttl_seconds": 9999, + "max_concurrent_activities": 3, + "create_if_missing": false, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 30, + "ttl_span": 600, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "admitted": false, + "error": "Worker session lease is owned by another worker.", + "reason": "session_owner_mismatch", + "status": 409, + "protocol_version": "1.20" + } + }, + "heartbeat_missing": { + "status": 404, + "response": { + "admitted": false, + "error": "Worker session was not found.", + "reason": "session_not_found", + "status": 404, + "protocol_version": "1.20" + } + }, + "heartbeat": { + "status": 200, + "response": { + "admitted": true, + "outcome": "heartbeat_recorded", + "session": { + "session_id": "{id}-wss-s1", + "namespace": "{id}-wss", + "connection": "c1", + "queue": "{id}-wss-q1", + "requirements": [], + "status": "active", + "lease_owner": "{id}-wss-w1", + "holder": { + "worker_id": "{id}-wss-w1", + "task_queue": "{id}-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 30, + "ttl_seconds": 9999, + "max_concurrent_activities": 3, + "create_if_missing": false, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 5, + "ttl_span": 600, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "activity_task_id": null, + "status": 200, + "reason": null, + "protocol_version": "1.20" + } + }, + "limit": { + "status": 409, + "response": { + "max_concurrent_worker_sessions": 2, + "admitted": false, + "error": "Worker has reached max_concurrent_worker_sessions.", + "reason": "worker_session_limit_exceeded", + "status": 409, + "protocol_version": "1.20" + } + }, + "short_created": { + "status": 201, + "response": { + "admitted": true, + "outcome": "created", + "session": { + "session_id": "{id}-wss-short", + "namespace": "{id}-wss", + "connection": null, + "queue": "{id}-wss-q1", + "requirements": [], + "status": "active", + "lease_owner": "{id}-wss-w2", + "holder": { + "worker_id": "{id}-wss-w2", + "task_queue": "{id}-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 2, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 2, + "ttl_span": 1800, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "activity_task_id": null, + "status": 201, + "reason": null, + "protocol_version": "1.20" + } + }, + "short_expired": { + "status": 200, + "response": { + "namespace": "{id}-wss", + "session": { + "session_id": "{id}-wss-short", + "namespace": "{id}-wss", + "connection": null, + "queue": "{id}-wss-q1", + "requirements": [], + "status": "expired", + "lease_owner": "{id}-wss-w2", + "holder": { + "worker_id": "{id}-wss-w2", + "task_queue": "{id}-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": "lease_expired", + "lease_seconds": 2, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 2, + "ttl_span": 1800, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + } + } + }, + "reacquired": { + "status": 200, + "response": { + "admitted": true, + "outcome": "reacquired", + "session": { + "session_id": "{id}-wss-short", + "namespace": "{id}-wss", + "connection": null, + "queue": "{id}-wss-q2", + "requirements": [], + "status": "active", + "lease_owner": "{id}-wss-w3", + "holder": { + "worker_id": "{id}-wss-w3", + "task_queue": "{id}-wss-q2", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 120, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 120, + "ttl_span": 1800, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "activity_task_id": null, + "status": 200, + "reason": null, + "protocol_version": "1.20" + } + }, + "reacquired_heartbeat": { + "status": 200, + "response": { + "admitted": true, + "outcome": "heartbeat_recorded", + "session": { + "session_id": "{id}-wss-short", + "namespace": "{id}-wss", + "connection": null, + "queue": "{id}-wss-q2", + "requirements": [], + "status": "active", + "lease_owner": "{id}-wss-w3", + "holder": { + "worker_id": "{id}-wss-w3", + "task_queue": "{id}-wss-q2", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 120, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 120, + "ttl_span": 1800, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "activity_task_id": null, + "status": 200, + "reason": null, + "protocol_version": "1.20" + } + }, + "close_by_old_owner": { + "status": 409, + "response": { + "session": { + "session_id": "{id}-wss-short", + "namespace": "{id}-wss", + "connection": null, + "queue": "{id}-wss-q2", + "requirements": [], + "status": "active", + "lease_owner": "{id}-wss-w3", + "holder": { + "worker_id": "{id}-wss-w3", + "task_queue": "{id}-wss-q2", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 120, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 120, + "ttl_span": 1800, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "admitted": false, + "error": "Worker session lease is owned by another worker.", + "reason": "session_owner_mismatch", + "status": 409, + "protocol_version": "1.20" + } + }, + "closed": { + "status": 200, + "response": { + "admitted": true, + "outcome": "closed", + "session": { + "session_id": "{id}-wss-short", + "namespace": "{id}-wss", + "connection": null, + "queue": "{id}-wss-q2", + "requirements": [], + "status": "closed", + "lease_owner": "{id}-wss-w3", + "holder": { + "worker_id": "{id}-wss-w3", + "task_queue": "{id}-wss-q2", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 120, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 120, + "ttl_span": 1800, + "closed": true, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "present", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "activity_task_id": null, + "status": 200, + "reason": null, + "protocol_version": "1.20" + } + }, + "closed_again": { + "status": 200, + "response": { + "admitted": true, + "outcome": "already_closed", + "session": { + "session_id": "{id}-wss-short", + "namespace": "{id}-wss", + "connection": null, + "queue": "{id}-wss-q2", + "requirements": [], + "status": "closed", + "lease_owner": "{id}-wss-w3", + "holder": { + "worker_id": "{id}-wss-w3", + "task_queue": "{id}-wss-q2", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 120, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 120, + "ttl_span": 1800, + "closed": true, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "present", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "activity_task_id": null, + "status": 200, + "reason": null, + "protocol_version": "1.20" + } + }, + "closed_by_other": { + "status": 409, + "response": { + "session": { + "session_id": "{id}-wss-short", + "namespace": "{id}-wss", + "connection": null, + "queue": "{id}-wss-q2", + "requirements": [], + "status": "closed", + "lease_owner": "{id}-wss-w3", + "holder": { + "worker_id": "{id}-wss-w3", + "task_queue": "{id}-wss-q2", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 120, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 120, + "ttl_span": 1800, + "closed": true, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "present", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "admitted": false, + "error": "Worker session lease is owned by another worker.", + "reason": "session_owner_mismatch", + "status": 409, + "protocol_version": "1.20" + } + }, + "closed_heartbeat": { + "status": 409, + "response": { + "session": { + "session_id": "{id}-wss-short", + "namespace": "{id}-wss", + "connection": null, + "queue": "{id}-wss-q2", + "requirements": [], + "status": "closed", + "lease_owner": "{id}-wss-w3", + "holder": { + "worker_id": "{id}-wss-w3", + "task_queue": "{id}-wss-q2", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 120, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 120, + "ttl_span": 1800, + "closed": true, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "present", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "admitted": false, + "error": "Worker session is not active.", + "reason": "session_not_active", + "status": 409, + "protocol_version": "1.20" + } + }, + "closed_create": { + "status": 409, + "response": { + "session": { + "session_id": "{id}-wss-short", + "namespace": "{id}-wss", + "connection": null, + "queue": "{id}-wss-q2", + "requirements": [], + "status": "closed", + "lease_owner": "{id}-wss-w3", + "holder": { + "worker_id": "{id}-wss-w3", + "task_queue": "{id}-wss-q2", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 120, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 120, + "ttl_span": 1800, + "closed": true, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "present", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "admitted": false, + "error": "Worker session was closed and cannot be reacquired.", + "reason": "session_closed", + "status": 409, + "protocol_version": "1.20" + } + }, + "closed_create_unmet": { + "status": 409, + "response": { + "missing_requirements": [ + "gpu" + ], + "admitted": false, + "error": "Worker does not advertise every worker-session requirement.", + "reason": "session_requirements_not_met", + "status": 409, + "protocol_version": "1.20" + } + }, + "reacquire_terminal": { + "status": 409, + "response": { + "session": { + "session_id": "{id}-wss-terminal", + "namespace": "{id}-wss", + "connection": null, + "queue": "{id}-wss-q1", + "requirements": [], + "status": "expired", + "lease_owner": "{id}-wss-w2", + "holder": { + "worker_id": "{id}-wss-w2", + "task_queue": "{id}-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": "lease_expired", + "lease_seconds": 2, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": false, + "active_activity_count": 0, + "lease_span": 2, + "ttl_span": 1800, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "admitted": false, + "error": "Worker session is terminal and cannot be reacquired.", + "reason": "session_reacquire_disallowed", + "status": 409, + "protocol_version": "1.20" + } + }, + "reacquire_disallowed": { + "status": 409, + "response": { + "session": { + "session_id": "{id}-wss-optional", + "namespace": "{id}-wss", + "connection": null, + "queue": "{id}-wss-q1", + "requirements": [], + "status": "expired", + "lease_owner": "{id}-wss-w2", + "holder": { + "worker_id": "{id}-wss-w2", + "task_queue": "{id}-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": "lease_expired", + "lease_seconds": 2, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 2, + "ttl_span": 1800, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "admitted": false, + "error": "Worker session options disallow reacquisition after holder failure.", + "reason": "session_reacquire_disallowed", + "status": 409, + "protocol_version": "1.20" + } + }, + "ttl_heartbeat": { + "status": 409, + "response": { + "session": { + "session_id": "{id}-wss-ttl", + "namespace": "{id}-wss", + "connection": null, + "queue": "{id}-wss-q1", + "requirements": [], + "status": "expired", + "lease_owner": "{id}-wss-w2", + "holder": { + "worker_id": "{id}-wss-w2", + "task_queue": "{id}-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": "ttl_expired", + "lease_seconds": 60, + "ttl_seconds": 2, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 60, + "ttl_span": 2, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "admitted": false, + "error": "Worker session is not active.", + "reason": "session_not_active", + "status": 409, + "protocol_version": "1.20" + } + }, + "ttl_create": { + "status": 409, + "response": { + "session": { + "session_id": "{id}-wss-ttl", + "namespace": "{id}-wss", + "connection": null, + "queue": "{id}-wss-q1", + "requirements": [], + "status": "expired", + "lease_owner": "{id}-wss-w2", + "holder": { + "worker_id": "{id}-wss-w2", + "task_queue": "{id}-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": "ttl_expired", + "lease_seconds": 60, + "ttl_seconds": 2, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 60, + "ttl_span": 2, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "admitted": false, + "error": "Worker session is terminal and cannot be reacquired.", + "reason": "session_reacquire_disallowed", + "status": 409, + "protocol_version": "1.20" + } + }, + "ttl_close": { + "status": 200, + "response": { + "admitted": true, + "outcome": "closed", + "session": { + "session_id": "{id}-wss-ttl", + "namespace": "{id}-wss", + "connection": null, + "queue": "{id}-wss-q1", + "requirements": [], + "status": "closed", + "lease_owner": "{id}-wss-w2", + "holder": { + "worker_id": "{id}-wss-w2", + "task_queue": "{id}-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 60, + "ttl_seconds": 2, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 60, + "ttl_span": 2, + "closed": true, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "present", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "activity_task_id": null, + "status": 200, + "reason": null, + "protocol_version": "1.20" + } + }, + "held_created": { + "status": 201, + "response": { + "admitted": true, + "outcome": "created", + "session": { + "session_id": "{id}-wss-held", + "namespace": "{id}-wss", + "connection": null, + "queue": "{id}-wss-q2", + "requirements": [], + "status": "active", + "lease_owner": "{id}-wss-w3", + "holder": { + "worker_id": "{id}-wss-w3", + "task_queue": "{id}-wss-q2", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 120, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 120, + "ttl_span": 1800, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "activity_task_id": null, + "status": 201, + "reason": null, + "protocol_version": "1.20" + } + }, + "holder_deregistered": 200, + "held_orphaned": { + "status": 200, + "response": { + "namespace": "{id}-wss", + "session": { + "session_id": "{id}-wss-held", + "namespace": "{id}-wss", + "connection": null, + "queue": "{id}-wss-q2", + "requirements": [], + "status": "orphaned", + "lease_owner": "{id}-wss-w3", + "holder": { + "worker_id": "{id}-wss-w3", + "task_queue": null, + "status": null, + "last_heartbeat_at": null, + "is_stale": true + }, + "failure_reason": "worker_heartbeat_stale", + "lease_seconds": 120, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 120, + "ttl_span": 1800, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + } + } + }, + "deregistered_create": { + "status": 412, + "response": { + "error": "Worker must be registered before creating a worker session.", + "reason": "worker_not_registered", + "worker_id": "{id}-wss-w3", + "protocol_version": "1.20" + } + }, + "orphan_reacquired": { + "status": 200, + "response": { + "admitted": true, + "outcome": "reacquired", + "session": { + "session_id": "{id}-wss-held", + "namespace": "{id}-wss", + "connection": null, + "queue": "{id}-wss-q1", + "requirements": [], + "status": "active", + "lease_owner": "{id}-wss-w2", + "holder": { + "worker_id": "{id}-wss-w2", + "task_queue": "{id}-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 120, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 120, + "ttl_span": 1800, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "activity_task_id": null, + "status": 200, + "reason": null, + "protocol_version": "1.20" + } + }, + "orphan_queue_mismatch": { + "status": 409, + "response": { + "error": "Worker [{id}-wss-w2] is registered for task queue [{id}-wss-q1], not session queue [{id}-wss-q2].", + "reason": "session_queue_mismatch", + "worker_id": "{id}-wss-w2", + "registered_task_queue": "{id}-wss-q1", + "requested_session_queue": "{id}-wss-q2", + "protocol_version": "1.20" + } + }, + "show_missing": { + "status": 404, + "response": { + "message": "Worker session [{id}-wss-none] not found in namespace [{id}-wss].", + "reason": "worker_session_not_found" + } + }, + "list_metrics": { + "total": 7, + "active": 2, + "closed": 2, + "expired": 3, + "failed": 0, + "orphaned": 0 + }, + "list_namespace": "{id}-wss", + "list_sessions": [ + { + "session_id": "{id}-wss-held", + "namespace": "{id}-wss", + "connection": null, + "queue": "{id}-wss-q1", + "requirements": [], + "status": "active", + "lease_owner": "{id}-wss-w2", + "holder": { + "worker_id": "{id}-wss-w2", + "task_queue": "{id}-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 120, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 120, + "ttl_span": 1800, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + { + "session_id": "{id}-wss-optional", + "namespace": "{id}-wss", + "connection": null, + "queue": "{id}-wss-q1", + "requirements": [], + "status": "expired", + "lease_owner": "{id}-wss-w2", + "holder": { + "worker_id": "{id}-wss-w2", + "task_queue": "{id}-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": "lease_expired", + "lease_seconds": 2, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 2, + "ttl_span": 1800, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + { + "session_id": "{id}-wss-s1", + "namespace": "{id}-wss", + "connection": "c1", + "queue": "{id}-wss-q1", + "requirements": [], + "status": "expired", + "lease_owner": "{id}-wss-w1", + "holder": { + "worker_id": "{id}-wss-w1", + "task_queue": "{id}-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": "lease_expired", + "lease_seconds": 30, + "ttl_seconds": 9999, + "max_concurrent_activities": 3, + "create_if_missing": false, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 5, + "ttl_span": 600, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + { + "session_id": "{id}-wss-s2", + "namespace": "{id}-wss", + "connection": null, + "queue": "{id}-wss-q1", + "requirements": [], + "status": "active", + "lease_owner": "{id}-wss-w1", + "holder": { + "worker_id": "{id}-wss-w1", + "task_queue": "{id}-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 120, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 120, + "ttl_span": 1800, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + { + "session_id": "{id}-wss-short", + "namespace": "{id}-wss", + "connection": null, + "queue": "{id}-wss-q2", + "requirements": [], + "status": "closed", + "lease_owner": "{id}-wss-w3", + "holder": { + "worker_id": "{id}-wss-w3", + "task_queue": null, + "status": null, + "last_heartbeat_at": null, + "is_stale": true + }, + "failure_reason": null, + "lease_seconds": 120, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 120, + "ttl_span": 1800, + "closed": true, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "present", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + { + "session_id": "{id}-wss-terminal", + "namespace": "{id}-wss", + "connection": null, + "queue": "{id}-wss-q1", + "requirements": [], + "status": "expired", + "lease_owner": "{id}-wss-w2", + "holder": { + "worker_id": "{id}-wss-w2", + "task_queue": "{id}-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": "lease_expired", + "lease_seconds": 2, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": false, + "active_activity_count": 0, + "lease_span": 2, + "ttl_span": 1800, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + { + "session_id": "{id}-wss-ttl", + "namespace": "{id}-wss", + "connection": null, + "queue": "{id}-wss-q1", + "requirements": [], + "status": "closed", + "lease_owner": "{id}-wss-w2", + "holder": { + "worker_id": "{id}-wss-w2", + "task_queue": "{id}-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 60, + "ttl_seconds": 2, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 60, + "ttl_span": 2, + "closed": true, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "present", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + } + ] + } + }, + "contract": "Worker sessions over the worker protocol and their visibility on the control plane, as published Server 2.11.1 serves them: the protocol gates (no header 400, below 1.8 409 worker_sessions_unavailable, above 1.20 400), validation, registration (412) and queue (409) refusals, requirements checked before the session lookup, create (201), reuse and reacquire (200, options the request omits reset to defaults, the TTL deadline never extended, the queue defaulting to the worker's own), heartbeat (lease renewed from now, stored lease unchanged), the per-worker session limit, lease and TTL expiry applied when the session is read, close and already_closed, the reacquire refusals, a deregistered holder orphaned on read, and the namespace list and metrics. Session times are whole seconds, so times are compared as spans. One published defect is bounded: a reacquired session keeps its old failure_reason (renewHeldSession filters out its own null clears); the published value or the intended null is accepted. Embedded applications have no worker protocol." +} diff --git a/tests/Fixtures/ServerParityExpected/worker-sessions-observation.json b/tests/Fixtures/ServerParityExpected/worker-sessions-observation.json new file mode 100644 index 000000000..e7b44937c --- /dev/null +++ b/tests/Fixtures/ServerParityExpected/worker-sessions-observation.json @@ -0,0 +1,1699 @@ +{ + "applicable": true, + "namespace": "pr1-worker-sessions-wss", + "workers": [ + "pr1-worker-sessions-wss-w1", + "pr1-worker-sessions-wss-w2", + "pr1-worker-sessions-wss-w3" + ], + "queues": [ + "pr1-worker-sessions-wss-q1", + "pr1-worker-sessions-wss-q2" + ], + "gate_no_protocol": { + "status": 400, + "response": { + "error": "Missing worker protocol version header.", + "reason": "missing_protocol_version", + "supported_version": "1.20", + "requested_version": null, + "remediation": "Send the X-Durable-Workflow-Protocol-Version: 1.20 header on worker protocol requests.", + "protocol_version": "1.20" + } + }, + "gate_protocol_1_7": { + "status": 409, + "response": { + "error": "Worker sessions require worker protocol 1.8 or newer.", + "reason": "worker_sessions_unavailable", + "supported_version": "1.20", + "requested_version": "1.7", + "minimum_protocol_version": "1.8", + "remediation": "Send the X-Durable-Workflow-Protocol-Version header with worker protocol 1.8 or newer and use a worker SDK that implements worker-session semantics.", + "protocol_version": "1.20" + } + }, + "gate_protocol_1_21": { + "status": 400, + "response": { + "error": "Unsupported worker protocol version.", + "reason": "unsupported_protocol_version", + "supported_version": "1.20", + "requested_version": "1.21", + "remediation": "Worker requested protocol version 1.21; this server supports 1.20. Workers may target any 1.x version with x ≤ 20. Upgrade the worker to a release that targets a compatible version, or connect to a server that matches.", + "protocol_version": "1.20" + } + }, + "gate_protocol_1_8_empty": { + "status": 422, + "response": { + "message": "The worker id field is required. (and 1 more error)", + "reason": "validation_failed", + "errors": { + "worker_id": [ + "The worker id field is required." + ], + "session_id": [ + "The session id field is required." + ] + }, + "validation_errors": { + "worker_id": [ + "The worker id field is required." + ], + "session_id": [ + "The session id field is required." + ] + }, + "protocol_version": "1.20" + } + }, + "gate_unknown_namespace": { + "status": 404, + "response": { + "message": "Namespace 'no-such-namespace-wss' does not exist.", + "reason": "namespace_not_found", + "namespace": "no-such-namespace-wss", + "remediation": "Register the namespace via POST /api/namespaces, or send an X-Namespace header naming an existing namespace.", + "protocol_version": "1.20" + } + }, + "gate_list_without_control_plane": { + "status": 400, + "response": { + "message": "Missing control-plane version header.", + "reason": "missing_control_plane_version", + "supported_version": "2", + "requested_version": null, + "remediation": "Send the X-Durable-Workflow-Control-Plane-Version: 2 header on control-plane requests." + } + }, + "gate_list_unknown_namespace": { + "status": 404, + "response": { + "message": "Namespace 'no-such-namespace-wss' does not exist.", + "reason": "namespace_not_found", + "namespace": "no-such-namespace-wss", + "remediation": "Register the namespace via POST /api/namespaces, or send an X-Namespace header naming an existing namespace." + } + }, + "invalid_empty": { + "status": 422, + "response": { + "message": "The worker id field is required. (and 1 more error)", + "reason": "validation_failed", + "errors": { + "worker_id": [ + "The worker id field is required." + ], + "session_id": [ + "The session id field is required." + ] + }, + "validation_errors": { + "worker_id": [ + "The worker id field is required." + ], + "session_id": [ + "The session id field is required." + ] + }, + "protocol_version": "1.20" + } + }, + "invalid_types": { + "status": 422, + "response": { + "message": "The worker id field must be a string. (and 9 more errors)", + "reason": "validation_failed", + "errors": { + "worker_id": [ + "The worker id field must be a string." + ], + "session_id": [ + "The session id field is required." + ], + "connection": [ + "The connection field must be a string." + ], + "queue": [ + "The queue field must be a string." + ], + "requirements": [ + "The requirements field must be an array." + ], + "lease_seconds": [ + "The lease seconds field must be at least 1." + ], + "ttl_seconds": [ + "The ttl seconds field must be an integer." + ], + "max_concurrent_activities": [ + "The max concurrent activities field must be an integer." + ], + "create_if_missing": [ + "The create if missing field must be true or false." + ], + "allow_reacquire_after_failure": [ + "The allow reacquire after failure field must be true or false." + ] + }, + "validation_errors": { + "worker_id": [ + "The worker id field must be a string." + ], + "session_id": [ + "The session id field is required." + ], + "connection": [ + "The connection field must be a string." + ], + "queue": [ + "The queue field must be a string." + ], + "requirements": [ + "The requirements field must be an array." + ], + "lease_seconds": [ + "The lease seconds field must be at least 1." + ], + "ttl_seconds": [ + "The ttl seconds field must be an integer." + ], + "max_concurrent_activities": [ + "The max concurrent activities field must be an integer." + ], + "create_if_missing": [ + "The create if missing field must be true or false." + ], + "allow_reacquire_after_failure": [ + "The allow reacquire after failure field must be true or false." + ] + }, + "protocol_version": "1.20" + } + }, + "invalid_heartbeat": { + "status": 422, + "response": { + "message": "The worker id field is required. (and 1 more error)", + "reason": "validation_failed", + "errors": { + "worker_id": [ + "The worker id field is required." + ], + "lease_seconds": [ + "The lease seconds field must be at least 1." + ] + }, + "validation_errors": { + "worker_id": [ + "The worker id field is required." + ], + "lease_seconds": [ + "The lease seconds field must be at least 1." + ] + }, + "protocol_version": "1.20" + } + }, + "invalid_close": { + "status": 422, + "response": { + "message": "The worker id field is required.", + "reason": "validation_failed", + "errors": { + "worker_id": [ + "The worker id field is required." + ] + }, + "validation_errors": { + "worker_id": [ + "The worker id field is required." + ] + }, + "protocol_version": "1.20" + } + }, + "unregistered": { + "status": 412, + "response": { + "error": "Worker must be registered before creating a worker session.", + "reason": "worker_not_registered", + "worker_id": "pr1-worker-sessions-wss-nobody", + "protocol_version": "1.20" + } + }, + "queue_mismatch": { + "status": 409, + "response": { + "error": "Worker [pr1-worker-sessions-wss-w1] is registered for task queue [pr1-worker-sessions-wss-q1], not session queue [pr1-worker-sessions-wss-q2].", + "reason": "session_queue_mismatch", + "worker_id": "pr1-worker-sessions-wss-w1", + "registered_task_queue": "pr1-worker-sessions-wss-q1", + "requested_session_queue": "pr1-worker-sessions-wss-q2", + "protocol_version": "1.20" + } + }, + "requirements_missing": { + "status": 409, + "response": { + "missing_requirements": [ + "tpu", + "fpga" + ], + "admitted": false, + "error": "Worker does not advertise every worker-session requirement.", + "reason": "session_requirements_not_met", + "status": 409, + "protocol_version": "1.20" + } + }, + "missing_without_create": { + "status": 409, + "response": { + "admitted": false, + "error": "Worker session does not exist and creation is disabled.", + "reason": "session_missing", + "status": 409, + "protocol_version": "1.20" + } + }, + "created": { + "status": 201, + "response": { + "admitted": true, + "outcome": "created", + "session": { + "session_id": "pr1-worker-sessions-wss-s1", + "namespace": "pr1-worker-sessions-wss", + "connection": "c1", + "queue": "pr1-worker-sessions-wss-q1", + "requirements": [ + "gpu" + ], + "status": "active", + "lease_owner": "pr1-worker-sessions-wss-w1", + "holder": { + "worker_id": "pr1-worker-sessions-wss-w1", + "task_queue": "pr1-worker-sessions-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 60, + "ttl_seconds": 600, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 60, + "ttl_span": 600, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "activity_task_id": null, + "status": 201, + "reason": null, + "protocol_version": "1.20" + } + }, + "float_lease": { + "status": 201, + "response": { + "admitted": true, + "outcome": "created", + "session": { + "session_id": "pr1-worker-sessions-wss-s2", + "namespace": "pr1-worker-sessions-wss", + "connection": null, + "queue": "pr1-worker-sessions-wss-q1", + "requirements": [], + "status": "active", + "lease_owner": "pr1-worker-sessions-wss-w1", + "holder": { + "worker_id": "pr1-worker-sessions-wss-w1", + "task_queue": "pr1-worker-sessions-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 120, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 120, + "ttl_span": 1800, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "activity_task_id": null, + "status": 201, + "reason": null, + "protocol_version": "1.20" + } + }, + "reused": { + "status": 200, + "response": { + "admitted": true, + "outcome": "reused", + "session": { + "session_id": "pr1-worker-sessions-wss-s1", + "namespace": "pr1-worker-sessions-wss", + "connection": "c1", + "queue": "pr1-worker-sessions-wss-q1", + "requirements": [], + "status": "active", + "lease_owner": "pr1-worker-sessions-wss-w1", + "holder": { + "worker_id": "pr1-worker-sessions-wss-w1", + "task_queue": "pr1-worker-sessions-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 30, + "ttl_seconds": 9999, + "max_concurrent_activities": 3, + "create_if_missing": false, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 30, + "ttl_span": 600, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "activity_task_id": null, + "status": 200, + "reason": null, + "protocol_version": "1.20" + } + }, + "foreign_create": { + "status": 409, + "response": { + "session": { + "session_id": "pr1-worker-sessions-wss-s1", + "namespace": "pr1-worker-sessions-wss", + "connection": "c1", + "queue": "pr1-worker-sessions-wss-q1", + "requirements": [], + "status": "active", + "lease_owner": "pr1-worker-sessions-wss-w1", + "holder": { + "worker_id": "pr1-worker-sessions-wss-w1", + "task_queue": "pr1-worker-sessions-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 30, + "ttl_seconds": 9999, + "max_concurrent_activities": 3, + "create_if_missing": false, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 30, + "ttl_span": 600, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "admitted": false, + "error": "Worker session is currently leased by another worker.", + "reason": "session_owned_by_another_worker", + "status": 409, + "protocol_version": "1.20" + } + }, + "foreign_heartbeat": { + "status": 409, + "response": { + "session": { + "session_id": "pr1-worker-sessions-wss-s1", + "namespace": "pr1-worker-sessions-wss", + "connection": "c1", + "queue": "pr1-worker-sessions-wss-q1", + "requirements": [], + "status": "active", + "lease_owner": "pr1-worker-sessions-wss-w1", + "holder": { + "worker_id": "pr1-worker-sessions-wss-w1", + "task_queue": "pr1-worker-sessions-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 30, + "ttl_seconds": 9999, + "max_concurrent_activities": 3, + "create_if_missing": false, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 30, + "ttl_span": 600, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "admitted": false, + "error": "Worker session lease is owned by another worker.", + "reason": "session_owner_mismatch", + "status": 409, + "protocol_version": "1.20" + } + }, + "foreign_close": { + "status": 409, + "response": { + "session": { + "session_id": "pr1-worker-sessions-wss-s1", + "namespace": "pr1-worker-sessions-wss", + "connection": "c1", + "queue": "pr1-worker-sessions-wss-q1", + "requirements": [], + "status": "active", + "lease_owner": "pr1-worker-sessions-wss-w1", + "holder": { + "worker_id": "pr1-worker-sessions-wss-w1", + "task_queue": "pr1-worker-sessions-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 30, + "ttl_seconds": 9999, + "max_concurrent_activities": 3, + "create_if_missing": false, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 30, + "ttl_span": 600, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "admitted": false, + "error": "Worker session lease is owned by another worker.", + "reason": "session_owner_mismatch", + "status": 409, + "protocol_version": "1.20" + } + }, + "heartbeat_missing": { + "status": 404, + "response": { + "admitted": false, + "error": "Worker session was not found.", + "reason": "session_not_found", + "status": 404, + "protocol_version": "1.20" + } + }, + "heartbeat": { + "status": 200, + "response": { + "admitted": true, + "outcome": "heartbeat_recorded", + "session": { + "session_id": "pr1-worker-sessions-wss-s1", + "namespace": "pr1-worker-sessions-wss", + "connection": "c1", + "queue": "pr1-worker-sessions-wss-q1", + "requirements": [], + "status": "active", + "lease_owner": "pr1-worker-sessions-wss-w1", + "holder": { + "worker_id": "pr1-worker-sessions-wss-w1", + "task_queue": "pr1-worker-sessions-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 30, + "ttl_seconds": 9999, + "max_concurrent_activities": 3, + "create_if_missing": false, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 5, + "ttl_span": 600, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "activity_task_id": null, + "status": 200, + "reason": null, + "protocol_version": "1.20" + } + }, + "limit": { + "status": 409, + "response": { + "max_concurrent_worker_sessions": 2, + "admitted": false, + "error": "Worker has reached max_concurrent_worker_sessions.", + "reason": "worker_session_limit_exceeded", + "status": 409, + "protocol_version": "1.20" + } + }, + "short_created": { + "status": 201, + "response": { + "admitted": true, + "outcome": "created", + "session": { + "session_id": "pr1-worker-sessions-wss-short", + "namespace": "pr1-worker-sessions-wss", + "connection": null, + "queue": "pr1-worker-sessions-wss-q1", + "requirements": [], + "status": "active", + "lease_owner": "pr1-worker-sessions-wss-w2", + "holder": { + "worker_id": "pr1-worker-sessions-wss-w2", + "task_queue": "pr1-worker-sessions-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 2, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 2, + "ttl_span": 1800, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "activity_task_id": null, + "status": 201, + "reason": null, + "protocol_version": "1.20" + } + }, + "short_expired": { + "status": 200, + "response": { + "namespace": "pr1-worker-sessions-wss", + "session": { + "session_id": "pr1-worker-sessions-wss-short", + "namespace": "pr1-worker-sessions-wss", + "connection": null, + "queue": "pr1-worker-sessions-wss-q1", + "requirements": [], + "status": "expired", + "lease_owner": "pr1-worker-sessions-wss-w2", + "holder": { + "worker_id": "pr1-worker-sessions-wss-w2", + "task_queue": "pr1-worker-sessions-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": "lease_expired", + "lease_seconds": 2, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 2, + "ttl_span": 1800, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + } + } + }, + "reacquired": { + "status": 200, + "response": { + "admitted": true, + "outcome": "reacquired", + "session": { + "session_id": "pr1-worker-sessions-wss-short", + "namespace": "pr1-worker-sessions-wss", + "connection": null, + "queue": "pr1-worker-sessions-wss-q2", + "requirements": [], + "status": "active", + "lease_owner": "pr1-worker-sessions-wss-w3", + "holder": { + "worker_id": "pr1-worker-sessions-wss-w3", + "task_queue": "pr1-worker-sessions-wss-q2", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": "lease_expired", + "lease_seconds": 120, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 120, + "ttl_span": 1800, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "activity_task_id": null, + "status": 200, + "reason": null, + "protocol_version": "1.20" + } + }, + "reacquired_heartbeat": { + "status": 200, + "response": { + "admitted": true, + "outcome": "heartbeat_recorded", + "session": { + "session_id": "pr1-worker-sessions-wss-short", + "namespace": "pr1-worker-sessions-wss", + "connection": null, + "queue": "pr1-worker-sessions-wss-q2", + "requirements": [], + "status": "active", + "lease_owner": "pr1-worker-sessions-wss-w3", + "holder": { + "worker_id": "pr1-worker-sessions-wss-w3", + "task_queue": "pr1-worker-sessions-wss-q2", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 120, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 120, + "ttl_span": 1800, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "activity_task_id": null, + "status": 200, + "reason": null, + "protocol_version": "1.20" + } + }, + "close_by_old_owner": { + "status": 409, + "response": { + "session": { + "session_id": "pr1-worker-sessions-wss-short", + "namespace": "pr1-worker-sessions-wss", + "connection": null, + "queue": "pr1-worker-sessions-wss-q2", + "requirements": [], + "status": "active", + "lease_owner": "pr1-worker-sessions-wss-w3", + "holder": { + "worker_id": "pr1-worker-sessions-wss-w3", + "task_queue": "pr1-worker-sessions-wss-q2", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 120, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 120, + "ttl_span": 1800, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "admitted": false, + "error": "Worker session lease is owned by another worker.", + "reason": "session_owner_mismatch", + "status": 409, + "protocol_version": "1.20" + } + }, + "closed": { + "status": 200, + "response": { + "admitted": true, + "outcome": "closed", + "session": { + "session_id": "pr1-worker-sessions-wss-short", + "namespace": "pr1-worker-sessions-wss", + "connection": null, + "queue": "pr1-worker-sessions-wss-q2", + "requirements": [], + "status": "closed", + "lease_owner": "pr1-worker-sessions-wss-w3", + "holder": { + "worker_id": "pr1-worker-sessions-wss-w3", + "task_queue": "pr1-worker-sessions-wss-q2", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 120, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 120, + "ttl_span": 1800, + "closed": true, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "present", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "activity_task_id": null, + "status": 200, + "reason": null, + "protocol_version": "1.20" + } + }, + "closed_again": { + "status": 200, + "response": { + "admitted": true, + "outcome": "already_closed", + "session": { + "session_id": "pr1-worker-sessions-wss-short", + "namespace": "pr1-worker-sessions-wss", + "connection": null, + "queue": "pr1-worker-sessions-wss-q2", + "requirements": [], + "status": "closed", + "lease_owner": "pr1-worker-sessions-wss-w3", + "holder": { + "worker_id": "pr1-worker-sessions-wss-w3", + "task_queue": "pr1-worker-sessions-wss-q2", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 120, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 120, + "ttl_span": 1800, + "closed": true, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "present", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "activity_task_id": null, + "status": 200, + "reason": null, + "protocol_version": "1.20" + } + }, + "closed_by_other": { + "status": 409, + "response": { + "session": { + "session_id": "pr1-worker-sessions-wss-short", + "namespace": "pr1-worker-sessions-wss", + "connection": null, + "queue": "pr1-worker-sessions-wss-q2", + "requirements": [], + "status": "closed", + "lease_owner": "pr1-worker-sessions-wss-w3", + "holder": { + "worker_id": "pr1-worker-sessions-wss-w3", + "task_queue": "pr1-worker-sessions-wss-q2", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 120, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 120, + "ttl_span": 1800, + "closed": true, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "present", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "admitted": false, + "error": "Worker session lease is owned by another worker.", + "reason": "session_owner_mismatch", + "status": 409, + "protocol_version": "1.20" + } + }, + "closed_heartbeat": { + "status": 409, + "response": { + "session": { + "session_id": "pr1-worker-sessions-wss-short", + "namespace": "pr1-worker-sessions-wss", + "connection": null, + "queue": "pr1-worker-sessions-wss-q2", + "requirements": [], + "status": "closed", + "lease_owner": "pr1-worker-sessions-wss-w3", + "holder": { + "worker_id": "pr1-worker-sessions-wss-w3", + "task_queue": "pr1-worker-sessions-wss-q2", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 120, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 120, + "ttl_span": 1800, + "closed": true, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "present", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "admitted": false, + "error": "Worker session is not active.", + "reason": "session_not_active", + "status": 409, + "protocol_version": "1.20" + } + }, + "closed_create": { + "status": 409, + "response": { + "session": { + "session_id": "pr1-worker-sessions-wss-short", + "namespace": "pr1-worker-sessions-wss", + "connection": null, + "queue": "pr1-worker-sessions-wss-q2", + "requirements": [], + "status": "closed", + "lease_owner": "pr1-worker-sessions-wss-w3", + "holder": { + "worker_id": "pr1-worker-sessions-wss-w3", + "task_queue": "pr1-worker-sessions-wss-q2", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 120, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 120, + "ttl_span": 1800, + "closed": true, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "present", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "admitted": false, + "error": "Worker session was closed and cannot be reacquired.", + "reason": "session_closed", + "status": 409, + "protocol_version": "1.20" + } + }, + "closed_create_unmet": { + "status": 409, + "response": { + "missing_requirements": [ + "gpu" + ], + "admitted": false, + "error": "Worker does not advertise every worker-session requirement.", + "reason": "session_requirements_not_met", + "status": 409, + "protocol_version": "1.20" + } + }, + "reacquire_terminal": { + "status": 409, + "response": { + "session": { + "session_id": "pr1-worker-sessions-wss-terminal", + "namespace": "pr1-worker-sessions-wss", + "connection": null, + "queue": "pr1-worker-sessions-wss-q1", + "requirements": [], + "status": "expired", + "lease_owner": "pr1-worker-sessions-wss-w2", + "holder": { + "worker_id": "pr1-worker-sessions-wss-w2", + "task_queue": "pr1-worker-sessions-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": "lease_expired", + "lease_seconds": 2, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": false, + "active_activity_count": 0, + "lease_span": 2, + "ttl_span": 1800, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "admitted": false, + "error": "Worker session is terminal and cannot be reacquired.", + "reason": "session_reacquire_disallowed", + "status": 409, + "protocol_version": "1.20" + } + }, + "reacquire_disallowed": { + "status": 409, + "response": { + "session": { + "session_id": "pr1-worker-sessions-wss-optional", + "namespace": "pr1-worker-sessions-wss", + "connection": null, + "queue": "pr1-worker-sessions-wss-q1", + "requirements": [], + "status": "expired", + "lease_owner": "pr1-worker-sessions-wss-w2", + "holder": { + "worker_id": "pr1-worker-sessions-wss-w2", + "task_queue": "pr1-worker-sessions-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": "lease_expired", + "lease_seconds": 2, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 2, + "ttl_span": 1800, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "admitted": false, + "error": "Worker session options disallow reacquisition after holder failure.", + "reason": "session_reacquire_disallowed", + "status": 409, + "protocol_version": "1.20" + } + }, + "ttl_heartbeat": { + "status": 409, + "response": { + "session": { + "session_id": "pr1-worker-sessions-wss-ttl", + "namespace": "pr1-worker-sessions-wss", + "connection": null, + "queue": "pr1-worker-sessions-wss-q1", + "requirements": [], + "status": "expired", + "lease_owner": "pr1-worker-sessions-wss-w2", + "holder": { + "worker_id": "pr1-worker-sessions-wss-w2", + "task_queue": "pr1-worker-sessions-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": "ttl_expired", + "lease_seconds": 60, + "ttl_seconds": 2, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 60, + "ttl_span": 2, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "admitted": false, + "error": "Worker session is not active.", + "reason": "session_not_active", + "status": 409, + "protocol_version": "1.20" + } + }, + "ttl_create": { + "status": 409, + "response": { + "session": { + "session_id": "pr1-worker-sessions-wss-ttl", + "namespace": "pr1-worker-sessions-wss", + "connection": null, + "queue": "pr1-worker-sessions-wss-q1", + "requirements": [], + "status": "expired", + "lease_owner": "pr1-worker-sessions-wss-w2", + "holder": { + "worker_id": "pr1-worker-sessions-wss-w2", + "task_queue": "pr1-worker-sessions-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": "ttl_expired", + "lease_seconds": 60, + "ttl_seconds": 2, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 60, + "ttl_span": 2, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "admitted": false, + "error": "Worker session is terminal and cannot be reacquired.", + "reason": "session_reacquire_disallowed", + "status": 409, + "protocol_version": "1.20" + } + }, + "ttl_close": { + "status": 200, + "response": { + "admitted": true, + "outcome": "closed", + "session": { + "session_id": "pr1-worker-sessions-wss-ttl", + "namespace": "pr1-worker-sessions-wss", + "connection": null, + "queue": "pr1-worker-sessions-wss-q1", + "requirements": [], + "status": "closed", + "lease_owner": "pr1-worker-sessions-wss-w2", + "holder": { + "worker_id": "pr1-worker-sessions-wss-w2", + "task_queue": "pr1-worker-sessions-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 60, + "ttl_seconds": 2, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 60, + "ttl_span": 2, + "closed": true, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "present", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "activity_task_id": null, + "status": 200, + "reason": null, + "protocol_version": "1.20" + } + }, + "held_created": { + "status": 201, + "response": { + "admitted": true, + "outcome": "created", + "session": { + "session_id": "pr1-worker-sessions-wss-held", + "namespace": "pr1-worker-sessions-wss", + "connection": null, + "queue": "pr1-worker-sessions-wss-q2", + "requirements": [], + "status": "active", + "lease_owner": "pr1-worker-sessions-wss-w3", + "holder": { + "worker_id": "pr1-worker-sessions-wss-w3", + "task_queue": "pr1-worker-sessions-wss-q2", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 120, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 120, + "ttl_span": 1800, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "activity_task_id": null, + "status": 201, + "reason": null, + "protocol_version": "1.20" + } + }, + "holder_deregistered": 200, + "held_orphaned": { + "status": 200, + "response": { + "namespace": "pr1-worker-sessions-wss", + "session": { + "session_id": "pr1-worker-sessions-wss-held", + "namespace": "pr1-worker-sessions-wss", + "connection": null, + "queue": "pr1-worker-sessions-wss-q2", + "requirements": [], + "status": "orphaned", + "lease_owner": "pr1-worker-sessions-wss-w3", + "holder": { + "worker_id": "pr1-worker-sessions-wss-w3", + "task_queue": null, + "status": null, + "last_heartbeat_at": null, + "is_stale": true + }, + "failure_reason": "worker_heartbeat_stale", + "lease_seconds": 120, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 120, + "ttl_span": 1800, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + } + } + }, + "deregistered_create": { + "status": 412, + "response": { + "error": "Worker must be registered before creating a worker session.", + "reason": "worker_not_registered", + "worker_id": "pr1-worker-sessions-wss-w3", + "protocol_version": "1.20" + } + }, + "orphan_reacquired": { + "status": 200, + "response": { + "admitted": true, + "outcome": "reacquired", + "session": { + "session_id": "pr1-worker-sessions-wss-held", + "namespace": "pr1-worker-sessions-wss", + "connection": null, + "queue": "pr1-worker-sessions-wss-q1", + "requirements": [], + "status": "active", + "lease_owner": "pr1-worker-sessions-wss-w2", + "holder": { + "worker_id": "pr1-worker-sessions-wss-w2", + "task_queue": "pr1-worker-sessions-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": "worker_heartbeat_stale", + "lease_seconds": 120, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 120, + "ttl_span": 1800, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "activity_task_id": null, + "status": 200, + "reason": null, + "protocol_version": "1.20" + } + }, + "orphan_queue_mismatch": { + "status": 409, + "response": { + "error": "Worker [pr1-worker-sessions-wss-w2] is registered for task queue [pr1-worker-sessions-wss-q1], not session queue [pr1-worker-sessions-wss-q2].", + "reason": "session_queue_mismatch", + "worker_id": "pr1-worker-sessions-wss-w2", + "registered_task_queue": "pr1-worker-sessions-wss-q1", + "requested_session_queue": "pr1-worker-sessions-wss-q2", + "protocol_version": "1.20" + } + }, + "show_missing": { + "status": 404, + "response": { + "message": "Worker session [pr1-worker-sessions-wss-none] not found in namespace [pr1-worker-sessions-wss].", + "reason": "worker_session_not_found" + } + }, + "list_metrics": { + "total": 7, + "active": 2, + "closed": 2, + "expired": 3, + "failed": 0, + "orphaned": 0 + }, + "list_namespace": "pr1-worker-sessions-wss", + "list_sessions": [ + { + "session_id": "pr1-worker-sessions-wss-held", + "namespace": "pr1-worker-sessions-wss", + "connection": null, + "queue": "pr1-worker-sessions-wss-q1", + "requirements": [], + "status": "active", + "lease_owner": "pr1-worker-sessions-wss-w2", + "holder": { + "worker_id": "pr1-worker-sessions-wss-w2", + "task_queue": "pr1-worker-sessions-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": "worker_heartbeat_stale", + "lease_seconds": 120, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 120, + "ttl_span": 1800, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + { + "session_id": "pr1-worker-sessions-wss-optional", + "namespace": "pr1-worker-sessions-wss", + "connection": null, + "queue": "pr1-worker-sessions-wss-q1", + "requirements": [], + "status": "expired", + "lease_owner": "pr1-worker-sessions-wss-w2", + "holder": { + "worker_id": "pr1-worker-sessions-wss-w2", + "task_queue": "pr1-worker-sessions-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": "lease_expired", + "lease_seconds": 2, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 2, + "ttl_span": 1800, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + { + "session_id": "pr1-worker-sessions-wss-s1", + "namespace": "pr1-worker-sessions-wss", + "connection": "c1", + "queue": "pr1-worker-sessions-wss-q1", + "requirements": [], + "status": "expired", + "lease_owner": "pr1-worker-sessions-wss-w1", + "holder": { + "worker_id": "pr1-worker-sessions-wss-w1", + "task_queue": "pr1-worker-sessions-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": "lease_expired", + "lease_seconds": 30, + "ttl_seconds": 9999, + "max_concurrent_activities": 3, + "create_if_missing": false, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 5, + "ttl_span": 600, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + { + "session_id": "pr1-worker-sessions-wss-s2", + "namespace": "pr1-worker-sessions-wss", + "connection": null, + "queue": "pr1-worker-sessions-wss-q1", + "requirements": [], + "status": "active", + "lease_owner": "pr1-worker-sessions-wss-w1", + "holder": { + "worker_id": "pr1-worker-sessions-wss-w1", + "task_queue": "pr1-worker-sessions-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 120, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 120, + "ttl_span": 1800, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + { + "session_id": "pr1-worker-sessions-wss-short", + "namespace": "pr1-worker-sessions-wss", + "connection": null, + "queue": "pr1-worker-sessions-wss-q2", + "requirements": [], + "status": "closed", + "lease_owner": "pr1-worker-sessions-wss-w3", + "holder": { + "worker_id": "pr1-worker-sessions-wss-w3", + "task_queue": null, + "status": null, + "last_heartbeat_at": null, + "is_stale": true + }, + "failure_reason": null, + "lease_seconds": 120, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 120, + "ttl_span": 1800, + "closed": true, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "present", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + { + "session_id": "pr1-worker-sessions-wss-terminal", + "namespace": "pr1-worker-sessions-wss", + "connection": null, + "queue": "pr1-worker-sessions-wss-q1", + "requirements": [], + "status": "expired", + "lease_owner": "pr1-worker-sessions-wss-w2", + "holder": { + "worker_id": "pr1-worker-sessions-wss-w2", + "task_queue": "pr1-worker-sessions-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": "lease_expired", + "lease_seconds": 2, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": false, + "active_activity_count": 0, + "lease_span": 2, + "ttl_span": 1800, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + { + "session_id": "pr1-worker-sessions-wss-ttl", + "namespace": "pr1-worker-sessions-wss", + "connection": null, + "queue": "pr1-worker-sessions-wss-q1", + "requirements": [], + "status": "closed", + "lease_owner": "pr1-worker-sessions-wss-w2", + "holder": { + "worker_id": "pr1-worker-sessions-wss-w2", + "task_queue": "pr1-worker-sessions-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 60, + "ttl_seconds": 2, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 60, + "ttl_span": 2, + "closed": true, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "present", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + } + ] +} diff --git a/tests/Unit/ServerParityWorkerSessionsContractTest.mjs b/tests/Unit/ServerParityWorkerSessionsContractTest.mjs new file mode 100644 index 000000000..615b7f7b8 --- /dev/null +++ b/tests/Unit/ServerParityWorkerSessionsContractTest.mjs @@ -0,0 +1,72 @@ +import assert from 'node:assert/strict'; +import {readFileSync} from 'node:fs'; +import test from 'node:test'; +import {checkWorkerSessions} from '../../scripts/conformance/server-parity/worker-sessions-contract.mjs'; + +const fixture = JSON.parse(readFileSync(new URL('../Fixtures/ServerParity/worker-sessions.json', import.meta.url))); +// The observation published Server 2.11.1 returned for this fixture. +const recorded = JSON.parse(readFileSync( + new URL('../Fixtures/ServerParityExpected/worker-sessions-observation.json', import.meta.url))); +const id = 'pr1-worker-sessions'; +const observation = (change = () => {}) => { + const state = structuredClone(recorded); + change(state); + return {mode: 'http', worker_sessions: state}; +}; +const check = state => checkWorkerSessions(fixture, state, id); + +test('the published 2.11.1 recording satisfies the contract', () => { + assert.deepStrictEqual(check(observation()), {scope: 'worker_sessions'}); +}); + +test('the intended clearing of failure_reason on reacquire is accepted too', () => { + assert.deepStrictEqual(check(observation(state => { + state.reacquired.response.session.failure_reason = null; + state.orphan_reacquired.response.session.failure_reason = null; + state.list_sessions.find(session => session.session_id.endsWith('-wss-held')).failure_reason = null; + })), {scope: 'worker_sessions'}); +}); + +test('embedded applications are explicitly inapplicable', () => { + assert.deepStrictEqual(check({mode: 'embedded', worker_sessions: + {applicable: false, reason: 'embedded_has_no_worker_protocol'}}), {scope: 'worker_sessions'}); +}); + +for (const [label, change, message] of [ + // Native answered 501 for every session route before this fixture. + ['an unimplemented create', state => { + state.created = {status: 501, response: {reason: 'rust_capability_not_implemented'}}; + }, /worker sessions created/], + ['a protocol gate that admits 1.7', state => { + state.gate_protocol_1_7 = structuredClone(state.gate_protocol_1_8_empty); + }, /worker sessions gate_protocol_1_7/], + ['requirements checked after the lookup', state => { + state.closed_create_unmet = structuredClone(state.closed_create); + }, /worker sessions closed_create_unmet/], + ['a heartbeat that rewrites the stored lease', state => { + state.heartbeat.response.session.lease_seconds = 5; + }, /worker sessions heartbeat/], + ['a reuse that extends the TTL deadline', state => { + state.reused.response.session.ttl_span = 9999; + }, /worker sessions reused/], + ['lease expiry not applied on read', state => { + state.short_expired.response.session.status = 'active'; + state.short_expired.response.session.failure_reason = null; + }, /worker sessions short_expired/], + ['an orphan not detected on read', state => { + state.held_orphaned.response.session.status = 'active'; + }, /worker sessions held_orphaned/], + ['a closed session that can be reacquired', state => { + state.closed_create = {status: 200, response: {outcome: 'reacquired'}}; + }, /worker sessions closed_create/], + ['a different failure_reason after reacquire', state => { + state.reacquired.response.session.failure_reason = 'ttl_expired'; + }, /worker sessions reacquired/], + ['a missing step', state => { + delete state.ttl_close; + }, /worker sessions ttl_close/], +]) { + test(`the contract refuses ${label}`, () => { + assert.throws(() => check(observation(change)), message); + }); +} From eea77b5a5d3edd39f3dea9a22f77546f6c8e47b0 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Sat, 24 Oct 2026 12:18:42 +0000 Subject: [PATCH 02/13] Rust #325: serve worker session lifecycle routes like published 2.11.1 Native answered 501 for POST /api/worker/sessions, its heartbeat and DELETE routes, and GET /api/worker-sessions and /{sessionId}. They now follow WorkerSessionController and WorkerSessionRegistry: - protocol 1.8 or newer (409 worker_sessions_unavailable below it), Laravel's rules after TrimStrings and ConvertEmptyStringsToNull, 412 for an unregistered worker and 409 for another session queue; - requirements checked before the session lookup; create (201), reuse and reacquire (200) with omitted options reset to their defaults, the TTL deadline never extended, an omitted connection kept and the queue defaulting to the worker's own; a float lease read as the default; - the per-worker session limit over live leases; the owner, closed, terminal and option reacquire refusals; - heartbeat renewing the lease from now without changing the stored lease; close, already_closed, and close from any status by the owner; - lease and TTL expiry and a lost holder (gone or silent past the stale window) applied whenever a session is read, including the list, which orders by last change then session ID and counts each status. Session times are written in whole seconds, as PHP keeps them. Native clears a reacquired session's failure reason, as PHP's code intends; published PHP drops that write (Server #737), and the fixture bounds the difference. Session-bound activities, capabilities and metrics follow in the next step. The worker-sessions fixture passes 3/3 against native on SQLite, and native execution tests pass on SQLite, PostgreSQL and MySQL. --- rust/src/runtime/execution.rs | 5 + rust/src/runtime/http.rs | 68 ++- rust/src/runtime/mod.rs | 1 + rust/src/runtime/worker_sessions.rs | 602 ++++++++++++++++++++++++ rust/tests/execution.rs | 3 + rust/tests/execution/worker_sessions.rs | 106 +++++ 6 files changed, 784 insertions(+), 1 deletion(-) create mode 100644 rust/src/runtime/worker_sessions.rs create mode 100644 rust/tests/execution/worker_sessions.rs diff --git a/rust/src/runtime/execution.rs b/rust/src/runtime/execution.rs index 0b1b8bb45..c2d9a86bb 100644 --- a/rust/src/runtime/execution.rs +++ b/rust/src/runtime/execution.rs @@ -1276,6 +1276,11 @@ impl Runtime { #[cfg(test)] delegate!(poll(body: Value, kind: &'static str, protocol_version: &str) -> Result); delegate!(poll_with_pages(body: Value, kind: &'static str, protocol_version: &str, pages: super::history_pages::HistoryPages, default_wait: u64) -> Result); + delegate!(create_worker_session(body: Value, protocol: &str) -> Result); + delegate!(heartbeat_worker_session(session_id: &str, body: Value, protocol: &str) -> Result); + delegate!(close_worker_session(session_id: &str, body: Value, protocol: &str) -> Result); + delegate!(list_worker_sessions() -> Result); + delegate!(show_worker_session(session_id: &str) -> Result); delegate!(heartbeat_task(task_id: &str, body: Value) -> Result); pub(crate) async fn deliver_cancellation(&self, task_id: &str, body: Value, protocol: &str) -> Result { let codec = self.transition_codec.clone(); diff --git a/rust/src/runtime/http.rs b/rust/src/runtime/http.rs index 1c6614aa0..e42f639d8 100644 --- a/rust/src/runtime/http.rs +++ b/rust/src/runtime/http.rs @@ -201,6 +201,11 @@ pub fn router(runtime: Runtime) -> Router { ) .route("/api/worker/register", post(register)) .route("/api/worker/heartbeat", post(worker_heartbeat)) + .route("/api/worker/sessions", post(create_worker_session)) + .route("/api/worker/sessions/{session_id}", delete(close_worker_session)) + .route("/api/worker/sessions/{session_id}/heartbeat", post(heartbeat_worker_session)) + .route("/api/worker-sessions", get(list_worker_sessions)) + .route("/api/worker-sessions/{session_id}", get(show_worker_session)) .route("/api/worker/registrations/{worker_id}", delete(deregister)) .route( "/api/worker/registrations/{worker_id}/deregister", @@ -635,7 +640,7 @@ enum Rest { } /// Route families a named namespace may use, by prefix. -const NAMESPACE_FAMILIES: [(&str, Rest, Allowed); 8] = [ +const NAMESPACE_FAMILIES: [(&str, Rest, Allowed); 9] = [ // Runtime credentials are server-wide; the namespace only scopes a tenant. ("/api/runtime-credentials/", Rest::Anything, Allowed::Any), ("/api/external-payloads/v1/", Rest::Anything, READ), @@ -651,6 +656,8 @@ const NAMESPACE_FAMILIES: [(&str, Rest, Allowed); 8] = [ ("/api/workers/", Rest::OneSegment, Allowed::Only(&["GET", "DELETE"])), // Deployments are the namespace's own build-id rollouts. ("/api/deployments/", Rest::NonEmpty, READ_WRITE), + // Worker session visibility reads the namespace's own sessions. + ("/api/worker-sessions/", Rest::OneSegment, READ), ]; fn named_namespace_operation(method: &str, path: &str) -> bool { @@ -692,6 +699,7 @@ fn exact_namespace_route(path: &str) -> Option { "/api/task-queues" | "/api/deployments" | "/api/workers" + | "/api/worker-sessions" | "/api/system/operator-metrics" | "/api/system/health" | "/api/system/operator-dashboard" @@ -757,6 +765,9 @@ fn worker_route(rest: &str) -> Option { ["register" | "heartbeat"] | ["registrations", _] | ["registrations", _, "deregister"] + | ["sessions"] + | ["sessions", _] + | ["sessions", _, "heartbeat"] | ["workflow-tasks" | "activity-tasks", "poll"] | ["query-tasks", "poll"] | ["query-tasks", _, "complete" | "fail" | "status"] @@ -1554,6 +1565,61 @@ fn worker_response(mut body: Value) -> Value { body["server_capabilities"] = capabilities(); body } +/// A worker session answer: PHP's status, its body repeating it, and the +/// worker protocol envelope. +fn session_answer(answer: super::worker_sessions::Answer) -> Response { + (answer.status, Json(worker_response(answer.body))).into_response() +} + +async fn create_worker_session( + Extension(runtime): Extension, + headers: HeaderMap, + Json(body): Json, +) -> Result { + let protocol = request_protocol(&headers)?; + runtime.create_worker_session(body, protocol).await.map(session_answer).map_err(worker_refusal) +} + +async fn heartbeat_worker_session( + Extension(runtime): Extension, + Path(session_id): Path, + headers: HeaderMap, + Json(body): Json, +) -> Result { + let protocol = request_protocol(&headers)?; + runtime.heartbeat_worker_session(&session_id, body, protocol).await.map(session_answer).map_err(worker_refusal) +} + +/// The holder may name itself in the JSON body or as `?worker_id=`. +async fn close_worker_session( + Extension(runtime): Extension, + Path(session_id): Path, + Query(query): Query>, + headers: HeaderMap, + body: axum::body::Bytes, +) -> Result { + let protocol = request_protocol(&headers)?; + let mut fields = if body.is_empty() { + json!({}) + } else { + serde_json::from_slice::(&body).ok().filter(Value::is_object).unwrap_or_else(|| json!({})) + }; + if let Some(worker_id) = query.get("worker_id") + && fields.get("worker_id").is_none() + { + fields["worker_id"] = json!(worker_id); + } + runtime.close_worker_session(&session_id, fields, protocol).await.map(session_answer).map_err(worker_refusal) +} + +async fn list_worker_sessions(Extension(runtime): Extension) -> Result> { + runtime.list_worker_sessions().await.map(Json) +} + +async fn show_worker_session(Extension(runtime): Extension, Path(session_id): Path) -> Result> { + runtime.show_worker_session(&session_id).await.map(Json) +} + async fn worker_heartbeat( Extension(runtime): Extension, headers: HeaderMap, diff --git a/rust/src/runtime/mod.rs b/rust/src/runtime/mod.rs index 399106960..05a0747cd 100644 --- a/rust/src/runtime/mod.rs +++ b/rust/src/runtime/mod.rs @@ -140,6 +140,7 @@ mod wait_timeouts; mod worker_contracts; mod worker_incarnations; mod worker_poll_fence; +mod worker_sessions; mod workflow_poll_requests; mod worker_management; mod worker_admission; diff --git a/rust/src/runtime/worker_sessions.rs b/rust/src/runtime/worker_sessions.rs new file mode 100644 index 000000000..0b550dc66 --- /dev/null +++ b/rust/src/runtime/worker_sessions.rs @@ -0,0 +1,602 @@ +//! Worker sessions, as published Server 2.11.1 serves them +//! (`WorkerSessionController` and `WorkerSessionRegistry`): a worker leases a +//! named session in its namespace, renews it with heartbeats, and closes it. +//! A session's lease and absolute TTL expire only when the session is read; +//! a holder whose registration is gone or silent leaves the session orphaned +//! on the next read. Session times are whole seconds, as PHP stores them. +//! +//! One published defect is not reproduced: PHP's `renewHeldSession` filters +//! its own `failure_reason => null` out, so a reused or reacquired session +//! keeps its old failure reason (Server #737). Native clears it, as that code +//! intends; the shared fixture bounds the difference explicitly. +use super::laravel_validation::{Rule, failure_messages, rule_errors}; +use super::transaction::Transaction; +use super::{Result, RuntimeError, store::now}; +use axum::http::StatusCode; +use chrono::{DateTime, Utc}; +use serde_json::{Map, Value, json}; + +const DEFAULT_LEASE_SECONDS: i64 = 120; +const DEFAULT_TTL_SECONDS: i64 = 1800; +const DEFAULT_MAX_CONCURRENT_ACTIVITIES: i64 = 1; +const MINIMUM_PROTOCOL: (u64, u64) = (1, 8); + +const SESSION_BY_ID: &str = "SELECT id,session_id,connection,queue,requirements,status,lease_owner,lease_expires_at,ttl_expires_at,closed_at,failure_reason,lease_seconds,ttl_seconds,max_concurrent_activities,CASE WHEN create_if_missing THEN 1 ELSE 0 END AS create_if_missing_flag,CASE WHEN allow_reacquire_after_failure THEN 1 ELSE 0 END AS allow_reacquire_flag,last_heartbeat_at,created_at,updated_at FROM workflow_worker_sessions WHERE namespace=$1 AND session_id=$2"; +const SESSIONS_TO_REFRESH: &str = "SELECT session_id FROM workflow_worker_sessions WHERE namespace=$1 AND (status='active' OR (status<>'closed' AND ttl_expires_at<=$2 AND (failure_reason IS NULL OR failure_reason<>'ttl_expired'))) ORDER BY id"; +const SESSIONS_LISTED: &str = "SELECT id,session_id,connection,queue,requirements,status,lease_owner,lease_expires_at,ttl_expires_at,closed_at,failure_reason,lease_seconds,ttl_seconds,max_concurrent_activities,CASE WHEN create_if_missing THEN 1 ELSE 0 END AS create_if_missing_flag,CASE WHEN allow_reacquire_after_failure THEN 1 ELSE 0 END AS allow_reacquire_flag,last_heartbeat_at,created_at,updated_at FROM workflow_worker_sessions WHERE namespace=$1 ORDER BY updated_at DESC,session_id ASC"; +const HELD_SESSIONS: &str = "SELECT COUNT(*) FROM workflow_worker_sessions WHERE namespace=$1 AND lease_owner=$2 AND status='active' AND (lease_expires_at IS NULL OR lease_expires_at>$3) AND session_id<>$4"; +const HOLDER: &str = "SELECT task_queue,status,last_heartbeat_at FROM workflow_worker_registrations WHERE namespace=$1 AND worker_id=$2"; + +/// A session time: PHP keeps whole seconds. +fn second(at: DateTime) -> DateTime { + DateTime::from_timestamp(at.timestamp(), 0).expect("whole-second timestamp") +} + +fn wire(at: Option>) -> Value { + at.map_or(Value::Null, |at| json!(super::cooperative::timestamp(at))) +} + +/// `rejectWorkerSessionsUnavailable`: worker sessions need protocol 1.8. +pub(super) fn sessions_gate(protocol: &str) -> Result<()> { + let version = protocol.split_once('.').and_then(|(major, minor)| { + let digits = |text: &str| !text.is_empty() && text.bytes().all(|byte| byte.is_ascii_digit()); + (digits(major) && digits(minor)).then(|| Some((major.parse::().ok()?, minor.parse::().ok()?))).flatten() + }); + if version.is_some_and(|version| version >= MINIMUM_PROTOCOL) { + return Ok(()); + } + Err(RuntimeError::Protocol { + status: StatusCode::CONFLICT, + response: json!({"error":"Worker sessions require worker protocol 1.8 or newer.","reason":"worker_sessions_unavailable", + "supported_version":super::WORKER_PROTOCOL_VERSION,"requested_version":protocol,"minimum_protocol_version":"1.8", + "remediation":"Send the X-Durable-Workflow-Protocol-Version header with worker protocol 1.8 or newer and use a worker SDK that implements worker-session semantics."}), + }) +} + +/// The controller's rules, after Laravel's TrimStrings and +/// ConvertEmptyStringsToNull. +fn validate(body: &Value, create: bool) -> Result { + use Rule::{Array, Boolean, Integer, Max, Min, Nullable, Required}; + let body = super::laravel_rules::normalize(body); + let mut rules: Vec<(&str, &[Rule])> = vec![("worker_id", &[Required, Rule::String, Max(255)])]; + if create { + rules.extend([ + ("session_id", &[Required, Rule::String, Max(255)][..]), + ("connection", &[Nullable, Rule::String, Max(255)][..]), + ("queue", &[Nullable, Rule::String, Max(255)][..]), + ("requirements", &[Nullable, Array][..]), + ]); + } + rules.push(("lease_seconds", &[Nullable, Integer, Min(1)])); + if create { + rules.extend([ + ("ttl_seconds", &[Nullable, Integer, Min(1)][..]), + ("max_concurrent_activities", &[Nullable, Integer, Min(1)][..]), + ("create_if_missing", &[Nullable, Boolean][..]), + ("allow_reacquire_after_failure", &[Nullable, Boolean][..]), + ]); + } + let mut errors = rule_errors(&body, &rules, |_, _| None); + // `requirements.*`: each a string of at most 255 characters, reported + // after `requirements` itself. + if let Some(items) = body.get("requirements").and_then(Value::as_array) { + let at = errors.iter().position(|(field, _)| field == "requirements").map_or(errors.len(), |at| at + 1); + let item_errors: Vec<(String, Vec)> = items + .iter() + .enumerate() + .map(|(index, item)| { + let field = format!("requirements.{index}"); + let mut messages = Vec::new(); + match item { + Value::Null => {} + Value::String(text) if text.chars().count() > 255 => { + messages.push(format!("The {field} field must not be greater than 255 characters.")); + } + Value::String(_) => {} + _ => messages.push(format!("The {field} field must be a string.")), + } + (field, messages) + }) + .collect(); + errors.splice(at..at, item_errors); + } + if let Some(error) = failure_messages(errors) { + return Err(error); + } + Ok(body) +} + +/// `positiveInt`: an integer above zero, or a string of digits. A number +/// written with a fraction, even an integral one, is not. +fn positive_int(value: &Value) -> Option { + match value { + Value::Number(number) => number.as_i64().filter(|value| *value > 0), + Value::String(text) if !text.is_empty() && text.bytes().all(|byte| byte.is_ascii_digit()) => { + text.parse::().ok().filter(|value| *value > 0) + } + _ => None, + } +} + +/// `boolValue`, as Laravel's `boolean` admits. +fn bool_value(value: &Value, default: bool) -> bool { + match value { + Value::Bool(flag) => *flag, + Value::Number(number) => number.as_i64().map_or(default, |value| value == 1), + Value::String(text) => match text.as_str() { + "1" => true, + "0" => false, + _ => default, + }, + _ => default, + } +} + +fn string_value(value: &Value) -> Option { + value.as_str().map(str::trim).filter(|text| !text.is_empty()).map(str::to_owned) +} + +/// `normalizeOptions`. +struct Options { + session_id: String, + connection: Option, + queue: Option, + requirements: Vec, + lease_seconds: i64, + ttl_seconds: i64, + max_concurrent_activities: i64, + create_if_missing: bool, + allow_reacquire_after_failure: bool, +} + +impl Options { + fn from(body: &Value, fallback_queue: Option) -> Option { + let mut requirements: Vec = Vec::new(); + for item in body["requirements"].as_array().into_iter().flatten() { + if let Some(text) = string_value(item) + && !requirements.contains(&text) + { + requirements.push(text); + } + } + Some(Self { + session_id: string_value(&body["session_id"])?, + connection: string_value(&body["connection"]), + queue: string_value(&body["queue"]).or(fallback_queue), + requirements, + lease_seconds: positive_int(&body["lease_seconds"]).unwrap_or(DEFAULT_LEASE_SECONDS), + ttl_seconds: positive_int(&body["ttl_seconds"]).unwrap_or(DEFAULT_TTL_SECONDS), + max_concurrent_activities: positive_int(&body["max_concurrent_activities"]).unwrap_or(DEFAULT_MAX_CONCURRENT_ACTIVITIES), + create_if_missing: bool_value(&body["create_if_missing"], true), + allow_reacquire_after_failure: bool_value(&body["allow_reacquire_after_failure"], true), + }) + } +} + +/// A session row. +struct Session { + id: i64, + session_id: String, + connection: Option, + queue: Option, + requirements: Vec, + status: String, + lease_owner: Option, + lease_expires_at: Option>, + ttl_expires_at: Option>, + closed_at: Option>, + failure_reason: Option, + lease_seconds: i64, + ttl_seconds: i64, + max_concurrent_activities: i64, + create_if_missing: bool, + allow_reacquire_after_failure: bool, + last_heartbeat_at: Option>, + created_at: Option>, + updated_at: Option>, +} + +/// A registry answer: the HTTP status and PHP's body, which repeats it. +pub(crate) struct Answer { + pub(crate) status: StatusCode, + pub(crate) body: Value, +} + +fn success(outcome: &str, session: Value, status: StatusCode) -> Answer { + Answer { status, body: json!({"admitted":true,"outcome":outcome,"session":session,"activity_task_id":null, + "status":status.as_u16(),"reason":null}) } +} + +fn failure(reason: &str, error: &str, status: StatusCode, context: Map) -> Answer { + let mut body = context; + body.insert("admitted".into(), json!(false)); + body.insert("error".into(), json!(error)); + body.insert("reason".into(), json!(reason)); + body.insert("status".into(), json!(status.as_u16())); + Answer { status, body: Value::Object(body) } +} + +fn with_session(session: Value) -> Map { + Map::from_iter([("session".to_owned(), session)]) +} + +store_methods! { + /// `POST /api/worker/sessions`. + pub(crate) async fn create_worker_session(&self, body: Value, protocol: &str) -> Result { + sessions_gate(protocol)?; + let body = validate(&body, true)?; + let worker_id = body["worker_id"].as_str().unwrap_or_default().to_owned(); + let Some(worker) = Self::query("SELECT task_queue,capabilities,max_concurrent_worker_sessions FROM workflow_worker_registrations WHERE namespace=$1 AND worker_id=$2") + .bind(self.namespace.as_ref()).bind(&worker_id).fetch_optional(&self.pool).await? + else { + return Err(RuntimeError::Protocol { status: StatusCode::PRECONDITION_FAILED, + response: json!({"error":"Worker must be registered before creating a worker session.","reason":"worker_not_registered","worker_id":worker_id}) }); + }; + let task_queue = DB::string(&worker, "task_queue")?; + if let Some(queue) = string_value(&body["queue"]).filter(|queue| *queue != task_queue) { + return Err(RuntimeError::Protocol { status: StatusCode::CONFLICT, + response: json!({"error":format!("Worker [{worker_id}] is registered for task queue [{task_queue}], not session queue [{queue}]."), + "reason":"session_queue_mismatch","worker_id":worker_id,"registered_task_queue":task_queue,"requested_session_queue":queue}) }); + } + let Some(options) = Options::from(&body, Some(task_queue.clone())) else { + return Err(RuntimeError::Protocol { status: StatusCode::UNPROCESSABLE_ENTITY, + response: json!({"error":"Worker session requires a non-empty session_id.","reason":"invalid_worker_session"}) }); + }; + let capabilities: Vec = DB::optional_document_row(&worker, "capabilities")? + .as_ref() + .and_then(Value::as_array) + .into_iter() + .flatten() + .filter_map(|value| value.as_str().map(str::to_owned)) + .collect(); + let limit = DB::optional_number(&worker, "max_concurrent_worker_sessions")?.unwrap_or(10); + let mut tx = self.begin().await?; + let answer = self.acquire(&mut tx, &worker_id, &task_queue, &capabilities, limit, &options).await?; + tx.commit().await?; + Ok(answer) + } + + /// `acquireLocked`. + async fn acquire( + &self, + tx: &mut Transaction<'_, DB>, + worker_id: &str, + task_queue: &str, + capabilities: &[String], + limit: i64, + options: &Options, + ) -> Result { + if options.queue.as_deref().is_some_and(|queue| queue != task_queue) { + return Ok(failure("session_queue_mismatch", "Worker session queue does not match the worker's task queue.", + StatusCode::CONFLICT, Map::new())); + } + // Requirements are checked before the session is looked up. + let missing: Vec<&String> = options.requirements.iter().filter(|requirement| !capabilities.contains(requirement)).collect(); + if !missing.is_empty() { + return Ok(failure("session_requirements_not_met", "Worker does not advertise every worker-session requirement.", + StatusCode::CONFLICT, Map::from_iter([("missing_requirements".to_owned(), json!(missing))]))); + } + let Some(mut session) = self.locked_session(tx, &options.session_id).await? else { + if !options.create_if_missing { + return Ok(failure("session_missing", "Worker session does not exist and creation is disabled.", + StatusCode::CONFLICT, Map::new())); + } + if let Some(refused) = self.session_limit(tx, worker_id, limit, &options.session_id).await? { + return Ok(refused); + } + let at = second(now()); + Self::query("INSERT INTO workflow_worker_sessions(namespace,session_id,connection,queue,requirements,status,lease_owner,lease_expires_at,ttl_expires_at,lease_seconds,ttl_seconds,max_concurrent_activities,create_if_missing,allow_reacquire_after_failure,last_heartbeat_at,created_at,updated_at) VALUES ($1,$2,$3,$4,$5,'active',$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16)") + .bind(self.namespace.as_ref()).bind(&options.session_id).bind(options.connection.as_deref()).bind(options.queue.as_deref()) + .bind(DB::document(&json!(options.requirements))).bind(worker_id) + .bind(DB::bind_time(at + chrono::Duration::seconds(options.lease_seconds))) + .bind(DB::bind_time(at + chrono::Duration::seconds(options.ttl_seconds))) + .bind(options.lease_seconds).bind(options.ttl_seconds).bind(options.max_concurrent_activities) + .bind(DB::optional_flag(Some(options.create_if_missing))).bind(DB::optional_flag(Some(options.allow_reacquire_after_failure))) + .bind(DB::bind_time(at)).bind(DB::bind_time(at)).bind(DB::bind_time(at)) + .execute(&mut **tx).await?; + let created = self.locked_session(tx, &options.session_id).await?.expect("the inserted session"); + return Ok(success("created", self.snapshot(tx, &created).await?, StatusCode::CREATED)); + }; + if session.status == "active" { + if session.lease_owner.as_deref() != Some(worker_id) { + let snapshot = self.snapshot(tx, &session).await?; + return Ok(failure("session_owned_by_another_worker", "Worker session is currently leased by another worker.", + StatusCode::CONFLICT, with_session(snapshot))); + } + self.renew(tx, &mut session, options, worker_id).await?; + return Ok(success("reused", self.snapshot(tx, &session).await?, StatusCode::OK)); + } + if session.status == "closed" { + let snapshot = self.snapshot(tx, &session).await?; + return Ok(failure("session_closed", "Worker session was closed and cannot be reacquired.", + StatusCode::CONFLICT, with_session(snapshot))); + } + if session.failure_reason.as_deref() == Some("ttl_expired") || !session.allow_reacquire_after_failure { + let snapshot = self.snapshot(tx, &session).await?; + return Ok(failure("session_reacquire_disallowed", "Worker session is terminal and cannot be reacquired.", + StatusCode::CONFLICT, with_session(snapshot))); + } + if !options.allow_reacquire_after_failure { + let snapshot = self.snapshot(tx, &session).await?; + return Ok(failure("session_reacquire_disallowed", "Worker session options disallow reacquisition after holder failure.", + StatusCode::CONFLICT, with_session(snapshot))); + } + if let Some(refused) = self.session_limit(tx, worker_id, limit, &options.session_id).await? { + return Ok(refused); + } + self.renew(tx, &mut session, options, worker_id).await?; + Ok(success("reacquired", self.snapshot(tx, &session).await?, StatusCode::OK)) + } + + /// The worker session limit, not counting this session. + async fn session_limit(&self, tx: &mut Transaction<'_, DB>, worker_id: &str, limit: i64, session_id: &str) -> Result> { + let held = Self::scalar(HELD_SESSIONS).bind(self.namespace.as_ref()).bind(worker_id).bind(DB::bind_time(now())).bind(session_id) + .fetch_one(&mut **tx).await?; + Ok((held >= limit).then(|| failure("worker_session_limit_exceeded", "Worker has reached max_concurrent_worker_sessions.", + StatusCode::CONFLICT, Map::from_iter([("max_concurrent_worker_sessions".to_owned(), json!(limit))])))) + } + + /// `renewHeldSession`. Options the request leaves out reset to their + /// defaults; the TTL deadline is never extended; an omitted connection + /// keeps the stored one. Unlike published PHP, the failure reason is + /// cleared (Server #737). + async fn renew(&self, tx: &mut Transaction<'_, DB>, session: &mut Session, options: &Options, worker_id: &str) -> Result<()> { + let at = second(now()); + if options.connection.is_some() { + session.connection.clone_from(&options.connection); + } + session.queue.clone_from(&options.queue); + session.requirements.clone_from(&options.requirements); + session.status = "active".into(); + session.lease_owner = Some(worker_id.to_owned()); + session.lease_expires_at = Some(at + chrono::Duration::seconds(options.lease_seconds)); + session.closed_at = None; + session.failure_reason = None; + session.lease_seconds = options.lease_seconds; + session.ttl_seconds = options.ttl_seconds; + session.max_concurrent_activities = options.max_concurrent_activities; + session.create_if_missing = options.create_if_missing; + session.allow_reacquire_after_failure = options.allow_reacquire_after_failure; + session.last_heartbeat_at = Some(at); + self.save(tx, session).await + } + + /// `POST /api/worker/sessions/{sessionId}/heartbeat`. + pub(crate) async fn heartbeat_worker_session(&self, session_id: &str, body: Value, protocol: &str) -> Result { + sessions_gate(protocol)?; + let body = validate(&body, false)?; + let worker_id = body["worker_id"].as_str().unwrap_or_default(); + let mut tx = self.begin().await?; + let Some(mut session) = self.locked_session(&mut tx, session_id).await? else { + tx.commit().await?; + return Ok(failure("session_not_found", "Worker session was not found.", StatusCode::NOT_FOUND, Map::new())); + }; + let answer = if session.status != "active" { + failure("session_not_active", "Worker session is not active.", StatusCode::CONFLICT, with_session(self.snapshot(&mut tx, &session).await?)) + } else if session.lease_owner.as_deref() != Some(worker_id) { + failure("session_owner_mismatch", "Worker session lease is owned by another worker.", StatusCode::CONFLICT, + with_session(self.snapshot(&mut tx, &session).await?)) + } else { + let at = second(now()); + let lease = super::laravel_validation::integer(&body["lease_seconds"]).unwrap_or(session.lease_seconds).max(1); + session.lease_expires_at = Some(at + chrono::Duration::seconds(lease)); + session.last_heartbeat_at = Some(at); + session.failure_reason = None; + self.save(&mut tx, &session).await?; + success("heartbeat_recorded", self.snapshot(&mut tx, &session).await?, StatusCode::OK) + }; + tx.commit().await?; + Ok(answer) + } + + /// `DELETE /api/worker/sessions/{sessionId}`. + pub(crate) async fn close_worker_session(&self, session_id: &str, body: Value, protocol: &str) -> Result { + sessions_gate(protocol)?; + let body = validate(&body, false)?; + let worker_id = body["worker_id"].as_str().unwrap_or_default(); + let mut tx = self.begin().await?; + let Some(mut session) = self.locked_session(&mut tx, session_id).await? else { + tx.commit().await?; + return Ok(failure("session_not_found", "Worker session was not found.", StatusCode::NOT_FOUND, Map::new())); + }; + let owner = session.lease_owner.as_deref() == Some(worker_id); + let answer = if session.status == "closed" && owner { + success("already_closed", self.snapshot(&mut tx, &session).await?, StatusCode::OK) + } else if !owner { + failure("session_owner_mismatch", "Worker session lease is owned by another worker.", StatusCode::CONFLICT, + with_session(self.snapshot(&mut tx, &session).await?)) + } else { + session.status = "closed".into(); + session.closed_at = Some(second(now())); + session.failure_reason = None; + self.save(&mut tx, &session).await?; + success("closed", self.snapshot(&mut tx, &session).await?, StatusCode::OK) + }; + tx.commit().await?; + Ok(answer) + } + + /// `GET /api/worker-sessions`: refreshes the namespace's sessions, then + /// lists them, newest change first, with the status counts. + pub(crate) async fn list_worker_sessions(&self) -> Result { + self.refresh_namespace_sessions().await?; + let mut tx = self.begin().await?; + let rows = Self::query(SESSIONS_LISTED).bind(self.namespace.as_ref()).fetch_all(&mut *tx).await?; + let mut sessions = Vec::new(); + let mut metrics = json!({"total":0,"active":0,"closed":0,"expired":0,"failed":0,"orphaned":0}); + for row in &rows { + let session = Self::session(row)?; + metrics["total"] = json!(metrics["total"].as_i64().unwrap_or(0) + 1); + if let Some(count) = metrics.get_mut(session.status.as_str()) { + *count = json!(count.as_i64().unwrap_or(0) + 1); + } + sessions.push(self.snapshot(&mut tx, &session).await?); + } + tx.commit().await?; + Ok(json!({"namespace":self.namespace.as_ref(),"metrics":metrics,"sessions":sessions})) + } + + /// `GET /api/worker-sessions/{sessionId}`. + pub(crate) async fn show_worker_session(&self, session_id: &str) -> Result { + self.refresh_namespace_sessions().await?; + let mut tx = self.begin().await?; + let row = Self::query(SESSION_BY_ID).bind(self.namespace.as_ref()).bind(session_id).fetch_optional(&mut *tx).await?; + let Some(row) = row else { + tx.commit().await?; + return Err(RuntimeError::Protocol { status: StatusCode::NOT_FOUND, + response: json!({"message":format!("Worker session [{session_id}] not found in namespace [{}].", self.namespace), + "reason":"worker_session_not_found"}) }); + }; + let session = Self::session(&row)?; + let snapshot = self.snapshot(&mut tx, &session).await?; + tx.commit().await?; + Ok(json!({"namespace":self.namespace.as_ref(),"session":snapshot})) + } + + /// `refreshNamespace`: each session that may have expired, in its own + /// transaction. + async fn refresh_namespace_sessions(&self) -> Result<()> { + let ids = Self::query(SESSIONS_TO_REFRESH).bind(self.namespace.as_ref()).bind(DB::bind_time(now())).fetch_all(&self.pool).await?; + for row in ids { + let mut tx = self.begin().await?; + self.locked_session(&mut tx, &DB::string(&row, "session_id")?).await?; + tx.commit().await?; + } + Ok(()) + } + + /// The session, refreshed (`refreshStatusLocked`). + async fn locked_session(&self, tx: &mut Transaction<'_, DB>, session_id: &str) -> Result> { + let Some(row) = Self::query(SESSION_BY_ID).bind(self.namespace.as_ref()).bind(session_id).fetch_optional(&mut **tx).await? else { + return Ok(None); + }; + let mut session = Self::session(&row)?; + if session.status == "closed" || session.failure_reason.as_deref() == Some("ttl_expired") { + return Ok(Some(session)); + } + let at = now(); + let change = if session.ttl_expires_at.is_some_and(|ttl| ttl <= at) { + Some(("expired", "ttl_expired")) + } else if session.status != "active" { + None + } else if session.lease_expires_at.is_some_and(|lease| lease <= at) { + Some(("expired", "lease_expired")) + } else if self.holder_stale(tx, session.lease_owner.as_deref()).await? { + Some(("orphaned", "worker_heartbeat_stale")) + } else { + None + }; + if let Some((status, reason)) = change { + session.status = status.into(); + session.failure_reason = Some(reason.into()); + self.save(tx, &session).await?; + } + Ok(Some(session)) + } + + /// `holderIsStale`: no registration, no heartbeat, or a heartbeat older + /// than the stale window. + async fn holder_stale(&self, tx: &mut Transaction<'_, DB>, owner: Option<&str>) -> Result { + let Some(owner) = owner else { + return Ok(true); + }; + let row = Self::query(HOLDER).bind(self.namespace.as_ref()).bind(owner).fetch_optional(&mut **tx).await?; + let heartbeat = match row { + Some(row) => DB::optional_instant(&row, "last_heartbeat_at")?, + None => None, + }; + Ok(heartbeat.is_none_or(|heartbeat| heartbeat < now() - chrono::Duration::seconds(self.worker_stale_after_seconds.max(1)))) + } + + async fn save(&self, tx: &mut Transaction<'_, DB>, session: &Session) -> Result<()> { + Self::query("UPDATE workflow_worker_sessions SET connection=$1,queue=$2,requirements=$3,status=$4,lease_owner=$5,lease_expires_at=$6,ttl_expires_at=$7,closed_at=$8,failure_reason=$9,lease_seconds=$10,ttl_seconds=$11,max_concurrent_activities=$12,create_if_missing=$13,allow_reacquire_after_failure=$14,last_heartbeat_at=$15,updated_at=$16 WHERE id=$17") + .bind(session.connection.as_deref()).bind(session.queue.as_deref()).bind(DB::document(&json!(session.requirements))) + .bind(session.status.as_str()).bind(session.lease_owner.as_deref()) + .bind(DB::optional_time(session.lease_expires_at)).bind(DB::optional_time(session.ttl_expires_at)).bind(DB::optional_time(session.closed_at)) + .bind(session.failure_reason.as_deref()).bind(session.lease_seconds).bind(session.ttl_seconds).bind(session.max_concurrent_activities) + .bind(DB::optional_flag(Some(session.create_if_missing))).bind(DB::optional_flag(Some(session.allow_reacquire_after_failure))) + .bind(DB::optional_time(session.last_heartbeat_at)).bind(DB::bind_time(second(now()))).bind(session.id) + .execute(&mut **tx).await?; + Ok(()) + } + + fn session(row: &DB::Row) -> Result { + let requirements = DB::optional_document_row(row, "requirements")? + .as_ref() + .and_then(Value::as_array) + .into_iter() + .flatten() + .filter_map(|value| value.as_str().map(str::to_owned)) + .collect(); + Ok(Session { + id: DB::number(row, "id")?, + session_id: DB::string(row, "session_id")?, + connection: DB::optional_string(row, "connection")?, + queue: DB::optional_string(row, "queue")?, + requirements, + status: DB::string(row, "status")?, + lease_owner: DB::optional_string(row, "lease_owner")?, + lease_expires_at: DB::optional_instant(row, "lease_expires_at")?, + ttl_expires_at: DB::optional_instant(row, "ttl_expires_at")?, + closed_at: DB::optional_instant(row, "closed_at")?, + failure_reason: DB::optional_string(row, "failure_reason")?, + lease_seconds: DB::number(row, "lease_seconds")?, + ttl_seconds: DB::number(row, "ttl_seconds")?, + max_concurrent_activities: DB::number(row, "max_concurrent_activities")?, + create_if_missing: DB::number(row, "create_if_missing_flag")? == 1, + allow_reacquire_after_failure: DB::number(row, "allow_reacquire_flag")? == 1, + last_heartbeat_at: DB::optional_instant(row, "last_heartbeat_at")?, + created_at: DB::optional_instant(row, "created_at")?, + updated_at: DB::optional_instant(row, "updated_at")?, + }) + } + + /// PHP's session snapshot, with its holder read from the registration. + async fn snapshot(&self, tx: &mut Transaction<'_, DB>, session: &Session) -> Result { + let holder = match session.lease_owner.as_deref() { + None => Value::Null, + Some(owner) => { + let row = Self::query(HOLDER).bind(self.namespace.as_ref()).bind(owner).fetch_optional(&mut **tx).await?; + let (task_queue, status, heartbeat) = match &row { + Some(row) => (DB::optional_string(row, "task_queue")?, DB::optional_string(row, "status")?, DB::optional_instant(row, "last_heartbeat_at")?), + None => (None, None, None), + }; + let stale = heartbeat.is_none_or(|heartbeat| heartbeat < now() - chrono::Duration::seconds(self.worker_stale_after_seconds.max(1))); + json!({"worker_id":owner,"task_queue":task_queue,"status":status,"last_heartbeat_at":wire(heartbeat),"is_stale":stale}) + } + }; + Ok(json!({"session_id":session.session_id,"namespace":self.namespace.as_ref(),"connection":session.connection,"queue":session.queue, + "requirements":session.requirements,"status":session.status,"lease_owner":session.lease_owner,"holder":holder, + "lease_expires_at":wire(session.lease_expires_at),"ttl_expires_at":wire(session.ttl_expires_at),"closed_at":wire(session.closed_at), + "failure_reason":session.failure_reason,"lease_seconds":session.lease_seconds,"ttl_seconds":session.ttl_seconds, + "max_concurrent_activities":session.max_concurrent_activities,"create_if_missing":session.create_if_missing, + "allow_reacquire_after_failure":session.allow_reacquire_after_failure,"active_activity_count":0, + "last_heartbeat_at":wire(session.last_heartbeat_at),"created_at":wire(session.created_at),"updated_at":wire(session.updated_at)})) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn sessions_need_protocol_one_eight() { + assert!(sessions_gate("1.8").is_ok()); + assert!(sessions_gate("1.20").is_ok()); + assert!(sessions_gate("1.10").is_ok()); + assert!(sessions_gate("1.7").is_err()); + assert!(sessions_gate("1.x").is_err()); + assert!(sessions_gate("+1.9").is_err()); + } + + #[test] + fn options_read_like_php() { + assert_eq!(positive_int(&json!("60")), Some(60)); + assert_eq!(positive_int(&json!(7)), Some(7)); + assert_eq!(positive_int(&serde_json::from_str::("7.0").unwrap()), None); + assert_eq!(positive_int(&json!("0")), None); + let options = Options::from(&json!({"session_id":"s","requirements":["gpu",null,"gpu","tpu"]}), Some("q".into())).unwrap(); + assert_eq!(options.requirements, ["gpu", "tpu"]); + assert_eq!((options.queue.as_deref(), options.lease_seconds, options.ttl_seconds), (Some("q"), 120, 1800)); + } +} diff --git a/rust/tests/execution.rs b/rust/tests/execution.rs index aba0ece83..f7abdac1b 100644 --- a/rust/tests/execution.rs +++ b/rust/tests/execution.rs @@ -295,6 +295,9 @@ mod workflow_poll_requests; #[path = "execution/worker_poll_fence.rs"] mod worker_poll_fence; +#[path = "execution/worker_sessions.rs"] +mod worker_sessions; + enum TestDatabase { Sqlite(tempfile::TempDir), Postgres { diff --git a/rust/tests/execution/worker_sessions.rs b/rust/tests/execution/worker_sessions.rs new file mode 100644 index 000000000..571d575a7 --- /dev/null +++ b/rust/tests/execution/worker_sessions.rs @@ -0,0 +1,106 @@ +//! Worker sessions as published 2.11.1 serves them (worker-sessions fixture): +//! create, reuse, heartbeat and close on the worker plane, lease expiry and a +//! holder's loss applied on read, and the namespace's list on the control +//! plane. +use super::*; +use std::time::Duration; + +async fn worker(app: &Router, worker: &str, capabilities: Value, sessions: Option) { + let mut body = json!({"worker_id":worker,"task_queue":"sessions","runtime":"php","supported_workflow_types":["echo"], + "supported_activity_types":["act"],"capabilities":capabilities}); + if let Some(sessions) = sessions { + body["max_concurrent_worker_sessions"] = json!(sessions); + } + let (status, answer) = request(app, "POST", "/api/worker/register", body).await; + assert_eq!(status, StatusCode::CREATED, "{answer}"); +} + +async fn create(app: &Router, body: Value) -> (StatusCode, Value) { + request(app, "POST", "/api/worker/sessions", body).await +} + +#[tokio::test] +async fn worker_sessions_lease_renew_expire_and_close_like_php() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + worker(&app, "holder", json!(["gpu"]), Some(2)).await; + worker(&app, "other", json!([]), None).await; + + // Create: the ID trimmed, requirements de-duplicated, a string lease read. + let (status, created) = create(&app, json!({"worker_id":"holder","session_id":" s1 ","requirements":["gpu"," gpu "], + "lease_seconds":"60","connection":"c1"})).await; + assert_eq!(status, StatusCode::CREATED, "{created}"); + let session = &created["session"]; + assert_eq!((created["outcome"].clone(), created["status"].clone()), (json!("created"), json!(201))); + assert_eq!((session["session_id"].clone(), session["requirements"].clone(), session["lease_seconds"].clone()), + (json!("s1"), json!(["gpu"]), json!(60)), "{session}"); + assert_eq!((session["create_if_missing"].clone(), session["allow_reacquire_after_failure"].clone()), (json!(true), json!(true))); + assert!(session["created_at"].as_str().unwrap().ends_with(".000000Z"), "whole seconds: {session}"); + // Requirements are checked before the lookup. + let (status, unmet) = create(&app, json!({"worker_id":"holder","session_id":"s1","requirements":["tpu"]})).await; + assert_eq!((status, unmet["missing_requirements"].clone()), (StatusCode::CONFLICT, json!(["tpu"])), "{unmet}"); + + // Reuse resets omitted options and keeps the TTL deadline. + let (status, reused) = create(&app, json!({"worker_id":"holder","session_id":"s1","ttl_seconds":9999,"create_if_missing":false})).await; + assert_eq!((status, reused["outcome"].clone()), (StatusCode::OK, json!("reused")), "{reused}"); + assert_eq!((reused["session"]["requirements"].clone(), reused["session"]["lease_seconds"].clone(), reused["session"]["create_if_missing"].clone(), + reused["session"]["connection"].clone(), reused["session"]["ttl_expires_at"].clone()), + (json!([]), json!(120), json!(false), json!("c1"), session["ttl_expires_at"].clone())); + let (status, foreign) = create(&app, json!({"worker_id":"other","session_id":"s1"})).await; + assert_eq!((status, foreign["reason"].clone()), (StatusCode::CONFLICT, json!("session_owned_by_another_worker"))); + + // The per-worker limit counts live leases. + let (status, _) = create(&app, json!({"worker_id":"holder","session_id":"s2","lease_seconds":2})).await; + assert_eq!(status, StatusCode::CREATED); + let (status, limited) = create(&app, json!({"worker_id":"holder","session_id":"s3"})).await; + assert_eq!((status, limited["max_concurrent_worker_sessions"].clone()), (StatusCode::CONFLICT, json!(2)), "{limited}"); + + // A lease expires on read; another worker reacquires with the failure + // reason cleared (published PHP keeps it: Server #737). + tokio::time::sleep(Duration::from_millis(3200)).await; + let (status, shown) = request(&app, "GET", "/api/worker-sessions/s2", Value::Null).await; + assert_eq!((status, shown["session"]["status"].clone(), shown["session"]["failure_reason"].clone()), + (StatusCode::OK, json!("expired"), json!("lease_expired")), "{shown}"); + let (status, reacquired) = create(&app, json!({"worker_id":"other","session_id":"s2"})).await; + assert_eq!((status, reacquired["outcome"].clone(), reacquired["session"]["failure_reason"].clone()), + (StatusCode::OK, json!("reacquired"), Value::Null), "{reacquired}"); + + // Heartbeat renews from now without changing the stored lease. + let (status, beat) = request(&app, "POST", "/api/worker/sessions/s1/heartbeat", json!({"worker_id":"holder","lease_seconds":5})).await; + assert_eq!((status, beat["outcome"].clone(), beat["session"]["lease_seconds"].clone()), (StatusCode::OK, json!("heartbeat_recorded"), json!(120))); + let (status, mismatch) = request(&app, "POST", "/api/worker/sessions/s1/heartbeat", json!({"worker_id":"other"})).await; + assert_eq!((status, mismatch["reason"].clone()), (StatusCode::CONFLICT, json!("session_owner_mismatch"))); + + // Close, close again, and a closed ID stays unusable. + let (status, closed) = request(&app, "DELETE", "/api/worker/sessions/s1?worker_id=holder", Value::Null).await; + assert_eq!((status, closed["outcome"].clone()), (StatusCode::OK, json!("closed")), "{closed}"); + let (status, again) = request(&app, "DELETE", "/api/worker/sessions/s1", json!({"worker_id":"holder"})).await; + assert_eq!((status, again["outcome"].clone(), again["session"]["closed_at"].clone()), (StatusCode::OK, json!("already_closed"), closed["session"]["closed_at"].clone())); + let (status, refused) = create(&app, json!({"worker_id":"holder","session_id":"s1"})).await; + assert_eq!((status, refused["reason"].clone()), (StatusCode::CONFLICT, json!("session_closed"))); + + // A deregistered holder leaves its session orphaned on read. + let (status, _) = request(&app, "DELETE", "/api/worker/registrations/other", Value::Null).await; + assert_eq!(status, StatusCode::OK); + let (status, listed) = request(&app, "GET", "/api/worker-sessions", Value::Null).await; + assert_eq!(status, StatusCode::OK, "{listed}"); + assert_eq!(listed["metrics"], json!({"total":2,"active":0,"closed":1,"expired":0,"failed":0,"orphaned":1}), "{listed}"); + let orphan = listed["sessions"].as_array().unwrap().iter().find(|session| session["session_id"] == "s2").unwrap(); + assert_eq!((orphan["status"].clone(), orphan["holder"]["task_queue"].clone(), orphan["holder"]["is_stale"].clone()), + (json!("orphaned"), Value::Null, json!(true)), "{orphan}"); +} + +#[tokio::test] +async fn worker_sessions_refuse_old_protocols_and_bad_requests() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + worker(&app, "gate", json!([]), None).await; + let (status, old) = request_protocol(&app, "POST", "/api/worker/sessions", json!({"worker_id":"gate","session_id":"g"}), "1.7").await; + assert_eq!((status, old["reason"].clone(), old["minimum_protocol_version"].clone()), (StatusCode::CONFLICT, json!("worker_sessions_unavailable"), json!("1.8")), "{old}"); + let (status, invalid) = request_protocol(&app, "POST", "/api/worker/sessions", json!({}), "1.8").await; + assert_eq!((status, invalid["errors"].as_object().unwrap().len()), (StatusCode::UNPROCESSABLE_ENTITY, 2), "{invalid}"); + let (status, unknown) = create(&app, json!({"worker_id":"nobody","session_id":"g"})).await; + assert_eq!((status, unknown["reason"].clone()), (StatusCode::PRECONDITION_FAILED, json!("worker_not_registered"))); + let (status, missing) = request(&app, "GET", "/api/worker-sessions/none", Value::Null).await; + assert_eq!((status, missing["reason"].clone()), (StatusCode::NOT_FOUND, json!("worker_session_not_found"))); +} From ffd60c33c1796d95bf9f8913851e5b6c0c5f5ce1 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Sat, 24 Oct 2026 12:24:23 +0000 Subject: [PATCH 03/13] Server parity: session-bound activities fixture, recorded from published 2.11.1 Fixture first for #736's activity half. In its own namespace, with an authoring worker, a worker without requirements, two GPU workers and one on another queue, the probe records: - schedule_activity with worker_session refused at protocol 1.7 and 1.17, on a timer command, without a session_id, and from a worker without the worker_sessions capability; then accepted, creating no session yet; - the first claim skipping a worker without the requirement and a 1.7 poll, admitted for a capable worker with the compact session on the task, and a second capable worker kept out while the session is held; - the activity heartbeat and renew-lease renewing the session by the command's lease (90 seconds) with the full snapshot, and status with the compact form; - after the holder closes the session: the heartbeat refused 409 session_not_active with worker_session_renewal and not recorded, status answering worker_session_unavailable, completion still 200; - a session's activity limit of one and of two, and a command queue other than the session queue that nobody can claim; - operator metrics counting the sessions, and the worker_sessions capability object with its verbs. Three published recordings are identical; published 2.11.1 and embedded pass. The unit test runs the recording and eight refused mutations, including native's current answers. --- scripts/conformance/server-parity.mjs | 2 +- .../conformance/server-parity/contract.mjs | 3 + scripts/conformance/server-parity/probe.php | 6 + .../worker-session-activities-contract.mjs | 23 ++ .../worker-session-activities-probe.php | 196 +++++++++ .../worker-session-activities.json | 372 ++++++++++++++++++ ...worker-session-activities-observation.json | 351 +++++++++++++++++ ...ityWorkerSessionActivitiesContractTest.mjs | 61 +++ 8 files changed, 1013 insertions(+), 1 deletion(-) create mode 100644 scripts/conformance/server-parity/worker-session-activities-contract.mjs create mode 100644 scripts/conformance/server-parity/worker-session-activities-probe.php create mode 100644 tests/Fixtures/ServerParity/worker-session-activities.json create mode 100644 tests/Fixtures/ServerParityExpected/worker-session-activities-observation.json create mode 100644 tests/Unit/ServerParityWorkerSessionActivitiesContractTest.mjs diff --git a/scripts/conformance/server-parity.mjs b/scripts/conformance/server-parity.mjs index d8537cb65..3ab7bdf71 100644 --- a/scripts/conformance/server-parity.mjs +++ b/scripts/conformance/server-parity.mjs @@ -103,7 +103,7 @@ try { for (const flag of ['url', 'application-root']) { if (values[flag]) arguments_.push(`--${flag}`, values[flag]); } - const probe = captureProbe('php', arguments_, {directory: dirname(resolve(values.output)), timeout: fixture.memo_replay ? fixture.memo_replay.overall_budget_seconds * 1000 + 5000 : fixture.corrected_markers ? fixture.corrected_markers.overall_budget_seconds * 1000 + 5000 : fixture.namespace_signal_pair || fixture.selected_run_signal_pair ? 75000 : fixture.update_validation ? 90000 : fixture.worker_poll_fence ? fixture.worker_poll_fence.overall_budget_seconds * 1000 + 5000 : fixture.worker_sessions ? fixture.worker_sessions.overall_budget_seconds * 1000 + 5000 : 45000, + const probe = captureProbe('php', arguments_, {directory: dirname(resolve(values.output)), timeout: fixture.memo_replay ? fixture.memo_replay.overall_budget_seconds * 1000 + 5000 : fixture.corrected_markers ? fixture.corrected_markers.overall_budget_seconds * 1000 + 5000 : fixture.namespace_signal_pair || fixture.selected_run_signal_pair ? 75000 : fixture.update_validation ? 90000 : fixture.worker_poll_fence ? fixture.worker_poll_fence.overall_budget_seconds * 1000 + 5000 : fixture.worker_sessions ? fixture.worker_sessions.overall_budget_seconds * 1000 + 5000 : fixture.worker_session_activities ? fixture.worker_session_activities.overall_budget_seconds * 1000 + 5000 : 45000, env: {...process.env, DW_PARITY_FAILURE_EVIDENCE_DIR: dirname(resolve(values.output))}, preserveStdout: join(sourceDirectory, name)}); const item = {fixture_id: fixture.id, fixture, outcome: 'runner-blocked'}; diff --git a/scripts/conformance/server-parity/contract.mjs b/scripts/conformance/server-parity/contract.mjs index b57439345..49f3922c4 100644 --- a/scripts/conformance/server-parity/contract.mjs +++ b/scripts/conformance/server-parity/contract.mjs @@ -16,6 +16,7 @@ import {checkActivityLeaseRenewal} from './activity-lease-renewal-contract.mjs'; import {checkWorkflowPollRequest} from './workflow-poll-request-contract.mjs'; import {checkWorkerPollFence} from './worker-poll-fence-contract.mjs'; import {checkWorkerSessions} from './worker-sessions-contract.mjs'; +import {checkWorkerSessionActivities} from './worker-session-activities-contract.mjs'; import {checkServiceCatalog} from './service-catalog-contract.mjs'; import {checkSystemMetrics} from './system-metrics-contract.mjs'; import {checkOperatorDashboardTypes} from './operator-dashboard-types-contract.mjs'; @@ -158,6 +159,7 @@ export function checkObservation(fixture, observation, workflowId, artifacts) { const workflowPollRequest = fixture.workflow_poll_request ? checkWorkflowPollRequest(fixture, observation, workflowId) : null; const workerPollFence = fixture.worker_poll_fence ? checkWorkerPollFence(fixture, observation, workflowId) : null; const workerSessions = fixture.worker_sessions ? checkWorkerSessions(fixture, observation, workflowId) : null; + const workerSessionActivities = fixture.worker_session_activities ? checkWorkerSessionActivities(fixture, observation, workflowId) : null; const serviceCatalog = fixture.service_catalog ? checkServiceCatalog(fixture, observation, workflowId) : null; const systemMetrics = fixture.system_metrics ? checkSystemMetrics(fixture, observation, workflowId) : null; const operatorDashboardTypes = fixture.operator_dashboard_types ? checkOperatorDashboardTypes(fixture, observation, workflowId) : null; @@ -717,6 +719,7 @@ export function checkObservation(fixture, observation, workflowId, artifacts) { ...(workflowPollRequest ? {workflow_poll_request: workflowPollRequest} : {}), ...(workerPollFence ? {worker_poll_fence: workerPollFence} : {}), ...(workerSessions ? {worker_sessions: workerSessions} : {}), + ...(workerSessionActivities ? {worker_session_activities: workerSessionActivities} : {}), ...(serviceCatalog ? {service_catalog: serviceCatalog} : {}), ...(systemMetrics ? {system_metrics: systemMetrics} : {}), ...(operatorDashboardTypes ? {operator_dashboard_types: operatorDashboardTypes} : {}), diff --git a/scripts/conformance/server-parity/probe.php b/scripts/conformance/server-parity/probe.php index 519d30bf9..13172fde2 100644 --- a/scripts/conformance/server-parity/probe.php +++ b/scripts/conformance/server-parity/probe.php @@ -434,6 +434,7 @@ function httpObservation(array $fixture, string $workflowId, string $namespace, $workflowPollRequest = isset($fixture['workflow_poll_request']) ? finishHttpWorkflowPollRequest($client, $fixture, $workflowId, $url) : null; $workerPollFence = isset($fixture['worker_poll_fence']) ? finishHttpWorkerPollFence($client, $fixture, $workflowId, $url) : null; $workerSessions = isset($fixture['worker_sessions']) ? finishHttpWorkerSessions($client, $fixture, $workflowId, $url) : null; + $workerSessionActivities = isset($fixture['worker_session_activities']) ? finishHttpWorkerSessionActivities($client, $fixture, $workflowId, $url) : null; $serviceCatalog = isset($fixture['service_catalog']) ? finishHttpServiceCatalog($client, $fixture, $workflowId, $url) : null; $systemMetrics = isset($fixture['system_metrics']) ? finishHttpSystemMetrics($client, $fixture, $workflowId, $url) : null; $operatorDashboardTypes = isset($fixture['operator_dashboard_types']) ? finishHttpOperatorDashboardTypes($client, $fixture, $workflowId, $url) : null; @@ -546,6 +547,7 @@ function httpObservation(array $fixture, string $workflowId, string $namespace, 'workflow_poll_request' => $workflowPollRequest, 'worker_poll_fence' => $workerPollFence, 'worker_sessions' => $workerSessions, + 'worker_session_activities' => $workerSessionActivities, 'service_catalog' => $serviceCatalog, 'system_metrics' => $systemMetrics, 'operator_dashboard_types' => $operatorDashboardTypes, @@ -862,6 +864,7 @@ function embeddedObservation(array $fixture, string $workflowId, string $namespa require __DIR__.'/workflow-poll-request-probe.php'; require __DIR__.'/worker-poll-fence-probe.php'; require __DIR__.'/worker-sessions-probe.php'; + require __DIR__.'/worker-session-activities-probe.php'; require __DIR__.'/service-catalog-probe.php'; require __DIR__.'/system-metrics-probe.php'; require __DIR__.'/operator-dashboard-types-probe.php'; @@ -1041,6 +1044,9 @@ function embeddedObservation(array $fixture, string $workflowId, string $namespa if ($mode === 'embedded' && isset($fixture['worker_sessions'])) { $observation['worker_sessions'] = ['applicable' => false, 'reason' => 'embedded_has_no_worker_protocol']; } + if ($mode === 'embedded' && isset($fixture['worker_session_activities'])) { + $observation['worker_session_activities'] = ['applicable' => false, 'reason' => 'embedded_has_no_worker_protocol']; + } if ($mode === 'embedded' && isset($fixture['service_catalog'])) { $observation['service_catalog'] = ['applicable' => false, 'reason' => 'embedded_has_no_http_service_catalog']; } diff --git a/scripts/conformance/server-parity/worker-session-activities-contract.mjs b/scripts/conformance/server-parity/worker-session-activities-contract.mjs new file mode 100644 index 000000000..1a9900562 --- /dev/null +++ b/scripts/conformance/server-parity/worker-session-activities-contract.mjs @@ -0,0 +1,23 @@ +import assert from 'node:assert/strict'; + +// Reviewed session-bound activities, recorded against published Server +// 2.11.1. The fixture's `expected` holds the whole normalized observation, +// with the probe's workflow prefix as `{id}`; session times are spans or +// presence flags. Every step compares exactly. +export function checkWorkerSessionActivities(fixture, observation, workflowId) { + const state = observation.worker_session_activities; + const projection = {scope: 'worker_session_activities'}; + assert.ok(state && typeof state === 'object', 'actual worker session activities observation'); + if (observation.mode === 'embedded') { + assert.deepStrictEqual(state, {applicable: false, reason: 'embedded_has_no_worker_protocol'}, + 'embedded worker session activities explicitly inapplicable'); + return projection; + } + const expected = fixture.worker_session_activities.expected; + const actual = JSON.parse(JSON.stringify(state).replaceAll(workflowId, '{id}')); + for (const [step, value] of Object.entries(expected)) { + assert.deepStrictEqual(actual[step], value, `worker session activities ${step}`); + } + assert.deepStrictEqual(Object.keys(actual).sort(), Object.keys(expected).sort(), 'worker session activities steps'); + return projection; +} diff --git a/scripts/conformance/server-parity/worker-session-activities-probe.php b/scripts/conformance/server-parity/worker-session-activities-probe.php new file mode 100644 index 000000000..dc55bf477 --- /dev/null +++ b/scripts/conformance/server-parity/worker-session-activities-probe.php @@ -0,0 +1,196 @@ +createNamespace($namespace, 'worker session activities λ', 30); + $token = getenv('DW_PARITY_TOKEN') ?: null; + $stack = HandlerStack::create(); + $stack->push(LockPressurePollRetries::middleware(), 'lock_pressure_poll_retry'); + $http = new WorkerSessionActivityHttp(['base_uri' => rtrim($url, '/'), 'http_errors' => false, + 'connect_timeout' => 1, 'timeout' => 30, 'handler' => $stack]); + $capabilityMap = null; + $raw = static function (string $method, string $path, ?array $body = null, ?string $protocol = '1.20', bool $control = false) + use ($http, $token, $namespace, &$capabilityMap): array { + $headers = array_filter(['Authorization' => $token === null ? null : 'Bearer '.$token, + 'Accept' => 'application/json', 'X-Namespace' => $namespace, + 'X-Durable-Workflow-Protocol-Version' => $control ? null : $protocol, + 'X-Durable-Workflow-Control-Plane-Version' => $control ? '2' : null]); + $response = $http->request($method, $path, ['headers' => $headers] + ($body === null ? [] : ['json' => $body])); + $decoded = json_decode((string) $response->getBody(), true); + if (is_array($decoded)) { + $capabilityMap ??= $decoded['server_capabilities'] ?? null; + unset($decoded['server_capabilities'], $decoded['control_plane'], $decoded['protocol_version']); + } + + return ['status' => $response->getStatusCode(), 'response' => $decoded]; + }; + $at = static fn (?string $value): ?int => $value === null ? null : (new DateTimeImmutable($value))->getTimestamp(); + $present = static fn (mixed $value): string => $value === null ? 'null' : 'present'; + // The full snapshot with its times as spans and presence flags. + $snapshot = static function (?array $session) use ($at, $present): ?array { + if ($session === null) { + return null; + } + $times = ['lease_expires_at', 'ttl_expires_at', 'closed_at', 'last_heartbeat_at', 'created_at', 'updated_at']; + $plain = array_diff_key($session, array_flip($times)); + if (is_array($plain['holder'] ?? null)) { + $plain['holder']['last_heartbeat_at'] = $present($plain['holder']['last_heartbeat_at']); + } + $heartbeat = $at($session['last_heartbeat_at'] ?? null); + + return $plain + ['lease_span' => isset($session['lease_expires_at']) && $heartbeat !== null ? $at($session['lease_expires_at']) - $heartbeat : null, + 'times' => array_map(static fn (string $key): string => $present($session[$key] ?? null), array_combine($times, $times))]; + }; + // The compact form a task or status carries: its times as presence flags. + $compact = static function (?array $session) use ($present): ?array { + if ($session === null) { + return null; + } + foreach (['lease_expires_at', 'ttl_expires_at'] as $key) { + if (array_key_exists($key, $session)) { + $session[$key] = $present($session[$key]); + } + } + + return $session; + }; + $ok = static function (array $recorded, int $status, string $operation): array { + if ($recorded['status'] !== $status) { + throw new RuntimeException('Worker session activities '.$operation.': '.json_encode($recorded)); + } + + return $recorded['response']; + }; + $empty = ['codec' => 'avro', 'blob' => $definition['empty_blob']]; + $queue = $workflowId.'-wsa-queue'; + $other = $workflowId.'-wsa-other-queue'; + $manifest = static fn (bool $sessions): array => ['local_activities' => ['supported' => false, 'minimum_protocol_version' => '1.18'], + 'sticky_execution' => ['supported' => false, 'minimum_protocol_version' => '1.18'], + 'worker_sessions' => ['supported' => $sessions, 'minimum_protocol_version' => '1.18']]; + $register = static function (string $worker, string $onQueue, array $capabilities) use ($raw, $ok, $definition, $manifest): void { + $ok($raw('POST', '/api/worker/register', ['worker_id' => $worker, 'task_queue' => $onQueue, 'runtime' => 'php', + 'sdk_version' => 'raw-http', 'supported_workflow_types' => [$definition['workflow_type']], + 'supported_activity_types' => [$definition['activity_type']], 'capabilities' => $capabilities, + 'capability_manifest' => $manifest(in_array('worker_sessions', $capabilities, true))]), 201, 'registration '.$worker); + }; + [$author, $plain, $gpu, $gpu2, $far] = array_map(static fn (string $name): string => $workflowId.'-wsa-'.$name, ['author', 'plain', 'gpu', 'gpu2', 'far']); + $register($author, $queue, ['worker_sessions']); + $register($plain, $queue, []); + $register($gpu, $queue, ['worker_sessions', 'gpu']); + $register($gpu2, $queue, ['worker_sessions', 'gpu']); + $register($far, $other, ['worker_sessions', 'gpu']); + $start = static fn (string $suffix): array => $ok($raw('POST', '/api/workflows', ['workflow_id' => $workflowId.'-wsa-'.$suffix, + 'workflow_type' => $definition['workflow_type'], 'task_queue' => $queue, 'input' => $empty], null, true), 201, 'start '.$suffix); + $workflowTask = static fn (string $worker): array => $raw('POST', '/api/worker/workflow-tasks/poll', + ['worker_id' => $worker, 'task_queue' => $queue, 'timeout_seconds' => 0])['response']['task'] ?? []; + $complete = static fn (array $task, array $commands, string $protocol = '1.20'): array => $raw('POST', + '/api/worker/workflow-tasks/'.($task['task_id'] ?? 'missing').'/complete', + ['lease_owner' => $task['lease_owner'] ?? null, 'workflow_task_attempt' => $task['workflow_task_attempt'] ?? null, 'commands' => $commands], $protocol); + $activity = static fn (array $session, array $extra = []): array => ['type' => 'schedule_activity', 'activity_type' => $definition['activity_type'], + 'arguments' => $empty, 'retry_policy' => ['max_attempts' => 1, 'backoff_seconds' => []], 'start_to_close_timeout' => 300, + 'worker_session' => $session] + $extra; + $refusal = static fn (array $recorded): array => ['status' => $recorded['status'], + 'reason' => $recorded['response']['reason'] ?? null, 'message' => $recorded['response']['message'] ?? $recorded['response']['error'] ?? null]; + $poll = static fn (string $worker, string $onQueue, string $protocol = '1.20'): array => $raw('POST', '/api/worker/activity-tasks/poll', + ['worker_id' => $worker, 'task_queue' => $onQueue, 'timeout_seconds' => 0], $protocol); + $leased = static fn (array $recorded): array => ['status' => $recorded['status'], 'poll_status' => $recorded['response']['poll_status'] ?? null, + 'worker_session' => $compact($recorded['response']['task']['worker_session'] ?? null), + 'has_session_key' => is_array($recorded['response']['task'] ?? null) && array_key_exists('worker_session', $recorded['response']['task'])]; + $own = static fn (array $task): array => ['activity_attempt_id' => $task['activity_attempt_id'] ?? null, 'lease_owner' => $task['lease_owner'] ?? null]; + $show = static fn (string $session): array => $raw('GET', '/api/worker-sessions/'.rawurlencode($session), null, null, true); + $state = ['applicable' => true, 'namespace' => $namespace]; + + // 17. Completion guards on one workflow task, then an accepted command. + $start('guards'); + $task = $workflowTask($author); + $lazy = $workflowId.'-wsa-lazy'; + $session = ['session_id' => $lazy, 'requirements' => ['gpu'], 'lease_seconds' => 90]; + $state['complete_protocol_1_7'] = $refusal($complete($task, [$activity($session)], '1.7')); + $state['complete_protocol_1_17'] = $refusal($complete($task, [$activity($session)], '1.17')); + $state['complete_on_timer'] = $refusal($complete($task, [['type' => 'start_timer', 'delay_seconds' => 60, 'worker_session' => $session]])); + $state['complete_without_session_id'] = $refusal($complete($task, [$activity(['queue' => $queue])])); + $state['complete_accepted'] = $complete($task, [$activity($session)])['status']; + $start('incapable'); + $incapable = $workflowTask($plain); + $state['complete_incapable'] = $refusal($complete($incapable, [$activity($session)])); + + // 18. The activity path: lazily created on the first admitted claim. + $state['show_before_claim'] = $refusal($show($lazy)); + $state['poll_without_requirement'] = $leased($poll($plain, $queue)); + $state['poll_protocol_1_7'] = $leased($poll($gpu, $queue, '1.7')); + $claimed = $poll($gpu, $queue); + $state['poll_capable'] = $leased($claimed); + $claim = $claimed['response']['task'] ?? []; + $shown = $show($lazy); + $state['show_after_claim'] = ['status' => $shown['status'], 'session' => $snapshot($shown['response']['session'] ?? null)]; + $state['poll_other_capable'] = $leased($poll($gpu2, $queue)); + $beat = $raw('POST', '/api/worker/activity-tasks/'.($claim['task_id'] ?? 'missing').'/heartbeat', $own($claim) + ['message' => 'half']); + $state['activity_heartbeat'] = ['status' => $beat['status'], 'heartbeat_recorded' => $beat['response']['heartbeat_recorded'] ?? null, + 'worker_session' => $snapshot($beat['response']['worker_session'] ?? null)]; + $status = $raw('POST', '/api/worker/activity-tasks/'.($claim['task_id'] ?? 'missing').'/status', $own($claim)); + $state['activity_status'] = ['status' => $status['status'], 'can_continue' => $status['response']['can_continue'] ?? null, + 'reason' => $status['response']['reason'] ?? null, 'worker_session' => $compact($status['response']['worker_session'] ?? null)]; + $renewed = $raw('POST', '/api/worker/activity-tasks/'.($claim['task_id'] ?? 'missing').'/renew-lease', $own($claim)); + $state['activity_renew'] = ['status' => $renewed['status'], 'lease_renewed' => $renewed['response']['lease_renewed'] ?? null, + 'worker_session' => $snapshot($renewed['response']['worker_session'] ?? null)]; + $state['session_closed'] = $raw('DELETE', '/api/worker/sessions/'.rawurlencode($lazy), ['worker_id' => $gpu])['status']; + $beat = $raw('POST', '/api/worker/activity-tasks/'.($claim['task_id'] ?? 'missing').'/heartbeat', $own($claim)); + $state['heartbeat_after_close'] = ['status' => $beat['status'], 'reason' => $beat['response']['reason'] ?? null, + 'can_continue' => $beat['response']['can_continue'] ?? null, 'heartbeat_recorded' => $beat['response']['heartbeat_recorded'] ?? null, + 'worker_session_renewal' => $beat['response']['worker_session_renewal'] ?? null, + 'worker_session_status' => $beat['response']['worker_session']['status'] ?? null]; + $status = $raw('POST', '/api/worker/activity-tasks/'.($claim['task_id'] ?? 'missing').'/status', $own($claim)); + $state['status_after_close'] = ['status' => $status['status'], 'can_continue' => $status['response']['can_continue'] ?? null, + 'reason' => $status['response']['reason'] ?? null]; + $state['complete_after_close'] = $raw('POST', '/api/worker/activity-tasks/'.($claim['task_id'] ?? 'missing').'/complete', + $own($claim) + ['result' => $empty])['status']; + + // 19. The session's activity limit: two activities on one session. + foreach ([1 => 'one', 2 => 'two'] as $limit => $name) { + $start('limit-'.$name); + $task = $workflowTask($author); + $shared = ['session_id' => $workflowId.'-wsa-shared-'.$name, 'requirements' => ['gpu'], 'max_concurrent_activities' => $limit]; + $complete($task, [$activity($shared), $activity($shared)]); + $first = $poll($gpu2, $queue); + $second = $poll($gpu2, $queue); + $state['limit_'.$name] = [$leased($first)['poll_status'], $leased($second)['poll_status']]; + foreach ([$first, $second] as $polled) { + $leasedTask = $polled['response']['task'] ?? null; + if (is_array($leasedTask)) { + $raw('POST', '/api/worker/activity-tasks/'.$leasedTask['task_id'].'/complete', $own($leasedTask) + ['result' => $empty]); + } + } + } + + // 20. A command queue other than the session's: nobody can claim it. + $start('queue'); + $task = $workflowTask($author); + $complete($task, [$activity(['session_id' => $workflowId.'-wsa-elsewhere', 'queue' => $queue], ['queue' => $other])]); + $state['queue_mismatch_polls'] = [$leased($poll($gpu, $queue))['poll_status'], $leased($poll($far, $other))['poll_status']]; + + // 21. Operator metrics count the namespace's sessions. + $metrics = $raw('GET', '/api/system/operator-metrics', null, null, true); + $state['operator_metrics'] = ['status' => $metrics['status'], + 'worker_sessions' => $metrics['response']['operator_metrics']['worker_sessions'] ?? null]; + + // The capability map, once. + $state['capabilities'] = ['worker_sessions' => $capabilityMap['worker_sessions'] ?? null, + 'worker_session_verbs' => $capabilityMap['worker_session_verbs'] ?? null]; + + return $state; +} diff --git a/tests/Fixtures/ServerParity/worker-session-activities.json b/tests/Fixtures/ServerParity/worker-session-activities.json new file mode 100644 index 000000000..fa0aa7df2 --- /dev/null +++ b/tests/Fixtures/ServerParity/worker-session-activities.json @@ -0,0 +1,372 @@ +{ + "schema": "durable-workflow.server-parity-fixture/v1", + "id": "worker-session-activities", + "review_state": "reviewed", + "workflow_type": "parity.v1.echo", + "input": {"message": "worker session activities λ", "count": 9007199254740993, "optional": null}, + "typed_value": {"type": "map", "value": { + "message": {"type": "string", "value": "worker session activities λ"}, + "count": {"type": "int64", "value": "9007199254740993"}, + "optional": {"type": "null", "value": null} + }}, + "activity": false, + "expected_events": ["StartAccepted", "WorkflowStarted", "WorkflowCompleted"], + "worker_session_activities": { + "workflow_type": "parity.v1.session_activities", + "activity_type": "parity.v1.session_activity", + "empty_blob": "wwHioz3/VYAiNwwA", + "overall_budget_seconds": 120, + "expected": { + "applicable": true, + "namespace": "{id}-wsa", + "complete_protocol_1_7": { + "status": 409, + "reason": "worker_sessions_unavailable", + "message": "Worker-session activity commands require worker protocol 1.8 or newer." + }, + "complete_protocol_1_17": { + "status": 409, + "reason": "portable_worker_affinity_protocol_floor_unavailable", + "message": null + }, + "complete_on_timer": { + "status": 422, + "reason": "validation_failed", + "message": "worker_session is only supported for schedule_activity commands." + }, + "complete_without_session_id": { + "status": 422, + "reason": "validation_failed", + "message": "Worker session commands require a non-empty session_id." + }, + "complete_accepted": 200, + "complete_incapable": { + "status": 409, + "reason": "worker_capability_not_supported", + "message": null + }, + "show_before_claim": { + "status": 404, + "reason": "worker_session_not_found", + "message": "Worker session [{id}-wsa-lazy] not found in namespace [{id}-wsa]." + }, + "poll_without_requirement": { + "status": 200, + "poll_status": "empty", + "worker_session": null, + "has_session_key": false + }, + "poll_protocol_1_7": { + "status": 200, + "poll_status": "empty", + "worker_session": null, + "has_session_key": false + }, + "poll_capable": { + "status": 200, + "poll_status": "leased", + "worker_session": { + "session_id": "{id}-wsa-lazy", + "queue": "{id}-wsa-queue", + "requirements": [ + "gpu" + ], + "lease_seconds": 90, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "status": "active", + "lease_owner": "{id}-wsa-gpu", + "lease_expires_at": "present", + "ttl_expires_at": "present" + }, + "has_session_key": true + }, + "show_after_claim": { + "status": 200, + "session": { + "session_id": "{id}-wsa-lazy", + "namespace": "{id}-wsa", + "connection": null, + "queue": "{id}-wsa-queue", + "requirements": [ + "gpu" + ], + "status": "active", + "lease_owner": "{id}-wsa-gpu", + "holder": { + "worker_id": "{id}-wsa-gpu", + "task_queue": "{id}-wsa-queue", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 90, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 1, + "lease_span": 90, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + } + }, + "poll_other_capable": { + "status": 200, + "poll_status": "empty", + "worker_session": null, + "has_session_key": false + }, + "activity_heartbeat": { + "status": 200, + "heartbeat_recorded": true, + "worker_session": { + "session_id": "{id}-wsa-lazy", + "namespace": "{id}-wsa", + "connection": null, + "queue": "{id}-wsa-queue", + "requirements": [ + "gpu" + ], + "status": "active", + "lease_owner": "{id}-wsa-gpu", + "holder": { + "worker_id": "{id}-wsa-gpu", + "task_queue": "{id}-wsa-queue", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 90, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 1, + "lease_span": 90, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + } + }, + "activity_status": { + "status": 200, + "can_continue": true, + "reason": null, + "worker_session": { + "session_id": "{id}-wsa-lazy", + "queue": "{id}-wsa-queue", + "requirements": [ + "gpu" + ], + "lease_seconds": 90, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "status": "active", + "lease_owner": "{id}-wsa-gpu", + "lease_expires_at": "present", + "ttl_expires_at": "present" + } + }, + "activity_renew": { + "status": 200, + "lease_renewed": true, + "worker_session": { + "session_id": "{id}-wsa-lazy", + "namespace": "{id}-wsa", + "connection": null, + "queue": "{id}-wsa-queue", + "requirements": [ + "gpu" + ], + "status": "active", + "lease_owner": "{id}-wsa-gpu", + "holder": { + "worker_id": "{id}-wsa-gpu", + "task_queue": "{id}-wsa-queue", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 90, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 1, + "lease_span": 90, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + } + }, + "session_closed": 200, + "heartbeat_after_close": { + "status": 409, + "reason": "session_not_active", + "can_continue": false, + "heartbeat_recorded": false, + "worker_session_renewal": { + "admitted": false, + "reason": "session_not_active", + "error": "Worker session is not active." + }, + "worker_session_status": "closed" + }, + "status_after_close": { + "status": 200, + "can_continue": false, + "reason": "worker_session_unavailable" + }, + "complete_after_close": 200, + "limit_one": [ + "leased", + "empty" + ], + "limit_two": [ + "leased", + "leased" + ], + "queue_mismatch_polls": [ + "empty", + "empty" + ], + "operator_metrics": { + "status": 200, + "worker_sessions": { + "total": 3, + "active": 2, + "closed": 1, + "expired": 0, + "failed": 0, + "orphaned": 0 + } + }, + "capabilities": { + "worker_sessions": { + "feature": "worker_sessions", + "contract_version": "1.0", + "minimum_protocol_version": "1.8", + "command_field": "worker_session", + "activity_options_field": "worker_session", + "verbs": [ + "create", + "heartbeat", + "close" + ], + "lifecycle": { + "creation": "lazy_create_on_first_admitted_activity_or_explicit_worker_create", + "renewal": "activity_heartbeat_or_explicit_session_heartbeat", + "close": "explicit_holder_close", + "lease_expiry": "session_expires_when_lease_is_not_renewed", + "ttl_expiry": "absolute_session_ttl_is_terminal_for_that_session_id" + }, + "ownership": "single_worker_lease_owner", + "lease": { + "scope": "namespace_session_id", + "owner": "registered_worker_id", + "activity_attempt_leases_remain_independent": true + }, + "admission": { + "queue_routing_first": true, + "requires_registered_worker": true, + "requires_capabilities": true, + "create_if_missing_default": true, + "allow_reacquire_after_failure_default": true + }, + "rollout_safety": { + "minimum_protocol_version": "1.8", + "mixed_server_rollout_fenced_by_protocol_version": true, + "servers_below_minimum_must_reject_worker_session_commands": true, + "servers_below_minimum_must_not_claim_worker_session_activity_tasks": true + }, + "limits": { + "max_concurrent_worker_sessions": "worker_registration", + "max_concurrent_activities": "session" + }, + "default_max_concurrent_activities": 1, + "renewal": { + "activity_heartbeat_renews_session": true, + "explicit_session_heartbeat": true + }, + "failure_detection": [ + "lease_expiry", + "registered_worker_heartbeat_staleness" + ], + "holder_loss": { + "in_flight_activities_keep_at_least_once_attempt_semantics": true, + "replacement_worker_must_reacquire_session": true, + "process_local_state_must_be_rebuilt_after_reacquire": true + }, + "cancellation": { + "workflow_cancellation_observed_through_activity_heartbeat": true, + "session_lease_does_not_override_activity_cancel_requested": true, + "planned_shutdown_should_close_sessions": true + }, + "routing": [ + "queue", + "connection", + "requirements" + ], + "visibility": [ + "active", + "closed", + "expired", + "failed", + "orphaned" + ], + "statuses": [ + "active", + "closed", + "expired", + "failed", + "orphaned" + ], + "terminal_statuses": [ + "closed" + ], + "terminal_conditions": [ + "explicit_close", + "ttl_expired", + "allow_reacquire_after_failure_false" + ], + "authoring_guidance": [ + "use_for_process_local_state_gpu_memory_or_filesystem_affinity", + "prefer_ordinary_queued_activities_for_independent_steps", + "prefer_one_larger_activity_for_atomic_side_effects" + ], + "supported": true + }, + "worker_session_verbs": [ + "create", + "heartbeat", + "close" + ] + } + } + }, + "contract": "Session-bound activities as published Server 2.11.1 serves them: schedule_activity with worker_session is refused below protocol 1.8 (worker_sessions_unavailable) and below 1.18 (portable_worker_affinity_protocol_floor_unavailable), on other commands and without a session_id (422), and from a worker without the worker_sessions capability (409 worker_capability_not_supported); an accepted command creates no session until its first admitted claim. A claim skips workers below 1.8, on another queue or missing a requirement, and a session held by one worker; the task carries the compact session. The activity heartbeat and renew-lease renew the session by the command's lease and answer the full snapshot; status answers the compact form. After the holder closes the session, the heartbeat is refused 409 session_not_active with worker_session_renewal, status answers worker_session_unavailable, and completion still succeeds. A session's max_concurrent_activities bounds its running activities; a command queue other than the session queue is unclaimable. Operator metrics count the namespace's sessions, and the capability map serves worker_sessions as an object with its verbs. Session times are compared as spans or presence. Embedded applications have no worker protocol." +} diff --git a/tests/Fixtures/ServerParityExpected/worker-session-activities-observation.json b/tests/Fixtures/ServerParityExpected/worker-session-activities-observation.json new file mode 100644 index 000000000..7cfdb5c46 --- /dev/null +++ b/tests/Fixtures/ServerParityExpected/worker-session-activities-observation.json @@ -0,0 +1,351 @@ +{ + "applicable": true, + "namespace": "pr1-worker-session-activities-wsa", + "complete_protocol_1_7": { + "status": 409, + "reason": "worker_sessions_unavailable", + "message": "Worker-session activity commands require worker protocol 1.8 or newer." + }, + "complete_protocol_1_17": { + "status": 409, + "reason": "portable_worker_affinity_protocol_floor_unavailable", + "message": null + }, + "complete_on_timer": { + "status": 422, + "reason": "validation_failed", + "message": "worker_session is only supported for schedule_activity commands." + }, + "complete_without_session_id": { + "status": 422, + "reason": "validation_failed", + "message": "Worker session commands require a non-empty session_id." + }, + "complete_accepted": 200, + "complete_incapable": { + "status": 409, + "reason": "worker_capability_not_supported", + "message": null + }, + "show_before_claim": { + "status": 404, + "reason": "worker_session_not_found", + "message": "Worker session [pr1-worker-session-activities-wsa-lazy] not found in namespace [pr1-worker-session-activities-wsa]." + }, + "poll_without_requirement": { + "status": 200, + "poll_status": "empty", + "worker_session": null, + "has_session_key": false + }, + "poll_protocol_1_7": { + "status": 200, + "poll_status": "empty", + "worker_session": null, + "has_session_key": false + }, + "poll_capable": { + "status": 200, + "poll_status": "leased", + "worker_session": { + "session_id": "pr1-worker-session-activities-wsa-lazy", + "queue": "pr1-worker-session-activities-wsa-queue", + "requirements": [ + "gpu" + ], + "lease_seconds": 90, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "status": "active", + "lease_owner": "pr1-worker-session-activities-wsa-gpu", + "lease_expires_at": "present", + "ttl_expires_at": "present" + }, + "has_session_key": true + }, + "show_after_claim": { + "status": 200, + "session": { + "session_id": "pr1-worker-session-activities-wsa-lazy", + "namespace": "pr1-worker-session-activities-wsa", + "connection": null, + "queue": "pr1-worker-session-activities-wsa-queue", + "requirements": [ + "gpu" + ], + "status": "active", + "lease_owner": "pr1-worker-session-activities-wsa-gpu", + "holder": { + "worker_id": "pr1-worker-session-activities-wsa-gpu", + "task_queue": "pr1-worker-session-activities-wsa-queue", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 90, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 1, + "lease_span": 90, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + } + }, + "poll_other_capable": { + "status": 200, + "poll_status": "empty", + "worker_session": null, + "has_session_key": false + }, + "activity_heartbeat": { + "status": 200, + "heartbeat_recorded": true, + "worker_session": { + "session_id": "pr1-worker-session-activities-wsa-lazy", + "namespace": "pr1-worker-session-activities-wsa", + "connection": null, + "queue": "pr1-worker-session-activities-wsa-queue", + "requirements": [ + "gpu" + ], + "status": "active", + "lease_owner": "pr1-worker-session-activities-wsa-gpu", + "holder": { + "worker_id": "pr1-worker-session-activities-wsa-gpu", + "task_queue": "pr1-worker-session-activities-wsa-queue", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 90, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 1, + "lease_span": 90, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + } + }, + "activity_status": { + "status": 200, + "can_continue": true, + "reason": null, + "worker_session": { + "session_id": "pr1-worker-session-activities-wsa-lazy", + "queue": "pr1-worker-session-activities-wsa-queue", + "requirements": [ + "gpu" + ], + "lease_seconds": 90, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "status": "active", + "lease_owner": "pr1-worker-session-activities-wsa-gpu", + "lease_expires_at": "present", + "ttl_expires_at": "present" + } + }, + "activity_renew": { + "status": 200, + "lease_renewed": true, + "worker_session": { + "session_id": "pr1-worker-session-activities-wsa-lazy", + "namespace": "pr1-worker-session-activities-wsa", + "connection": null, + "queue": "pr1-worker-session-activities-wsa-queue", + "requirements": [ + "gpu" + ], + "status": "active", + "lease_owner": "pr1-worker-session-activities-wsa-gpu", + "holder": { + "worker_id": "pr1-worker-session-activities-wsa-gpu", + "task_queue": "pr1-worker-session-activities-wsa-queue", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 90, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 1, + "lease_span": 90, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + } + }, + "session_closed": 200, + "heartbeat_after_close": { + "status": 409, + "reason": "session_not_active", + "can_continue": false, + "heartbeat_recorded": false, + "worker_session_renewal": { + "admitted": false, + "reason": "session_not_active", + "error": "Worker session is not active." + }, + "worker_session_status": "closed" + }, + "status_after_close": { + "status": 200, + "can_continue": false, + "reason": "worker_session_unavailable" + }, + "complete_after_close": 200, + "limit_one": [ + "leased", + "empty" + ], + "limit_two": [ + "leased", + "leased" + ], + "queue_mismatch_polls": [ + "empty", + "empty" + ], + "operator_metrics": { + "status": 200, + "worker_sessions": { + "total": 3, + "active": 2, + "closed": 1, + "expired": 0, + "failed": 0, + "orphaned": 0 + } + }, + "capabilities": { + "worker_sessions": { + "feature": "worker_sessions", + "contract_version": "1.0", + "minimum_protocol_version": "1.8", + "command_field": "worker_session", + "activity_options_field": "worker_session", + "verbs": [ + "create", + "heartbeat", + "close" + ], + "lifecycle": { + "creation": "lazy_create_on_first_admitted_activity_or_explicit_worker_create", + "renewal": "activity_heartbeat_or_explicit_session_heartbeat", + "close": "explicit_holder_close", + "lease_expiry": "session_expires_when_lease_is_not_renewed", + "ttl_expiry": "absolute_session_ttl_is_terminal_for_that_session_id" + }, + "ownership": "single_worker_lease_owner", + "lease": { + "scope": "namespace_session_id", + "owner": "registered_worker_id", + "activity_attempt_leases_remain_independent": true + }, + "admission": { + "queue_routing_first": true, + "requires_registered_worker": true, + "requires_capabilities": true, + "create_if_missing_default": true, + "allow_reacquire_after_failure_default": true + }, + "rollout_safety": { + "minimum_protocol_version": "1.8", + "mixed_server_rollout_fenced_by_protocol_version": true, + "servers_below_minimum_must_reject_worker_session_commands": true, + "servers_below_minimum_must_not_claim_worker_session_activity_tasks": true + }, + "limits": { + "max_concurrent_worker_sessions": "worker_registration", + "max_concurrent_activities": "session" + }, + "default_max_concurrent_activities": 1, + "renewal": { + "activity_heartbeat_renews_session": true, + "explicit_session_heartbeat": true + }, + "failure_detection": [ + "lease_expiry", + "registered_worker_heartbeat_staleness" + ], + "holder_loss": { + "in_flight_activities_keep_at_least_once_attempt_semantics": true, + "replacement_worker_must_reacquire_session": true, + "process_local_state_must_be_rebuilt_after_reacquire": true + }, + "cancellation": { + "workflow_cancellation_observed_through_activity_heartbeat": true, + "session_lease_does_not_override_activity_cancel_requested": true, + "planned_shutdown_should_close_sessions": true + }, + "routing": [ + "queue", + "connection", + "requirements" + ], + "visibility": [ + "active", + "closed", + "expired", + "failed", + "orphaned" + ], + "statuses": [ + "active", + "closed", + "expired", + "failed", + "orphaned" + ], + "terminal_statuses": [ + "closed" + ], + "terminal_conditions": [ + "explicit_close", + "ttl_expired", + "allow_reacquire_after_failure_false" + ], + "authoring_guidance": [ + "use_for_process_local_state_gpu_memory_or_filesystem_affinity", + "prefer_ordinary_queued_activities_for_independent_steps", + "prefer_one_larger_activity_for_atomic_side_effects" + ], + "supported": true + }, + "worker_session_verbs": [ + "create", + "heartbeat", + "close" + ] + } +} diff --git a/tests/Unit/ServerParityWorkerSessionActivitiesContractTest.mjs b/tests/Unit/ServerParityWorkerSessionActivitiesContractTest.mjs new file mode 100644 index 000000000..049a2ebcd --- /dev/null +++ b/tests/Unit/ServerParityWorkerSessionActivitiesContractTest.mjs @@ -0,0 +1,61 @@ +import assert from 'node:assert/strict'; +import {readFileSync} from 'node:fs'; +import test from 'node:test'; +import {checkWorkerSessionActivities} from '../../scripts/conformance/server-parity/worker-session-activities-contract.mjs'; + +const fixture = JSON.parse(readFileSync(new URL('../Fixtures/ServerParity/worker-session-activities.json', import.meta.url))); +// The observation published Server 2.11.1 returned for this fixture. +const recorded = JSON.parse(readFileSync( + new URL('../Fixtures/ServerParityExpected/worker-session-activities-observation.json', import.meta.url))); +const id = 'pr1-worker-session-activities'; +const observation = (change = () => {}) => { + const state = structuredClone(recorded); + change(state); + return {mode: 'http', worker_session_activities: state}; +}; +const check = state => checkWorkerSessionActivities(fixture, state, id); + +test('the published 2.11.1 recording satisfies the contract', () => { + assert.deepStrictEqual(check(observation()), {scope: 'worker_session_activities'}); +}); + +test('embedded applications are explicitly inapplicable', () => { + assert.deepStrictEqual(check({mode: 'embedded', worker_session_activities: + {applicable: false, reason: 'embedded_has_no_worker_protocol'}}), {scope: 'worker_session_activities'}); +}); + +for (const [label, change, message] of [ + // Native before this fixture: worker_session refused as an unknown field, + // sessions advertised as a bare false, and metrics fixed at zero. + ['worker_session refused as an unknown command field', state => { + state.complete_accepted = 422; + }, /worker session activities complete_accepted/], + ['a claim that ignores the session requirements', state => { + state.poll_without_requirement.poll_status = 'leased'; + }, /worker session activities poll_without_requirement/], + ['a task without its session', state => { + state.poll_capable.worker_session = null; + state.poll_capable.has_session_key = false; + }, /worker session activities poll_capable/], + ['a heartbeat that does not renew the session', state => { + state.activity_heartbeat.worker_session = null; + }, /worker session activities activity_heartbeat/], + ['a heartbeat recorded after the session closed', state => { + state.heartbeat_after_close.status = 200; + state.heartbeat_after_close.heartbeat_recorded = true; + }, /worker session activities heartbeat_after_close/], + ['a session limit that is not enforced', state => { + state.limit_one = ['leased', 'leased']; + }, /worker session activities limit_one/], + ['sessions advertised as a bare flag', state => { + state.capabilities.worker_sessions = false; + state.capabilities.worker_session_verbs = null; + }, /worker session activities capabilities/], + ['metrics fixed at zero', state => { + state.operator_metrics.worker_sessions = {total: 0, active: 0, closed: 0, expired: 0, failed: 0, orphaned: 0}; + }, /worker session activities operator_metrics/], +]) { + test(`the contract refuses ${label}`, () => { + assert.throws(() => check(observation(change)), message); + }); +} From 17f48c4b470c1a4635452391c1e03baf94dd487c Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Sat, 24 Oct 2026 12:40:16 +0000 Subject: [PATCH 04/13] Rust #325: session-bound activities, the sessions capability and metrics Native refused worker_session on schedule_activity as an unknown field, ignored sessions when claiming and renewing activities, advertised worker_sessions as a bare false and counted no sessions in operator metrics. It now follows published 2.11.1: - completion: schedule_activity accepts connection and worker_session; with any worker_session, PHP's guards in order (409 worker_sessions_unavailable below protocol 1.8, 409 portable_worker_affinity_protocol_floor_unavailable below 1.18, 409 worker_capability_not_supported without the flat capability and a supported manifest entry), then the normalizer's 422 checks; the activity inherits the session's queue and connection when it names none and stores WorkerSessionOptions' snapshot in its options; - claim: a session-bound activity goes only to a worker on 1.8 or newer, on the session queue, with every requirement; the session then admits the claim (created lazily, reused within its activity limit, or reacquired) or the claim rolls back; the task carries the compact session; - the activity heartbeat and renew-lease renew the session by the command's lease first and answer the full snapshot; a refused renewal answers its status with worker_session_renewal and records nothing; status shows the compact form, read without a refresh, and answers worker_session_unavailable once the session is not the holder's live lease; - the session snapshot counts its running activities; server_capabilities.worker_sessions is the published contract object with worker_session_verbs; operator metrics and dashboards count the namespace's sessions after a refresh. The worker-session-activities, worker-sessions and worker-poll-fence fixtures pass against native locally, and native execution tests cover admission, the held session, the activity limit, heartbeat renewal and the closed-session refusals on SQLite, PostgreSQL and MySQL. --- rust/src/runtime/activity_heartbeats.rs | 45 ++- rust/src/runtime/activity_lease_renewal.rs | 24 +- rust/src/runtime/backend.rs | 6 + rust/src/runtime/operator_metrics.rs | 6 +- rust/src/runtime/store.rs | 68 +++- rust/src/runtime/worker_sessions.rs | 359 ++++++++++++++++++++- rust/tests/execution/worker_sessions.rs | 61 ++++ 7 files changed, 551 insertions(+), 18 deletions(-) diff --git a/rust/src/runtime/activity_heartbeats.rs b/rust/src/runtime/activity_heartbeats.rs index 3d1560ded..f6d3fbc96 100644 --- a/rust/src/runtime/activity_heartbeats.rs +++ b/rust/src/runtime/activity_heartbeats.rs @@ -43,7 +43,7 @@ where async fn observe_activity(&self, task_id: &str, body: Value, protocol: &str) -> Result { let mut tx = self.begin_read().await?; let (response, _) = self - .activity_observation(&mut tx, task_id, body, protocol) + .activity_observation(&mut tx, task_id, body, protocol, true) .await?; tx.commit().await?; Ok(response) @@ -55,6 +55,7 @@ where task_id: &str, body: Value, protocol: &str, + check_session: bool, ) -> Result<(Value, bool)> { reject_fields(&body, &["activity_attempt_id", "lease_owner"])?; let attempt_id = text(&body, "activity_attempt_id")?; @@ -127,6 +128,24 @@ where } else { (true, false, None) }; + // A session-bound attempt shows its session's compact form, read + // without a refresh; on status, a session that is no longer the + // holder's live lease stops an attempt that could otherwise continue. + let session = DB::optional_document_row(&activity, "activity_options")? + .and_then(|options| super::worker_sessions::CommandSession::from_snapshot(&options["worker_session"])); + let (worker_session, can_continue, reason) = match &session { + Some(session) => { + let queue = DB::optional_string(&activity, "queue")?.unwrap_or_default(); + let connection = DB::optional_string(&activity, "connection")?; + let (view, usable) = self.session_for_status(tx, session, &queue, connection.as_deref(), owner).await?; + if check_session && can_continue && !usable { + (view, false, Some("worker_session_unavailable")) + } else { + (view, can_continue, reason) + } + } + None => (Value::Null, can_continue, reason), + }; // Observation preserves the existing lease and never records progress. let response = json!({"task_id":task_id,"activity_attempt_id":attempt_id,"lease_owner":owner, "can_continue":can_continue,"cancel_requested":cancel_requested,"reason":reason,"heartbeat_recorded":false, @@ -135,7 +154,7 @@ where "last_heartbeat_at":DB::optional_instant(&attempt,"last_heartbeat_at")?.or(DB::optional_instant(&activity,"last_heartbeat_at")?).map(super::cooperative::timestamp), "run_status":run_status,"run_closed_reason":DB::optional_string(&run,"closed_reason")?,"run_closed_at":DB::optional_instant(&run,"closed_at")?.map(super::cooperative::timestamp), "activity_status":activity_status,"attempt_status":attempt_status,"task_status":task_status, - "deadlines":super::activity_deadlines::deadlines_view::(&activity)?,"worker_session":null,"protocol_version":protocol,"server_capabilities":super::store::capabilities()}); + "deadlines":super::activity_deadlines::deadlines_view::(&activity)?,"worker_session":worker_session,"protocol_version":protocol,"server_capabilities":super::store::capabilities()}); Ok((response, Self::cancellation_request(&run)?.is_none())) } @@ -185,6 +204,7 @@ where task_id, json!({"activity_attempt_id":attempt_id,"lease_owner":text(&body,"lease_owner")?}), protocol, + false, ) .await .map_err(|error| match error { @@ -245,6 +265,25 @@ where return Err(refuse(StatusCode::CONFLICT, "stale_attempt")); } super::activity_deadlines::fence::(&activity)?; + // `heartbeatForAttempt`: a session-bound attempt renews its session + // by the command's lease first. A refused renewal records nothing. + let mut worker_session = Value::Null; + if let Some(session) = DB::optional_document_row(&activity, "activity_options")? + .and_then(|options| super::worker_sessions::CommandSession::from_snapshot(&options["worker_session"])) + { + match self.renew_session_for_attempt(&mut tx, &session, text(&body, "lease_owner")?).await? { + Ok(snapshot) => worker_session = snapshot, + Err((status, snapshot, renewal)) => { + return Err(super::RuntimeError::Protocol { status, + response: json!({"task_id":task_id,"activity_attempt_id":attempt_id,"lease_owner":body["lease_owner"], + "cancel_requested":observed["cancel_requested"],"can_continue":false,"reason":renewal["reason"], + "run_closed_reason":observed["run_closed_reason"],"run_closed_at":observed["run_closed_at"], + "heartbeat_recorded":false,"lease_expires_at":observed["lease_expires_at"], + "last_heartbeat_at":observed["last_heartbeat_at"],"error":renewal["error"],"worker_session":snapshot, + "worker_session_renewal":renewal,"protocol_version":protocol,"server_capabilities":super::store::capabilities()}) }); + } + } + } let at = now(); let expires = if Self::cancellation_request(&run)?.is_none() { at + chrono::Duration::seconds(self.activity_task_lease_seconds) @@ -303,7 +342,7 @@ where Ok( json!({"task_id":task_id,"activity_attempt_id":attempt_id,"lease_owner":body["lease_owner"], "cancel_requested":false,"can_continue":true,"reason":null,"run_closed_reason":null,"run_closed_at":null, - "heartbeat_recorded":true,"lease_expires_at":expires,"last_heartbeat_at":at,"worker_session":null, + "heartbeat_recorded":true,"lease_expires_at":expires,"last_heartbeat_at":at,"worker_session":worker_session, "protocol_version":protocol,"server_capabilities":super::store::capabilities()}), ) } diff --git a/rust/src/runtime/activity_lease_renewal.rs b/rust/src/runtime/activity_lease_renewal.rs index ac1a84f9e..ea30fd056 100644 --- a/rust/src/runtime/activity_lease_renewal.rs +++ b/rust/src/runtime/activity_lease_renewal.rs @@ -92,6 +92,28 @@ store_methods! { Some(task) => DB::optional_string(task, "lease_owner")?.as_deref() == Some(owner), None => false, }; + // `heartbeatForAttempt`: a session-bound attempt renews its session + // before its own lease; a refused renewal renews nothing. + let mut worker_session = Value::Null; + let session = match &rows.activity { + Some(activity) => DB::optional_document_row(activity, "activity_options")? + .and_then(|options| super::worker_sessions::CommandSession::from_snapshot(&options["worker_session"])), + None => None, + }; + if leased_by_owner && let Some(session) = &session { + match self.renew_session_for_attempt(&mut tx, session, owner).await? { + Ok(snapshot) => worker_session = snapshot, + Err((status, snapshot, renewal)) => { + let mut response = observed; + response["lease_renewed"] = json!(false); + response["can_continue"] = json!(false); + response["reason"] = renewal["reason"].clone(); + response["worker_session"] = snapshot; + response["worker_session_renewal"] = renewal; + return Err(worker(status, response)); + } + } + } let at = now(); let mut status = if leased_by_owner { let task = rows.task.as_ref().expect("task checked above"); @@ -118,7 +140,7 @@ store_methods! { status["lease_renewed"] = status["can_continue"].clone(); status["server_time"] = json!(micros(at)); status["task_id"] = json!(task_id); - status["worker_session"] = Value::Null; + status["worker_session"] = worker_session; let code = outcome_status(status["reason"].as_str()); if code != StatusCode::OK { return Err(worker(code, status)); diff --git a/rust/src/runtime/backend.rs b/rust/src/runtime/backend.rs index cd2d7360d..1561518f8 100644 --- a/rust/src/runtime/backend.rs +++ b/rust/src/runtime/backend.rs @@ -57,6 +57,9 @@ pub(super) trait Backend: Database { const WORKFLOW_POLL_EXPIRED: &str = "SELECT \"key\" FROM dw_query_cache WHERE \"key\" LIKE 'dw-native-workflow-poll:%' AND expiration<=$1 ORDER BY \"key\" LIMIT 32"; /// Add a cache row, or lock the existing one with a no-op update, so /// transactions over one poll request ID run one at a time. + /// A worker session's running activity attempts held by its owner, + /// other than one task (`activeActivityCount`). + const SESSION_RUNNING_ATTEMPTS: &str = "SELECT COUNT(*) FROM activity_attempts p JOIN activity_executions e ON e.id=p.activity_execution_id JOIN workflow_tasks t ON t.id=p.workflow_task_id WHERE t.namespace=$1 AND p.status='running' AND p.lease_expires_at>$2 AND p.lease_owner=$3 AND e.activity_options->'worker_session'->>'session_id'=$4 AND p.workflow_task_id<>$5"; const QUERY_CACHE_LOCK: &str = "INSERT INTO dw_query_cache(\"key\",value,expiration) VALUES ($1,$2,$3) ON CONFLICT (\"key\") DO UPDATE SET expiration=dw_query_cache.expiration"; /// `WorkflowTaskKindSelector`: create the queue's cursor at /// `update_validation` if it is missing, and lock it. @@ -159,6 +162,7 @@ SELECT id FROM clocks ORDER BY CASE WHEN opened_at IS NULL THEN 1 ELSE 0 END, const OBSERVATION_FAILURE_EVENT: &str = "SELECT id,sequence,event_type,recorded_at FROM workflow_history_events WHERE workflow_run_id=$1 AND json_extract(payload,'$.failure_id') COLLATE BINARY=$2 AND event_type IN ('ActivityFailed','ActivityTimedOut','ChildRunFailed','ChildRunCancelled','ChildRunTerminated','WorkflowFailed','WorkflowTimedOut','WorkflowCancelled','WorkflowTerminated','UpdateCompleted') ORDER BY sequence DESC LIMIT 1"; const ACTIVITY_RESULT_EVENT_SQL: &str = "SELECT sequence,event_type,payload FROM workflow_history_events WHERE workflow_run_id=$1 AND event_type IN ('ActivityScheduled','ActivityStarted','ActivityHeartbeatRecorded','ActivityRetryScheduled','ActivityCompleted','ActivityFailed','ActivityTimedOut','ActivityCancelled') AND (json_extract(payload,'$.activity_execution_id') COLLATE BINARY=$2 OR json_extract(payload,'$.activity.id') COLLATE BINARY=$3) ORDER BY sequence DESC LIMIT 1"; const QUERY_CACHE_SCAN: &str = "SELECT \"key\",value,expiration FROM dw_query_cache WHERE \"key\" LIKE 'dw-native-query:%' AND json_extract(value,'$.namespace')=$1 ORDER BY \"key\" LIMIT 65"; + const SESSION_RUNNING_ATTEMPTS: &str = "SELECT COUNT(*) FROM activity_attempts p JOIN activity_executions e ON e.id=p.activity_execution_id JOIN workflow_tasks t ON t.id=p.workflow_task_id WHERE t.namespace=$1 AND p.status='running' AND p.lease_expires_at>$2 AND p.lease_owner=$3 AND json_extract(e.activity_options,'$.worker_session.session_id')=$4 AND p.workflow_task_id<>$5"; const QUERY_CACHE_PENDING_FOR_RUN: &str = "SELECT json_extract(value,'$.query_admitted_at') AS query_admitted_at,json_extract(value,'$.query_deadline_at') AS query_deadline_at,json_extract(value,'$.receipt_expires_at') AS receipt_expires_at FROM dw_query_cache WHERE \"key\" LIKE 'dw-native-query:%' AND expiration>$2 AND json_extract(value,'$.run_id')=$1 AND json_extract(value,'$.status') IN ('pending','leased') ORDER BY \"key\" LIMIT 65"; const POLL_WAKE_QUERIES_SQL: &str = "SELECT \"key\" AS id FROM dw_query_cache WHERE \"key\" LIKE 'dw-native-query:%' AND expiration>$1 AND json_extract(value,'$.namespace')=$2 AND json_extract(value,'$.task_queue')=$3 AND json_extract(value,'$.query_deadline_at')>$4 AND (json_extract(value,'$.status')='pending' OR (json_extract(value,'$.status')='leased' AND json_extract(value,'$.lease_expires_at')<=$5)) ORDER BY \"key\" LIMIT 65"; const POLL_WAKE_TASKS_SQL: &str = r#" @@ -463,6 +467,7 @@ impl Backend for MySql { const QUERY_POLL_EXPIRED: &str = "SELECT `key` FROM dw_query_cache WHERE `key` LIKE 'dw-native-query-poll:%' AND expiration<=$1 ORDER BY `key` LIMIT 32"; const WORKFLOW_POLL_EXPIRED: &str = "SELECT `key` FROM dw_query_cache WHERE `key` LIKE 'dw-native-workflow-poll:%' AND expiration<=$1 ORDER BY `key` LIMIT 32"; const QUERY_CACHE_LOCK: &str = "INSERT INTO dw_query_cache(`key`,value,expiration) VALUES ($1,$2,$3) ON DUPLICATE KEY UPDATE expiration=expiration"; + const SESSION_RUNNING_ATTEMPTS: &str = "SELECT COUNT(*) FROM activity_attempts p JOIN activity_executions e ON e.id=p.activity_execution_id JOIN workflow_tasks t ON t.id=p.workflow_task_id WHERE t.namespace=? AND p.status='running' AND p.lease_expires_at>? AND p.lease_owner=? AND JSON_UNQUOTE(JSON_EXTRACT(e.activity_options,'$.worker_session.session_id'))=? AND p.workflow_task_id<>?"; const TASK_KIND_CURSOR_LOCK: &str = "INSERT INTO workflow_task_poll_cursors(namespace,task_queue,next_task_kind,created_at,updated_at) VALUES ($1,$2,'update_validation',$3,$4) ON DUPLICATE KEY UPDATE next_task_kind=next_task_kind"; const TIMER_TASK_SQL: &str = "SELECT id,payload FROM workflow_tasks WHERE workflow_run_id=? AND task_type='timer' AND status='ready' AND JSON_UNQUOTE(JSON_EXTRACT(payload,'$.timer_id'))=? AND BINARY namespace=BINARY ?"; const TIMER_SCHEDULE_SQL: &str = "SELECT payload FROM workflow_history_events WHERE workflow_run_id=? AND event_type='TimerScheduled' AND JSON_UNQUOTE(JSON_EXTRACT(payload,'$.timer_id'))=? ORDER BY sequence DESC LIMIT 1"; @@ -489,6 +494,7 @@ impl Backend for MySql { || sql == Self::POLL_WAKE_QUERIES_SQL || sql == Self::QUERY_CACHE_PENDING_FOR_RUN || sql == Self::QUERY_CACHE_SCAN + || sql == Self::SESSION_RUNNING_ATTEMPTS { return Cow::Borrowed(sql); } diff --git a/rust/src/runtime/operator_metrics.rs b/rust/src/runtime/operator_metrics.rs index 240a718f6..02fa06405 100644 --- a/rust/src/runtime/operator_metrics.rs +++ b/rust/src/runtime/operator_metrics.rs @@ -283,8 +283,7 @@ where /// server adds. pub(super) async fn operator_metrics(&self, config: &MetricsConfig) -> Result { let mut metrics = self.operator_metrics_snapshot(config).await?; - metrics["worker_sessions"] = - json!({"total": 0, "active": 0, "closed": 0, "expired": 0, "failed": 0, "orphaned": 0}); + metrics["worker_sessions"] = self.worker_session_metrics().await?; metrics["capacity_evidence"] = json!({"available": false, "reason": "native_capacity_evidence_unavailable"}); metrics["runtime_external_payload_cleanup"] = @@ -309,8 +308,7 @@ where metrics["capacity_evidence"] = json!({"available": false, "reason": "native_capacity_evidence_unavailable"}); } - metrics["worker_sessions"] = - json!({"total": 0, "active": 0, "closed": 0, "expired": 0, "failed": 0, "orphaned": 0}); + metrics["worker_sessions"] = self.worker_session_metrics().await?; metrics["runtime_external_payload_cleanup"] = self.external_payload_cleanup_metrics(config).await?; let filter = types.map(|types| Value::from(types.to_vec()).to_string()); let filter = filter.as_deref(); diff --git a/rust/src/runtime/store.rs b/rust/src/runtime/store.rs index d310338c9..64a55706e 100644 --- a/rust/src/runtime/store.rs +++ b/rust/src/runtime/store.rs @@ -1763,7 +1763,46 @@ where { continue; } + // `workerCanSatisfySession`: a session-bound activity goes only to + // a worker on protocol 1.8 or newer, on the session's queue, with + // every requirement among its capabilities. + let session = match activity.as_ref() { + Some(activity) => DB::optional_document_row(activity, "activity_options")? + .and_then(|options| super::worker_sessions::CommandSession::from_snapshot(&options["worker_session"])), + None => None, + }; + let activity_queue = match activity.as_ref() { + Some(activity) => DB::optional_string(activity, "queue")?.unwrap_or_else(|| queue.to_owned()), + None => queue.to_owned(), + }; + if let Some(session) = &session { + let capabilities: Vec = DB::optional_document_row(&worker, "capabilities")? + .as_ref() + .and_then(Value::as_array) + .into_iter() + .flatten() + .filter_map(|value| value.as_str().map(str::to_owned)) + .collect(); + if !session.claimable_by(protocol_version, &DB::string(&worker, "task_queue")?, &activity_queue, &capabilities) { + continue; + } + } let task_id = DB::string(&task, "id")?; + // `admitActivity`: the session admits the claim or the claim is + // rolled back, so a held or full session leaves the task ready. + let session_view = match &session { + Some(session) => { + let connection = match activity.as_ref() { + Some(activity) => DB::optional_string(activity, "connection")?, + None => None, + }; + match self.admit_session_activity(tx, &worker, session, &activity_queue, connection.as_deref(), &task_id).await? { + Some(view) => Some(view), + None => continue, + } + } + None => None, + }; let attempt = DB::number(&task, "attempt_count")? .checked_add(1) .ok_or_else(|| refuse(StatusCode::CONFLICT, "task_attempt_exhausted"))?; @@ -1853,6 +1892,9 @@ where claim["attempt_number"] = json!(attempt); claim["activity_type"] = json!(type_name); claim["idempotency_key"] = json!(activity_id); + if let Some(view) = session_view.clone() { + claim["worker_session"] = view; + } // A standalone activity started without input has none. if let Some(arguments) = DB::optional_string(&activity, "arguments")? { claim["arguments"] = wire(&arguments); @@ -2270,7 +2312,9 @@ where "type", "activity_type", "arguments", + "connection", "queue", + "worker_session", "retry_policy", "start_to_close_timeout", "schedule_to_start_timeout", @@ -2393,6 +2437,7 @@ where ); } Self::workflow_task_guard(&mut tx, &task, &body).await?; + self.guard_worker_session_commands(&mut tx, &task, task_id, &body, protocol, commands).await?; // `childCancellationPolicyGuard`: a child that requests cancellation // on parent close, or propagates the parent's cancellation, needs a // cooperative worker's claim and a protocol 1.20 request. @@ -2533,11 +2578,16 @@ where let metadata = super::selections::metadata(command, "activity")?; super::selections::check_sequence(&metadata, sequence)?; let activity_id = id(); - let queue = command - .get("queue") - .map_or(Ok(DB::string(&run, "queue")?), |_| { - text(command, "queue").map(str::to_owned) - })?; + // A session-bound activity without its own queue or + // connection takes the session's (`applyWorkerSessionRouting`). + let session = super::worker_sessions::command_session(command); + let queue = match command.get("queue").filter(|queue| !queue.is_null()) { + Some(_) => text(command, "queue")?.to_owned(), + None => session.as_ref().and_then(|session| session.queue.clone()).map_or_else(|| DB::string(&run, "queue"), Ok)?, + }; + let connection = command["connection"].as_str().map(str::trim).filter(|connection| !connection.is_empty()).map(str::to_owned) + .or_else(|| session.as_ref().and_then(|session| session.connection.clone())); + let activity_options = session.as_ref().map(|session| json!({"worker_session": session.snapshot()})); let activity_type = text(command, "activity_type")?; let arguments = match external.get(&index) { Some(prepared) => { @@ -2562,11 +2612,12 @@ where // The preserved SQL schema defaults this counter to one. // Published authored scheduling explicitly starts at zero; // only the original worker claim consumes its first attempt. - Self::query("INSERT INTO activity_executions(id,workflow_run_id,sequence,activity_type,activity_class,status,attempt_count,payload_codec,arguments,queue,retry_policy,parallel_group_path,schedule_deadline_at,schedule_to_close_deadline_at) VALUES ($1,$2,$3,$4,$5,'pending',0,'avro',$6,$7,$8,$9,$10,$11)") + Self::query("INSERT INTO activity_executions(id,workflow_run_id,sequence,activity_type,activity_class,status,attempt_count,payload_codec,arguments,queue,retry_policy,parallel_group_path,schedule_deadline_at,schedule_to_close_deadline_at,connection,activity_options) VALUES ($1,$2,$3,$4,$5,'pending',0,'avro',$6,$7,$8,$9,$10,$11,$12,$13)") .bind(&activity_id).bind(&run_id).bind(sequence).bind(activity_type).bind(activity_type) .bind(&arguments).bind(&queue).bind(policy.as_ref().map(DB::document)) .bind(DB::optional_document(&metadata["parallel_group_path"])) - .bind(DB::optional_time(schedule_deadline)).bind(DB::optional_time(lifecycle_deadline)).execute(&mut *tx).await?; + .bind(DB::optional_time(schedule_deadline)).bind(DB::optional_time(lifecycle_deadline)) + .bind(connection.as_deref()).bind(activity_options.as_ref().map(DB::document)).execute(&mut *tx).await?; let activity = Self::query("SELECT * FROM activity_executions WHERE id=$1") .bind(&activity_id) .fetch_one(&mut *tx) @@ -4118,7 +4169,8 @@ pub(crate) fn capabilities() -> Value { let mut value = json!({"supported_workflow_task_commands": ["schedule_activity", "start_timer", "start_child_workflow", "open_condition_wait", "open_signal_wait", "record_version_marker", "record_side_effect", "complete_workflow", "fail_workflow", "continue_as_new", "complete_update", "fail_update"], "worker_deregistration_fencing":{"schema":"durable-workflow.v2.worker-deregistration.v1","supported":true,"receipt_retention_seconds":600,"endpoint":"/worker/registrations/{workerId}/deregister"}, "workflow_memo_updates": false, "cooperative_cancellation": true, "prepared_local_activities": false, - "worker_sessions": false, "sticky_execution": false, "local_activities": false, "message_streams": false, + "worker_sessions": super::worker_sessions::capability(), "worker_session_verbs": ["create", "heartbeat", "close"], + "sticky_execution": false, "local_activities": false, "message_streams": false, "update_validation_tasks": true, "synchronous_update_validation": super::update_validation::capability(), "workflow_updates": true, "query_tasks": true, "query_task_authority":true, "query_task_poll_request_idempotency": true, "activity_lease_renewal": true}); diff --git a/rust/src/runtime/worker_sessions.rs b/rust/src/runtime/worker_sessions.rs index 0b550dc66..06c46b47c 100644 --- a/rust/src/runtime/worker_sessions.rs +++ b/rust/src/runtime/worker_sessions.rs @@ -36,6 +36,38 @@ fn wire(at: Option>) -> Value { at.map_or(Value::Null, |at| json!(super::cooperative::timestamp(at))) } +/// `server_capabilities.worker_sessions`: the published contract +/// (`WorkerProtocolVersion::workerSessionSemantics`) and its support. +pub(super) fn capability() -> Value { + json!({"feature":"worker_sessions","contract_version":"1.0","minimum_protocol_version":"1.8","command_field":"worker_session", + "activity_options_field":"worker_session","verbs":["create","heartbeat","close"], + "lifecycle":{"creation":"lazy_create_on_first_admitted_activity_or_explicit_worker_create","renewal":"activity_heartbeat_or_explicit_session_heartbeat", + "close":"explicit_holder_close","lease_expiry":"session_expires_when_lease_is_not_renewed", + "ttl_expiry":"absolute_session_ttl_is_terminal_for_that_session_id"}, + "ownership":"single_worker_lease_owner", + "lease":{"scope":"namespace_session_id","owner":"registered_worker_id","activity_attempt_leases_remain_independent":true}, + "admission":{"queue_routing_first":true,"requires_registered_worker":true,"requires_capabilities":true,"create_if_missing_default":true, + "allow_reacquire_after_failure_default":true}, + "rollout_safety":{"minimum_protocol_version":"1.8","mixed_server_rollout_fenced_by_protocol_version":true, + "servers_below_minimum_must_reject_worker_session_commands":true,"servers_below_minimum_must_not_claim_worker_session_activity_tasks":true}, + "limits":{"max_concurrent_worker_sessions":"worker_registration","max_concurrent_activities":"session"}, + "default_max_concurrent_activities":1, + "renewal":{"activity_heartbeat_renews_session":true,"explicit_session_heartbeat":true}, + "failure_detection":["lease_expiry","registered_worker_heartbeat_staleness"], + "holder_loss":{"in_flight_activities_keep_at_least_once_attempt_semantics":true,"replacement_worker_must_reacquire_session":true, + "process_local_state_must_be_rebuilt_after_reacquire":true}, + "cancellation":{"workflow_cancellation_observed_through_activity_heartbeat":true,"session_lease_does_not_override_activity_cancel_requested":true, + "planned_shutdown_should_close_sessions":true}, + "routing":["queue","connection","requirements"], + "visibility":["active","closed","expired","failed","orphaned"], + "statuses":["active","closed","expired","failed","orphaned"], + "terminal_statuses":["closed"], + "terminal_conditions":["explicit_close","ttl_expired","allow_reacquire_after_failure_false"], + "authoring_guidance":["use_for_process_local_state_gpu_memory_or_filesystem_affinity","prefer_ordinary_queued_activities_for_independent_steps", + "prefer_one_larger_activity_for_atomic_side_effects"], + "supported":true}) +} + /// `rejectWorkerSessionsUnavailable`: worker sessions need protocol 1.8. pub(super) fn sessions_gate(protocol: &str) -> Result<()> { let version = protocol.split_once('.').and_then(|(major, minor)| { @@ -173,6 +205,143 @@ impl Options { } } +/// A `schedule_activity` command's `worker_session`, as the command +/// normalizer leaves it and `WorkerSessionOptions::toSnapshot` stores it in +/// the activity's options: omitted values stay null. +pub(super) struct CommandSession { + pub(super) session_id: String, + pub(super) connection: Option, + pub(super) queue: Option, + requirements: Vec, + lease_seconds: Option, + ttl_seconds: Option, + max_concurrent_activities: Option, + create_if_missing: bool, + allow_reacquire_after_failure: bool, +} + +impl CommandSession { + pub(super) fn snapshot(&self) -> Value { + json!({"session_id":self.session_id,"connection":self.connection,"queue":self.queue,"requirements":self.requirements, + "lease_seconds":self.lease_seconds,"ttl_seconds":self.ttl_seconds,"max_concurrent_activities":self.max_concurrent_activities, + "create_if_missing":self.create_if_missing,"allow_reacquire_after_failure":self.allow_reacquire_after_failure}) + } + + /// The stored snapshot, from an activity's options. + pub(super) fn from_snapshot(value: &Value) -> Option { + let mut requirements: Vec = Vec::new(); + for item in value["requirements"].as_array().into_iter().flatten() { + if let Some(text) = string_value(item) + && !requirements.contains(&text) + { + requirements.push(text); + } + } + Some(Self { + session_id: string_value(&value["session_id"])?, + connection: string_value(&value["connection"]), + queue: string_value(&value["queue"]), + requirements, + lease_seconds: value["lease_seconds"].as_i64().filter(|value| *value > 0), + ttl_seconds: value["ttl_seconds"].as_i64().filter(|value| *value > 0), + max_concurrent_activities: value["max_concurrent_activities"].as_i64().filter(|value| *value > 0), + create_if_missing: value["create_if_missing"].as_bool().unwrap_or(true), + allow_reacquire_after_failure: value["allow_reacquire_after_failure"].as_bool().unwrap_or(true), + }) + } + + /// `normalizeOptions` with the activity's queue and connection as the + /// fallbacks. + fn options(&self, queue: Option<&str>, connection: Option<&str>) -> Options { + Options { + session_id: self.session_id.clone(), + connection: self.connection.clone().or_else(|| connection.map(str::to_owned)), + queue: self.queue.clone().or_else(|| queue.map(str::to_owned)), + requirements: self.requirements.clone(), + lease_seconds: self.lease_seconds.unwrap_or(DEFAULT_LEASE_SECONDS), + ttl_seconds: self.ttl_seconds.unwrap_or(DEFAULT_TTL_SECONDS), + max_concurrent_activities: self.max_concurrent_activities.unwrap_or(DEFAULT_MAX_CONCURRENT_ACTIVITIES), + create_if_missing: self.create_if_missing, + allow_reacquire_after_failure: self.allow_reacquire_after_failure, + } + } + + /// `workerCanSatisfySession`: the claiming worker's protocol, task queue + /// and capabilities. + pub(super) fn claimable_by(&self, protocol: &str, worker_queue: &str, activity_queue: &str, capabilities: &[String]) -> bool { + sessions_gate(protocol).is_ok() + && self.queue.as_deref().unwrap_or(activity_queue) == worker_queue + && self.requirements.iter().all(|requirement| capabilities.contains(requirement)) + } + + /// The lease an activity heartbeat renews the session by. + pub(super) fn heartbeat_lease(&self) -> i64 { + self.lease_seconds.unwrap_or(DEFAULT_LEASE_SECONDS) + } +} + +/// A validated command's `worker_session`, when it names a session. +pub(super) fn command_session(command: &Value) -> Option { + let raw = command.get("worker_session").filter(|value| value.is_object())?; + CommandSession::from_snapshot(raw) +} + +/// The command normalizer's `worker_session` checks, with its messages. +fn normalizer_errors(index: usize, command: &Value) -> Vec<(String, Vec)> { + let Some(raw) = command.get("worker_session").filter(|value| !value.is_null()) else { + return Vec::new(); + }; + let at = |field: &str| format!("commands.{index}.worker_session{field}"); + let Some(raw) = raw.as_object() else { + return vec![(at(""), vec!["Workflow task command field [worker_session] must be an object when provided.".into()])]; + }; + if raw.get("session_id").and_then(Value::as_str).is_none_or(|id| id.trim().is_empty()) { + return vec![(at(".session_id"), vec!["Worker session commands require a non-empty session_id.".into()])]; + } + let mut errors = Vec::new(); + for field in ["connection", "queue"] { + if let Some(value) = raw.get(field).filter(|value| !value.is_null()) + && value.as_str().is_none_or(|text| text.trim().is_empty()) + { + errors.push((at(&format!(".{field}")), vec![format!("Worker session field [{field}] must be a non-empty string when provided.")])); + } + } + for field in ["lease_seconds", "ttl_seconds", "max_concurrent_activities"] { + if let Some(value) = raw.get(field).filter(|value| !value.is_null()) + && value.as_i64().is_none_or(|value| value < 1) + { + errors.push((at(&format!(".{field}")), vec![format!("Worker session field [{field}] must be a positive integer when provided.")])); + } + } + for field in ["create_if_missing", "allow_reacquire_after_failure"] { + if let Some(value) = raw.get(field).filter(|value| !value.is_null()) + && !value.is_boolean() + { + errors.push((at(&format!(".{field}")), vec![format!("Worker session field [{field}] must be boolean when provided.")])); + } + } + if let Some(requirements) = raw.get("requirements").filter(|value| !value.is_null()) { + match requirements.as_array() { + None => errors.push((at(".requirements"), vec!["Worker session requirements must be a list of non-empty strings.".into()])), + Some(items) => { + for (position, item) in items.iter().enumerate() { + if item.as_str().is_none_or(|text| text.trim().is_empty()) { + errors.push((at(&format!(".requirements.{position}")), vec!["Worker session requirements must be non-empty strings.".into()])); + } + } + } + } + } + errors +} + +/// A protocol version as `major.minor`. +fn version(protocol: Option<&str>) -> Option<(u64, u64)> { + let (major, minor) = protocol?.split_once('.')?; + let digits = |text: &str| !text.is_empty() && text.bytes().all(|byte| byte.is_ascii_digit()); + (digits(major) && digits(minor)).then(|| Some((major.parse().ok()?, minor.parse().ok()?))).flatten() +} + /// A session row. struct Session { id: i64, @@ -220,6 +389,25 @@ fn with_session(session: Value) -> Map { Map::from_iter([("session".to_owned(), session)]) } +/// The compact form a task or status carries (`activitySessionPayload`): +/// the normalized options with the row's lease, without null values. +fn compact(options: &Options, session: Option<&Session>) -> Value { + let mut fields = json!({"session_id":options.session_id,"connection":options.connection,"queue":options.queue, + "requirements":options.requirements,"lease_seconds":options.lease_seconds,"ttl_seconds":options.ttl_seconds, + "max_concurrent_activities":options.max_concurrent_activities,"create_if_missing":options.create_if_missing, + "allow_reacquire_after_failure":options.allow_reacquire_after_failure}); + if let Some(session) = session { + fields["status"] = json!(session.status); + fields["lease_owner"] = json!(session.lease_owner); + fields["lease_expires_at"] = wire(session.lease_expires_at); + fields["ttl_expires_at"] = wire(session.ttl_expires_at); + } + if let Some(map) = fields.as_object_mut() { + map.retain(|_, value| !value.is_null()); + } + fields +} + store_methods! { /// `POST /api/worker/sessions`. pub(crate) async fn create_worker_session(&self, body: Value, protocol: &str) -> Result { @@ -251,7 +439,7 @@ store_methods! { .collect(); let limit = DB::optional_number(&worker, "max_concurrent_worker_sessions")?.unwrap_or(10); let mut tx = self.begin().await?; - let answer = self.acquire(&mut tx, &worker_id, &task_queue, &capabilities, limit, &options).await?; + let answer = self.acquire(&mut tx, &worker_id, &task_queue, &capabilities, limit, &options, None).await?; tx.commit().await?; Ok(answer) } @@ -265,6 +453,7 @@ store_methods! { capabilities: &[String], limit: i64, options: &Options, + activity_task: Option<&str>, ) -> Result { if options.queue.as_deref().is_some_and(|queue| queue != task_queue) { return Ok(failure("session_queue_mismatch", "Worker session queue does not match the worker's task queue.", @@ -303,6 +492,15 @@ store_methods! { return Ok(failure("session_owned_by_another_worker", "Worker session is currently leased by another worker.", StatusCode::CONFLICT, with_session(snapshot))); } + // An activity claim also respects the session's activity limit. + if let Some(task) = activity_task { + let running = self.running_session_activities(tx, &session, task).await?; + if running >= session.max_concurrent_activities.max(1) { + let snapshot = self.snapshot(tx, &session).await?; + return Ok(failure("session_activity_limit_exceeded", "Worker session has reached max_concurrent_activities.", + StatusCode::CONFLICT, with_session(snapshot))); + } + } self.renew(tx, &mut session, options, worker_id).await?; return Ok(success("reused", self.snapshot(tx, &session).await?, StatusCode::OK)); } @@ -361,6 +559,163 @@ store_methods! { self.save(tx, session).await } + /// `activeActivityCount`: the owner's running, unexpired attempts of + /// activities bound to this session, other than one task. + async fn running_session_activities(&self, tx: &mut Transaction<'_, DB>, session: &Session, except_task: &str) -> Result { + let Some(owner) = session.lease_owner.as_deref() else { + return Ok(0); + }; + Ok(Self::scalar(DB::SESSION_RUNNING_ATTEMPTS).bind(self.namespace.as_ref()).bind(DB::bind_time(now())).bind(owner) + .bind(session.session_id.as_str()).bind(except_task).fetch_one(&mut **tx).await?) + } + + /// `guardWorkerSessionCommandsAvailable`, then + /// `guardPortableWorkerAffinityCompletion`, then the command normalizer's + /// `worker_session` checks, in PHP's order. + pub(super) async fn guard_worker_session_commands( + &self, + tx: &mut Transaction<'_, DB>, + task: &DB::Row, + task_id: &str, + body: &Value, + protocol: Option<&str>, + commands: &[Value], + ) -> Result<()> { + if !commands.iter().any(|command| command.get("worker_session").is_some_and(|value| !value.is_null())) { + return Ok(()); + } + let attempt = body["workflow_task_attempt"].clone(); + let version = version(protocol); + if version.is_none_or(|version| version < MINIMUM_PROTOCOL) { + return Err(RuntimeError::Protocol { status: StatusCode::CONFLICT, + response: json!({"task_id":task_id,"workflow_task_attempt":attempt,"outcome":"rejected","recorded":false, + "reason":"worker_sessions_unavailable","error":"Worker-session activity commands require worker protocol 1.8 or newer.", + "requested_version":protocol,"minimum_protocol_version":"1.8", + "remediation":"Complete worker-session workflow tasks through a server node advertising worker protocol 1.8 or newer."}) }); + } + let rejection = |reason: &str| RuntimeError::Protocol { status: StatusCode::CONFLICT, + response: json!({"task_id":task_id,"workflow_task_attempt":attempt,"outcome":"rejected","recorded":false,"reason":reason, + "capability":"worker_sessions","requested_version":protocol,"minimum_protocol_version":"1.18", + "remediation":"Use cold durable replay unless the registered worker manifest explicitly supports the required capability and exact cache identity."}) }; + if version.is_none_or(|version| version < (1, 18)) { + return Err(rejection("portable_worker_affinity_protocol_floor_unavailable")); + } + let owner = DB::optional_string(task, "lease_owner")?.unwrap_or_default(); + let worker = Self::query("SELECT capabilities,capability_manifest FROM workflow_worker_registrations WHERE namespace=$1 AND worker_id=$2") + .bind(self.namespace.as_ref()).bind(&owner).fetch_optional(&mut **tx).await?; + let capable = match &worker { + Some(row) => { + let flat = DB::optional_document_row(row, "capabilities")? + .and_then(|capabilities| capabilities.as_array().map(|items| items.iter().any(|item| item == "worker_sessions"))) + .unwrap_or(false); + let manifest = DB::optional_document_row(row, "capability_manifest")? + .is_some_and(|manifest| manifest["worker_sessions"]["supported"] == json!(true)); + flat && manifest + } + None => false, + }; + if !capable { + return Err(rejection("worker_capability_not_supported")); + } + let errors: Vec<_> = commands.iter().enumerate().flat_map(|(index, command)| normalizer_errors(index, command)).collect(); + failure_messages(errors).map_or(Ok(()), Err) + } + + /// `admitActivity` inside a claim: acquire or reuse the activity's + /// session for the claiming worker, within its activity limit. Answers + /// the compact form, or `None` when the session refuses the claim. + pub(super) async fn admit_session_activity( + &self, + tx: &mut Transaction<'_, DB>, + worker: &DB::Row, + session: &CommandSession, + activity_queue: &str, + activity_connection: Option<&str>, + task_id: &str, + ) -> Result> { + let worker_id = DB::string(worker, "worker_id")?; + let task_queue = DB::string(worker, "task_queue")?; + let capabilities: Vec = DB::optional_document_row(worker, "capabilities")? + .as_ref() + .and_then(Value::as_array) + .into_iter() + .flatten() + .filter_map(|value| value.as_str().map(str::to_owned)) + .collect(); + let limit = DB::optional_number(worker, "max_concurrent_worker_sessions")?.unwrap_or(10); + let options = session.options(Some(activity_queue), activity_connection); + let answer = self.acquire(tx, &worker_id, &task_queue, &capabilities, limit, &options, Some(task_id)).await?; + if answer.body["admitted"] != json!(true) { + return Ok(None); + } + let row = self.locked_session(tx, &options.session_id).await?; + Ok(Some(compact(&options, row.as_ref()))) + } + + /// `heartbeatForAttempt`: an activity heartbeat or lease renewal renews + /// its session by the command's lease. Answers the full snapshot, or the + /// refusal's status, its snapshot (if any) and the renewal's failure. + pub(super) async fn renew_session_for_attempt( + &self, + tx: &mut Transaction<'_, DB>, + session: &CommandSession, + lease_owner: &str, + ) -> Result> { + let Some(mut row) = self.locked_session(tx, &session.session_id).await? else { + return Ok(Err((StatusCode::NOT_FOUND, Value::Null, + json!({"admitted":false,"reason":"session_not_found","error":"Worker session was not found."})))); + }; + if row.status != "active" { + let snapshot = self.snapshot(tx, &row).await?; + return Ok(Err((StatusCode::CONFLICT, snapshot, + json!({"admitted":false,"reason":"session_not_active","error":"Worker session is not active."})))); + } + if row.lease_owner.as_deref() != Some(lease_owner) { + let snapshot = self.snapshot(tx, &row).await?; + return Ok(Err((StatusCode::CONFLICT, snapshot, + json!({"admitted":false,"reason":"session_owner_mismatch","error":"Worker session lease is owned by another worker."})))); + } + let at = second(now()); + row.lease_expires_at = Some(at + chrono::Duration::seconds(session.heartbeat_lease().max(1))); + row.last_heartbeat_at = Some(at); + row.failure_reason = None; + self.save(tx, &row).await?; + Ok(Ok(self.snapshot(tx, &row).await?)) + } + + /// The compact form an attempt's status carries, read without a + /// refresh, and whether the session still lets the attempt continue. + pub(super) async fn session_for_status( + &self, + tx: &mut Transaction<'_, DB>, + session: &CommandSession, + activity_queue: &str, + activity_connection: Option<&str>, + lease_owner: &str, + ) -> Result<(Value, bool)> { + let row = Self::query(SESSION_BY_ID).bind(self.namespace.as_ref()).bind(session.session_id.as_str()).fetch_optional(&mut **tx).await?; + let row = row.as_ref().map(Self::session).transpose()?; + let at = now(); + let usable = row.as_ref().is_some_and(|row| row.status == "active" && row.lease_owner.as_deref() == Some(lease_owner) + && row.lease_expires_at.is_some_and(|lease| lease > at) && row.ttl_expires_at.is_some_and(|ttl| ttl > at)); + Ok((compact(&session.options(Some(activity_queue), activity_connection), row.as_ref()), usable)) + } + + /// `metrics`: the namespace's sessions by status, after a refresh. + pub(crate) async fn worker_session_metrics(&self) -> Result { + self.refresh_namespace_sessions().await?; + let rows = Self::query("SELECT status FROM workflow_worker_sessions WHERE namespace=$1").bind(self.namespace.as_ref()).fetch_all(&self.pool).await?; + let mut metrics = json!({"total":0,"active":0,"closed":0,"expired":0,"failed":0,"orphaned":0}); + for row in rows { + let status = DB::string(&row, "status")?; + metrics["total"] = json!(metrics["total"].as_i64().unwrap_or(0) + 1); + if let Some(count) = metrics.get_mut(status.as_str()) { + *count = json!(count.as_i64().unwrap_or(0) + 1); + } + } + Ok(metrics) + } + /// `POST /api/worker/sessions/{sessionId}/heartbeat`. pub(crate) async fn heartbeat_worker_session(&self, session_id: &str, body: Value, protocol: &str) -> Result { sessions_gate(protocol)?; @@ -570,7 +925,7 @@ store_methods! { "lease_expires_at":wire(session.lease_expires_at),"ttl_expires_at":wire(session.ttl_expires_at),"closed_at":wire(session.closed_at), "failure_reason":session.failure_reason,"lease_seconds":session.lease_seconds,"ttl_seconds":session.ttl_seconds, "max_concurrent_activities":session.max_concurrent_activities,"create_if_missing":session.create_if_missing, - "allow_reacquire_after_failure":session.allow_reacquire_after_failure,"active_activity_count":0, + "allow_reacquire_after_failure":session.allow_reacquire_after_failure,"active_activity_count":self.running_session_activities(tx, session, "").await?, "last_heartbeat_at":wire(session.last_heartbeat_at),"created_at":wire(session.created_at),"updated_at":wire(session.updated_at)})) } } diff --git a/rust/tests/execution/worker_sessions.rs b/rust/tests/execution/worker_sessions.rs index 571d575a7..661eca750 100644 --- a/rust/tests/execution/worker_sessions.rs +++ b/rust/tests/execution/worker_sessions.rs @@ -104,3 +104,64 @@ async fn worker_sessions_refuse_old_protocols_and_bad_requests() { let (status, missing) = request(&app, "GET", "/api/worker-sessions/none", Value::Null).await; assert_eq!((status, missing["reason"].clone()), (StatusCode::NOT_FOUND, json!("worker_session_not_found"))); } + +async fn session_worker(app: &Router, worker: &str, capabilities: Value) { + let manifest = json!({"local_activities":{"supported":false,"minimum_protocol_version":"1.18"}, + "sticky_execution":{"supported":false,"minimum_protocol_version":"1.18"}, + "worker_sessions":{"supported":capabilities.as_array().unwrap().iter().any(|item| item == "worker_sessions"),"minimum_protocol_version":"1.18"}}); + let (status, answer) = request(app, "POST", "/api/worker/register", json!({"worker_id":worker,"task_queue":"test","runtime":"php", + "supported_workflow_types":["echo"],"supported_activity_types":["act"],"capabilities":capabilities,"capability_manifest":manifest})).await; + assert_eq!(status, StatusCode::CREATED, "{answer}"); +} + +fn bound(session: Value) -> Value { + json!({"type":"schedule_activity","activity_type":"act","arguments":envelope(Payload::Array(vec![])),"worker_session":session}) +} + +// Session-bound activities (worker-session-activities fixture): lazily +// created on the first admitted claim, held against other workers, bounded +// by the session's activity limit, and renewed by the activity heartbeat. +#[tokio::test] +async fn session_bound_activities_are_admitted_held_limited_and_renewed() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + session_worker(&app, "author", json!(["worker_sessions"])).await; + session_worker(&app, "gpu-a", json!(["worker_sessions", "gpu"])).await; + session_worker(&app, "gpu-b", json!(["worker_sessions", "gpu"])).await; + start(&app, "session-run").await; + let task = poll(&app, "author", "workflow").await; + let session = json!({"session_id":"lazy","requirements":["gpu"],"lease_seconds":90,"max_concurrent_activities":1}); + let (status, done) = finish_task(&app, &task, json!([bound(session.clone()), bound(session)])).await; + assert_eq!(status, StatusCode::OK, "{done}"); + let (status, _) = request(&app, "GET", "/api/worker-sessions/lazy", Value::Null).await; + assert_eq!(status, StatusCode::NOT_FOUND, "no session before the first claim"); + + let first = poll(&app, "gpu-a", "activity").await; + assert_eq!((first["worker_session"]["session_id"].clone(), first["worker_session"]["lease_owner"].clone()), + (json!("lazy"), json!("gpu-a")), "{first}"); + // The session holds one activity: neither worker gets the second. + assert!(poll(&app, "gpu-a", "activity").await.is_null()); + assert!(poll(&app, "gpu-b", "activity").await.is_null()); + let (status, shown) = request(&app, "GET", "/api/worker-sessions/lazy", Value::Null).await; + assert_eq!((status, shown["session"]["active_activity_count"].clone()), (StatusCode::OK, json!(1)), "{shown}"); + + // The heartbeat renews the session by the command's lease. + let own = json!({"activity_attempt_id":first["activity_attempt_id"],"lease_owner":"gpu-a"}); + let path = format!("/api/worker/activity-tasks/{}", first["task_id"].as_str().unwrap()); + let (status, beat) = request(&app, "POST", &format!("{path}/heartbeat"), own.clone()).await; + assert_eq!(status, StatusCode::OK, "{beat}"); + let lease = |value: &Value| chrono::DateTime::parse_from_rfc3339(value.as_str().unwrap()).unwrap().timestamp(); + assert_eq!(lease(&beat["worker_session"]["lease_expires_at"]) - lease(&beat["worker_session"]["last_heartbeat_at"]), 90, "{beat}"); + + // Once the holder closes the session, the heartbeat is refused and the + // attempt still completes; the second activity is then claimable again. + let (status, _) = request(&app, "DELETE", "/api/worker/sessions/lazy", json!({"worker_id":"gpu-a"})).await; + assert_eq!(status, StatusCode::OK); + let (status, refused) = request(&app, "POST", &format!("{path}/heartbeat"), own.clone()).await; + assert_eq!((status, refused["worker_session_renewal"]["reason"].clone()), (StatusCode::CONFLICT, json!("session_not_active")), "{refused}"); + let (status, observed) = request_protocol(&app, "POST", &format!("{path}/status"), own.clone(), "1.20").await; + assert_eq!((status, observed["reason"].clone()), (StatusCode::OK, json!("worker_session_unavailable")), "{observed}"); + let mut completion = own.clone(); + completion["result"] = envelope(Payload::Null); + assert_eq!(request(&app, "POST", &format!("{path}/complete"), completion).await.0, StatusCode::OK); +} From fed2ec2e52a1567f302097043558868f8ed816ef Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Sat, 24 Oct 2026 22:27:18 +0000 Subject: [PATCH 05/13] Server #740: fence worker sessions by registration, bound their numbers, keep requirements linear Security's three R2 findings on #740, in both servers. Session holders are fenced by registration incarnation. Each create and renewal records a SHA-256 digest of the holder registration's token: PHP in a new lease_owner_incarnation column, native in a native-owned dw_worker_session_holders table (schema version 5). Wherever holder loss is applied, a session whose holder incarnation is no longer the worker's current registration becomes orphaned with worker_registration_replaced. A same-ID replacement then reacquires (never reuses) the session through create or a session-bound claim, and heartbeats are refused with session_not_active until it does. A worker heartbeat keeps the registration, so the original holder still reuses. Create, heartbeat and close accept an optional registration_token. One that is not the worker's current token is refused with 409 stale_worker_registration and changes nothing. Upgrading keeps running holders: the PHP migration backfills active sessions from their holder's current token, and native takeover seeds the holder table the same way. The compiled native catalogs gain only the new table's rows. Session numbers have portable maxima: lease_seconds 86,400, ttl_seconds 2,592,000 and max_concurrent_activities 1,000. They are enforced as Laravel max rules on the create and heartbeat routes and in schedule_activity commands, before arithmetic or writes. Native also builds every session time with checked chrono, so an out-of-range stored value answers an error instead of panicking. The runtime schema's request fields carry the same maximum and document the token and new reasons. Requirement lists are linear. One first-occurrence helper backed by a hash set replaces both Vec::contains dedupes, and capability membership uses a set built once per check. A comparison-counting unit test fails on a quadratic scan of 8,000 near-limit entries. Co-Authored-By: Claude Opus 5.5 --- .../Api/WorkerSessionController.php | 19 +- app/Models/WorkerSessionLease.php | 1 + app/Support/WorkerSessionRegistry.php | 118 ++++++++- .../WorkflowTaskCompletionValidation.php | 6 +- ..._add_worker_session_holder_incarnation.php | 46 ++++ .../worker-sessions-runtime.schema.json | 25 +- .../{0004_catalog.json => 0005_catalog.json} | 4 + .../mariadb/0005_worker_session_holders.sql | 10 + .../{0004_catalog.json => 0005_catalog.json} | 4 + .../mysql/0005_worker_session_holders.sql | 10 + .../{0004_catalog.json => 0005_catalog.json} | 88 +++++++ .../postgres/0005_worker_session_holders.sql | 11 + .../sqlite/0005_worker_session_holders.sql | 11 + rust/src/runtime/completion_rules.rs | 6 +- rust/src/runtime/mysql.rs | 32 ++- rust/src/runtime/namespace_cleanup.rs | 1 + rust/src/runtime/postgres.rs | 20 +- rust/src/runtime/schema.rs | 14 +- rust/src/runtime/upgrade.rs | 2 + rust/src/runtime/upgrade/takeover.rs | 15 ++ rust/src/runtime/upgrade_tests.rs | 5 +- rust/src/runtime/worker_sessions.rs | 243 +++++++++++++++--- rust/tests/execution/worker_sessions.rs | 155 +++++++++++ .../server-parity/mysql-schema.php | 2 +- .../server-parity/postgres-schema.php | 4 +- scripts/conformance/server-parity/schema.php | 2 +- .../WorkerSessionHolderMigrationTest.php | 69 +++++ tests/Feature/WorkerSessionProtocolTest.php | 197 +++++++++++++- 28 files changed, 1026 insertions(+), 94 deletions(-) create mode 100644 database/migrations/2026_10_24_000100_add_worker_session_holder_incarnation.php rename rust/migrations/mariadb/{0004_catalog.json => 0005_catalog.json} (99%) create mode 100644 rust/migrations/mariadb/0005_worker_session_holders.sql rename rust/migrations/mysql/{0004_catalog.json => 0005_catalog.json} (99%) create mode 100644 rust/migrations/mysql/0005_worker_session_holders.sql rename rust/migrations/postgres/{0004_catalog.json => 0005_catalog.json} (99%) create mode 100644 rust/migrations/postgres/0005_worker_session_holders.sql create mode 100644 rust/migrations/sqlite/0005_worker_session_holders.sql create mode 100644 tests/Feature/WorkerSessionHolderMigrationTest.php diff --git a/app/Http/Controllers/Api/WorkerSessionController.php b/app/Http/Controllers/Api/WorkerSessionController.php index b88ae8c35..7c895a0fe 100644 --- a/app/Http/Controllers/Api/WorkerSessionController.php +++ b/app/Http/Controllers/Api/WorkerSessionController.php @@ -11,6 +11,9 @@ class WorkerSessionController { + /** The token registration returned, as fenced deregistration accepts it. */ + private const REGISTRATION_TOKEN_RULES = ['nullable', 'string', 'regex:/^[a-f0-9]{32}$/D']; + public function __construct( private readonly WorkerSessionRegistry $sessions, ) {} @@ -52,7 +55,7 @@ public function create(Request $request): JsonResponse ], 422); } - $result = $this->sessions->createOrReacquire($namespace, $worker, $options); + $result = $this->sessions->createOrReacquire($namespace, $worker, $options, $validated['registration_token'] ?? null); return WorkerProtocol::json($result, (int) $result['status']); } @@ -71,7 +74,8 @@ public function heartbeat(Request $request, string $sessionId): JsonResponse $validated = $request->validate([ 'worker_id' => ['required', 'string', 'max:255'], - 'lease_seconds' => ['nullable', 'integer', 'min:1'], + 'lease_seconds' => ['nullable', 'integer', 'min:1', 'max:'.WorkerSessionRegistry::MAX_LEASE_SECONDS], + 'registration_token' => self::REGISTRATION_TOKEN_RULES, ]); $result = $this->sessions->heartbeat( @@ -79,6 +83,7 @@ public function heartbeat(Request $request, string $sessionId): JsonResponse $validated['worker_id'], $sessionId, $validated['lease_seconds'] ?? null, + $validated['registration_token'] ?? null, ); return WorkerProtocol::json($result, (int) $result['status']); @@ -98,9 +103,10 @@ public function close(Request $request, string $sessionId): JsonResponse $validated = $request->validate([ 'worker_id' => ['required', 'string', 'max:255'], + 'registration_token' => self::REGISTRATION_TOKEN_RULES, ]); - $result = $this->sessions->close($namespace, $validated['worker_id'], $sessionId); + $result = $this->sessions->close($namespace, $validated['worker_id'], $sessionId, $validated['registration_token'] ?? null); return WorkerProtocol::json($result, (int) $result['status']); } @@ -154,11 +160,12 @@ private function validateSessionRequest(Request $request, bool $workerIdRequired 'queue' => ['nullable', 'string', 'max:255'], 'requirements' => ['nullable', 'array'], 'requirements.*' => ['string', 'max:255'], - 'lease_seconds' => ['nullable', 'integer', 'min:1'], - 'ttl_seconds' => ['nullable', 'integer', 'min:1'], - 'max_concurrent_activities' => ['nullable', 'integer', 'min:1'], + 'lease_seconds' => ['nullable', 'integer', 'min:1', 'max:'.WorkerSessionRegistry::MAX_LEASE_SECONDS], + 'ttl_seconds' => ['nullable', 'integer', 'min:1', 'max:'.WorkerSessionRegistry::MAX_TTL_SECONDS], + 'max_concurrent_activities' => ['nullable', 'integer', 'min:1', 'max:'.WorkerSessionRegistry::MAX_CONCURRENT_ACTIVITIES], 'create_if_missing' => ['nullable', 'boolean'], 'allow_reacquire_after_failure' => ['nullable', 'boolean'], + 'registration_token' => self::REGISTRATION_TOKEN_RULES, ]); } diff --git a/app/Models/WorkerSessionLease.php b/app/Models/WorkerSessionLease.php index 1ad4e866c..0c0ab034d 100644 --- a/app/Models/WorkerSessionLease.php +++ b/app/Models/WorkerSessionLease.php @@ -22,6 +22,7 @@ class WorkerSessionLease extends Model 'requirements', 'status', 'lease_owner', + 'lease_owner_incarnation', 'lease_expires_at', 'ttl_expires_at', 'closed_at', diff --git a/app/Support/WorkerSessionRegistry.php b/app/Support/WorkerSessionRegistry.php index 2dce3b529..a8dcc3a7f 100644 --- a/app/Support/WorkerSessionRegistry.php +++ b/app/Support/WorkerSessionRegistry.php @@ -13,6 +13,17 @@ final class WorkerSessionRegistry { private const DEFAULT_LEASE_SECONDS = 120; + /** + * Portable maxima, the same in native (Server #740). A heartbeat renews + * the lease, so a longer lease only delays holder-loss detection; thirty + * days of TTL keeps ttl_expires_at representable in a MySQL TIMESTAMP. + */ + public const MAX_LEASE_SECONDS = 86400; + + public const MAX_TTL_SECONDS = 2592000; + + public const MAX_CONCURRENT_ACTIVITIES = 1000; + private const DEFAULT_TTL_SECONDS = 1800; private const DEFAULT_MAX_CONCURRENT_ACTIVITIES = 1; @@ -63,9 +74,11 @@ public function createOrReacquire( string $namespace, WorkerRegistration $worker, array $options, + ?string $registrationToken = null, ): array { - return DB::transaction(function () use ($namespace, $worker, $options): array { - return $this->acquireLocked($namespace, $worker, $options, null); + return DB::transaction(function () use ($namespace, $worker, $options, $registrationToken): array { + return $this->staleRegistration($namespace, $worker->worker_id, $registrationToken) + ?? $this->acquireLocked($namespace, $worker, $options, null); }); } @@ -87,9 +100,18 @@ public function admitActivity( /** * @return array */ - public function heartbeat(string $namespace, string $workerId, string $sessionId, ?int $leaseSeconds = null): array - { - return DB::transaction(function () use ($namespace, $workerId, $sessionId, $leaseSeconds): array { + public function heartbeat( + string $namespace, + string $workerId, + string $sessionId, + ?int $leaseSeconds = null, + ?string $registrationToken = null, + ): array { + return DB::transaction(function () use ($namespace, $workerId, $sessionId, $leaseSeconds, $registrationToken): array { + if ($stale = $this->staleRegistration($namespace, $workerId, $registrationToken)) { + return $stale; + } + /** @var WorkerSessionLease|null $session */ $session = WorkerSessionLease::query() ->where('namespace', $namespace) @@ -176,9 +198,13 @@ public function heartbeatForAttempt(string $namespace, string $attemptId, string /** * @return array */ - public function close(string $namespace, string $workerId, string $sessionId): array + public function close(string $namespace, string $workerId, string $sessionId, ?string $registrationToken = null): array { - return DB::transaction(function () use ($namespace, $workerId, $sessionId): array { + return DB::transaction(function () use ($namespace, $workerId, $sessionId, $registrationToken): array { + if ($stale = $this->staleRegistration($namespace, $workerId, $registrationToken)) { + return $stale; + } + /** @var WorkerSessionLease|null $session */ $session = WorkerSessionLease::query() ->where('namespace', $namespace) @@ -374,6 +400,7 @@ private function acquireLocked( 'requirements' => $options['requirements'], 'status' => WorkerSessionLease::STATUS_ACTIVE, 'lease_owner' => $worker->worker_id, + 'lease_owner_incarnation' => $this->incarnation($worker), 'lease_expires_at' => $now->copy()->addSeconds($options['lease_seconds']), 'ttl_expires_at' => $now->copy()->addSeconds($options['ttl_seconds']), 'lease_seconds' => $options['lease_seconds'], @@ -404,7 +431,7 @@ private function acquireLocked( } } - $session = $this->renewHeldSession($session, $options, $worker->worker_id); + $session = $this->renewHeldSession($session, $options, $worker); return $this->success($session, 'reused', $activityTaskId); } @@ -441,7 +468,7 @@ private function acquireLocked( return $limit; } - $session = $this->renewHeldSession($session, $options, $worker->worker_id); + $session = $this->renewHeldSession($session, $options, $worker); return $this->success($session, 'reacquired', $activityTaskId); } @@ -452,11 +479,12 @@ private function acquireLocked( private function renewHeldSession( WorkerSessionLease $session, array $options, - string $workerId, + WorkerRegistration $worker, ): WorkerSessionLease { $now = now(); + $workerId = $worker->worker_id; - $session->forceFill(array_filter([ + $session->forceFill([...array_filter([ 'connection' => $options['connection'], 'queue' => $options['queue'], 'requirements' => $options['requirements'], @@ -471,7 +499,10 @@ private function renewHeldSession( 'create_if_missing' => $options['create_if_missing'], 'allow_reacquire_after_failure' => $options['allow_reacquire_after_failure'], 'last_heartbeat_at' => $now, - ], static fn (mixed $value): bool => $value !== null))->save(); + ], static fn (mixed $value): bool => $value !== null), + // Set even when null: the holder is now this registration. + 'lease_owner_incarnation' => $this->incarnation($worker), + ])->save(); return $session; } @@ -506,10 +537,11 @@ private function refreshStatusLocked(WorkerSessionLease $session): WorkerSession return $session; } - if ($session->lease_owner !== null && $this->workerIsStale($session->namespace, $session->lease_owner)) { + $holderLoss = $session->lease_owner === null ? null : $this->holderLoss($session); + if ($holderLoss !== null) { $session->forceFill([ 'status' => WorkerSessionLease::STATUS_ORPHANED, - 'failure_reason' => 'worker_heartbeat_stale', + 'failure_reason' => $holderLoss, ])->save(); } @@ -688,6 +720,64 @@ private function activeActivityCount( return (int) $query->count(); } + /** + * Why the holder no longer holds an active session: its registration is + * gone or silent, or the worker ID re-registered since it acquired the + * session. A replacement process must reacquire and rebuild its + * process-local state, so it never inherits the session as reused. + */ + private function holderLoss(WorkerSessionLease $session): ?string + { + if ($this->workerIsStale($session->namespace, (string) $session->lease_owner)) { + return 'worker_heartbeat_stale'; + } + + $worker = WorkerRegistration::query() + ->where('namespace', $session->namespace) + ->where('worker_id', $session->lease_owner) + ->first(); + + return $worker instanceof WorkerRegistration && $this->incarnation($worker) === $session->lease_owner_incarnation + ? null : 'worker_registration_replaced'; + } + + /** + * The digest a session keeps of its holder's registration, as the + * cooperative claim does, so the deregistration credential itself is + * never stored on the session. + */ + private function incarnation(WorkerRegistration $worker): ?string + { + return is_string($worker->registration_token) ? hash('sha256', $worker->registration_token) : null; + } + + /** + * A registration token the request sends must be the worker's current + * one. Without it, a stale process and its same-ID replacement are + * indistinguishable once the replacement has reacquired. + * + * @return array|null + */ + private function staleRegistration(string $namespace, string $workerId, ?string $registrationToken): ?array + { + if ($registrationToken === null) { + return null; + } + + $current = WorkerRegistration::query() + ->where('namespace', $namespace) + ->where('worker_id', $workerId) + ->value('registration_token'); + + return is_string($current) && hash_equals($current, $registrationToken) + ? null + : $this->failure( + 'stale_worker_registration', + 'The registration token is not the worker\'s current registration. Register again and reacquire the session.', + 409, + ); + } + private function workerIsStale(string $namespace, string $workerId): bool { /** @var WorkerRegistration|null $worker */ diff --git a/app/Support/WorkflowTaskCompletionValidation.php b/app/Support/WorkflowTaskCompletionValidation.php index 591defb2c..15067ca14 100644 --- a/app/Support/WorkflowTaskCompletionValidation.php +++ b/app/Support/WorkflowTaskCompletionValidation.php @@ -71,9 +71,9 @@ public static function rules(): array 'commands.*.worker_session.queue' => ['nullable', 'string', 'max:255'], 'commands.*.worker_session.requirements' => ['nullable', 'array'], 'commands.*.worker_session.requirements.*' => ['string', 'max:255'], - 'commands.*.worker_session.lease_seconds' => ['nullable', 'integer', 'min:1'], - 'commands.*.worker_session.ttl_seconds' => ['nullable', 'integer', 'min:1'], - 'commands.*.worker_session.max_concurrent_activities' => ['nullable', 'integer', 'min:1'], + 'commands.*.worker_session.lease_seconds' => ['nullable', 'integer', 'min:1', 'max:'.WorkerSessionRegistry::MAX_LEASE_SECONDS], + 'commands.*.worker_session.ttl_seconds' => ['nullable', 'integer', 'min:1', 'max:'.WorkerSessionRegistry::MAX_TTL_SECONDS], + 'commands.*.worker_session.max_concurrent_activities' => ['nullable', 'integer', 'min:1', 'max:'.WorkerSessionRegistry::MAX_CONCURRENT_ACTIVITIES], 'commands.*.worker_session.create_if_missing' => ['nullable', 'boolean'], 'commands.*.worker_session.allow_reacquire_after_failure' => ['nullable', 'boolean'], 'commands.*.execution_timeout_seconds' => ['nullable', 'integer', 'min:1'], diff --git a/database/migrations/2026_10_24_000100_add_worker_session_holder_incarnation.php b/database/migrations/2026_10_24_000100_add_worker_session_holder_incarnation.php new file mode 100644 index 000000000..111411bf1 --- /dev/null +++ b/database/migrations/2026_10_24_000100_add_worker_session_holder_incarnation.php @@ -0,0 +1,46 @@ +char('lease_owner_incarnation', 64)->nullable()->after('lease_owner'); + }); + + // Each active session stays with the registration that holds it now, + // so upgrading does not orphan sessions whose workers keep running. + DB::table('workflow_worker_sessions') + ->where('status', 'active') + ->whereNotNull('lease_owner') + ->orderBy('id') + ->chunkById(500, function ($sessions): void { + foreach ($sessions as $session) { + $token = DB::table('workflow_worker_registrations') + ->where('namespace', $session->namespace) + ->where('worker_id', $session->lease_owner) + ->value('registration_token'); + + if (is_string($token)) { + DB::table('workflow_worker_sessions') + ->where('id', $session->id) + ->update(['lease_owner_incarnation' => hash('sha256', $token)]); + } + } + }); + } + + public function down(): void + { + Schema::table('workflow_worker_sessions', function (Blueprint $table) { + $table->dropColumn('lease_owner_incarnation'); + }); + } +}; diff --git a/resources/platform-protocol-specs/worker-sessions-runtime.schema.json b/resources/platform-protocol-specs/worker-sessions-runtime.schema.json index d52225008..8f8a1cf45 100644 --- a/resources/platform-protocol-specs/worker-sessions-runtime.schema.json +++ b/resources/platform-protocol-specs/worker-sessions-runtime.schema.json @@ -223,9 +223,9 @@ "items": { "type": "string", "minLength": 1, "maxLength": 255 }, "uniqueItems": true }, - "lease_seconds": { "type": "integer", "minimum": 1, "default": 120 }, - "ttl_seconds": { "type": "integer", "minimum": 1, "default": 1800 }, - "max_concurrent_activities": { "type": "integer", "minimum": 1, "default": 1 }, + "lease_seconds": { "type": "integer", "minimum": 1, "maximum": 86400, "default": 120 }, + "ttl_seconds": { "type": "integer", "minimum": 1, "maximum": 2592000, "default": 1800 }, + "max_concurrent_activities": { "type": "integer", "minimum": 1, "maximum": 1000, "default": 1 }, "create_if_missing": { "type": "boolean", "default": true }, "allow_reacquire_after_failure": { "type": "boolean", "default": true } } @@ -238,18 +238,25 @@ "required": ["worker_id"], "additionalProperties": true, "properties": { - "worker_id": { "type": "string", "minLength": 1, "maxLength": 255 } + "worker_id": { "type": "string", "minLength": 1, "maxLength": 255 }, + "registration_token": { "$ref": "#/$defs/workerRegistrationToken" } } } ] }, + "workerRegistrationToken": { + "type": "string", + "pattern": "^[a-f0-9]{32}$", + "description": "The token worker registration returned. When sent, it must be the worker's current registration, otherwise the request is refused with stale_worker_registration and changes nothing. A same-ID re-registration orphans the sessions the previous registration held (worker_registration_replaced) until they are reacquired." + }, "workerSessionHeartbeatRequest": { "type": "object", "required": ["worker_id"], "additionalProperties": true, "properties": { "worker_id": { "type": "string", "minLength": 1, "maxLength": 255 }, - "lease_seconds": { "type": "integer", "minimum": 1 } + "lease_seconds": { "type": "integer", "minimum": 1, "maximum": 86400 }, + "registration_token": { "$ref": "#/$defs/workerRegistrationToken" } } }, "workerSessionCloseRequest": { @@ -257,7 +264,8 @@ "required": ["worker_id"], "additionalProperties": true, "properties": { - "worker_id": { "type": "string", "minLength": 1, "maxLength": 255 } + "worker_id": { "type": "string", "minLength": 1, "maxLength": 255 }, + "registration_token": { "$ref": "#/$defs/workerRegistrationToken" } } }, "workerSessionOperationResult": { @@ -289,7 +297,8 @@ "session_owned_by_another_worker", "session_activity_limit_exceeded", "session_closed", - "session_reacquire_disallowed" + "session_reacquire_disallowed", + "stale_worker_registration" ] }, "error": { "type": "string" }, @@ -425,7 +434,7 @@ }, "workerSessionFailureReason": { "type": ["string", "null"], - "enum": [null, "ttl_expired", "lease_expired", "worker_heartbeat_stale"] + "enum": [null, "ttl_expired", "lease_expired", "worker_heartbeat_stale", "worker_registration_replaced"] } } } diff --git a/rust/migrations/mariadb/0004_catalog.json b/rust/migrations/mariadb/0005_catalog.json similarity index 99% rename from rust/migrations/mariadb/0004_catalog.json rename to rust/migrations/mariadb/0005_catalog.json index def952353..57d64e99a 100644 --- a/rust/migrations/mariadb/0004_catalog.json +++ b/rust/migrations/mariadb/0005_catalog.json @@ -27,6 +27,10 @@ "dw_worker_registration_incarnations", "CREATE TABLE `dw_worker_registration_incarnations` (\n `token` char(32) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,\n `namespace` varchar(255) NOT NULL,\n `worker_id` varchar(255) NOT NULL,\n `status` varchar(32) NOT NULL,\n `recovered_workflow_task_count` bigint(20) DEFAULT NULL,\n `finished_at` datetime(6) DEFAULT NULL,\n `created_at` datetime(6) NOT NULL,\n `updated_at` datetime(6) NOT NULL,\n PRIMARY KEY (`token`),\n KEY `dw_worker_incarnation_authority` (`namespace`,`worker_id`,`status`),\n KEY `dw_worker_incarnation_retention` (`status`,`finished_at`),\n CONSTRAINT `dw_worker_incarnation_state` CHECK (`status` = 'active' and `finished_at` is null and `recovered_workflow_task_count` is null or `status` = 'superseded' and `finished_at` is not null and `recovered_workflow_task_count` is null or `status` = 'deregistered' and `finished_at` is not null and `recovered_workflow_task_count` is not null and `recovered_workflow_task_count` >= 0)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci" ], + [ + "dw_worker_session_holders", + "CREATE TABLE `dw_worker_session_holders` (\n `namespace` varchar(128) NOT NULL,\n `session_id` varchar(255) NOT NULL,\n `incarnation` char(64) COLLATE ascii_bin NOT NULL,\n `updated_at` datetime(6) NOT NULL,\n PRIMARY KEY (`namespace`,`session_id`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci" + ], [ "failed_jobs", "CREATE TABLE `failed_jobs` (\n `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT,\n `uuid` varchar(255) NOT NULL,\n `connection` text NOT NULL,\n `queue` text NOT NULL,\n `payload` longtext NOT NULL,\n `exception` longtext NOT NULL,\n `failed_at` timestamp NOT NULL DEFAULT current_timestamp(),\n PRIMARY KEY (`id`),\n UNIQUE KEY `failed_jobs_uuid_unique` (`uuid`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci" diff --git a/rust/migrations/mariadb/0005_worker_session_holders.sql b/rust/migrations/mariadb/0005_worker_session_holders.sql new file mode 100644 index 000000000..ad3089f47 --- /dev/null +++ b/rust/migrations/mariadb/0005_worker_session_holders.sql @@ -0,0 +1,10 @@ +-- Native development bootstrap version 5: the registration incarnation that +-- holds each worker session (Server #740). A same-ID re-registration rotates +-- the incarnation, which orphans the session until it is reacquired. +CREATE TABLE dw_worker_session_holders ( + namespace VARCHAR(128) NOT NULL, + session_id VARCHAR(255) NOT NULL, + incarnation CHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + updated_at DATETIME(6) NOT NULL, + PRIMARY KEY (namespace, session_id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; diff --git a/rust/migrations/mysql/0004_catalog.json b/rust/migrations/mysql/0005_catalog.json similarity index 99% rename from rust/migrations/mysql/0004_catalog.json rename to rust/migrations/mysql/0005_catalog.json index 5a4a6319e..4f67db5a6 100644 --- a/rust/migrations/mysql/0004_catalog.json +++ b/rust/migrations/mysql/0005_catalog.json @@ -31,6 +31,10 @@ "dw_worker_registration_incarnations", "CREATE TABLE `dw_worker_registration_incarnations` (\n `token` char(32) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,\n `namespace` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL,\n `worker_id` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL,\n `status` varchar(32) COLLATE utf8mb4_unicode_ci NOT NULL,\n `recovered_workflow_task_count` bigint DEFAULT NULL,\n `finished_at` datetime(6) DEFAULT NULL,\n `created_at` datetime(6) NOT NULL,\n `updated_at` datetime(6) NOT NULL,\n PRIMARY KEY (`token`),\n KEY `dw_worker_incarnation_authority` (`namespace`,`worker_id`,`status`),\n KEY `dw_worker_incarnation_retention` (`status`,`finished_at`),\n CONSTRAINT `dw_worker_incarnation_state` CHECK ((((`status` = _utf8mb4'active') and (`finished_at` is null) and (`recovered_workflow_task_count` is null)) or ((`status` = _utf8mb4'superseded') and (`finished_at` is not null) and (`recovered_workflow_task_count` is null)) or ((`status` = _utf8mb4'deregistered') and (`finished_at` is not null) and (`recovered_workflow_task_count` is not null) and (`recovered_workflow_task_count` >= 0))))\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci" ], + [ + "dw_worker_session_holders", + "CREATE TABLE `dw_worker_session_holders` (\n `namespace` varchar(128) COLLATE utf8mb4_unicode_ci NOT NULL,\n `session_id` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL,\n `incarnation` char(64) COLLATE ascii_bin NOT NULL,\n `updated_at` datetime(6) NOT NULL,\n PRIMARY KEY (`namespace`,`session_id`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci" + ], [ "failed_jobs", "CREATE TABLE `failed_jobs` (\n `id` bigint unsigned NOT NULL AUTO_INCREMENT,\n `uuid` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL,\n `connection` text COLLATE utf8mb4_unicode_ci NOT NULL,\n `queue` text COLLATE utf8mb4_unicode_ci NOT NULL,\n `payload` longtext COLLATE utf8mb4_unicode_ci NOT NULL,\n `exception` longtext COLLATE utf8mb4_unicode_ci NOT NULL,\n `failed_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,\n PRIMARY KEY (`id`),\n UNIQUE KEY `failed_jobs_uuid_unique` (`uuid`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci" diff --git a/rust/migrations/mysql/0005_worker_session_holders.sql b/rust/migrations/mysql/0005_worker_session_holders.sql new file mode 100644 index 000000000..ad3089f47 --- /dev/null +++ b/rust/migrations/mysql/0005_worker_session_holders.sql @@ -0,0 +1,10 @@ +-- Native development bootstrap version 5: the registration incarnation that +-- holds each worker session (Server #740). A same-ID re-registration rotates +-- the incarnation, which orphans the session until it is reacquired. +CREATE TABLE dw_worker_session_holders ( + namespace VARCHAR(128) NOT NULL, + session_id VARCHAR(255) NOT NULL, + incarnation CHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + updated_at DATETIME(6) NOT NULL, + PRIMARY KEY (namespace, session_id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; diff --git a/rust/migrations/postgres/0004_catalog.json b/rust/migrations/postgres/0005_catalog.json similarity index 99% rename from rust/migrations/postgres/0004_catalog.json rename to rust/migrations/postgres/0005_catalog.json index 3ad9fe814..4dd16a521 100644 --- a/rust/migrations/postgres/0004_catalog.json +++ b/rust/migrations/postgres/0005_catalog.json @@ -896,6 +896,58 @@ "generated": "" } ], + [ + "column", + "dw_worker_session_holders.1", + { + "name": "namespace", + "type": "text", + "default": null, + "identity": "", + "not_null": true, + "collation": "default", + "generated": "" + } + ], + [ + "column", + "dw_worker_session_holders.2", + { + "name": "session_id", + "type": "text", + "default": null, + "identity": "", + "not_null": true, + "collation": "default", + "generated": "" + } + ], + [ + "column", + "dw_worker_session_holders.3", + { + "name": "incarnation", + "type": "text", + "default": null, + "identity": "", + "not_null": true, + "collation": "default", + "generated": "" + } + ], + [ + "column", + "dw_worker_session_holders.4", + { + "name": "updated_at", + "type": "timestamp(6) without time zone", + "default": null, + "identity": "", + "not_null": true, + "collation": null, + "generated": "" + } + ], [ "column", "failed_jobs.1", @@ -11975,6 +12027,16 @@ "definition": "PRIMARY KEY (token)" } ], + [ + "constraint", + "dw_worker_session_holders.dw_worker_session_holders_pkey", + { + "deferred": false, + "validated": true, + "deferrable": false, + "definition": "PRIMARY KEY (namespace, session_id)" + } + ], [ "constraint", "failed_jobs.failed_jobs_pkey", @@ -13064,6 +13126,17 @@ "definition": "CREATE UNIQUE INDEX dw_worker_registration_incarnations_pkey ON public.dw_worker_registration_incarnations USING btree (token)" } ], + [ + "index", + "dw_worker_session_holders_pkey", + { + "ready": true, + "valid": true, + "unique": true, + "primary": true, + "definition": "CREATE UNIQUE INDEX dw_worker_session_holders_pkey ON public.dw_worker_session_holders USING btree (namespace, session_id)" + } + ], [ "index", "dw_workflow_tasks_poll", @@ -17265,6 +17338,21 @@ "force_row_security": false } ], + [ + "relation", + "dw_worker_session_holders", + { + "kind": "r", + "view": null, + "options": null, + "partition": false, + "persistence": "p", + "row_security": false, + "access_method": "heap", + "partition_key": null, + "force_row_security": false + } + ], [ "relation", "failed_jobs", diff --git a/rust/migrations/postgres/0005_worker_session_holders.sql b/rust/migrations/postgres/0005_worker_session_holders.sql new file mode 100644 index 000000000..8609300a6 --- /dev/null +++ b/rust/migrations/postgres/0005_worker_session_holders.sql @@ -0,0 +1,11 @@ +-- Native development bootstrap version 5: the registration incarnation that +-- holds each worker session (Server #740). A same-ID re-registration rotates +-- the incarnation, which orphans the session until it is reacquired. +CREATE TABLE dw_worker_session_holders ( + namespace text NOT NULL, + session_id text NOT NULL, + incarnation text NOT NULL, + updated_at timestamp(6) without time zone NOT NULL, + PRIMARY KEY (namespace, session_id) +); +UPDATE dw_server_schema SET version=5 WHERE engine='rust-development' AND version=4; diff --git a/rust/migrations/sqlite/0005_worker_session_holders.sql b/rust/migrations/sqlite/0005_worker_session_holders.sql new file mode 100644 index 000000000..d7e099452 --- /dev/null +++ b/rust/migrations/sqlite/0005_worker_session_holders.sql @@ -0,0 +1,11 @@ +-- Native development bootstrap version 5: the registration incarnation that +-- holds each worker session (Server #740). A same-ID re-registration rotates +-- the incarnation, which orphans the session until it is reacquired. +CREATE TABLE dw_worker_session_holders ( + namespace TEXT NOT NULL, + session_id TEXT NOT NULL, + incarnation TEXT NOT NULL, + updated_at TEXT NOT NULL, + PRIMARY KEY (namespace, session_id) +); +UPDATE dw_server_schema SET version=5 WHERE engine='rust-development' AND version=4; diff --git a/rust/src/runtime/completion_rules.rs b/rust/src/runtime/completion_rules.rs index dc887b57b..15f7de134 100644 --- a/rust/src/runtime/completion_rules.rs +++ b/rust/src/runtime/completion_rules.rs @@ -119,9 +119,9 @@ const COMMAND: &[(&str, &[Check])] = &[ ("worker_session.queue", &[NUL, STR, max(255)]), ("worker_session.requirements", &[NUL, ARR]), ("worker_session.requirements.*", &[STR, max(255)]), - ("worker_session.lease_seconds", &[NUL, INT, min(1)]), - ("worker_session.ttl_seconds", &[NUL, INT, min(1)]), - ("worker_session.max_concurrent_activities", &[NUL, INT, min(1)]), + ("worker_session.lease_seconds", &[NUL, INT, min(1), max(super::worker_sessions::MAX_LEASE_SECONDS)]), + ("worker_session.ttl_seconds", &[NUL, INT, min(1), max(super::worker_sessions::MAX_TTL_SECONDS)]), + ("worker_session.max_concurrent_activities", &[NUL, INT, min(1), max(super::worker_sessions::MAX_CONCURRENT_ACTIVITIES)]), ("worker_session.create_if_missing", &[NUL, BOOL]), ("worker_session.allow_reacquire_after_failure", &[NUL, BOOL]), ("execution_timeout_seconds", &[NUL, INT, min(1)]), diff --git a/rust/src/runtime/mysql.rs b/rust/src/runtime/mysql.rs index 60b63da41..025f571e0 100644 --- a/rust/src/runtime/mysql.rs +++ b/rust/src/runtime/mysql.rs @@ -9,7 +9,7 @@ use sqlx::{ }; use std::{collections::BTreeMap, sync::OnceLock, time::Duration}; -pub const VERSION: i64 = 4; +pub const VERSION: i64 = 5; type Catalog = Vec<(String, String)>; pub const MARKER_SQL: &str = "CREATE TABLE dw_server_schema (engine VARCHAR(64) NOT NULL PRIMARY KEY,version BIGINT NOT NULL,bootstrap_checksum VARBINARY(48) NOT NULL) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin"; @@ -44,7 +44,9 @@ impl Flavor { concat!( include_str!("../../migrations/mysql/0003_full_schema.sql"), "\n", - include_str!("../../migrations/mysql/0004_worker_incarnations.sql") + include_str!("../../migrations/mysql/0004_worker_incarnations.sql"), + "\n", + include_str!("../../migrations/mysql/0005_worker_session_holders.sql") ), ), Self::MariaDb => ( @@ -52,14 +54,16 @@ impl Flavor { concat!( include_str!("../../migrations/mariadb/0003_full_schema.sql"), "\n", - include_str!("../../migrations/mariadb/0004_worker_incarnations.sql") + include_str!("../../migrations/mariadb/0004_worker_incarnations.sql"), + "\n", + include_str!("../../migrations/mariadb/0005_worker_session_holders.sql") ), ), }; slot.get_or_init(|| { let mut migrations = Migrator::with_migrations(vec![Migration::new( VERSION, - "full frozen PHP schema and native worker incarnations".into(), + "full frozen PHP schema, native worker incarnations and session holders".into(), MigrationType::Simple, sql.into_sql_str(), false, @@ -72,8 +76,8 @@ impl Flavor { } fn expected(self) -> Result { Ok(serde_json::from_str(&canonical(match self { - Self::MySql => include_str!("../../migrations/mysql/0004_catalog.json"), - Self::MariaDb => include_str!("../../migrations/mariadb/0004_catalog.json"), + Self::MySql => include_str!("../../migrations/mysql/0005_catalog.json"), + Self::MariaDb => include_str!("../../migrations/mariadb/0005_catalog.json"), }))?) } /// The single native migration, which a takeover records as applied. @@ -87,11 +91,19 @@ impl Flavor { let (frozen, incarnations) = match self { Self::MySql => ( include_str!("../../migrations/mysql/0003_full_schema.sql"), - include_str!("../../migrations/mysql/0004_worker_incarnations.sql"), + concat!( + include_str!("../../migrations/mysql/0004_worker_incarnations.sql"), + "\n", + include_str!("../../migrations/mysql/0005_worker_session_holders.sql") + ), ), Self::MariaDb => ( include_str!("../../migrations/mariadb/0003_full_schema.sql"), - include_str!("../../migrations/mariadb/0004_worker_incarnations.sql"), + concat!( + include_str!("../../migrations/mariadb/0004_worker_incarnations.sql"), + "\n", + include_str!("../../migrations/mariadb/0005_worker_session_holders.sql") + ), ), }; Ok(super::upgrade::native_relations( @@ -102,6 +114,10 @@ impl Flavor { .replace( "CREATE TABLE dw_worker_registration_incarnations", "CREATE TABLE IF NOT EXISTS dw_worker_registration_incarnations", + ) + .replace( + "CREATE TABLE dw_worker_session_holders", + "CREATE TABLE IF NOT EXISTS dw_worker_session_holders", )) } pub(super) fn checksum(self) -> &'static [u8] { diff --git a/rust/src/runtime/namespace_cleanup.rs b/rust/src/runtime/namespace_cleanup.rs index 3fbd980f1..5dd4eb923 100644 --- a/rust/src/runtime/namespace_cleanup.rs +++ b/rust/src/runtime/namespace_cleanup.rs @@ -148,6 +148,7 @@ const CLEANUP: &[(&str, &str, usize)] = &[ const NATIVE_CLEANUP: &[&str] = &[ "DELETE FROM dw_poll_receipts WHERE namespace=$1", "DELETE FROM dw_worker_registration_incarnations WHERE namespace=$1", + "DELETE FROM dw_worker_session_holders WHERE namespace=$1", ]; /// Registered external payloads and payload columns holding a reference. diff --git a/rust/src/runtime/postgres.rs b/rust/src/runtime/postgres.rs index 0744d1ac0..a89775873 100644 --- a/rust/src/runtime/postgres.rs +++ b/rust/src/runtime/postgres.rs @@ -13,7 +13,7 @@ use sqlx::{ use super::{Result, refuse}; -pub const VERSION: i64 = 4; +pub const VERSION: i64 = 5; pub const CATALOG_QUERY: &str = include_str!("postgres-catalog.sql"); type Catalog = Vec<(String, String, Json)>; @@ -22,12 +22,14 @@ fn migrator() -> &'static Migrator { MIGRATOR.get_or_init(|| { let mut migrations = Migrator::with_migrations(vec![Migration::new( VERSION, - "full frozen PHP PostgreSQL schema and native worker incarnations".into(), + "full frozen PHP PostgreSQL schema, native worker incarnations and session holders".into(), MigrationType::Simple, concat!( include_str!("../../migrations/postgres/0003_full_schema.sql"), "\n", - include_str!("../../migrations/postgres/0004_worker_incarnations.sql") + include_str!("../../migrations/postgres/0004_worker_incarnations.sql"), + "\n", + include_str!("../../migrations/postgres/0005_worker_session_holders.sql") ) .into_sql_str(), false, @@ -52,7 +54,11 @@ pub(super) fn native_relations_sql() -> Result { super::upgrade::native_relations( include_str!("../../migrations/postgres/0003_full_schema.sql"), "CREATE TABLE public.dw_server_schema ", - include_str!("../../migrations/postgres/0004_worker_incarnations.sql"), + concat!( + include_str!("../../migrations/postgres/0004_worker_incarnations.sql"), + "\n", + include_str!("../../migrations/postgres/0005_worker_session_holders.sql") + ), ) } @@ -85,7 +91,7 @@ pub(super) async fn php_source_catalog(connection: &mut PgConnection) -> Result< )); } let native: Catalog = - serde_json::from_str(include_str!("../../migrations/postgres/0004_catalog.json"))?; + serde_json::from_str(include_str!("../../migrations/postgres/0005_catalog.json"))?; let expected: Catalog = native .into_iter() .filter(|(kind, name, _)| { @@ -180,7 +186,7 @@ pub(super) async fn verify(connection: &mut PgConnection) -> Result<()> { // from SQL text. It retains types, precision, defaults, indexes, foreign // keys and sequence ownership while excluding live rows/sequence values. let expected: Catalog = - serde_json::from_str(include_str!("../../migrations/postgres/0004_catalog.json"))?; + serde_json::from_str(include_str!("../../migrations/postgres/0005_catalog.json"))?; let actual = catalog(connection).await?; if actual != expected { return Err(refuse( @@ -345,7 +351,7 @@ mod tests { #[test] fn compiled_catalog_preserves_long_object_identities_and_int64_limits() { let rows: Catalog = - serde_json::from_str(include_str!("../../migrations/postgres/0004_catalog.json")) + serde_json::from_str(include_str!("../../migrations/postgres/0005_catalog.json")) .unwrap(); let identities: std::collections::BTreeSet<_> = rows.iter().map(|(kind, name, _)| (kind, name)).collect(); diff --git a/rust/src/runtime/schema.rs b/rust/src/runtime/schema.rs index 9b602f319..55db13331 100644 --- a/rust/src/runtime/schema.rs +++ b/rust/src/runtime/schema.rs @@ -11,19 +11,21 @@ use tokio::sync::OnceCell; use super::{Result, refuse}; -pub(super) const VERSION: i64 = 4; +pub(super) const VERSION: i64 = 5; fn migrator() -> &'static Migrator { static MIGRATOR: OnceLock = OnceLock::new(); MIGRATOR.get_or_init(|| { Migrator::with_migrations(vec![Migration::new( VERSION, - "full frozen PHP schema and native worker incarnations".into(), + "full frozen PHP schema, native worker incarnations and session holders".into(), MigrationType::Simple, concat!( include_str!("../../migrations/sqlite/0003_full_schema.sql"), "\n", - include_str!("../../migrations/sqlite/0004_worker_incarnations.sql") + include_str!("../../migrations/sqlite/0004_worker_incarnations.sql"), + "\n", + include_str!("../../migrations/sqlite/0005_worker_session_holders.sql") ) .into_sql_str(), false, @@ -38,7 +40,11 @@ pub(super) fn native_relations_sql() -> Result { super::upgrade::native_relations( include_str!("../../migrations/sqlite/0003_full_schema.sql"), "CREATE TABLE dw_server_schema ", - include_str!("../../migrations/sqlite/0004_worker_incarnations.sql"), + concat!( + include_str!("../../migrations/sqlite/0004_worker_incarnations.sql"), + "\n", + include_str!("../../migrations/sqlite/0005_worker_session_holders.sql") + ), ) } diff --git a/rust/src/runtime/upgrade.rs b/rust/src/runtime/upgrade.rs index 0995ed343..612e0977c 100644 --- a/rust/src/runtime/upgrade.rs +++ b/rust/src/runtime/upgrade.rs @@ -105,6 +105,7 @@ pub(super) fn native_relation(name: &str) -> bool { | "dw_poll_receipts" | "dw_query_cache" | "dw_worker_registration_incarnations" + | "dw_worker_session_holders" ) } @@ -120,6 +121,7 @@ pub(super) fn native_object(name: &str) -> bool { | "dw_worker_registration_incarnations_pkey" | "dw_worker_incarnation_authority" | "dw_worker_incarnation_retention" + | "dw_worker_session_holders_pkey" ) } diff --git a/rust/src/runtime/upgrade/takeover.rs b/rust/src/runtime/upgrade/takeover.rs index 4412cfb71..6997340f7 100644 --- a/rust/src/runtime/upgrade/takeover.rs +++ b/rust/src/runtime/upgrade/takeover.rs @@ -469,6 +469,21 @@ where .execute(&mut *tx) .await?; } + // Each active session stays with the registration that holds it now, so a + // takeover does not orphan sessions whose workers keep running. A resumed + // takeover skips sessions it already seeded. + let held = query::("SELECT s.namespace,s.session_id,i.token FROM workflow_worker_sessions s JOIN dw_worker_registration_incarnations i ON i.namespace=s.namespace AND i.worker_id=s.lease_owner AND i.status='active' WHERE s.status='active' AND NOT EXISTS (SELECT 1 FROM dw_worker_session_holders h WHERE h.namespace=s.namespace AND h.session_id=s.session_id) ORDER BY s.namespace,s.session_id") + .fetch_all(&mut *tx) + .await?; + for row in &held { + query::("INSERT INTO dw_worker_session_holders(namespace,session_id,incarnation,updated_at) VALUES ($1,$2,$3,$4)") + .bind(DB::string(row, "namespace")?) + .bind(DB::string(row, "session_id")?) + .bind(super::super::worker_sessions::incarnation_digest(&DB::string(row, "token")?)) + .bind(DB::bind_time(now)) + .execute(&mut *tx) + .await?; + } Ok((copied, missing.len())) } diff --git a/rust/src/runtime/upgrade_tests.rs b/rust/src/runtime/upgrade_tests.rs index c55082a5a..3c69d31df 100644 --- a/rust/src/runtime/upgrade_tests.rs +++ b/rust/src/runtime/upgrade_tests.rs @@ -25,12 +25,15 @@ async fn php_layout() -> (tempfile::TempDir, String) { sqlx::raw_sql(concat!( include_str!("../../migrations/sqlite/0003_full_schema.sql"), "\n", - include_str!("../../migrations/sqlite/0004_worker_incarnations.sql") + include_str!("../../migrations/sqlite/0004_worker_incarnations.sql"), + "\n", + include_str!("../../migrations/sqlite/0005_worker_session_holders.sql") )) .execute(&mut connection) .await .unwrap(); for table in [ + "dw_worker_session_holders", "dw_worker_registration_incarnations", "dw_task_completions", "dw_poll_receipts", diff --git a/rust/src/runtime/worker_sessions.rs b/rust/src/runtime/worker_sessions.rs index 06c46b47c..092cf1c64 100644 --- a/rust/src/runtime/worker_sessions.rs +++ b/rust/src/runtime/worker_sessions.rs @@ -15,8 +15,17 @@ use super::{Result, RuntimeError, store::now}; use axum::http::StatusCode; use chrono::{DateTime, Utc}; use serde_json::{Map, Value, json}; +use std::collections::HashSet; +use std::hash::Hash; const DEFAULT_LEASE_SECONDS: i64 = 120; +/// Portable session maxima, the same in PHP and native (Server #740). A lease +/// is renewed by heartbeats, so a longer one only delays holder-loss +/// detection. Thirty days of TTL keeps `ttl_expires_at` representable in a +/// MySQL `TIMESTAMP` for years, and every value fits PostgreSQL's `integer`. +pub(super) const MAX_LEASE_SECONDS: i64 = 86_400; +pub(super) const MAX_TTL_SECONDS: i64 = 2_592_000; +pub(super) const MAX_CONCURRENT_ACTIVITIES: i64 = 1_000; const DEFAULT_TTL_SECONDS: i64 = 1800; const DEFAULT_MAX_CONCURRENT_ACTIVITIES: i64 = 1; const MINIMUM_PROTOCOL: (u64, u64) = (1, 8); @@ -26,12 +35,26 @@ const SESSIONS_TO_REFRESH: &str = "SELECT session_id FROM workflow_worker_sessio const SESSIONS_LISTED: &str = "SELECT id,session_id,connection,queue,requirements,status,lease_owner,lease_expires_at,ttl_expires_at,closed_at,failure_reason,lease_seconds,ttl_seconds,max_concurrent_activities,CASE WHEN create_if_missing THEN 1 ELSE 0 END AS create_if_missing_flag,CASE WHEN allow_reacquire_after_failure THEN 1 ELSE 0 END AS allow_reacquire_flag,last_heartbeat_at,created_at,updated_at FROM workflow_worker_sessions WHERE namespace=$1 ORDER BY updated_at DESC,session_id ASC"; const HELD_SESSIONS: &str = "SELECT COUNT(*) FROM workflow_worker_sessions WHERE namespace=$1 AND lease_owner=$2 AND status='active' AND (lease_expires_at IS NULL OR lease_expires_at>$3) AND session_id<>$4"; const HOLDER: &str = "SELECT task_queue,status,last_heartbeat_at FROM workflow_worker_registrations WHERE namespace=$1 AND worker_id=$2"; +/// The worker's current registration token, as the poll fence reads it. +const CURRENT_TOKEN: &str = "SELECT token FROM dw_worker_registration_incarnations WHERE namespace=$1 AND worker_id=$2 AND status='active' ORDER BY created_at DESC LIMIT 1"; +/// The registration incarnation that acquired or last renewed a session. +const SESSION_HOLDER: &str = "SELECT incarnation FROM dw_worker_session_holders WHERE namespace=$1 AND session_id=$2"; /// A session time: PHP keeps whole seconds. fn second(at: DateTime) -> DateTime { DateTime::from_timestamp(at.timestamp(), 0).expect("whole-second timestamp") } +/// `at` plus a session's seconds. Validation bounds request values, so this +/// fails only for a stored value no current server accepts; that answers an +/// error instead of panicking inside a durable transition. +fn after(at: DateTime, seconds: i64) -> Result> { + chrono::Duration::try_seconds(seconds) + .and_then(|span| at.checked_add_signed(span)) + .ok_or_else(|| RuntimeError::Protocol { status: StatusCode::UNPROCESSABLE_ENTITY, + response: json!({"error":"Worker session duration is out of range.","reason":"invalid_worker_session"}) }) +} + fn wire(at: Option>) -> Value { at.map_or(Value::Null, |at| json!(super::cooperative::timestamp(at))) } @@ -99,15 +122,16 @@ fn validate(body: &Value, create: bool) -> Result { ("requirements", &[Nullable, Array][..]), ]); } - rules.push(("lease_seconds", &[Nullable, Integer, Min(1)])); + rules.push(("lease_seconds", &[Nullable, Integer, Min(1), Max(MAX_LEASE_SECONDS)])); if create { rules.extend([ - ("ttl_seconds", &[Nullable, Integer, Min(1)][..]), - ("max_concurrent_activities", &[Nullable, Integer, Min(1)][..]), + ("ttl_seconds", &[Nullable, Integer, Min(1), Max(MAX_TTL_SECONDS)][..]), + ("max_concurrent_activities", &[Nullable, Integer, Min(1), Max(MAX_CONCURRENT_ACTIVITIES)][..]), ("create_if_missing", &[Nullable, Boolean][..]), ("allow_reacquire_after_failure", &[Nullable, Boolean][..]), ]); } + rules.push(("registration_token", REGISTRATION_TOKEN_RULES)); let mut errors = rule_errors(&body, &rules, |_, _| None); // `requirements.*`: each a string of at most 255 characters, reported // after `requirements` itself. @@ -164,10 +188,52 @@ fn bool_value(value: &Value, default: bool) -> bool { } } +/// The token registration returns (`bin2hex(random_bytes(16))`). +fn registration_token_format(text: &str) -> bool { + text.len() == 32 && text.bytes().all(|byte| matches!(byte, b'0'..=b'9' | b'a'..=b'f')) +} + +/// An optional `registration_token` on create, heartbeat and close. +const REGISTRATION_TOKEN_RULES: &[Rule] = &[Rule::Nullable, Rule::String, Rule::Pattern(registration_token_format)]; + +/// The digest a session keeps of its holder's registration, as PHP keeps +/// `hash('sha256', token)`, so the deregistration credential itself is never +/// stored with the session. +pub(super) fn incarnation_digest(token: &str) -> String { + use sha2::{Digest, Sha256}; + Sha256::digest(token.as_bytes()).iter().map(|byte| format!("{byte:02x}")).collect() +} + +/// Equal tokens, compared without an early exit (`hash_equals`). +fn same_token(left: &str, right: &str) -> bool { + left.len() == right.len() && left.bytes().zip(right.bytes()).fold(0, |differs, (a, b)| differs | (a ^ b)) == 0 +} + fn string_value(value: &Value) -> Option { value.as_str().map(str::trim).filter(|text| !text.is_empty()).map(str::to_owned) } +/// Each item once, in order of first occurrence (`array_unique`). The seen +/// index is a hash set, so a request-sized list stays linear even while a +/// durable transition holds the server-wide lock. +fn first_occurrences(items: impl IntoIterator) -> Vec { + let mut seen = HashSet::new(); + items.into_iter().filter(|item| seen.insert(item.clone())).collect() +} + +/// The required items missing from `available`, in required order, looked up +/// in a set built once (`array_flip`), not by scanning `available` per item. +fn missing_from<'a, T: Hash + Eq>(required: &'a [T], available: &[T]) -> Vec<&'a T> { + let available: HashSet<&T> = available.iter().collect(); + required.iter().filter(|item| !available.contains(item)).collect() +} + +/// A `requirements` list as `normalizeOptions` and the stored snapshot read +/// it: trimmed, non-empty strings, each once. +fn requirement_list(value: &Value) -> Vec { + first_occurrences(value.as_array().into_iter().flatten().filter_map(string_value)) +} + /// `normalizeOptions`. struct Options { session_id: String, @@ -183,14 +249,7 @@ struct Options { impl Options { fn from(body: &Value, fallback_queue: Option) -> Option { - let mut requirements: Vec = Vec::new(); - for item in body["requirements"].as_array().into_iter().flatten() { - if let Some(text) = string_value(item) - && !requirements.contains(&text) - { - requirements.push(text); - } - } + let requirements = requirement_list(&body["requirements"]); Some(Self { session_id: string_value(&body["session_id"])?, connection: string_value(&body["connection"]), @@ -229,14 +288,7 @@ impl CommandSession { /// The stored snapshot, from an activity's options. pub(super) fn from_snapshot(value: &Value) -> Option { - let mut requirements: Vec = Vec::new(); - for item in value["requirements"].as_array().into_iter().flatten() { - if let Some(text) = string_value(item) - && !requirements.contains(&text) - { - requirements.push(text); - } - } + let requirements = requirement_list(&value["requirements"]); Some(Self { session_id: string_value(&value["session_id"])?, connection: string_value(&value["connection"]), @@ -271,7 +323,7 @@ impl CommandSession { pub(super) fn claimable_by(&self, protocol: &str, worker_queue: &str, activity_queue: &str, capabilities: &[String]) -> bool { sessions_gate(protocol).is_ok() && self.queue.as_deref().unwrap_or(activity_queue) == worker_queue - && self.requirements.iter().all(|requirement| capabilities.contains(requirement)) + && missing_from(&self.requirements, capabilities).is_empty() } /// The lease an activity heartbeat renews the session by. @@ -439,7 +491,10 @@ store_methods! { .collect(); let limit = DB::optional_number(&worker, "max_concurrent_worker_sessions")?.unwrap_or(10); let mut tx = self.begin().await?; - let answer = self.acquire(&mut tx, &worker_id, &task_queue, &capabilities, limit, &options, None).await?; + let answer = match self.stale_registration(&mut tx, &worker_id, body["registration_token"].as_str()).await? { + Some(stale) => stale, + None => self.acquire(&mut tx, &worker_id, &task_queue, &capabilities, limit, &options, None).await?, + }; tx.commit().await?; Ok(answer) } @@ -460,7 +515,7 @@ store_methods! { StatusCode::CONFLICT, Map::new())); } // Requirements are checked before the session is looked up. - let missing: Vec<&String> = options.requirements.iter().filter(|requirement| !capabilities.contains(requirement)).collect(); + let missing = missing_from(&options.requirements, capabilities); if !missing.is_empty() { return Ok(failure("session_requirements_not_met", "Worker does not advertise every worker-session requirement.", StatusCode::CONFLICT, Map::from_iter([("missing_requirements".to_owned(), json!(missing))]))); @@ -477,12 +532,13 @@ store_methods! { Self::query("INSERT INTO workflow_worker_sessions(namespace,session_id,connection,queue,requirements,status,lease_owner,lease_expires_at,ttl_expires_at,lease_seconds,ttl_seconds,max_concurrent_activities,create_if_missing,allow_reacquire_after_failure,last_heartbeat_at,created_at,updated_at) VALUES ($1,$2,$3,$4,$5,'active',$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16)") .bind(self.namespace.as_ref()).bind(&options.session_id).bind(options.connection.as_deref()).bind(options.queue.as_deref()) .bind(DB::document(&json!(options.requirements))).bind(worker_id) - .bind(DB::bind_time(at + chrono::Duration::seconds(options.lease_seconds))) - .bind(DB::bind_time(at + chrono::Duration::seconds(options.ttl_seconds))) + .bind(DB::bind_time(after(at, options.lease_seconds)?)) + .bind(DB::bind_time(after(at, options.ttl_seconds)?)) .bind(options.lease_seconds).bind(options.ttl_seconds).bind(options.max_concurrent_activities) .bind(DB::optional_flag(Some(options.create_if_missing))).bind(DB::optional_flag(Some(options.allow_reacquire_after_failure))) .bind(DB::bind_time(at)).bind(DB::bind_time(at)).bind(DB::bind_time(at)) .execute(&mut **tx).await?; + self.record_holder(tx, &options.session_id, worker_id).await?; let created = self.locked_session(tx, &options.session_id).await?.expect("the inserted session"); return Ok(success("created", self.snapshot(tx, &created).await?, StatusCode::CREATED)); }; @@ -547,7 +603,7 @@ store_methods! { session.requirements.clone_from(&options.requirements); session.status = "active".into(); session.lease_owner = Some(worker_id.to_owned()); - session.lease_expires_at = Some(at + chrono::Duration::seconds(options.lease_seconds)); + session.lease_expires_at = Some(after(at, options.lease_seconds)?); session.closed_at = None; session.failure_reason = None; session.lease_seconds = options.lease_seconds; @@ -556,7 +612,9 @@ store_methods! { session.create_if_missing = options.create_if_missing; session.allow_reacquire_after_failure = options.allow_reacquire_after_failure; session.last_heartbeat_at = Some(at); - self.save(tx, session).await + self.save(tx, session).await?; + // The renewing registration now holds the session. + self.record_holder(tx, &session.session_id, worker_id).await } /// `activeActivityCount`: the owner's running, unexpired attempts of @@ -676,7 +734,7 @@ store_methods! { json!({"admitted":false,"reason":"session_owner_mismatch","error":"Worker session lease is owned by another worker."})))); } let at = second(now()); - row.lease_expires_at = Some(at + chrono::Duration::seconds(session.heartbeat_lease().max(1))); + row.lease_expires_at = Some(after(at, session.heartbeat_lease().max(1))?); row.last_heartbeat_at = Some(at); row.failure_reason = None; self.save(tx, &row).await?; @@ -722,6 +780,10 @@ store_methods! { let body = validate(&body, false)?; let worker_id = body["worker_id"].as_str().unwrap_or_default(); let mut tx = self.begin().await?; + if let Some(stale) = self.stale_registration(&mut tx, worker_id, body["registration_token"].as_str()).await? { + tx.commit().await?; + return Ok(stale); + } let Some(mut session) = self.locked_session(&mut tx, session_id).await? else { tx.commit().await?; return Ok(failure("session_not_found", "Worker session was not found.", StatusCode::NOT_FOUND, Map::new())); @@ -734,7 +796,7 @@ store_methods! { } else { let at = second(now()); let lease = super::laravel_validation::integer(&body["lease_seconds"]).unwrap_or(session.lease_seconds).max(1); - session.lease_expires_at = Some(at + chrono::Duration::seconds(lease)); + session.lease_expires_at = Some(after(at, lease)?); session.last_heartbeat_at = Some(at); session.failure_reason = None; self.save(&mut tx, &session).await?; @@ -750,6 +812,10 @@ store_methods! { let body = validate(&body, false)?; let worker_id = body["worker_id"].as_str().unwrap_or_default(); let mut tx = self.begin().await?; + if let Some(stale) = self.stale_registration(&mut tx, worker_id, body["registration_token"].as_str()).await? { + tx.commit().await?; + return Ok(stale); + } let Some(mut session) = self.locked_session(&mut tx, session_id).await? else { tx.commit().await?; return Ok(failure("session_not_found", "Worker session was not found.", StatusCode::NOT_FOUND, Map::new())); @@ -836,10 +902,8 @@ store_methods! { None } else if session.lease_expires_at.is_some_and(|lease| lease <= at) { Some(("expired", "lease_expired")) - } else if self.holder_stale(tx, session.lease_owner.as_deref()).await? { - Some(("orphaned", "worker_heartbeat_stale")) } else { - None + self.holder_loss(tx, &session).await?.map(|reason| ("orphaned", reason)) }; if let Some((status, reason)) = change { session.status = status.into(); @@ -849,6 +913,59 @@ store_methods! { Ok(Some(session)) } + /// Why an active session's holder no longer holds it: its registration + /// is gone or silent, or the worker ID registered again since it acquired + /// or renewed the session. A replacement process must reacquire it and + /// rebuild its process-local state, so it never inherits it as reused. + async fn holder_loss(&self, tx: &mut Transaction<'_, DB>, session: &Session) -> Result> { + if self.holder_stale(tx, session.lease_owner.as_deref()).await? { + return Ok(Some("worker_heartbeat_stale")); + } + let current = self.current_incarnation(tx, session.lease_owner.as_deref().unwrap_or_default()).await?; + let holder = Self::query(SESSION_HOLDER).bind(self.namespace.as_ref()).bind(&session.session_id) + .fetch_optional(&mut **tx).await? + .map(|row| DB::string(&row, "incarnation")) + .transpose()?; + Ok((current != holder).then_some("worker_registration_replaced")) + } + + async fn current_token(&self, tx: &mut Transaction<'_, DB>, worker_id: &str) -> Result> { + Self::query(CURRENT_TOKEN).bind(self.namespace.as_ref()).bind(worker_id).fetch_optional(&mut **tx).await? + .map(|row| DB::string(&row, "token")) + .transpose() + } + + async fn current_incarnation(&self, tx: &mut Transaction<'_, DB>, worker_id: &str) -> Result> { + Ok(self.current_token(tx, worker_id).await?.as_deref().map(incarnation_digest)) + } + + /// Records the worker's current registration as the session's holder. A + /// registration without a token, such as one kept from a PHP takeover, + /// records none. + async fn record_holder(&self, tx: &mut Transaction<'_, DB>, session_id: &str, worker_id: &str) -> Result<()> { + Self::query("DELETE FROM dw_worker_session_holders WHERE namespace=$1 AND session_id=$2") + .bind(self.namespace.as_ref()).bind(session_id).execute(&mut **tx).await?; + if let Some(incarnation) = self.current_incarnation(tx, worker_id).await? { + Self::query("INSERT INTO dw_worker_session_holders(namespace,session_id,incarnation,updated_at) VALUES ($1,$2,$3,$4)") + .bind(self.namespace.as_ref()).bind(session_id).bind(incarnation).bind(DB::bind_time(now())) + .execute(&mut **tx).await?; + } + Ok(()) + } + + /// A registration token the request sent must be the worker's current + /// one. Without it, a stale process and its same-ID replacement are + /// indistinguishable once the replacement has reacquired. + async fn stale_registration(&self, tx: &mut Transaction<'_, DB>, worker_id: &str, token: Option<&str>) -> Result> { + let Some(token) = token else { + return Ok(None); + }; + let current = self.current_token(tx, worker_id).await?; + Ok((!current.is_some_and(|current| same_token(¤t, token))).then(|| failure("stale_worker_registration", + "The registration token is not the worker's current registration. Register again and reacquire the session.", + StatusCode::CONFLICT, Map::new()))) + } + /// `holderIsStale`: no registration, no heartbeat, or a heartbeat older /// than the stale window. async fn holder_stale(&self, tx: &mut Transaction<'_, DB>, owner: Option<&str>) -> Result { @@ -954,4 +1071,68 @@ mod tests { assert_eq!(options.requirements, ["gpu", "tpu"]); assert_eq!((options.queue.as_deref(), options.lease_seconds, options.ttl_seconds), (Some("q"), 120, 1800)); } + + /// A string whose equality checks are counted, so the tests below fail + /// on a quadratic scan rather than on a timing threshold. + #[derive(Clone, Hash)] + struct Counted(String); + + thread_local! { + static COMPARISONS: std::cell::Cell = const { std::cell::Cell::new(0) }; + } + + impl Counted { + fn eq_count() -> usize { + COMPARISONS.with(std::cell::Cell::get) + } + } + + impl PartialEq for Counted { + fn eq(&self, other: &Self) -> bool { + COMPARISONS.with(|count| count.set(count.get() + 1)); + self.0 == other.0 + } + } + + impl Eq for Counted {} + + /// Distinct near-limit strings with a shared 248-character prefix, as a + /// 2 MiB request can carry about 8,000 of them. + fn near_limit(count: usize) -> Vec { + let prefix = "r".repeat(248); + (0..count).map(|index| Counted(format!("{prefix}{index:07}"))).collect() + } + + #[test] + fn requirement_lists_stay_linear_at_the_request_limit() { + let distinct = near_limit(8_000); + let mut doubled = distinct.clone(); + doubled.extend(distinct.iter().rev().cloned()); + let before = Counted::eq_count(); + let unique = first_occurrences(doubled); + assert_eq!(unique.len(), 8_000); + assert!(unique.iter().zip(&distinct).all(|(left, right)| left.0 == right.0), "first occurrence order"); + let missing = missing_from(&unique, &distinct); + assert!(missing.is_empty()); + let mut partial = distinct.clone(); + partial.truncate(4_000); + assert_eq!(missing_from(&unique, &partial).len(), 4_000); + // A Vec scan would compare about n^2/2 = 32 million times. + assert!(Counted::eq_count() - before < 64_000, "{} comparisons", Counted::eq_count() - before); + } + + #[test] + fn near_limit_requirements_normalize_in_order() { + let items: Vec = near_limit(8_000).into_iter().map(|item| json!(item.0)).collect(); + let mut body = json!({"session_id":"s"}); + body["requirements"] = Value::Array(items.iter().chain(items.iter()).cloned().collect()); + let options = Options::from(&body, None).unwrap(); + assert_eq!(options.requirements.len(), 8_000); + assert_eq!(options.requirements.first().map(String::as_str), items[0].as_str()); + let stored = CommandSession::from_snapshot(&json!({"session_id":"s","requirements":body["requirements"]})).unwrap(); + assert_eq!(stored.requirements, options.requirements); + let capabilities = options.requirements.clone(); + assert!(stored.claimable_by("1.20", "q", "q", &capabilities)); + assert!(!stored.claimable_by("1.20", "q", "q", &capabilities[1..])); + } } diff --git a/rust/tests/execution/worker_sessions.rs b/rust/tests/execution/worker_sessions.rs index 661eca750..4f01728c9 100644 --- a/rust/tests/execution/worker_sessions.rs +++ b/rust/tests/execution/worker_sessions.rs @@ -165,3 +165,158 @@ async fn session_bound_activities_are_admitted_held_limited_and_renewed() { completion["result"] = envelope(Payload::Null); assert_eq!(request(&app, "POST", &format!("{path}/complete"), completion).await.0, StatusCode::OK); } + +// Session numbers have portable maxima, the same in PHP (Server #740). One +// above a maximum, and i64::MAX, are refused with 422 before any arithmetic +// or write, on the create and heartbeat routes and in a schedule_activity +// command; the maxima themselves are accepted. +#[tokio::test] +async fn session_numbers_are_bounded_on_every_path() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + worker(&app, "bounded", json!(["gpu"]), None).await; + let maxima = [("lease_seconds", 86_400_i64), ("ttl_seconds", 2_592_000), ("max_concurrent_activities", 1_000)]; + let message = |field: &str, maximum: i64| json!(format!("The {} field must not be greater than {maximum}.", field.replace('_', " "))); + for (field, maximum) in maxima { + for value in [maximum + 1, i64::MAX] { + let mut body = json!({"worker_id":"bounded","session_id":format!("too-large-{field}")}); + body[field] = json!(value); + let (status, refused) = create(&app, body).await; + assert_eq!((status, refused["errors"][field][0].clone()), (StatusCode::UNPROCESSABLE_ENTITY, message(field, maximum)), "{refused}"); + } + } + assert_eq!(task_retry::rows(&database, "SELECT COUNT(*) FROM workflow_worker_sessions").await, 0, "refused creates record nothing"); + + let mut at_maxima = json!({"worker_id":"bounded","session_id":"at-maxima"}); + for (field, maximum) in maxima { + at_maxima[field] = json!(maximum); + } + let (status, created) = create(&app, at_maxima).await; + assert_eq!(status, StatusCode::CREATED, "{created}"); + assert_eq!((created["session"]["lease_seconds"].clone(), created["session"]["ttl_seconds"].clone(), + created["session"]["max_concurrent_activities"].clone()), (json!(86_400), json!(2_592_000), json!(1_000)), "{created}"); + let lease = || async { + request(&app, "GET", "/api/worker-sessions/at-maxima", Value::Null).await.1["session"]["lease_expires_at"].clone() + }; + let before = lease().await; + for value in [86_401, i64::MAX] { + let (status, refused) = request(&app, "POST", "/api/worker/sessions/at-maxima/heartbeat", + json!({"worker_id":"bounded","lease_seconds":value})).await; + assert_eq!((status, refused["errors"]["lease_seconds"][0].clone()), (StatusCode::UNPROCESSABLE_ENTITY, + message("lease_seconds", 86_400)), "{refused}"); + } + assert_eq!(lease().await, before, "a refused heartbeat renews nothing"); + let (status, beat) = request(&app, "POST", "/api/worker/sessions/at-maxima/heartbeat", + json!({"worker_id":"bounded","lease_seconds":86_400})).await; + assert_eq!((status, beat["outcome"].clone()), (StatusCode::OK, json!("heartbeat_recorded")), "{beat}"); + + session_worker(&app, "author", json!(["worker_sessions"])).await; + start(&app, "bounded-run").await; + let task = poll(&app, "author", "workflow").await; + for (field, maximum) in maxima { + let mut session = json!({"session_id":"command-maxima"}); + session[field] = json!(i64::MAX); + let (status, refused) = finish_task(&app, &task, json!([bound(session)])).await; + let key = format!("commands.0.worker_session.{field}"); + assert_eq!((status, refused["errors"][&key][0].clone()), (StatusCode::UNPROCESSABLE_ENTITY, + json!(format!("The {key} field must not be greater than {maximum}."))), "{refused}"); + } + assert_eq!(task_retry::rows(&database, "SELECT COUNT(*) FROM activity_executions").await, 0, "refused commands schedule nothing"); +} + +async fn register_holder(app: &Router, worker: &str, queue: &str) -> String { + let manifest = json!({"local_activities":{"supported":false,"minimum_protocol_version":"1.18"}, + "sticky_execution":{"supported":false,"minimum_protocol_version":"1.18"}, + "worker_sessions":{"supported":true,"minimum_protocol_version":"1.18"}}); + let (status, answer) = request(app, "POST", "/api/worker/register", json!({"worker_id":worker,"task_queue":queue,"runtime":"php", + "supported_workflow_types":["echo"],"supported_activity_types":["act"],"capabilities":["worker_sessions","gpu"], + "capability_manifest":manifest})).await; + assert_eq!(status, StatusCode::CREATED, "{answer}"); + answer["registration_token"].as_str().expect("a registration token").to_owned() +} + +// A same-ID re-registration orphans the sessions the previous registration +// held (Server #740): the replacement reacquires, never reuses, and a request +// that sends the replaced registration's token is refused without changing +// anything. A worker heartbeat keeps the registration, so its holder reuses. +#[tokio::test] +async fn a_same_id_replacement_reacquires_the_session_and_fences_the_old_process() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + let create = |token: Option<&str>| { + let mut body = json!({"worker_id":"gpu-worker","session_id":"gpu-state","requirements":["gpu"]}); + if let Some(token) = token { + body["registration_token"] = json!(token); + } + create(&app, body) + }; + let heartbeat = |token: Option<&str>| { + let mut body = json!({"worker_id":"gpu-worker"}); + if let Some(token) = token { + body["registration_token"] = json!(token); + } + request(&app, "POST", "/api/worker/sessions/gpu-state/heartbeat", body) + }; + + let first = register_holder(&app, "gpu-worker", "sessions").await; + let (status, created) = create(Some(&first)).await; + assert_eq!((status, created["outcome"].clone()), (StatusCode::CREATED, json!("created")), "{created}"); + let (status, beat) = request(&app, "POST", "/api/worker/heartbeat", json!({"worker_id":"gpu-worker"})).await; + assert_eq!(status, StatusCode::OK, "{beat}"); + let (status, reused) = create(None).await; + assert_eq!((status, reused["outcome"].clone()), (StatusCode::OK, json!("reused")), "{reused}"); + + let second = register_holder(&app, "gpu-worker", "sessions").await; + assert_ne!(first, second); + let (status, orphaned) = heartbeat(None).await; + assert_eq!((status, orphaned["reason"].clone(), orphaned["session"]["status"].clone(), orphaned["session"]["failure_reason"].clone()), + (StatusCode::CONFLICT, json!("session_not_active"), json!("orphaned"), json!("worker_registration_replaced")), "{orphaned}"); + for refused in [heartbeat(Some(&first)).await, create(Some(&first)).await] { + assert_eq!((refused.0, refused.1["reason"].clone()), (StatusCode::CONFLICT, json!("stale_worker_registration")), "{}", refused.1); + } + let (status, reacquired) = create(Some(&second)).await; + assert_eq!((status, reacquired["outcome"].clone(), reacquired["session"]["status"].clone()), + (StatusCode::OK, json!("reacquired"), json!("active")), "{reacquired}"); + let (status, stale) = heartbeat(Some(&first)).await; + assert_eq!((status, stale["reason"].clone()), (StatusCode::CONFLICT, json!("stale_worker_registration")), "{stale}"); + for current in [heartbeat(Some(&second)).await, heartbeat(None).await] { + assert_eq!((current.0, current.1["outcome"].clone()), (StatusCode::OK, json!("heartbeat_recorded")), "{}", current.1); + } + let (status, refused) = request(&app, "DELETE", "/api/worker/sessions/gpu-state", + json!({"worker_id":"gpu-worker","registration_token":first})).await; + assert_eq!((status, refused["reason"].clone()), (StatusCode::CONFLICT, json!("stale_worker_registration")), "{refused}"); + let (status, invalid) = request(&app, "DELETE", "/api/worker/sessions/gpu-state", + json!({"worker_id":"gpu-worker","registration_token":"not-a-token"})).await; + assert_eq!((status, invalid["errors"]["registration_token"][0].clone()), + (StatusCode::UNPROCESSABLE_ENTITY, json!("The registration token field format is invalid.")), "{invalid}"); + let (status, closed) = request(&app, "DELETE", "/api/worker/sessions/gpu-state", + json!({"worker_id":"gpu-worker","registration_token":second})).await; + assert_eq!((status, closed["outcome"].clone()), (StatusCode::OK, json!("closed")), "{closed}"); +} + +// A session-bound activity claim by the replacement takes the orphaned +// session through reacquisition, so the claim names the replacement's +// registration as the holder. +#[tokio::test] +async fn a_same_id_replacement_reacquires_a_session_through_an_activity_claim() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + register_holder(&app, "gpu-worker", "test").await; + let (status, created) = create(&app, json!({"worker_id":"gpu-worker","session_id":"gpu-claim","requirements":["gpu"]})).await; + assert_eq!(status, StatusCode::CREATED, "{created}"); + start(&app, "replacement-run").await; + let task = poll(&app, "gpu-worker", "workflow").await; + let (status, done) = finish_task(&app, &task, json!([bound(json!({"session_id":"gpu-claim","requirements":["gpu"]}))])).await; + assert_eq!(status, StatusCode::OK, "{done}"); + + register_holder(&app, "gpu-worker", "test").await; + let (status, shown) = request(&app, "GET", "/api/worker-sessions/gpu-claim", Value::Null).await; + assert_eq!((status, shown["session"]["status"].clone(), shown["session"]["failure_reason"].clone()), + (StatusCode::OK, json!("orphaned"), json!("worker_registration_replaced")), "{shown}"); + let claimed = poll(&app, "gpu-worker", "activity").await; + assert_eq!((claimed["worker_session"]["session_id"].clone(), claimed["worker_session"]["lease_owner"].clone()), + (json!("gpu-claim"), json!("gpu-worker")), "{claimed}"); + let (status, shown) = request(&app, "GET", "/api/worker-sessions/gpu-claim", Value::Null).await; + assert_eq!((status, shown["session"]["status"].clone(), shown["session"]["failure_reason"].clone()), + (StatusCode::OK, json!("active"), Value::Null), "{shown}"); +} diff --git a/scripts/conformance/server-parity/mysql-schema.php b/scripts/conformance/server-parity/mysql-schema.php index 20e2bca6e..1a71793d7 100644 --- a/scripts/conformance/server-parity/mysql-schema.php +++ b/scripts/conformance/server-parity/mysql-schema.php @@ -106,7 +106,7 @@ function catalog(PDO $pdo): array } $pdo->exec('ROLLBACK'); $pdo = connection($argv[3]); -$native = array_values(array_filter(catalog($pdo), static fn (array $row): bool => ! in_array($row[0], ['dw_server_schema', '_sqlx_migrations', 'dw_task_completions', 'dw_poll_receipts', 'dw_query_cache', 'dw_worker_registration_incarnations'], true))); +$native = array_values(array_filter(catalog($pdo), static fn (array $row): bool => ! in_array($row[0], ['dw_server_schema', '_sqlx_migrations', 'dw_task_completions', 'dw_poll_receipts', 'dw_query_cache', 'dw_worker_registration_incarnations', 'dw_worker_session_holders'], true))); $pdo->exec('ROLLBACK'); // Native polling uses an additional index on an existing table. Retain all // original column, index and constraint definitions in the comparison. diff --git a/scripts/conformance/server-parity/postgres-schema.php b/scripts/conformance/server-parity/postgres-schema.php index e5da9aaf5..7ea05ff49 100644 --- a/scripts/conformance/server-parity/postgres-schema.php +++ b/scripts/conformance/server-parity/postgres-schema.php @@ -70,8 +70,8 @@ function catalog(string $database): array throw new InvalidArgumentException('Compare requires two independent databases.'); } $native = catalog($argv[3]); -$tables = ['_sqlx_migrations', 'dw_server_schema', 'dw_task_completions', 'dw_poll_receipts', 'dw_query_cache', 'dw_worker_registration_incarnations']; -$indexes = ['_sqlx_migrations_pkey', 'dw_task_completions_pkey', 'dw_poll_receipts_pkey', 'dw_workflow_tasks_poll', 'dw_query_cache_pkey', 'dw_worker_registration_incarnations_pkey', 'dw_worker_incarnation_authority', 'dw_worker_incarnation_retention']; +$tables = ['_sqlx_migrations', 'dw_server_schema', 'dw_task_completions', 'dw_poll_receipts', 'dw_query_cache', 'dw_worker_registration_incarnations', 'dw_worker_session_holders']; +$indexes = ['_sqlx_migrations_pkey', 'dw_task_completions_pkey', 'dw_poll_receipts_pkey', 'dw_workflow_tasks_poll', 'dw_query_cache_pkey', 'dw_worker_registration_incarnations_pkey', 'dw_worker_incarnation_authority', 'dw_worker_incarnation_retention', 'dw_worker_session_holders_pkey']; $native = array_values(array_filter($native, static function (array $row) use ($tables, $indexes): bool { if (in_array($row[0], ['column', 'constraint'], true)) { return ! in_array(explode('.', $row[1], 2)[0], $tables, true); diff --git a/scripts/conformance/server-parity/schema.php b/scripts/conformance/server-parity/schema.php index 4dd30342e..5929ce8db 100644 --- a/scripts/conformance/server-parity/schema.php +++ b/scripts/conformance/server-parity/schema.php @@ -38,7 +38,7 @@ function catalog(string $filename): array $php = catalog($argv[1]); $native = catalog($argv[2]); -$nativeOnly = ['_sqlx_migrations', 'dw_server_schema', 'dw_task_completions', 'dw_poll_receipts', 'dw_workflow_tasks_poll', 'dw_query_cache', 'dw_worker_registration_incarnations', 'dw_worker_incarnation_authority', 'dw_worker_incarnation_retention']; +$nativeOnly = ['_sqlx_migrations', 'dw_server_schema', 'dw_task_completions', 'dw_poll_receipts', 'dw_workflow_tasks_poll', 'dw_query_cache', 'dw_worker_registration_incarnations', 'dw_worker_incarnation_authority', 'dw_worker_incarnation_retention', 'dw_worker_session_holders']; $native = array_values(array_filter($native, static fn (array $row): bool => ! in_array($row['name'], $nativeOnly, true))); if ($php !== $native) { $byName = static fn (array $rows): array => array_column($rows, null, 'name'); diff --git a/tests/Feature/WorkerSessionHolderMigrationTest.php b/tests/Feature/WorkerSessionHolderMigrationTest.php new file mode 100644 index 000000000..ccd9f1196 --- /dev/null +++ b/tests/Feature/WorkerSessionHolderMigrationTest.php @@ -0,0 +1,69 @@ +down(); + + $token = bin2hex(random_bytes(16)); + $this->seedWorker('holder', $token); + $this->seedWorker('tokenless', null); + $this->seedSession('held', 'holder', WorkerSessionLease::STATUS_ACTIVE); + $this->seedSession('closed', 'holder', WorkerSessionLease::STATUS_CLOSED); + $this->seedSession('tokenless-held', 'tokenless', WorkerSessionLease::STATUS_ACTIVE); + $this->seedSession('unregistered', 'gone', WorkerSessionLease::STATUS_ACTIVE); + + $migration->up(); + + $holders = WorkerSessionLease::query()->pluck('lease_owner_incarnation', 'session_id')->all(); + $this->assertSame([ + 'held' => hash('sha256', $token), + 'closed' => null, + 'tokenless-held' => null, + 'unregistered' => null, + ], $holders); + } + + private function seedWorker(string $workerId, ?string $token): void + { + WorkerRegistration::query()->create([ + 'worker_id' => $workerId, + 'namespace' => 'default', + 'task_queue' => 'gpu-activities', + 'runtime' => 'php', + 'last_heartbeat_at' => now(), + 'status' => 'active', + ])->forceFill(['registration_token' => $token])->save(); + } + + private function seedSession(string $sessionId, string $owner, string $status): void + { + WorkerSessionLease::query()->create([ + 'namespace' => 'default', + 'session_id' => $sessionId, + 'status' => $status, + 'lease_owner' => $owner, + 'lease_expires_at' => now()->addMinutes(2), + 'ttl_expires_at' => now()->addMinutes(30), + 'lease_seconds' => 120, + 'ttl_seconds' => 1800, + 'max_concurrent_activities' => 1, + 'create_if_missing' => true, + 'allow_reacquire_after_failure' => true, + 'last_heartbeat_at' => now(), + ]); + } +} diff --git a/tests/Feature/WorkerSessionProtocolTest.php b/tests/Feature/WorkerSessionProtocolTest.php index 80525a7db..3bcb2b179 100644 --- a/tests/Feature/WorkerSessionProtocolTest.php +++ b/tests/Feature/WorkerSessionProtocolTest.php @@ -9,6 +9,7 @@ use App\Support\ActivityTaskPollRequestStore; use App\Support\ServerPollingCache; use App\Support\WorkerProtocol; +use App\Support\WorkerSessionRegistry; use Illuminate\Foundation\Testing\RefreshDatabase; use Illuminate\Support\Facades\Queue; use Mockery\MockInterface; @@ -748,17 +749,21 @@ public function test_activity_heartbeat_reports_worker_session_renewal_failure() ->assertJsonPath('task.worker_session.session_id', 'gpu-heartbeat-renewal') ->assertJsonPath('task.worker_session.lease_owner', 'gpu-heartbeat-renewal-worker'); - $this->registerWorkerThroughProtocol( + $otherToken = $this->registerWorkerThroughProtocol( workerId: 'gpu-heartbeat-renewal-other-worker', taskQueue: 'gpu-activities', supportedActivityTypes: ['tests.external-greeting-activity'], capabilities: ['gpu:nvidia-l4'], ); + // Hand the session to the other worker's current registration. WorkerSessionLease::query() ->where('namespace', 'default') ->where('session_id', 'gpu-heartbeat-renewal') - ->update(['lease_owner' => 'gpu-heartbeat-renewal-other-worker']); + ->update([ + 'lease_owner' => 'gpu-heartbeat-renewal-other-worker', + 'lease_owner_incarnation' => hash('sha256', (string) $otherToken), + ]); $taskId = (string) $activityPoll->json('task.task_id'); $attemptId = (string) $activityPoll->json('task.activity_attempt_id'); @@ -1020,6 +1025,187 @@ public function test_absolute_ttl_remains_terminal_after_a_lease_was_already_mar * @param list $supportedActivityTypes * @param list $capabilities */ + public function test_session_numbers_have_portable_maxima_on_every_path(): void + { + Queue::fake(); + $this->registerWorkerThroughProtocol( + workerId: 'bounded-worker', + taskQueue: 'gpu-activities', + supportedWorkflowTypes: ['tests.external-greeting-workflow'], + capabilities: ['gpu:nvidia-l4'], + ); + $maxima = [ + 'lease_seconds' => WorkerSessionRegistry::MAX_LEASE_SECONDS, + 'ttl_seconds' => WorkerSessionRegistry::MAX_TTL_SECONDS, + 'max_concurrent_activities' => WorkerSessionRegistry::MAX_CONCURRENT_ACTIVITIES, + ]; + $this->assertSame([86400, 2592000, 1000], array_values($maxima)); + + foreach ($maxima as $field => $maximum) { + foreach ([$maximum + 1, PHP_INT_MAX] as $value) { + $this->postJson('/api/worker/sessions', [ + 'worker_id' => 'bounded-worker', 'session_id' => "too-large-{$field}", $field => $value, + ], $this->workerHeaders()) + ->assertStatus(422) + ->assertJsonPath("errors.{$field}.0", sprintf( + 'The %s field must not be greater than %d.', str_replace('_', ' ', $field), $maximum, + )); + } + } + $this->assertSame(0, WorkerSessionLease::query()->count(), 'refused creates record nothing'); + + $this->postJson('/api/worker/sessions', ['worker_id' => 'bounded-worker', 'session_id' => 'at-maxima', ...$maxima], + $this->workerHeaders()) + ->assertCreated() + ->assertJsonPath('session.lease_seconds', 86400) + ->assertJsonPath('session.ttl_seconds', 2592000) + ->assertJsonPath('session.max_concurrent_activities', 1000); + $leaseExpiresAt = WorkerSessionLease::query()->where('session_id', 'at-maxima')->value('lease_expires_at'); + + foreach ([86401, PHP_INT_MAX] as $value) { + $this->postJson('/api/worker/sessions/at-maxima/heartbeat', ['worker_id' => 'bounded-worker', 'lease_seconds' => $value], + $this->workerHeaders()) + ->assertStatus(422) + ->assertJsonPath('errors.lease_seconds.0', 'The lease seconds field must not be greater than 86400.'); + } + $this->assertEquals($leaseExpiresAt, WorkerSessionLease::query()->where('session_id', 'at-maxima')->value('lease_expires_at'), + 'a refused heartbeat renews nothing'); + $this->postJson('/api/worker/sessions/at-maxima/heartbeat', ['worker_id' => 'bounded-worker', 'lease_seconds' => 86400], + $this->workerHeaders()) + ->assertOk() + ->assertJsonPath('outcome', 'heartbeat_recorded'); + + $this->configureWorkflowTypes(['tests.external-greeting-workflow' => ExternalGreetingWorkflow::class]); + $this->postJson('/api/workflows', [ + 'workflow_id' => 'wf-worker-session-maxima', 'workflow_type' => 'tests.external-greeting-workflow', + 'task_queue' => 'gpu-activities', 'input' => ['Ada'], + ], $this->apiHeaders())->assertCreated(); + $poll = $this->postJson('/api/worker/workflow-tasks/poll', ['worker_id' => 'bounded-worker', 'task_queue' => 'gpu-activities'], + $this->workerHeaders()); + $poll->assertOk(); + foreach ($maxima as $field => $maximum) { + $errors = $this->postJson(sprintf('/api/worker/workflow-tasks/%s/complete', $poll->json('task.task_id')), [ + 'lease_owner' => $poll->json('task.lease_owner'), + 'workflow_task_attempt' => $poll->json('task.workflow_task_attempt'), + 'commands' => [[ + 'type' => 'schedule_activity', + 'activity_type' => 'tests.external-greeting-activity', + 'arguments' => Serializer::serializeWithCodec((string) config('workflows.serializer'), ['Ada']), + 'worker_session' => ['session_id' => 'command-maxima', $field => PHP_INT_MAX], + ]], + ], $this->workerHeaders()) + ->assertStatus(422) + ->json('errors'); + $this->assertSame( + sprintf('The commands.0.worker_session.%s field must not be greater than %d.', $field, $maximum), + $errors["commands.0.worker_session.{$field}"][0] ?? null, + json_encode($errors), + ); + } + $this->assertSame(0, ActivityExecution::query()->count(), 'refused commands schedule nothing'); + } + + public function test_a_same_id_replacement_reacquires_the_session_and_fences_the_old_process(): void + { + $register = fn (): ?string => $this->registerWorkerThroughProtocol( + workerId: 'gpu-worker', + taskQueue: 'gpu-activities', + supportedActivityTypes: ['tests.external-greeting-activity'], + capabilities: ['gpu:nvidia-l4'], + ); + $holder = static fn (): ?string => WorkerSessionLease::query()->where('session_id', 'gpu-state')->value('lease_owner_incarnation'); + $create = fn (?string $token = null) => $this->postJson('/api/worker/sessions', array_filter([ + 'worker_id' => 'gpu-worker', 'session_id' => 'gpu-state', 'requirements' => ['gpu:nvidia-l4'], 'registration_token' => $token, + ]), $this->workerHeaders()); + $heartbeat = fn (?string $token = null) => $this->postJson('/api/worker/sessions/gpu-state/heartbeat', + array_filter(['worker_id' => 'gpu-worker', 'registration_token' => $token]), $this->workerHeaders()); + + $first = $register(); + $this->assertIsString($first); + $create($first)->assertCreated()->assertJsonPath('outcome', 'created'); + $this->assertSame(hash('sha256', $first), $holder(), 'the session keeps a digest, never the token'); + + // A worker heartbeat keeps the registration, so the holder reuses it. + $this->postJson('/api/worker/heartbeat', ['worker_id' => 'gpu-worker'], $this->workerHeaders())->assertOk(); + $create()->assertOk()->assertJsonPath('outcome', 'reused'); + + // A same-ID re-registration rotates the token: the session is orphaned. + $second = $register(); + $this->assertIsString($second); + $this->assertNotSame($first, $second); + $heartbeat() + ->assertStatus(409) + ->assertJsonPath('reason', 'session_not_active') + ->assertJsonPath('session.status', 'orphaned') + ->assertJsonPath('session.failure_reason', 'worker_registration_replaced'); + $heartbeat($first)->assertStatus(409)->assertJsonPath('reason', 'stale_worker_registration'); + $create($first)->assertStatus(409)->assertJsonPath('reason', 'stale_worker_registration'); + $this->assertSame(hash('sha256', $first), $holder(), 'a stale token changes nothing'); + + // The replacement reacquires, never reuses, and becomes the holder. + $create($second)->assertOk()->assertJsonPath('outcome', 'reacquired')->assertJsonPath('session.status', 'active'); + $this->assertSame(hash('sha256', $second), $holder()); + $heartbeat($first)->assertStatus(409)->assertJsonPath('reason', 'stale_worker_registration'); + $heartbeat($second)->assertOk()->assertJsonPath('outcome', 'heartbeat_recorded'); + $heartbeat()->assertOk()->assertJsonPath('outcome', 'heartbeat_recorded'); + $this->deleteJson('/api/worker/sessions/gpu-state', ['worker_id' => 'gpu-worker', 'registration_token' => $first], + $this->workerHeaders())->assertStatus(409)->assertJsonPath('reason', 'stale_worker_registration'); + $this->deleteJson('/api/worker/sessions/gpu-state', ['worker_id' => 'gpu-worker', 'registration_token' => $second], + $this->workerHeaders())->assertOk()->assertJsonPath('outcome', 'closed'); + } + + public function test_a_same_id_replacement_reacquires_a_session_through_an_activity_claim(): void + { + Queue::fake(); + $this->configureWorkflowTypes(['tests.external-greeting-workflow' => ExternalGreetingWorkflow::class]); + $register = fn (): ?string => $this->registerWorkerThroughProtocol( + workerId: 'gpu-worker', + taskQueue: 'gpu-activities', + supportedWorkflowTypes: ['tests.external-greeting-workflow'], + supportedActivityTypes: ['tests.external-greeting-activity'], + capabilities: ['gpu:nvidia-l4'], + ); + $first = $register(); + $this->postJson('/api/worker/sessions', ['worker_id' => 'gpu-worker', 'session_id' => 'gpu-claim', 'requirements' => ['gpu:nvidia-l4']], + $this->workerHeaders())->assertCreated(); + $this->postJson('/api/workflows', [ + 'workflow_id' => 'wf-worker-session-replacement', 'workflow_type' => 'tests.external-greeting-workflow', + 'task_queue' => 'gpu-activities', 'input' => ['Ada'], + ], $this->apiHeaders())->assertCreated(); + $poll = $this->postJson('/api/worker/workflow-tasks/poll', ['worker_id' => 'gpu-worker', 'task_queue' => 'gpu-activities'], + $this->workerHeaders()); + $this->postJson(sprintf('/api/worker/workflow-tasks/%s/complete', $poll->json('task.task_id')), [ + 'lease_owner' => $poll->json('task.lease_owner'), + 'workflow_task_attempt' => $poll->json('task.workflow_task_attempt'), + 'commands' => [[ + 'type' => 'schedule_activity', + 'activity_type' => 'tests.external-greeting-activity', + 'arguments' => Serializer::serializeWithCodec((string) config('workflows.serializer'), ['Ada']), + 'worker_session' => ['session_id' => 'gpu-claim', 'requirements' => ['gpu:nvidia-l4']], + ]], + ], $this->workerHeaders())->assertOk(); + + $second = $register(); + $this->assertNotSame($first, $second); + $this->getJson('/api/worker-sessions/gpu-claim', $this->apiHeaders()) + ->assertOk() + ->assertJsonPath('session.status', 'orphaned') + ->assertJsonPath('session.failure_reason', 'worker_registration_replaced'); + $this->postJson('/api/worker/activity-tasks/poll', ['worker_id' => 'gpu-worker', 'task_queue' => 'gpu-activities'], + $this->workerHeaders()) + ->assertOk() + ->assertJsonPath('poll_status', 'leased') + ->assertJsonPath('task.worker_session.session_id', 'gpu-claim') + ->assertJsonPath('task.worker_session.lease_owner', 'gpu-worker'); + $this->assertSame(hash('sha256', (string) $second), + WorkerSessionLease::query()->where('session_id', 'gpu-claim')->value('lease_owner_incarnation')); + // Published PHP keeps the old failure reason on a reacquired session + // (Server #737), so only the status is asserted here. + $this->getJson('/api/worker-sessions/gpu-claim', $this->apiHeaders()) + ->assertOk() + ->assertJsonPath('session.status', 'active'); + } + private function registerWorkerThroughProtocol( string $workerId, string $taskQueue, @@ -1028,7 +1214,7 @@ private function registerWorkerThroughProtocol( array $capabilities = [], int $maxConcurrentWorkerSessions = 10, ?string $protocolVersion = null, - ): void { + ): ?string { $requestedProtocol = $protocolVersion ?? WorkerProtocol::VERSION; $workerSupportsSessions = version_compare( $requestedProtocol, @@ -1061,10 +1247,11 @@ private function registerWorkerThroughProtocol( ); } - $this->postJson('/api/worker/register', $registration, $protocolVersion === null + return $this->postJson('/api/worker/register', $registration, $protocolVersion === null ? $this->workerHeaders() : $this->workerProtocolHeaders($protocolVersion)) - ->assertCreated(); + ->assertCreated() + ->json('registration_token'); } /** From bdeb19268ca9bd8fe076d118dae5c950a0112cf7 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Sat, 24 Oct 2026 22:29:36 +0000 Subject: [PATCH 06/13] CI: run the worker-session contract tests with the other fixture contract tests The worker-sessions and session-activities contracts are checked while recording, but their own unit tests (the recorded observation plus each mutation the contract must refuse) were never run. They now run in the comparator step and trigger the fixtures on change, like the others. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/server-parity.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/server-parity.yml b/.github/workflows/server-parity.yml index 1b25ed7a7..00d7c399e 100644 --- a/.github/workflows/server-parity.yml +++ b/.github/workflows/server-parity.yml @@ -49,6 +49,8 @@ on: - tests/Unit/ServerParityRepairPassContractTest.mjs - tests/Unit/ServerParityClusterInfoContractTest.mjs - tests/Unit/ServerParityNamespaceDeleteContractTest.mjs + - tests/Unit/ServerParityWorkerSessionsContractTest.mjs + - tests/Unit/ServerParityWorkerSessionActivitiesContractTest.mjs - tests/Unit/ServerParityUpdateValidationContractTest.mjs - tests/Unit/ServerParityFixtureSelectionTest.mjs - tests/Unit/ServerParityWaitingHistoryContractTest.mjs @@ -1532,7 +1534,7 @@ jobs: -e COMPOSER_HOME=/tmp/composer -v "$PWD:/source" -w /source "$image" \ install --working-dir=scripts/conformance/server-parity --no-interaction --no-progress docker run --rm --user=1000:1000 --entrypoint node -v "$PWD:/source:ro" -w /source \ - "$image" --test tests/Unit/ServerParityRunnerTest.mjs tests/Unit/ServerParitySdkDirectionLanesTest.mjs tests/Unit/ServerParityCooperativeContractTest.mjs tests/Unit/ServerParityChildCancellationContractTest.mjs tests/Unit/ServerParityScheduleContractTest.mjs tests/Unit/ServerParityVisibilityContractTest.mjs tests/Unit/ServerParityAdmissionContractTest.mjs tests/Unit/ServerParityRoleTokenContractTest.mjs tests/Unit/ServerParityWorkerDeregistrationContractTest.mjs tests/Unit/ServerParitySearchAttributeContractTest.mjs tests/Unit/ServerParityWorkerListingContractTest.mjs tests/Unit/ServerParityWorkerReplacementContractTest.mjs tests/Unit/ServerParityPatchDeploymentContractTest.mjs tests/Unit/ServerParityWaitingHistoryContractTest.mjs tests/Unit/ServerParityTaskRetryContractTest.mjs tests/Unit/ServerParityPublishedPatchContractTest.mjs tests/Unit/ServerParityPublishedPatchTransportTest.mjs tests/Unit/ServerParityRecordFileTest.mjs tests/Unit/ServerParityLateWorkflowCompletionContractTest.mjs tests/Unit/ServerParityStorageTestContractTest.mjs tests/Unit/ServerParitySchedulePoliciesContractTest.mjs tests/Unit/ServerParitySignalsBeforeWorkerContractTest.mjs tests/Unit/ServerParityScheduleCatalogContractTest.mjs tests/Unit/ServerParityOperatorMetricsContractTest.mjs tests/Unit/ServerParityActivityLeaseRenewalContractTest.mjs tests/Unit/ServerParityWorkflowPollRequestContractTest.mjs tests/Unit/ServerParityServiceCatalogContractTest.mjs tests/Unit/ServerParitySystemMetricsContractTest.mjs tests/Unit/ServerParityOperatorDashboardTypesContractTest.mjs tests/Unit/ServerParityPrometheusMetricsContractTest.mjs tests/Unit/ServerParityPayloadCleanupContractTest.mjs tests/Unit/ServerParityRepairPassContractTest.mjs tests/Unit/ServerParityClusterInfoContractTest.mjs tests/Unit/ServerParityNamespaceDeleteContractTest.mjs tests/Unit/ServerParityUpdateValidationContractTest.mjs tests/Unit/ServerParityFixtureSelectionTest.mjs > parity-evidence/comparator-tests.txt + "$image" --test tests/Unit/ServerParityRunnerTest.mjs tests/Unit/ServerParitySdkDirectionLanesTest.mjs tests/Unit/ServerParityCooperativeContractTest.mjs tests/Unit/ServerParityChildCancellationContractTest.mjs tests/Unit/ServerParityScheduleContractTest.mjs tests/Unit/ServerParityVisibilityContractTest.mjs tests/Unit/ServerParityAdmissionContractTest.mjs tests/Unit/ServerParityRoleTokenContractTest.mjs tests/Unit/ServerParityWorkerDeregistrationContractTest.mjs tests/Unit/ServerParitySearchAttributeContractTest.mjs tests/Unit/ServerParityWorkerListingContractTest.mjs tests/Unit/ServerParityWorkerReplacementContractTest.mjs tests/Unit/ServerParityPatchDeploymentContractTest.mjs tests/Unit/ServerParityWaitingHistoryContractTest.mjs tests/Unit/ServerParityTaskRetryContractTest.mjs tests/Unit/ServerParityPublishedPatchContractTest.mjs tests/Unit/ServerParityPublishedPatchTransportTest.mjs tests/Unit/ServerParityRecordFileTest.mjs tests/Unit/ServerParityLateWorkflowCompletionContractTest.mjs tests/Unit/ServerParityStorageTestContractTest.mjs tests/Unit/ServerParitySchedulePoliciesContractTest.mjs tests/Unit/ServerParitySignalsBeforeWorkerContractTest.mjs tests/Unit/ServerParityScheduleCatalogContractTest.mjs tests/Unit/ServerParityOperatorMetricsContractTest.mjs tests/Unit/ServerParityActivityLeaseRenewalContractTest.mjs tests/Unit/ServerParityWorkflowPollRequestContractTest.mjs tests/Unit/ServerParityServiceCatalogContractTest.mjs tests/Unit/ServerParitySystemMetricsContractTest.mjs tests/Unit/ServerParityOperatorDashboardTypesContractTest.mjs tests/Unit/ServerParityPrometheusMetricsContractTest.mjs tests/Unit/ServerParityPayloadCleanupContractTest.mjs tests/Unit/ServerParityRepairPassContractTest.mjs tests/Unit/ServerParityClusterInfoContractTest.mjs tests/Unit/ServerParityNamespaceDeleteContractTest.mjs tests/Unit/ServerParityWorkerSessionsContractTest.mjs tests/Unit/ServerParityWorkerSessionActivitiesContractTest.mjs tests/Unit/ServerParityUpdateValidationContractTest.mjs tests/Unit/ServerParityFixtureSelectionTest.mjs > parity-evidence/comparator-tests.txt docker run --rm --user=1000:1000 --cpus=1 --memory=256m --memory-swap=256m \ --network=none --entrypoint node -v "$PWD:/source:ro" -w /source \ "$image" --test tests/Unit/ServerParityCodecDeadlineLoadTest.mjs > parity-evidence/codec-load-tests.txt From 1ef0c644e564e460e70f6340bec38b16413dd1bd Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Sat, 24 Oct 2026 22:50:18 +0000 Subject: [PATCH 07/13] Server #740: drop the unreachable takeover seeding; read the schema version from its constant Native takeover still refuses any PHP source with worker-session rows (upgrade_source_worker_sessions_unsupported), so seeding session holders during takeover could never run. It is removed rather than kept as untested code. When that refusal is lifted, the takeover must seed dw_worker_session_holders from each active session's current registration, as the PHP migration backfills lease_owner_incarnation. The PHP 2.9.7 takeover test asserted marker version 4. It now reads the schema version constant. Co-Authored-By: Claude Opus 5.5 --- rust/src/runtime/upgrade/takeover.rs | 15 --------------- rust/src/runtime/upgrade_tests.rs | 2 +- rust/src/runtime/worker_sessions.rs | 2 +- 3 files changed, 2 insertions(+), 17 deletions(-) diff --git a/rust/src/runtime/upgrade/takeover.rs b/rust/src/runtime/upgrade/takeover.rs index 6997340f7..4412cfb71 100644 --- a/rust/src/runtime/upgrade/takeover.rs +++ b/rust/src/runtime/upgrade/takeover.rs @@ -469,21 +469,6 @@ where .execute(&mut *tx) .await?; } - // Each active session stays with the registration that holds it now, so a - // takeover does not orphan sessions whose workers keep running. A resumed - // takeover skips sessions it already seeded. - let held = query::("SELECT s.namespace,s.session_id,i.token FROM workflow_worker_sessions s JOIN dw_worker_registration_incarnations i ON i.namespace=s.namespace AND i.worker_id=s.lease_owner AND i.status='active' WHERE s.status='active' AND NOT EXISTS (SELECT 1 FROM dw_worker_session_holders h WHERE h.namespace=s.namespace AND h.session_id=s.session_id) ORDER BY s.namespace,s.session_id") - .fetch_all(&mut *tx) - .await?; - for row in &held { - query::("INSERT INTO dw_worker_session_holders(namespace,session_id,incarnation,updated_at) VALUES ($1,$2,$3,$4)") - .bind(DB::string(row, "namespace")?) - .bind(DB::string(row, "session_id")?) - .bind(super::super::worker_sessions::incarnation_digest(&DB::string(row, "token")?)) - .bind(DB::bind_time(now)) - .execute(&mut *tx) - .await?; - } Ok((copied, missing.len())) } diff --git a/rust/src/runtime/upgrade_tests.rs b/rust/src/runtime/upgrade_tests.rs index 3c69d31df..a76db6497 100644 --- a/rust/src/runtime/upgrade_tests.rs +++ b/rust/src/runtime/upgrade_tests.rs @@ -947,7 +947,7 @@ async fn takeover_converts_a_php_297_database_that_native_then_opens() { .fetch_one(&mut connection) .await .unwrap(); - assert_eq!(marker, ("rust-development".to_owned(), 4)); + assert_eq!(marker, ("rust-development".to_owned(), super::super::schema::VERSION)); connection.close().await.unwrap(); } diff --git a/rust/src/runtime/worker_sessions.rs b/rust/src/runtime/worker_sessions.rs index 092cf1c64..a6d89439d 100644 --- a/rust/src/runtime/worker_sessions.rs +++ b/rust/src/runtime/worker_sessions.rs @@ -199,7 +199,7 @@ const REGISTRATION_TOKEN_RULES: &[Rule] = &[Rule::Nullable, Rule::String, Rule:: /// The digest a session keeps of its holder's registration, as PHP keeps /// `hash('sha256', token)`, so the deregistration credential itself is never /// stored with the session. -pub(super) fn incarnation_digest(token: &str) -> String { +fn incarnation_digest(token: &str) -> String { use sha2::{Digest, Sha256}; Sha256::digest(token.as_bytes()).iter().map(|byte| format!("{byte:02x}")).collect() } From 4b746ec0550795bf481d6f464033b02b4c6a6a47 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Sun, 25 Oct 2026 04:04:40 +0000 Subject: [PATCH 08/13] Server #740: record the worker-session routes as served The unserved-route list arrived with #745 and still named the five worker-session routes this branch serves, so route_coverage failed after the merge. Native now serves 138 of the 175 published route methods. Co-Authored-By: Claude Opus 5.5 --- docs/rust-server-port.md | 11 +++++------ rust/tests/fixtures/unserved-php-routes.v2.11.1.txt | 5 ----- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/docs/rust-server-port.md b/docs/rust-server-port.md index 0afd87729..d45b88ed6 100644 --- a/docs/rust-server-port.md +++ b/docs/rust-server-port.md @@ -82,13 +82,12 @@ Updated October 24, 2026. ([#737](https://github.com/durable-workflow/server/issues/737)). No expectation is edited to make native pass. -- **Parity surface.** Native serves 133 of the 175 API route methods in - published 2.11.1's route table, with message streams, update validation - and activity cancellation acknowledgement. An execution test sends each - one through the native router, and the ones still answering - `501 rust_capability_not_implemented` must match +- **Parity surface.** Native serves 138 of the 175 API route methods in + published 2.11.1's route table, with message streams, update validation, + activity cancellation acknowledgement and worker sessions. An execution + test sends each one through the native router, and the ones still + answering `501 rust_capability_not_implemented` must match [`unserved-php-routes.v2.11.1.txt`](../rust/tests/fixtures/unserved-php-routes.v2.11.1.txt). - The work in review serves 5 more: worker sessions. - **Next action.** Land the items above in order, keep porting the unserved routes fixture first, and package the native binary into the server image. diff --git a/rust/tests/fixtures/unserved-php-routes.v2.11.1.txt b/rust/tests/fixtures/unserved-php-routes.v2.11.1.txt index e62882383..63b6590ca 100644 --- a/rust/tests/fixtures/unserved-php-routes.v2.11.1.txt +++ b/rust/tests/fixtures/unserved-php-routes.v2.11.1.txt @@ -20,11 +20,6 @@ DELETE /api/service-endpoints/{endpointName}/services/{serviceName}/operations/{ POST /api/service-endpoints/{endpointName}/services/{serviceName}/operations/{operationName}/execute GET /api/service-endpoints/{endpointName}/services/{serviceName}/operations/{operationName}/service-calls/{serviceCallId} POST /api/service-endpoints/{endpointName}/services/{serviceName}/operations/{operationName}/service-calls/{serviceCallId}/cancel -GET /api/worker-sessions -GET /api/worker-sessions/{sessionId} -POST /api/worker/sessions -DELETE /api/worker/sessions/{sessionId} -POST /api/worker/sessions/{sessionId}/heartbeat POST /api/worker/workflow-tasks/{taskId}/cancellation-scopes/checkpoint POST /api/worker/workflow-tasks/{taskId}/cancellation-scopes/deliver POST /api/worker/workflow-tasks/{taskId}/cancellation-scopes/open From 8b606429001e446c7b2fae98bda13a56be7b902d Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Sun, 25 Oct 2026 07:39:09 +0000 Subject: [PATCH 09/13] Server #740: hash the counted test string by hand, as its equality reads it SonarCloud rule rust:S7424 refused a derived Hash beside a hand-written PartialEq on the test-only Counted helper. Both read only the inner string, so they agreed; implementing Hash by hand makes that explicit. Co-Authored-By: Claude Opus 5.5 --- rust/src/runtime/worker_sessions.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/rust/src/runtime/worker_sessions.rs b/rust/src/runtime/worker_sessions.rs index a6d89439d..05d57c588 100644 --- a/rust/src/runtime/worker_sessions.rs +++ b/rust/src/runtime/worker_sessions.rs @@ -1074,9 +1074,16 @@ mod tests { /// A string whose equality checks are counted, so the tests below fail /// on a quadratic scan rather than on a timing threshold. - #[derive(Clone, Hash)] + #[derive(Clone)] struct Counted(String); + // Hash and equality both read only the string, so equal values hash alike. + impl std::hash::Hash for Counted { + fn hash(&self, state: &mut H) { + self.0.hash(state); + } + } + thread_local! { static COMPARISONS: std::cell::Cell = const { std::cell::Cell::new(0) }; } From ad7ad3aea7189f2b300b3a2783683dc7e64a9d30 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Sun, 25 Oct 2026 08:10:59 +0000 Subject: [PATCH 10/13] Server #740: require the registration token from workers that declare they send it Security's R2 found that an explicit worker session create, heartbeat or close without a registration token was accepted even after a same-ID replacement had reacquired the session, so a stale process could reuse, renew or close its replacement's session. The published PHP SDK sends no token, so requiring it from every worker would break published clients. A worker now opts in with the worker_session_registration_token capability at registration. When the worker ID's current registration declares it, every explicit create, heartbeat and close must carry the current token. A missing token or a non-current one gets 409 stale_worker_registration and changes nothing. Without the capability, a request without a token acts for the current registration, as published. PHP and native apply the same fence first in each request's transaction, and the runtime schema states the conditional contract. Tests in both: after reacquisition the stale process, with its old token or none, is refused on all three verbs and the session is unchanged; the published tokenless path stays explicit. With the old fence restored, PHP's test sees 200 for 409 and native's stale create reuses the session. Co-Authored-By: Claude Opus 5.5 --- app/Support/WorkerSessionRegistry.php | 37 ++++-- .../worker-sessions-runtime.schema.json | 2 +- rust/src/runtime/worker_sessions.rs | 22 +++- rust/tests/execution/worker_sessions.rs | 107 +++++++++++++----- tests/Feature/WorkerSessionProtocolTest.php | 60 ++++++++-- 5 files changed, 178 insertions(+), 50 deletions(-) diff --git a/app/Support/WorkerSessionRegistry.php b/app/Support/WorkerSessionRegistry.php index a8dcc3a7f..57e363c41 100644 --- a/app/Support/WorkerSessionRegistry.php +++ b/app/Support/WorkerSessionRegistry.php @@ -24,6 +24,13 @@ final class WorkerSessionRegistry public const MAX_CONCURRENT_ACTIVITIES = 1000; + /** + * A worker that declares this capability at registration sends its + * registration token on every explicit worker session create, heartbeat + * and close, and the server refuses those requests without it. + */ + public const REGISTRATION_TOKEN_CAPABILITY = 'worker_session_registration_token'; + private const DEFAULT_TTL_SECONDS = 1800; private const DEFAULT_MAX_CONCURRENT_ACTIVITIES = 1; @@ -753,23 +760,37 @@ private function incarnation(WorkerRegistration $worker): ?string /** * A registration token the request sends must be the worker's current - * one. Without it, a stale process and its same-ID replacement are - * indistinguishable once the replacement has reacquired. + * one. Without a token, a stale process and its same-ID replacement are + * indistinguishable once the replacement has reacquired, so a worker + * whose current registration declares REGISTRATION_TOKEN_CAPABILITY must + * send it on every explicit create, heartbeat and close. A registration + * without the capability keeps the published contract: a request without + * a token acts for the worker ID's current registration. * * @return array|null */ private function staleRegistration(string $namespace, string $workerId, ?string $registrationToken): ?array { - if ($registrationToken === null) { - return null; - } - + /** @var WorkerRegistration|null $current */ $current = WorkerRegistration::query() ->where('namespace', $namespace) ->where('worker_id', $workerId) - ->value('registration_token'); + ->first(['registration_token', 'capabilities']); + + if ($registrationToken === null) { + return $current instanceof WorkerRegistration + && in_array(self::REGISTRATION_TOKEN_CAPABILITY, $current->capabilities ?? [], true) + ? $this->failure( + 'stale_worker_registration', + 'The worker\'s current registration requires its registration token on worker session requests. Send it, or register again and reacquire the session.', + 409, + ) + : null; + } - return is_string($current) && hash_equals($current, $registrationToken) + return $current instanceof WorkerRegistration + && is_string($current->registration_token) + && hash_equals($current->registration_token, $registrationToken) ? null : $this->failure( 'stale_worker_registration', diff --git a/resources/platform-protocol-specs/worker-sessions-runtime.schema.json b/resources/platform-protocol-specs/worker-sessions-runtime.schema.json index 8f8a1cf45..301d56f21 100644 --- a/resources/platform-protocol-specs/worker-sessions-runtime.schema.json +++ b/resources/platform-protocol-specs/worker-sessions-runtime.schema.json @@ -247,7 +247,7 @@ "workerRegistrationToken": { "type": "string", "pattern": "^[a-f0-9]{32}$", - "description": "The token worker registration returned. When sent, it must be the worker's current registration, otherwise the request is refused with stale_worker_registration and changes nothing. A same-ID re-registration orphans the sessions the previous registration held (worker_registration_replaced) until they are reacquired." + "description": "The token worker registration returned. When sent, it must be the worker's current registration, otherwise the request is refused with stale_worker_registration and changes nothing. A worker whose current registration declared the worker_session_registration_token capability must send it on every explicit create, heartbeat and close: without it the request is refused the same way, so a replaced process can neither reuse, renew nor close its replacement's session. Without that capability a request without a token acts for the worker ID's current registration, as before the token existed. A same-ID re-registration orphans the sessions the previous registration held (worker_registration_replaced) until they are reacquired. Session-bound activity claims and their heartbeats keep their own attempt leases." }, "workerSessionHeartbeatRequest": { "type": "object", diff --git a/rust/src/runtime/worker_sessions.rs b/rust/src/runtime/worker_sessions.rs index 05d57c588..8cd3ae865 100644 --- a/rust/src/runtime/worker_sessions.rs +++ b/rust/src/runtime/worker_sessions.rs @@ -36,6 +36,12 @@ const SESSIONS_LISTED: &str = "SELECT id,session_id,connection,queue,requirement const HELD_SESSIONS: &str = "SELECT COUNT(*) FROM workflow_worker_sessions WHERE namespace=$1 AND lease_owner=$2 AND status='active' AND (lease_expires_at IS NULL OR lease_expires_at>$3) AND session_id<>$4"; const HOLDER: &str = "SELECT task_queue,status,last_heartbeat_at FROM workflow_worker_registrations WHERE namespace=$1 AND worker_id=$2"; /// The worker's current registration token, as the poll fence reads it. +/// The capabilities the worker's current registration declared. +const REGISTRATION_CAPABILITIES: &str = "SELECT capabilities FROM workflow_worker_registrations WHERE namespace=$1 AND worker_id=$2"; +/// `WorkerSessionRegistry::REGISTRATION_TOKEN_CAPABILITY`: a worker that +/// declares it sends its registration token on every explicit session +/// create, heartbeat and close, and requests without it are refused. +const REGISTRATION_TOKEN_CAPABILITY: &str = "worker_session_registration_token"; const CURRENT_TOKEN: &str = "SELECT token FROM dw_worker_registration_incarnations WHERE namespace=$1 AND worker_id=$2 AND status='active' ORDER BY created_at DESC LIMIT 1"; /// The registration incarnation that acquired or last renewed a session. const SESSION_HOLDER: &str = "SELECT incarnation FROM dw_worker_session_holders WHERE namespace=$1 AND session_id=$2"; @@ -954,11 +960,21 @@ store_methods! { } /// A registration token the request sent must be the worker's current - /// one. Without it, a stale process and its same-ID replacement are - /// indistinguishable once the replacement has reacquired. + /// one. Without a token, a stale process and its same-ID replacement are + /// indistinguishable once the replacement has reacquired, so a worker + /// whose current registration declares [`REGISTRATION_TOKEN_CAPABILITY`] + /// must send it on every explicit create, heartbeat and close. Without + /// the capability, a request without a token acts for the worker ID's + /// current registration, as published PHP's does. async fn stale_registration(&self, tx: &mut Transaction<'_, DB>, worker_id: &str, token: Option<&str>) -> Result> { let Some(token) = token else { - return Ok(None); + let declared = Self::query(REGISTRATION_CAPABILITIES) + .bind(self.namespace.as_ref()).bind(worker_id).fetch_optional(&mut **tx).await? + .map(|row| DB::optional_document_row(&row, "capabilities")).transpose()?.flatten() + .is_some_and(|flags| flags.as_array().is_some_and(|flags| flags.iter().any(|flag| flag == REGISTRATION_TOKEN_CAPABILITY))); + return Ok(declared.then(|| failure("stale_worker_registration", + "The worker's current registration requires its registration token on worker session requests. Send it, or register again and reacquire the session.", + StatusCode::CONFLICT, Map::new()))); }; let current = self.current_token(tx, worker_id).await?; Ok((!current.is_some_and(|current| same_token(¤t, token))).then(|| failure("stale_worker_registration", diff --git a/rust/tests/execution/worker_sessions.rs b/rust/tests/execution/worker_sessions.rs index 4f01728c9..a26fa3a2b 100644 --- a/rust/tests/execution/worker_sessions.rs +++ b/rust/tests/execution/worker_sessions.rs @@ -225,11 +225,21 @@ async fn session_numbers_are_bounded_on_every_path() { } async fn register_holder(app: &Router, worker: &str, queue: &str) -> String { + register_with(app, worker, queue, json!(["worker_sessions", "gpu"])).await +} + +/// A holder that declares it sends its registration token on every +/// explicit session request, so the server requires it. +async fn register_token_holder(app: &Router, worker: &str, queue: &str) -> String { + register_with(app, worker, queue, json!(["worker_sessions", "gpu", "worker_session_registration_token"])).await +} + +async fn register_with(app: &Router, worker: &str, queue: &str, capabilities: Value) -> String { let manifest = json!({"local_activities":{"supported":false,"minimum_protocol_version":"1.18"}, "sticky_execution":{"supported":false,"minimum_protocol_version":"1.18"}, "worker_sessions":{"supported":true,"minimum_protocol_version":"1.18"}}); let (status, answer) = request(app, "POST", "/api/worker/register", json!({"worker_id":worker,"task_queue":queue,"runtime":"php", - "supported_workflow_types":["echo"],"supported_activity_types":["act"],"capabilities":["worker_sessions","gpu"], + "supported_workflow_types":["echo"],"supported_activity_types":["act"],"capabilities":capabilities, "capability_manifest":manifest})).await; assert_eq!(status, StatusCode::CREATED, "{answer}"); answer["registration_token"].as_str().expect("a registration token").to_owned() @@ -237,60 +247,99 @@ async fn register_holder(app: &Router, worker: &str, queue: &str) -> String { // A same-ID re-registration orphans the sessions the previous registration // held (Server #740): the replacement reacquires, never reuses, and a request -// that sends the replaced registration's token is refused without changing -// anything. A worker heartbeat keeps the registration, so its holder reuses. +// from the replaced process, with its old token or with none, is refused +// without changing anything. A worker heartbeat keeps the registration, so +// its holder reuses. The worker declared that it sends its token. #[tokio::test] async fn a_same_id_replacement_reacquires_the_session_and_fences_the_old_process() { let database = TestDatabase::new().await; let app = router(database.open().await.unwrap()); - let create = |token: Option<&str>| { - let mut body = json!({"worker_id":"gpu-worker","session_id":"gpu-state","requirements":["gpu"]}); + let with_token = |mut body: Value, token: Option<&str>| { if let Some(token) = token { body["registration_token"] = json!(token); } - create(&app, body) + body }; - let heartbeat = |token: Option<&str>| { - let mut body = json!({"worker_id":"gpu-worker"}); - if let Some(token) = token { - body["registration_token"] = json!(token); + let create = |token: Option<&str>, options: Value| { + let mut body = json!({"worker_id":"gpu-worker","session_id":"gpu-state","requirements":["gpu"]}); + for (key, value) in options.as_object().into_iter().flatten() { + body[key] = value.clone(); } - request(&app, "POST", "/api/worker/sessions/gpu-state/heartbeat", body) + create(&app, with_token(body, token)) }; + let heartbeat = |token: Option<&str>| request(&app, "POST", "/api/worker/sessions/gpu-state/heartbeat", + with_token(json!({"worker_id":"gpu-worker","lease_seconds":900}), token)); + let close = |token: Option<&str>| request(&app, "DELETE", "/api/worker/sessions/gpu-state", + with_token(json!({"worker_id":"gpu-worker"}), token)); + let shown = || request(&app, "GET", "/api/worker-sessions/gpu-state", Value::Null); - let first = register_holder(&app, "gpu-worker", "sessions").await; - let (status, created) = create(Some(&first)).await; + let first = register_token_holder(&app, "gpu-worker", "sessions").await; + let (status, created) = create(Some(&first), json!({})).await; assert_eq!((status, created["outcome"].clone()), (StatusCode::CREATED, json!("created")), "{created}"); let (status, beat) = request(&app, "POST", "/api/worker/heartbeat", json!({"worker_id":"gpu-worker"})).await; assert_eq!(status, StatusCode::OK, "{beat}"); - let (status, reused) = create(None).await; + let (status, reused) = create(Some(&first), json!({})).await; assert_eq!((status, reused["outcome"].clone()), (StatusCode::OK, json!("reused")), "{reused}"); - let second = register_holder(&app, "gpu-worker", "sessions").await; + let second = register_token_holder(&app, "gpu-worker", "sessions").await; assert_ne!(first, second); - let (status, orphaned) = heartbeat(None).await; + let (status, orphaned) = heartbeat(Some(&second)).await; assert_eq!((status, orphaned["reason"].clone(), orphaned["session"]["status"].clone(), orphaned["session"]["failure_reason"].clone()), (StatusCode::CONFLICT, json!("session_not_active"), json!("orphaned"), json!("worker_registration_replaced")), "{orphaned}"); - for refused in [heartbeat(Some(&first)).await, create(Some(&first)).await] { + for refused in [heartbeat(Some(&first)).await, create(Some(&first), json!({})).await] { assert_eq!((refused.0, refused.1["reason"].clone()), (StatusCode::CONFLICT, json!("stale_worker_registration")), "{}", refused.1); } - let (status, reacquired) = create(Some(&second)).await; + let (status, reacquired) = create(Some(&second), json!({})).await; assert_eq!((status, reacquired["outcome"].clone(), reacquired["session"]["status"].clone()), (StatusCode::OK, json!("reacquired"), json!("active")), "{reacquired}"); - let (status, stale) = heartbeat(Some(&first)).await; - assert_eq!((status, stale["reason"].clone()), (StatusCode::CONFLICT, json!("stale_worker_registration")), "{stale}"); - for current in [heartbeat(Some(&second)).await, heartbeat(None).await] { - assert_eq!((current.0, current.1["outcome"].clone()), (StatusCode::OK, json!("heartbeat_recorded")), "{}", current.1); + let (status, current) = heartbeat(Some(&second)).await; + assert_eq!((status, current["outcome"].clone()), (StatusCode::OK, json!("heartbeat_recorded")), "{current}"); + + // The replaced process can neither reuse, renew nor close the session, + // with its old token or with none, and nothing changes. + let before = shown().await; + assert_eq!(before.0, StatusCode::OK, "{}", before.1); + for stale in [Some(first.as_str()), None] { + for refused in [create(stale, json!({"lease_seconds":5,"max_concurrent_activities":9})).await, heartbeat(stale).await, close(stale).await] { + assert_eq!((refused.0, refused.1["reason"].clone()), (StatusCode::CONFLICT, json!("stale_worker_registration")), "{}", refused.1); + } + assert_eq!(shown().await, before, "a refused request changes nothing"); } - let (status, refused) = request(&app, "DELETE", "/api/worker/sessions/gpu-state", - json!({"worker_id":"gpu-worker","registration_token":first})).await; - assert_eq!((status, refused["reason"].clone()), (StatusCode::CONFLICT, json!("stale_worker_registration")), "{refused}"); - let (status, invalid) = request(&app, "DELETE", "/api/worker/sessions/gpu-state", - json!({"worker_id":"gpu-worker","registration_token":"not-a-token"})).await; + let (status, invalid) = close(Some("not-a-token")).await; assert_eq!((status, invalid["errors"]["registration_token"][0].clone()), (StatusCode::UNPROCESSABLE_ENTITY, json!("The registration token field format is invalid.")), "{invalid}"); - let (status, closed) = request(&app, "DELETE", "/api/worker/sessions/gpu-state", - json!({"worker_id":"gpu-worker","registration_token":second})).await; + let (status, closed) = close(Some(&second)).await; + assert_eq!((status, closed["outcome"].clone()), (StatusCode::OK, json!("closed")), "{closed}"); +} + +// Published workers send no registration token. Without the token +// capability, a request without one acts for the worker ID's current +// registration, as published PHP's does; a token that is sent must be current. +#[tokio::test] +async fn without_the_token_capability_a_tokenless_request_acts_for_the_current_registration() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + let body = json!({"worker_id":"gpu-worker","session_id":"gpu-legacy","requirements":["gpu"]}); + let first = register_holder(&app, "gpu-worker", "sessions").await; + let (status, created) = create(&app, body.clone()).await; + assert_eq!((status, created["outcome"].clone()), (StatusCode::CREATED, json!("created")), "{created}"); + register_holder(&app, "gpu-worker", "sessions").await; + let heartbeat = |token: Option<&str>| { + let mut body = json!({"worker_id":"gpu-worker"}); + if let Some(token) = token { + body["registration_token"] = json!(token); + } + request(&app, "POST", "/api/worker/sessions/gpu-legacy/heartbeat", body) + }; + let (status, orphaned) = heartbeat(None).await; + assert_eq!((status, orphaned["session"]["failure_reason"].clone()), (StatusCode::CONFLICT, json!("worker_registration_replaced")), "{orphaned}"); + let (status, reacquired) = create(&app, body).await; + assert_eq!((status, reacquired["outcome"].clone()), (StatusCode::OK, json!("reacquired")), "{reacquired}"); + let (status, beat) = heartbeat(None).await; + assert_eq!((status, beat["outcome"].clone()), (StatusCode::OK, json!("heartbeat_recorded")), "{beat}"); + let (status, stale) = heartbeat(Some(&first)).await; + assert_eq!((status, stale["reason"].clone()), (StatusCode::CONFLICT, json!("stale_worker_registration")), "{stale}"); + let (status, closed) = request(&app, "DELETE", "/api/worker/sessions/gpu-legacy", json!({"worker_id":"gpu-worker"})).await; assert_eq!((status, closed["outcome"].clone()), (StatusCode::OK, json!("closed")), "{closed}"); } diff --git a/tests/Feature/WorkerSessionProtocolTest.php b/tests/Feature/WorkerSessionProtocolTest.php index 3bcb2b179..120b82726 100644 --- a/tests/Feature/WorkerSessionProtocolTest.php +++ b/tests/Feature/WorkerSessionProtocolTest.php @@ -1107,17 +1107,23 @@ public function test_session_numbers_have_portable_maxima_on_every_path(): void public function test_a_same_id_replacement_reacquires_the_session_and_fences_the_old_process(): void { + // The worker declares that it sends its registration token on every + // explicit session request, so the server requires it. $register = fn (): ?string => $this->registerWorkerThroughProtocol( workerId: 'gpu-worker', taskQueue: 'gpu-activities', supportedActivityTypes: ['tests.external-greeting-activity'], - capabilities: ['gpu:nvidia-l4'], + capabilities: ['gpu:nvidia-l4', WorkerSessionRegistry::REGISTRATION_TOKEN_CAPABILITY], ); $holder = static fn (): ?string => WorkerSessionLease::query()->where('session_id', 'gpu-state')->value('lease_owner_incarnation'); - $create = fn (?string $token = null) => $this->postJson('/api/worker/sessions', array_filter([ + $row = static fn (): array => WorkerSessionLease::query()->where('session_id', 'gpu-state')->firstOrFail()->getAttributes(); + $create = fn (?string $token = null, array $options = []) => $this->postJson('/api/worker/sessions', array_filter([ 'worker_id' => 'gpu-worker', 'session_id' => 'gpu-state', 'requirements' => ['gpu:nvidia-l4'], 'registration_token' => $token, + ...$options, ]), $this->workerHeaders()); $heartbeat = fn (?string $token = null) => $this->postJson('/api/worker/sessions/gpu-state/heartbeat', + array_filter(['worker_id' => 'gpu-worker', 'registration_token' => $token, 'lease_seconds' => 900]), $this->workerHeaders()); + $close = fn (?string $token = null) => $this->deleteJson('/api/worker/sessions/gpu-state', array_filter(['worker_id' => 'gpu-worker', 'registration_token' => $token]), $this->workerHeaders()); $first = $register(); @@ -1127,13 +1133,13 @@ public function test_a_same_id_replacement_reacquires_the_session_and_fences_the // A worker heartbeat keeps the registration, so the holder reuses it. $this->postJson('/api/worker/heartbeat', ['worker_id' => 'gpu-worker'], $this->workerHeaders())->assertOk(); - $create()->assertOk()->assertJsonPath('outcome', 'reused'); + $create($first)->assertOk()->assertJsonPath('outcome', 'reused'); // A same-ID re-registration rotates the token: the session is orphaned. $second = $register(); $this->assertIsString($second); $this->assertNotSame($first, $second); - $heartbeat() + $heartbeat($second) ->assertStatus(409) ->assertJsonPath('reason', 'session_not_active') ->assertJsonPath('session.status', 'orphaned') @@ -1145,13 +1151,49 @@ public function test_a_same_id_replacement_reacquires_the_session_and_fences_the // The replacement reacquires, never reuses, and becomes the holder. $create($second)->assertOk()->assertJsonPath('outcome', 'reacquired')->assertJsonPath('session.status', 'active'); $this->assertSame(hash('sha256', $second), $holder()); - $heartbeat($first)->assertStatus(409)->assertJsonPath('reason', 'stale_worker_registration'); $heartbeat($second)->assertOk()->assertJsonPath('outcome', 'heartbeat_recorded'); + + // The stale process, with its old token or with none, can neither + // reuse, renew nor close the replacement's session, and changes nothing. + $before = $row(); + foreach ([$first, null] as $stale) { + $create($stale, ['lease_seconds' => 5, 'max_concurrent_activities' => 9]) + ->assertStatus(409)->assertJsonPath('reason', 'stale_worker_registration'); + $heartbeat($stale)->assertStatus(409)->assertJsonPath('reason', 'stale_worker_registration'); + $close($stale)->assertStatus(409)->assertJsonPath('reason', 'stale_worker_registration'); + $this->assertSame($before, $row(), 'a refused request changes nothing'); + } + $close($second)->assertOk()->assertJsonPath('outcome', 'closed'); + } + + public function test_without_the_token_capability_a_tokenless_request_acts_for_the_current_registration(): void + { + // Published workers do not send a registration token. For them the + // published contract holds: a request without one acts for the worker + // ID's current registration, while a token that is sent must be current. + $register = fn (): ?string => $this->registerWorkerThroughProtocol( + workerId: 'gpu-worker', + taskQueue: 'gpu-activities', + supportedActivityTypes: ['tests.external-greeting-activity'], + capabilities: ['gpu:nvidia-l4'], + ); + $create = fn (?string $token = null) => $this->postJson('/api/worker/sessions', array_filter([ + 'worker_id' => 'gpu-worker', 'session_id' => 'gpu-legacy', 'requirements' => ['gpu:nvidia-l4'], 'registration_token' => $token, + ]), $this->workerHeaders()); + $heartbeat = fn (?string $token = null) => $this->postJson('/api/worker/sessions/gpu-legacy/heartbeat', + array_filter(['worker_id' => 'gpu-worker', 'registration_token' => $token]), $this->workerHeaders()); + + $first = $register(); + $create()->assertCreated()->assertJsonPath('outcome', 'created'); + $second = $register(); + $heartbeat()->assertStatus(409)->assertJsonPath('session.failure_reason', 'worker_registration_replaced'); + $create()->assertOk()->assertJsonPath('outcome', 'reacquired'); + $this->assertSame(hash('sha256', (string) $second), + WorkerSessionLease::query()->where('session_id', 'gpu-legacy')->value('lease_owner_incarnation')); $heartbeat()->assertOk()->assertJsonPath('outcome', 'heartbeat_recorded'); - $this->deleteJson('/api/worker/sessions/gpu-state', ['worker_id' => 'gpu-worker', 'registration_token' => $first], - $this->workerHeaders())->assertStatus(409)->assertJsonPath('reason', 'stale_worker_registration'); - $this->deleteJson('/api/worker/sessions/gpu-state', ['worker_id' => 'gpu-worker', 'registration_token' => $second], - $this->workerHeaders())->assertOk()->assertJsonPath('outcome', 'closed'); + $heartbeat($first)->assertStatus(409)->assertJsonPath('reason', 'stale_worker_registration'); + $this->deleteJson('/api/worker/sessions/gpu-legacy', ['worker_id' => 'gpu-worker'], $this->workerHeaders()) + ->assertOk()->assertJsonPath('outcome', 'closed'); } public function test_a_same_id_replacement_reacquires_a_session_through_an_activity_claim(): void From 50f87f6adb5a6803f641d3c88e122a04ebb61f94 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Sun, 25 Oct 2026 09:01:17 +0000 Subject: [PATCH 11/13] Server #740: hold the worker registration through each session write Security's R3 found a PHP check-then-write race. The registration fence was a plain read, while same-ID re-registration locks that row before it rotates the token. A session request could pass the fence, let a replacement rotate and reacquire, and then still write. Create could even record the replaced model's incarnation, since it used the registration the controller read before its transaction. Each explicit create, heartbeat and close transaction now first locks the worker's current registration row (lockForUpdate). On SQLite, which ignores FOR UPDATE, it first takes the writer lock with a no-op update, as ExternalPayloadBackupHold does. The token fence checks that locked row, and create admits against it for routing, capabilities and the recorded holder. Native already serializes writes in begin(); its create now also re-reads the registration inside that transaction. WorkerSessionRegistrationRaceTest forks two real MySQL and PostgreSQL connections. A session request paused after its registration lock makes the re-registration wait; a create paused after its first read admits against the replacement's registration. With the previous registry the six lock cases and both published-worker create cases fail. Co-Authored-By: Claude Opus 5.5 --- app/Support/WorkerSessionRegistry.php | 53 ++- rust/src/runtime/worker_sessions.rs | 34 +- .../WorkerSessionRegistrationRaceTest.php | 329 ++++++++++++++++++ 3 files changed, 395 insertions(+), 21 deletions(-) create mode 100644 tests/Feature/WorkerSessionRegistrationRaceTest.php diff --git a/app/Support/WorkerSessionRegistry.php b/app/Support/WorkerSessionRegistry.php index 57e363c41..b17e1b65e 100644 --- a/app/Support/WorkerSessionRegistry.php +++ b/app/Support/WorkerSessionRegistry.php @@ -84,8 +84,18 @@ public function createOrReacquire( ?string $registrationToken = null, ): array { return DB::transaction(function () use ($namespace, $worker, $options, $registrationToken): array { - return $this->staleRegistration($namespace, $worker->worker_id, $registrationToken) - ?? $this->acquireLocked($namespace, $worker, $options, null); + // Admit against the registration as it is now, locked, not as the + // controller read it before the transaction. + $current = $this->lockedRegistration($namespace, $worker->worker_id); + if ($stale = $this->staleRegistration($current, $registrationToken)) { + return $stale; + } + if (! $current instanceof WorkerRegistration) { + return $this->failure('worker_not_registered', 'Worker must be registered before creating a worker session.', 412, + ['worker_id' => $worker->worker_id]); + } + + return $this->acquireLocked($namespace, $current, $options, null); }); } @@ -115,7 +125,7 @@ public function heartbeat( ?string $registrationToken = null, ): array { return DB::transaction(function () use ($namespace, $workerId, $sessionId, $leaseSeconds, $registrationToken): array { - if ($stale = $this->staleRegistration($namespace, $workerId, $registrationToken)) { + if ($stale = $this->staleRegistration($this->lockedRegistration($namespace, $workerId), $registrationToken)) { return $stale; } @@ -208,7 +218,7 @@ public function heartbeatForAttempt(string $namespace, string $attemptId, string public function close(string $namespace, string $workerId, string $sessionId, ?string $registrationToken = null): array { return DB::transaction(function () use ($namespace, $workerId, $sessionId, $registrationToken): array { - if ($stale = $this->staleRegistration($namespace, $workerId, $registrationToken)) { + if ($stale = $this->staleRegistration($this->lockedRegistration($namespace, $workerId), $registrationToken)) { return $stale; } @@ -758,6 +768,33 @@ private function incarnation(WorkerRegistration $worker): ?string return is_string($worker->registration_token) ? hash('sha256', $worker->registration_token) : null; } + /** + * The worker's current registration, locked until the session request + * commits. A same-ID re-registration locks this row before it rotates the + * token, so the request either finishes before the rotation or sees it; + * it never checks one incarnation and writes for the next. SQLite ignores + * FOR UPDATE, so its writer lock is taken first, before any read, as + * ExternalPayloadBackupHold does. + */ + private function lockedRegistration(string $namespace, string $workerId): ?WorkerRegistration + { + if (DB::getDriverName() === 'sqlite') { + DB::table((new WorkerRegistration)->getTable()) + ->where('namespace', $namespace) + ->where('worker_id', $workerId) + ->update(['worker_id' => $workerId]); + } + + /** @var WorkerRegistration|null $current */ + $current = WorkerRegistration::query() + ->where('namespace', $namespace) + ->where('worker_id', $workerId) + ->lockForUpdate() + ->first(); + + return $current; + } + /** * A registration token the request sends must be the worker's current * one. Without a token, a stale process and its same-ID replacement are @@ -769,14 +806,8 @@ private function incarnation(WorkerRegistration $worker): ?string * * @return array|null */ - private function staleRegistration(string $namespace, string $workerId, ?string $registrationToken): ?array + private function staleRegistration(?WorkerRegistration $current, ?string $registrationToken): ?array { - /** @var WorkerRegistration|null $current */ - $current = WorkerRegistration::query() - ->where('namespace', $namespace) - ->where('worker_id', $workerId) - ->first(['registration_token', 'capabilities']); - if ($registrationToken === null) { return $current instanceof WorkerRegistration && in_array(self::REGISTRATION_TOKEN_CAPABILITY, $current->capabilities ?? [], true) diff --git a/rust/src/runtime/worker_sessions.rs b/rust/src/runtime/worker_sessions.rs index 8cd3ae865..4524c58ac 100644 --- a/rust/src/runtime/worker_sessions.rs +++ b/rust/src/runtime/worker_sessions.rs @@ -36,6 +36,8 @@ const SESSIONS_LISTED: &str = "SELECT id,session_id,connection,queue,requirement const HELD_SESSIONS: &str = "SELECT COUNT(*) FROM workflow_worker_sessions WHERE namespace=$1 AND lease_owner=$2 AND status='active' AND (lease_expires_at IS NULL OR lease_expires_at>$3) AND session_id<>$4"; const HOLDER: &str = "SELECT task_queue,status,last_heartbeat_at FROM workflow_worker_registrations WHERE namespace=$1 AND worker_id=$2"; /// The worker's current registration token, as the poll fence reads it. +/// The registration terms a worker session is admitted against. +const SESSION_REGISTRATION: &str = "SELECT task_queue,capabilities,max_concurrent_worker_sessions FROM workflow_worker_registrations WHERE namespace=$1 AND worker_id=$2"; /// The capabilities the worker's current registration declared. const REGISTRATION_CAPABILITIES: &str = "SELECT capabilities FROM workflow_worker_registrations WHERE namespace=$1 AND worker_id=$2"; /// `WorkerSessionRegistry::REGISTRATION_TOKEN_CAPABILITY`: a worker that @@ -472,7 +474,7 @@ store_methods! { sessions_gate(protocol)?; let body = validate(&body, true)?; let worker_id = body["worker_id"].as_str().unwrap_or_default().to_owned(); - let Some(worker) = Self::query("SELECT task_queue,capabilities,max_concurrent_worker_sessions FROM workflow_worker_registrations WHERE namespace=$1 AND worker_id=$2") + let Some(worker) = Self::query(SESSION_REGISTRATION) .bind(self.namespace.as_ref()).bind(&worker_id).fetch_optional(&self.pool).await? else { return Err(RuntimeError::Protocol { status: StatusCode::PRECONDITION_FAILED, @@ -488,18 +490,30 @@ store_methods! { return Err(RuntimeError::Protocol { status: StatusCode::UNPROCESSABLE_ENTITY, response: json!({"error":"Worker session requires a non-empty session_id.","reason":"invalid_worker_session"}) }); }; - let capabilities: Vec = DB::optional_document_row(&worker, "capabilities")? - .as_ref() - .and_then(Value::as_array) - .into_iter() - .flatten() - .filter_map(|value| value.as_str().map(str::to_owned)) - .collect(); - let limit = DB::optional_number(&worker, "max_concurrent_worker_sessions")?.unwrap_or(10); let mut tx = self.begin().await?; + // Admit against the registration as it is inside the transaction, not + // as it was read above, as PHP's createOrReacquire does under its row + // lock; begin() serializes this with every re-registration. + let current = Self::query(SESSION_REGISTRATION) + .bind(self.namespace.as_ref()).bind(&worker_id).fetch_optional(&mut *tx).await?; let answer = match self.stale_registration(&mut tx, &worker_id, body["registration_token"].as_str()).await? { Some(stale) => stale, - None => self.acquire(&mut tx, &worker_id, &task_queue, &capabilities, limit, &options, None).await?, + None => match current { + None => failure("worker_not_registered", "Worker must be registered before creating a worker session.", + StatusCode::PRECONDITION_FAILED, Map::from_iter([("worker_id".to_owned(), json!(worker_id))])), + Some(current) => { + let capabilities: Vec = DB::optional_document_row(¤t, "capabilities")? + .as_ref() + .and_then(Value::as_array) + .into_iter() + .flatten() + .filter_map(|value| value.as_str().map(str::to_owned)) + .collect(); + let limit = DB::optional_number(¤t, "max_concurrent_worker_sessions")?.unwrap_or(10); + let queue = DB::string(¤t, "task_queue")?; + self.acquire(&mut tx, &worker_id, &queue, &capabilities, limit, &options, None).await? + } + }, }; tx.commit().await?; Ok(answer) diff --git a/tests/Feature/WorkerSessionRegistrationRaceTest.php b/tests/Feature/WorkerSessionRegistrationRaceTest.php new file mode 100644 index 000000000..a19378aeb --- /dev/null +++ b/tests/Feature/WorkerSessionRegistrationRaceTest.php @@ -0,0 +1,329 @@ +initializeDatabase($driver); + $first = $this->register(); + $this->postJson('/api/worker/sessions', ['worker_id' => 'race-worker', 'session_id' => 'race-state', + 'requirements' => ['gpu'], 'registration_token' => $first], $this->workerHeaders())->assertCreated(); + $actors = []; + DB::purge(); + try { + $actors['session'] = $this->forkActor(fn (): array => $this->sessionRequest($verb, $first), pause: 'lock'); + $this->readMessage($actors['session']); + fwrite($actors['session']['socket'], "go\n"); + $this->assertTrue($this->readMessage($actors['session'])['registration_locked']); + + $actors['register'] = $this->forkActor(fn (): array => ['token' => $this->register()]); + $replacement = $this->readMessage($actors['register']); + fwrite($actors['register']['socket'], "go\n"); + // The re-registration waits on the row the session request holds. + $this->awaitLockWait($replacement['connection_id']); + + fwrite($actors['session']['socket'], "continue\n"); + $session = $this->finishActor($actors['session']); + unset($actors['session']); + $second = $this->finishActor($actors['register'])['token']; + unset($actors['register']); + } finally { + foreach ($actors as $actor) { + posix_kill($actor['pid'], SIGKILL); + pcntl_waitpid($actor['pid'], $status); + fclose($actor['socket']); + } + DB::purge(); + DB::reconnect(); + } + + // The request ran wholly before the rotation, for the incarnation + // that was current then. + $this->assertSame(200, $session['status'], json_encode($session)); + $this->assertSame(match ($verb) { + 'create' => 'reused', 'heartbeat' => 'heartbeat_recorded', 'close' => 'closed' + }, + $session['outcome'], json_encode($session)); + $this->assertIsString($second); + $this->assertNotSame($first, $second); + $this->assertSame(hash('sha256', $first), + WorkerSessionLease::query()->where('session_id', 'race-state')->value('lease_owner_incarnation')); + + // After the rotation the old process is refused and changes nothing. + $before = WorkerSessionLease::query()->where('session_id', 'race-state')->firstOrFail()->getAttributes(); + $this->sessionResponse($verb, $first)->assertStatus(409)->assertJsonPath('reason', 'stale_worker_registration'); + $this->assertSame($before, WorkerSessionLease::query()->where('session_id', 'race-state')->firstOrFail()->getAttributes()); + } + + public static function creates(): array + { + $cases = []; + foreach (['mysql', 'pgsql'] as $driver) { + $cases[$driver.' token worker'] = [$driver, true]; + $cases[$driver.' published worker'] = [$driver, false]; + } + + return $cases; + } + + /** + * A create read the worker's registration before a same-ID replacement + * registered and reacquired the session, and continues only afterwards. + * It is admitted against the registration as it is when it commits: a + * worker that sends its token is refused with its old token and changes + * nothing, and a published tokenless create reuses the replacement's + * session. Neither restores the old incarnation. + */ + #[DataProvider('creates')] + public function test_a_create_admits_against_the_registration_current_when_it_commits(string $driver, bool $sendsToken): void + { + $this->initializeDatabase($driver); + $first = $this->register($sendsToken); + $this->sessionResponse('create', $sendsToken ? $first : null)->assertCreated(); + $actors = []; + DB::purge(); + try { + $actors['session'] = $this->forkActor(fn (): array => $this->sessionRequest('create', $sendsToken ? $first : null), pause: 'read'); + $this->readMessage($actors['session']); + fwrite($actors['session']['socket'], "go\n"); + $this->assertTrue($this->readMessage($actors['session'])['registration_locked']); + + // The replacement registers and reacquires while the create waits + // between its first read and its transaction. + $second = $this->register($sendsToken); + $this->sessionResponse('heartbeat', $sendsToken ? $second : null)->assertStatus(409) + ->assertJsonPath('session.failure_reason', 'worker_registration_replaced'); + $this->sessionResponse('create', $sendsToken ? $second : null)->assertOk()->assertJsonPath('outcome', 'reacquired'); + $before = WorkerSessionLease::query()->where('session_id', 'race-state')->firstOrFail()->getAttributes(); + + fwrite($actors['session']['socket'], "continue\n"); + $session = $this->finishActor($actors['session']); + unset($actors['session']); + } finally { + foreach ($actors as $actor) { + posix_kill($actor['pid'], SIGKILL); + pcntl_waitpid($actor['pid'], $status); + fclose($actor['socket']); + } + DB::purge(); + DB::reconnect(); + } + + $this->assertNotSame($first, $second); + $after = WorkerSessionLease::query()->where('session_id', 'race-state')->firstOrFail()->getAttributes(); + if ($sendsToken) { + $this->assertSame([409, 'stale_worker_registration'], [$session['status'], $session['reason']], json_encode($session)); + $this->assertSame($before, $after, 'the stale create changes nothing'); + } else { + $this->assertSame([200, 'reused'], [$session['status'], $session['outcome']], json_encode($session)); + } + $this->assertSame(hash('sha256', $second), $after['lease_owner_incarnation'], 'the session keeps the replacement as its holder'); + } + + private function initializeDatabase(string $driver): void + { + $prefix = $driver === 'pgsql' ? 'DW_TEST_COMPLETION_PGSQL_' : 'DW_TEST_COMPLETION_MYSQL_'; + $host = getenv($prefix.'HOST'); + if (! $host) { + $this->markTestSkipped('Set '.$prefix.'HOST to a disposable database test server.'); + } + if (! function_exists('pcntl_fork') || ! function_exists('posix_kill') || ! function_exists('stream_socket_pair')) { + $this->markTestSkipped('Process control and local sockets are required.'); + } + $port = getenv($prefix.'PORT') ?: ($driver === 'pgsql' ? '5432' : '3306'); + $user = getenv($prefix.'USER') ?: ($driver === 'pgsql' ? 'postgres' : 'root'); + $password = getenv($prefix.'PASSWORD') ?: ''; + $this->databaseAdmin = [$driver.':host='.$host.';port='.$port.($driver === 'pgsql' ? ';dbname=postgres' : ''), $user, $password]; + $this->databaseName = 'dw_session_race_'.bin2hex(random_bytes(6)); + $admin = new PDO(...$this->databaseAdmin); + $admin->exec('CREATE DATABASE '.$this->databaseName); + unset($admin); + config(['database.default' => $driver, 'database.connections.'.$driver.'.host' => $host, + 'database.connections.'.$driver.'.port' => $port, 'database.connections.'.$driver.'.database' => $this->databaseName, + 'database.connections.'.$driver.'.username' => $user, 'database.connections.'.$driver.'.password' => $password, + 'database.connections.'.$driver.'.url' => null]); + DB::purge($driver); + $this->artisan('migrate:fresh', ['--force' => true])->assertExitCode(0); + $this->createNamespace('default'); + } + + protected function tearDown(): void + { + if ($this->databaseAdmin !== null) { + DB::disconnect(); + $admin = new PDO(...$this->databaseAdmin); + $admin->exec('DROP DATABASE '.$this->databaseName); + } + parent::tearDown(); + } + + /** Registers race-worker, by default declaring that it sends its token; returns the token. */ + private function register(bool $sendsToken = true): string + { + return (string) $this->postJson('/api/worker/register', [ + 'worker_id' => 'race-worker', + 'task_queue' => 'race-queue', + 'runtime' => 'php', + 'supported_activity_types' => ['tests.race-activity'], + 'capabilities' => ['gpu', 'worker_sessions', ...($sendsToken ? [WorkerSessionRegistry::REGISTRATION_TOKEN_CAPABILITY] : [])], + 'capability_manifest' => array_replace($this->portableWorkerAffinityRefusalManifest(), [ + 'worker_sessions' => ['supported' => true, + 'minimum_protocol_version' => WorkerProtocol::PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION, + 'implementation' => 'worker_session_registration_race_test'], + ]), + ], $this->workerHeaders())->assertCreated()->json('registration_token'); + } + + private function sessionResponse(string $verb, ?string $token) + { + return match ($verb) { + 'create' => $this->postJson('/api/worker/sessions', array_filter(['worker_id' => 'race-worker', 'session_id' => 'race-state', + 'requirements' => ['gpu'], 'registration_token' => $token]), $this->workerHeaders()), + 'heartbeat' => $this->postJson('/api/worker/sessions/race-state/heartbeat', + ['worker_id' => 'race-worker', 'registration_token' => $token, 'lease_seconds' => 900], $this->workerHeaders()), + 'close' => $this->deleteJson('/api/worker/sessions/race-state', + ['worker_id' => 'race-worker', 'registration_token' => $token], $this->workerHeaders()), + }; + } + + /** @return array */ + private function sessionRequest(string $verb, ?string $token): array + { + $response = $this->sessionResponse($verb, $token); + + return ['status' => $response->status(), 'outcome' => $response->json('outcome'), 'reason' => $response->json('reason')]; + } + + /** + * Runs $work in a child process on its own database connection. The child + * reports its connection id, waits for "go", and with $pause stops until + * "continue" right after it first reads the worker's registration ('read') + * or locks it ('lock'). + * + * @return array{pid: int, socket: resource} + */ + private function forkActor(callable $work, ?string $pause = null): array + { + $sockets = stream_socket_pair(STREAM_PF_UNIX, STREAM_SOCK_STREAM, STREAM_IPPROTO_IP); + $this->assertIsArray($sockets); + $pid = pcntl_fork(); + $this->assertNotSame(-1, $pid); + if ($pid === 0) { + fclose($sockets[0]); + stream_set_timeout($sockets[1], 15); + $ok = false; + try { + DB::reconnect(); + $idQuery = DB::connection()->getDriverName() === 'mysql' + ? 'SELECT CONNECTION_ID() AS id' : 'SELECT pg_backend_pid() AS id'; + fwrite($sockets[1], json_encode(['connection_id' => (int) DB::selectOne($idQuery)->id]).PHP_EOL); + if (fgets($sockets[1]) !== "go\n") { + throw new RuntimeException('Actor was not released.'); + } + if ($pause !== null) { + $paused = false; + DB::listen(static function (QueryExecuted $query) use (&$paused, $sockets, $pause): void { + if (! $paused && str_contains($query->sql, 'workflow_worker_registrations') + && ($pause === 'read' || str_contains(strtolower($query->sql), 'for update'))) { + $paused = true; + fwrite($sockets[1], json_encode(['registration_locked' => true]).PHP_EOL); + if (fgets($sockets[1]) !== "continue\n") { + throw new RuntimeException('Session request was not released.'); + } + } + }); + } + fwrite($sockets[1], json_encode(['result' => $work()], JSON_THROW_ON_ERROR).PHP_EOL); + $ok = true; + } catch (Throwable $error) { + fwrite($sockets[1], json_encode(['error' => $error::class.': '.$error->getMessage()]).PHP_EOL); + } finally { + DB::disconnect(); + fclose($sockets[1]); + } + exit($ok ? 0 : 1); + } + fclose($sockets[1]); + stream_set_timeout($sockets[0], 15); + + return ['pid' => $pid, 'socket' => $sockets[0]]; + } + + private function awaitLockWait(int $connectionId): void + { + $query = DB::connection()->getDriverName() === 'mysql' + ? 'SELECT COUNT(*) AS waiting FROM performance_schema.data_lock_waits w JOIN performance_schema.threads t ON t.THREAD_ID = w.REQUESTING_THREAD_ID WHERE t.PROCESSLIST_ID = ?' + : 'SELECT COUNT(*) AS waiting FROM pg_locks WHERE pid = ? AND NOT granted'; + $until = microtime(true) + 5; + do { + if ((int) DB::selectOne($query, [$connectionId])->waiting > 0) { + $this->addToAssertionCount(1); + + return; + } + usleep(10_000); + } while (microtime(true) < $until); + $this->fail('The re-registration did not wait on the session request\'s registration lock, connection '.$connectionId.'.'); + } + + private function readMessage(array $actor): array + { + $line = fgets($actor['socket']); + $this->assertIsString($line, 'Actor did not respond.'); + $message = json_decode($line, true, flags: JSON_THROW_ON_ERROR); + $this->assertArrayNotHasKey('error', $message, $message['error'] ?? ''); + + return $message; + } + + private function finishActor(array $actor): array + { + $message = $this->readMessage($actor); + pcntl_waitpid($actor['pid'], $status); + $this->assertTrue(pcntl_wifexited($status)); + $this->assertSame(0, pcntl_wexitstatus($status)); + fclose($actor['socket']); + + return $message['result']; + } +} From 7f79034a21b70cb567883ee37e78cef93fe3725c Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Sun, 25 Oct 2026 09:32:43 +0000 Subject: [PATCH 12/13] Server #740: refuse session requests from a worker with no current registration Security's R4: after deregistration deletes the worker's registration, an explicit heartbeat or close without a token passed the fence (no registration, so no declared token capability) and reached the old session, which it could renew, mark or close. Each explicit create, heartbeat and close now requires the worker's current registration inside its fenced transaction, before the token rules: without one it answers 412 worker_not_registered and changes nothing, with or without a token. Tokenless compatibility applies only while a current registration exists without the token capability. PHP and native match, with the same messages. Tests: PHP and native deregister a token worker (fenced route) and a published worker (plain route), then send heartbeat and close with the old token and without; each is refused and the session is unchanged. With the R3 code PHP answers 409 instead of 412 and native reaches the session (409 session_not_active). The two-connection race test adds a deregistration that waits on a paused heartbeat or close's registration lock, after which the same request is refused. Co-Authored-By: Claude Opus 5.5 --- app/Support/WorkerSessionRegistry.php | 40 ++++++++---- rust/src/runtime/worker_sessions.rs | 40 +++++++++--- rust/tests/execution/worker_sessions.rs | 54 ++++++++++++++++ tests/Feature/WorkerSessionProtocolTest.php | 41 +++++++++++++ .../WorkerSessionRegistrationRaceTest.php | 61 +++++++++++++++++++ 5 files changed, 215 insertions(+), 21 deletions(-) diff --git a/app/Support/WorkerSessionRegistry.php b/app/Support/WorkerSessionRegistry.php index b17e1b65e..a579920a5 100644 --- a/app/Support/WorkerSessionRegistry.php +++ b/app/Support/WorkerSessionRegistry.php @@ -87,13 +87,12 @@ public function createOrReacquire( // Admit against the registration as it is now, locked, not as the // controller read it before the transaction. $current = $this->lockedRegistration($namespace, $worker->worker_id); + if (! $current instanceof WorkerRegistration) { + return $this->notRegistered($worker->worker_id, 'creating'); + } if ($stale = $this->staleRegistration($current, $registrationToken)) { return $stale; } - if (! $current instanceof WorkerRegistration) { - return $this->failure('worker_not_registered', 'Worker must be registered before creating a worker session.', 412, - ['worker_id' => $worker->worker_id]); - } return $this->acquireLocked($namespace, $current, $options, null); }); @@ -125,7 +124,11 @@ public function heartbeat( ?string $registrationToken = null, ): array { return DB::transaction(function () use ($namespace, $workerId, $sessionId, $leaseSeconds, $registrationToken): array { - if ($stale = $this->staleRegistration($this->lockedRegistration($namespace, $workerId), $registrationToken)) { + $current = $this->lockedRegistration($namespace, $workerId); + if (! $current instanceof WorkerRegistration) { + return $this->notRegistered($workerId, 'renewing'); + } + if ($stale = $this->staleRegistration($current, $registrationToken)) { return $stale; } @@ -218,7 +221,11 @@ public function heartbeatForAttempt(string $namespace, string $attemptId, string public function close(string $namespace, string $workerId, string $sessionId, ?string $registrationToken = null): array { return DB::transaction(function () use ($namespace, $workerId, $sessionId, $registrationToken): array { - if ($stale = $this->staleRegistration($this->lockedRegistration($namespace, $workerId), $registrationToken)) { + $current = $this->lockedRegistration($namespace, $workerId); + if (! $current instanceof WorkerRegistration) { + return $this->notRegistered($workerId, 'closing'); + } + if ($stale = $this->staleRegistration($current, $registrationToken)) { return $stale; } @@ -795,6 +802,19 @@ private function lockedRegistration(string $namespace, string $workerId): ?Worke return $current; } + /** + * A session request needs the worker's current registration, with or + * without a token: once deregistration deletes it, the former process can + * no longer renew, close or recreate the session it held. + * + * @return array + */ + private function notRegistered(string $workerId, string $action): array + { + return $this->failure('worker_not_registered', sprintf('Worker must be registered before %s a worker session.', $action), 412, + ['worker_id' => $workerId]); + } + /** * A registration token the request sends must be the worker's current * one. Without a token, a stale process and its same-ID replacement are @@ -806,11 +826,10 @@ private function lockedRegistration(string $namespace, string $workerId): ?Worke * * @return array|null */ - private function staleRegistration(?WorkerRegistration $current, ?string $registrationToken): ?array + private function staleRegistration(WorkerRegistration $current, ?string $registrationToken): ?array { if ($registrationToken === null) { - return $current instanceof WorkerRegistration - && in_array(self::REGISTRATION_TOKEN_CAPABILITY, $current->capabilities ?? [], true) + return in_array(self::REGISTRATION_TOKEN_CAPABILITY, $current->capabilities ?? [], true) ? $this->failure( 'stale_worker_registration', 'The worker\'s current registration requires its registration token on worker session requests. Send it, or register again and reacquire the session.', @@ -819,8 +838,7 @@ private function staleRegistration(?WorkerRegistration $current, ?string $regist : null; } - return $current instanceof WorkerRegistration - && is_string($current->registration_token) + return is_string($current->registration_token) && hash_equals($current->registration_token, $registrationToken) ? null : $this->failure( diff --git a/rust/src/runtime/worker_sessions.rs b/rust/src/runtime/worker_sessions.rs index 4524c58ac..886700603 100644 --- a/rust/src/runtime/worker_sessions.rs +++ b/rust/src/runtime/worker_sessions.rs @@ -436,6 +436,13 @@ fn success(outcome: &str, session: Value, status: StatusCode) -> Answer { "status":status.as_u16(),"reason":null}) } } +/// `notRegistered`: an explicit session request from a worker without a +/// current registration. +fn not_registered(worker_id: &str, action: &str) -> Answer { + failure("worker_not_registered", &format!("Worker must be registered before {action} a worker session."), + StatusCode::PRECONDITION_FAILED, Map::from_iter([("worker_id".to_owned(), json!(worker_id))])) +} + fn failure(reason: &str, error: &str, status: StatusCode, context: Map) -> Answer { let mut body = context; body.insert("admitted".into(), json!(false)); @@ -496,12 +503,11 @@ store_methods! { // lock; begin() serializes this with every re-registration. let current = Self::query(SESSION_REGISTRATION) .bind(self.namespace.as_ref()).bind(&worker_id).fetch_optional(&mut *tx).await?; - let answer = match self.stale_registration(&mut tx, &worker_id, body["registration_token"].as_str()).await? { - Some(stale) => stale, - None => match current { - None => failure("worker_not_registered", "Worker must be registered before creating a worker session.", - StatusCode::PRECONDITION_FAILED, Map::from_iter([("worker_id".to_owned(), json!(worker_id))])), - Some(current) => { + let answer = match current { + None => not_registered(&worker_id, "creating"), + Some(current) => match self.stale_registration(&mut tx, &worker_id, body["registration_token"].as_str()).await? { + Some(stale) => stale, + None => { let capabilities: Vec = DB::optional_document_row(¤t, "capabilities")? .as_ref() .and_then(Value::as_array) @@ -800,9 +806,9 @@ store_methods! { let body = validate(&body, false)?; let worker_id = body["worker_id"].as_str().unwrap_or_default(); let mut tx = self.begin().await?; - if let Some(stale) = self.stale_registration(&mut tx, worker_id, body["registration_token"].as_str()).await? { + if let Some(refused) = self.session_request_fence(&mut tx, worker_id, body["registration_token"].as_str(), "renewing").await? { tx.commit().await?; - return Ok(stale); + return Ok(refused); } let Some(mut session) = self.locked_session(&mut tx, session_id).await? else { tx.commit().await?; @@ -832,9 +838,9 @@ store_methods! { let body = validate(&body, false)?; let worker_id = body["worker_id"].as_str().unwrap_or_default(); let mut tx = self.begin().await?; - if let Some(stale) = self.stale_registration(&mut tx, worker_id, body["registration_token"].as_str()).await? { + if let Some(refused) = self.session_request_fence(&mut tx, worker_id, body["registration_token"].as_str(), "closing").await? { tx.commit().await?; - return Ok(stale); + return Ok(refused); } let Some(mut session) = self.locked_session(&mut tx, session_id).await? else { tx.commit().await?; @@ -973,6 +979,20 @@ store_methods! { Ok(()) } + /// The fence of an explicit heartbeat or close: the worker must have a + /// current registration, with or without a token, so a process whose + /// registration was deleted can no longer renew or close the session it + /// held; then the registration token rules apply. + async fn session_request_fence(&self, tx: &mut Transaction<'_, DB>, worker_id: &str, token: Option<&str>, action: &str) + -> Result> { + let registered = Self::query(SESSION_REGISTRATION) + .bind(self.namespace.as_ref()).bind(worker_id).fetch_optional(&mut **tx).await?; + if registered.is_none() { + return Ok(Some(not_registered(worker_id, action))); + } + self.stale_registration(tx, worker_id, token).await + } + /// A registration token the request sent must be the worker's current /// one. Without a token, a stale process and its same-ID replacement are /// indistinguishable once the replacement has reacquired, so a worker diff --git a/rust/tests/execution/worker_sessions.rs b/rust/tests/execution/worker_sessions.rs index a26fa3a2b..2b03dc3b0 100644 --- a/rust/tests/execution/worker_sessions.rs +++ b/rust/tests/execution/worker_sessions.rs @@ -312,6 +312,60 @@ async fn a_same_id_replacement_reacquires_the_session_and_fences_the_old_process assert_eq!((status, closed["outcome"].clone()), (StatusCode::OK, json!("closed")), "{closed}"); } +// Deregistration deletes the registration and leaves the session for +// holder-loss handling. The former process, with its token or without, can +// neither renew nor close the session, and nothing changes: a token worker +// deregistered through the fenced route, a published one through the plain one. +#[tokio::test] +async fn a_deregistered_worker_can_neither_renew_nor_close_its_session() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + for sends_token in [true, false] { + let worker = if sends_token { "gpu-token-worker" } else { "gpu-published-worker" }; + let session = format!("{worker}-state"); + let token = if sends_token { + register_token_holder(&app, worker, "sessions").await + } else { + register_holder(&app, worker, "sessions").await + }; + let mut body = json!({"worker_id":worker,"session_id":session,"requirements":["gpu"]}); + if sends_token { + body["registration_token"] = json!(token); + } + let (status, created) = create(&app, body).await; + assert_eq!(status, StatusCode::CREATED, "{created}"); + let shown_path = format!("/api/worker-sessions/{session}"); + let shown = || request(&app, "GET", &shown_path, Value::Null); + let (status, deregistered) = if sends_token { + request(&app, "POST", &format!("/api/worker/registrations/{worker}/deregister"), json!({"registration_token":token})).await + } else { + request(&app, "DELETE", &format!("/api/worker/registrations/{worker}"), json!({})).await + }; + assert_eq!(status, StatusCode::OK, "{deregistered}"); + // The session shows its holder lost on read, as PHP's does; that read + // settles it before the refused requests are compared with it. + let before = shown().await; + assert_eq!((before.0, before.1["session"]["status"].clone()), (StatusCode::OK, json!("orphaned")), "{}", before.1); + for sent in [None, Some(token.as_str())] { + let with_token = |mut body: Value| { + if let Some(sent) = sent { + body["registration_token"] = json!(sent); + } + body + }; + let (status, renewed) = request(&app, "POST", &format!("/api/worker/sessions/{session}/heartbeat"), + with_token(json!({"worker_id":worker,"lease_seconds":900}))).await; + assert_eq!((status, renewed["reason"].clone(), renewed["error"].clone()), (StatusCode::PRECONDITION_FAILED, + json!("worker_not_registered"), json!("Worker must be registered before renewing a worker session.")), "{renewed}"); + let (status, closed) = request(&app, "DELETE", &format!("/api/worker/sessions/{session}"), + with_token(json!({"worker_id":worker}))).await; + assert_eq!((status, closed["reason"].clone(), closed["error"].clone()), (StatusCode::PRECONDITION_FAILED, + json!("worker_not_registered"), json!("Worker must be registered before closing a worker session.")), "{closed}"); + assert_eq!(shown().await, before, "a refused request changes nothing"); + } + } +} + // Published workers send no registration token. Without the token // capability, a request without one acts for the worker ID's current // registration, as published PHP's does; a token that is sent must be current. diff --git a/tests/Feature/WorkerSessionProtocolTest.php b/tests/Feature/WorkerSessionProtocolTest.php index 120b82726..700cb88f7 100644 --- a/tests/Feature/WorkerSessionProtocolTest.php +++ b/tests/Feature/WorkerSessionProtocolTest.php @@ -1166,6 +1166,47 @@ public function test_a_same_id_replacement_reacquires_the_session_and_fences_the $close($second)->assertOk()->assertJsonPath('outcome', 'closed'); } + public function test_a_deregistered_worker_can_neither_renew_nor_close_its_session(): void + { + // Deregistration deletes the registration and leaves the session for + // holder-loss handling. The former process, with its token or without, + // is refused and changes nothing: a token worker deregistered through + // the fenced route, a published worker through the plain one. + foreach ([true, false] as $sendsToken) { + $workerId = $sendsToken ? 'gpu-token-worker' : 'gpu-published-worker'; + $sessionId = $workerId.'-state'; + $token = $this->registerWorkerThroughProtocol( + workerId: $workerId, + taskQueue: 'gpu-activities', + supportedActivityTypes: ['tests.external-greeting-activity'], + capabilities: ['gpu:nvidia-l4', ...($sendsToken ? [WorkerSessionRegistry::REGISTRATION_TOKEN_CAPABILITY] : [])], + ); + $this->assertIsString($token); + $this->postJson('/api/worker/sessions', array_filter(['worker_id' => $workerId, 'session_id' => $sessionId, + 'requirements' => ['gpu:nvidia-l4'], 'registration_token' => $sendsToken ? $token : null]), $this->workerHeaders()) + ->assertCreated(); + $row = static fn (): array => WorkerSessionLease::query()->where('session_id', $sessionId)->firstOrFail()->getAttributes(); + $before = $row(); + if ($sendsToken) { + $this->postJson("/api/worker/registrations/{$workerId}/deregister", ['registration_token' => $token], + $this->workerHeaders())->assertOk(); + } else { + $this->deleteJson("/api/worker/registrations/{$workerId}", [], $this->workerHeaders())->assertOk(); + } + foreach ([null, $token] as $sent) { + $this->postJson("/api/worker/sessions/{$sessionId}/heartbeat", + array_filter(['worker_id' => $workerId, 'registration_token' => $sent, 'lease_seconds' => 900]), $this->workerHeaders()) + ->assertStatus(412)->assertJsonPath('reason', 'worker_not_registered') + ->assertJsonPath('error', 'Worker must be registered before renewing a worker session.'); + $this->deleteJson("/api/worker/sessions/{$sessionId}", + array_filter(['worker_id' => $workerId, 'registration_token' => $sent]), $this->workerHeaders()) + ->assertStatus(412)->assertJsonPath('reason', 'worker_not_registered') + ->assertJsonPath('error', 'Worker must be registered before closing a worker session.'); + $this->assertSame($before, $row(), 'a refused request changes nothing'); + } + } + } + public function test_without_the_token_capability_a_tokenless_request_acts_for_the_current_registration(): void { // Published workers do not send a registration token. For them the diff --git a/tests/Feature/WorkerSessionRegistrationRaceTest.php b/tests/Feature/WorkerSessionRegistrationRaceTest.php index a19378aeb..3c9a848b2 100644 --- a/tests/Feature/WorkerSessionRegistrationRaceTest.php +++ b/tests/Feature/WorkerSessionRegistrationRaceTest.php @@ -95,6 +95,67 @@ public function test_a_session_request_holds_the_registration_until_it_commits(s $this->assertSame($before, WorkerSessionLease::query()->where('session_id', 'race-state')->firstOrFail()->getAttributes()); } + public static function deregistrations(): array + { + $cases = []; + foreach (['mysql', 'pgsql'] as $driver) { + foreach (['heartbeat', 'close'] as $verb) { + $cases[$driver.' '.$verb] = [$driver, $verb]; + } + } + + return $cases; + } + + /** + * A published tokenless heartbeat or close and the worker's deregistration + * serialize on the registration row: the request either commits while the + * registration exists, as here, or sees none and is refused, as it is + * afterwards, without changing the session. + */ + #[DataProvider('deregistrations')] + public function test_a_session_request_and_a_deregistration_serialize(string $driver, string $verb): void + { + $this->initializeDatabase($driver); + $this->register(sendsToken: false); + $this->sessionResponse('create', null)->assertCreated(); + $actors = []; + DB::purge(); + try { + $actors['session'] = $this->forkActor(fn (): array => $this->sessionRequest($verb, null), pause: 'lock'); + $this->readMessage($actors['session']); + fwrite($actors['session']['socket'], "go\n"); + $this->assertTrue($this->readMessage($actors['session'])['registration_locked']); + + $actors['deregister'] = $this->forkActor(fn (): array => ['status' => $this->deleteJson('/api/worker/registrations/race-worker', + [], $this->workerHeaders())->status()]); + $deregistration = $this->readMessage($actors['deregister']); + fwrite($actors['deregister']['socket'], "go\n"); + // The deregistration waits on the row the session request holds. + $this->awaitLockWait($deregistration['connection_id']); + + fwrite($actors['session']['socket'], "continue\n"); + $session = $this->finishActor($actors['session']); + unset($actors['session']); + $this->assertSame(200, $this->finishActor($actors['deregister'])['status']); + unset($actors['deregister']); + } finally { + foreach ($actors as $actor) { + posix_kill($actor['pid'], SIGKILL); + pcntl_waitpid($actor['pid'], $status); + fclose($actor['socket']); + } + DB::purge(); + DB::reconnect(); + } + + $this->assertSame([200, $verb === 'heartbeat' ? 'heartbeat_recorded' : 'closed'], [$session['status'], $session['outcome']], + json_encode($session)); + $before = WorkerSessionLease::query()->where('session_id', 'race-state')->firstOrFail()->getAttributes(); + $this->sessionResponse($verb, null)->assertStatus(412)->assertJsonPath('reason', 'worker_not_registered'); + $this->assertSame($before, WorkerSessionLease::query()->where('session_id', 'race-state')->firstOrFail()->getAttributes()); + } + public static function creates(): array { $cases = []; From cd8beca3dbc7b73c9bb2987043dca3bee54682d4 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Sun, 25 Oct 2026 10:13:02 +0000 Subject: [PATCH 13/13] Server #740: refuse session requests from a superseded registration Security's R5: plain deregistration commits the registration as superseded, recovers its leases and only then deletes the row. A published tokenless create, heartbeat or close in that window still passed the fence and could reacquire, renew or close the session. Each explicit create, heartbeat and close now refuses a locked registration whose status is superseded, right after the absent-row check and before the token rules, with the refusal the worker heartbeat already gives: 409 worker_registration_superseded and its remediation. Native, whose plain deregistration is atomic, answers identically. Tests: PHP and native refuse create, heartbeat and close from a superseded registration of a token worker and of a published worker, with the session unchanged. The race test pauses a real deregistration after its superseded commit, before the delete, and shows tokenless create, heartbeat and close refused in that window, then 412 once the row is gone. With the R4 code each of these is accepted (200). Co-Authored-By: Claude Opus 5.5 --- app/Support/WorkerSessionRegistry.php | 26 +++++++ rust/src/runtime/worker_sessions.rs | 21 +++++- rust/tests/execution/worker_sessions.rs | 43 +++++++++++ tests/Feature/WorkerSessionProtocolTest.php | 35 +++++++++ .../WorkerSessionRegistrationRaceTest.php | 75 +++++++++++++++++-- 5 files changed, 193 insertions(+), 7 deletions(-) diff --git a/app/Support/WorkerSessionRegistry.php b/app/Support/WorkerSessionRegistry.php index a579920a5..f59c95333 100644 --- a/app/Support/WorkerSessionRegistry.php +++ b/app/Support/WorkerSessionRegistry.php @@ -90,6 +90,9 @@ public function createOrReacquire( if (! $current instanceof WorkerRegistration) { return $this->notRegistered($worker->worker_id, 'creating'); } + if ($superseded = $this->supersededRegistration($current)) { + return $superseded; + } if ($stale = $this->staleRegistration($current, $registrationToken)) { return $stale; } @@ -128,6 +131,9 @@ public function heartbeat( if (! $current instanceof WorkerRegistration) { return $this->notRegistered($workerId, 'renewing'); } + if ($superseded = $this->supersededRegistration($current)) { + return $superseded; + } if ($stale = $this->staleRegistration($current, $registrationToken)) { return $stale; } @@ -225,6 +231,9 @@ public function close(string $namespace, string $workerId, string $sessionId, ?s if (! $current instanceof WorkerRegistration) { return $this->notRegistered($workerId, 'closing'); } + if ($superseded = $this->supersededRegistration($current)) { + return $superseded; + } if ($stale = $this->staleRegistration($current, $registrationToken)) { return $stale; } @@ -815,6 +824,23 @@ private function notRegistered(string $workerId, string $action): array ['worker_id' => $workerId]); } + /** + * A registration fenced as superseded, by deregistration before its row is + * deleted or by lease recovery, has lost its authority, as the worker + * heartbeat and polls already treat it. + * + * @return array|null + */ + private function supersededRegistration(WorkerRegistration $current): ?array + { + return $current->status === WorkerRegistration::STATUS_SUPERSEDED + ? $this->failure('worker_registration_superseded', 'Worker registration was fenced after its leases were recovered.', 409, [ + 'worker_id' => $current->worker_id, + 'remediation' => 'Restart and register the worker with a new worker_id.', + ]) + : null; + } + /** * A registration token the request sends must be the worker's current * one. Without a token, a stale process and its same-ID replacement are diff --git a/rust/src/runtime/worker_sessions.rs b/rust/src/runtime/worker_sessions.rs index 886700603..57f8bfdd3 100644 --- a/rust/src/runtime/worker_sessions.rs +++ b/rust/src/runtime/worker_sessions.rs @@ -37,7 +37,7 @@ const HELD_SESSIONS: &str = "SELECT COUNT(*) FROM workflow_worker_sessions WHERE const HOLDER: &str = "SELECT task_queue,status,last_heartbeat_at FROM workflow_worker_registrations WHERE namespace=$1 AND worker_id=$2"; /// The worker's current registration token, as the poll fence reads it. /// The registration terms a worker session is admitted against. -const SESSION_REGISTRATION: &str = "SELECT task_queue,capabilities,max_concurrent_worker_sessions FROM workflow_worker_registrations WHERE namespace=$1 AND worker_id=$2"; +const SESSION_REGISTRATION: &str = "SELECT task_queue,capabilities,max_concurrent_worker_sessions,status FROM workflow_worker_registrations WHERE namespace=$1 AND worker_id=$2"; /// The capabilities the worker's current registration declared. const REGISTRATION_CAPABILITIES: &str = "SELECT capabilities FROM workflow_worker_registrations WHERE namespace=$1 AND worker_id=$2"; /// `WorkerSessionRegistry::REGISTRATION_TOKEN_CAPABILITY`: a worker that @@ -436,6 +436,14 @@ fn success(outcome: &str, session: Value, status: StatusCode) -> Answer { "status":status.as_u16(),"reason":null}) } } +/// `supersededRegistration`: a registration fenced as superseded has lost +/// its authority, as PHP's worker heartbeat treats it. +fn superseded_answer(worker_id: &str) -> Answer { + failure("worker_registration_superseded", "Worker registration was fenced after its leases were recovered.", + StatusCode::CONFLICT, Map::from_iter([("worker_id".to_owned(), json!(worker_id)), + ("remediation".to_owned(), json!("Restart and register the worker with a new worker_id."))])) +} + /// `notRegistered`: an explicit session request from a worker without a /// current registration. fn not_registered(worker_id: &str, action: &str) -> Answer { @@ -505,6 +513,7 @@ store_methods! { .bind(self.namespace.as_ref()).bind(&worker_id).fetch_optional(&mut *tx).await?; let answer = match current { None => not_registered(&worker_id, "creating"), + Some(current) if Self::registration_superseded(¤t)? => superseded_answer(&worker_id), Some(current) => match self.stale_registration(&mut tx, &worker_id, body["registration_token"].as_str()).await? { Some(stale) => stale, None => { @@ -979,6 +988,11 @@ store_methods! { Ok(()) } + /// Whether a registration row is fenced as superseded. + fn registration_superseded(row: &DB::Row) -> Result { + Ok(DB::optional_string(row, "status")?.as_deref() == Some("superseded")) + } + /// The fence of an explicit heartbeat or close: the worker must have a /// current registration, with or without a token, so a process whose /// registration was deleted can no longer renew or close the session it @@ -987,8 +1001,11 @@ store_methods! { -> Result> { let registered = Self::query(SESSION_REGISTRATION) .bind(self.namespace.as_ref()).bind(worker_id).fetch_optional(&mut **tx).await?; - if registered.is_none() { + let Some(registered) = registered else { return Ok(Some(not_registered(worker_id, action))); + }; + if Self::registration_superseded(®istered)? { + return Ok(Some(superseded_answer(worker_id))); } self.stale_registration(tx, worker_id, token).await } diff --git a/rust/tests/execution/worker_sessions.rs b/rust/tests/execution/worker_sessions.rs index 2b03dc3b0..c52bfbc6d 100644 --- a/rust/tests/execution/worker_sessions.rs +++ b/rust/tests/execution/worker_sessions.rs @@ -366,6 +366,49 @@ async fn a_deregistered_worker_can_neither_renew_nor_close_its_session() { } } +// A registration fenced as superseded (plain deregistration commits that +// before deleting the row, and lease recovery supersedes a stale holder) can +// neither create, renew nor close a session until the worker registers again. +#[tokio::test] +async fn a_superseded_registration_can_neither_create_renew_nor_close_its_session() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + for sends_token in [true, false] { + let worker = if sends_token { "gpu-token-worker" } else { "gpu-published-worker" }; + let session = format!("{worker}-state"); + let token = if sends_token { + register_token_holder(&app, worker, "sessions").await + } else { + register_holder(&app, worker, "sessions").await + }; + let sent = sends_token.then_some(token.as_str()); + let with_token = |mut body: Value| { + if let Some(sent) = sent { + body["registration_token"] = json!(sent); + } + body + }; + let (status, created) = create(&app, with_token(json!({"worker_id":worker,"session_id":session,"requirements":["gpu"]}))).await; + assert_eq!(status, StatusCode::CREATED, "{created}"); + // Test-owned literal only. + database.execute(sqlx::AssertSqlSafe(format!( + "UPDATE workflow_worker_registrations SET status='superseded' WHERE worker_id='{worker}'"))).await; + let shown_path = format!("/api/worker-sessions/{session}"); + let shown = || request(&app, "GET", &shown_path, Value::Null); + let before = shown().await; + assert_eq!(before.0, StatusCode::OK, "{}", before.1); + for refused in [ + create(&app, with_token(json!({"worker_id":worker,"session_id":session,"requirements":["gpu"],"lease_seconds":5}))).await, + request(&app, "POST", &format!("/api/worker/sessions/{session}/heartbeat"), with_token(json!({"worker_id":worker,"lease_seconds":900}))).await, + request(&app, "DELETE", &format!("/api/worker/sessions/{session}"), with_token(json!({"worker_id":worker}))).await, + ] { + assert_eq!((refused.0, refused.1["reason"].clone(), refused.1["remediation"].clone()), (StatusCode::CONFLICT, + json!("worker_registration_superseded"), json!("Restart and register the worker with a new worker_id.")), "{}", refused.1); + } + assert_eq!(shown().await, before, "a refused request changes nothing"); + } +} + // Published workers send no registration token. Without the token // capability, a request without one acts for the worker ID's current // registration, as published PHP's does; a token that is sent must be current. diff --git a/tests/Feature/WorkerSessionProtocolTest.php b/tests/Feature/WorkerSessionProtocolTest.php index 700cb88f7..242993d0d 100644 --- a/tests/Feature/WorkerSessionProtocolTest.php +++ b/tests/Feature/WorkerSessionProtocolTest.php @@ -1207,6 +1207,41 @@ public function test_a_deregistered_worker_can_neither_renew_nor_close_its_sessi } } + public function test_a_superseded_registration_can_neither_create_renew_nor_close_its_session(): void + { + // Plain deregistration commits the registration as superseded before it + // deletes the row, and lease recovery supersedes a stale holder. Until + // the worker registers again, its session requests are refused. + foreach ([true, false] as $sendsToken) { + $workerId = $sendsToken ? 'gpu-token-worker' : 'gpu-published-worker'; + $sessionId = $workerId.'-state'; + $token = $this->registerWorkerThroughProtocol( + workerId: $workerId, + taskQueue: 'gpu-activities', + supportedActivityTypes: ['tests.external-greeting-activity'], + capabilities: ['gpu:nvidia-l4', ...($sendsToken ? [WorkerSessionRegistry::REGISTRATION_TOKEN_CAPABILITY] : [])], + ); + $sent = $sendsToken ? $token : null; + $this->postJson('/api/worker/sessions', array_filter(['worker_id' => $workerId, 'session_id' => $sessionId, + 'requirements' => ['gpu:nvidia-l4'], 'registration_token' => $sent]), $this->workerHeaders())->assertCreated(); + WorkerRegistration::query()->where('worker_id', $workerId)->update(['status' => WorkerRegistration::STATUS_SUPERSEDED]); + $row = static fn (): array => WorkerSessionLease::query()->where('session_id', $sessionId)->firstOrFail()->getAttributes(); + $before = $row(); + foreach ([ + $this->postJson('/api/worker/sessions', array_filter(['worker_id' => $workerId, 'session_id' => $sessionId, + 'requirements' => ['gpu:nvidia-l4'], 'registration_token' => $sent, 'lease_seconds' => 5]), $this->workerHeaders()), + $this->postJson("/api/worker/sessions/{$sessionId}/heartbeat", + array_filter(['worker_id' => $workerId, 'registration_token' => $sent, 'lease_seconds' => 900]), $this->workerHeaders()), + $this->deleteJson("/api/worker/sessions/{$sessionId}", + array_filter(['worker_id' => $workerId, 'registration_token' => $sent]), $this->workerHeaders()), + ] as $refused) { + $refused->assertStatus(409)->assertJsonPath('reason', 'worker_registration_superseded') + ->assertJsonPath('remediation', 'Restart and register the worker with a new worker_id.'); + } + $this->assertSame($before, $row(), 'a refused request changes nothing'); + } + } + public function test_without_the_token_capability_a_tokenless_request_acts_for_the_current_registration(): void { // Published workers do not send a registration token. For them the diff --git a/tests/Feature/WorkerSessionRegistrationRaceTest.php b/tests/Feature/WorkerSessionRegistrationRaceTest.php index 3c9a848b2..595995527 100644 --- a/tests/Feature/WorkerSessionRegistrationRaceTest.php +++ b/tests/Feature/WorkerSessionRegistrationRaceTest.php @@ -2,11 +2,14 @@ namespace Tests\Feature; +use App\Models\WorkerRegistration; use App\Models\WorkerSessionLease; use App\Support\WorkerProtocol; use App\Support\WorkerSessionRegistry; use Illuminate\Database\Events\QueryExecuted; +use Illuminate\Database\Events\TransactionCommitted; use Illuminate\Support\Facades\DB; +use Illuminate\Support\Facades\Event; use PDO; use PHPUnit\Framework\Attributes\DataProvider; use RuntimeException; @@ -54,7 +57,7 @@ public function test_a_session_request_holds_the_registration_until_it_commits(s $actors['session'] = $this->forkActor(fn (): array => $this->sessionRequest($verb, $first), pause: 'lock'); $this->readMessage($actors['session']); fwrite($actors['session']['socket'], "go\n"); - $this->assertTrue($this->readMessage($actors['session'])['registration_locked']); + $this->assertNotNull($this->readMessage($actors['session'])['paused']); $actors['register'] = $this->forkActor(fn (): array => ['token' => $this->register()]); $replacement = $this->readMessage($actors['register']); @@ -125,7 +128,7 @@ public function test_a_session_request_and_a_deregistration_serialize(string $dr $actors['session'] = $this->forkActor(fn (): array => $this->sessionRequest($verb, null), pause: 'lock'); $this->readMessage($actors['session']); fwrite($actors['session']['socket'], "go\n"); - $this->assertTrue($this->readMessage($actors['session'])['registration_locked']); + $this->assertNotNull($this->readMessage($actors['session'])['paused']); $actors['deregister'] = $this->forkActor(fn (): array => ['status' => $this->deleteJson('/api/worker/registrations/race-worker', [], $this->workerHeaders())->status()]); @@ -156,6 +159,54 @@ public function test_a_session_request_and_a_deregistration_serialize(string $dr $this->assertSame($before, WorkerSessionLease::query()->where('session_id', 'race-state')->firstOrFail()->getAttributes()); } + public static function drivers(): array + { + return ['mysql' => ['mysql'], 'pgsql' => ['pgsql']]; + } + + /** + * Plain deregistration commits the registration as superseded, recovers + * its leases and only then deletes the row. Inside that window a + * published tokenless create, heartbeat or close is refused and changes + * nothing; once the row is gone it is refused as unregistered. + */ + #[DataProvider('drivers')] + public function test_session_requests_inside_a_deregistrations_superseded_window_are_refused(string $driver): void + { + $this->initializeDatabase($driver); + $this->register(sendsToken: false); + $this->sessionResponse('create', null)->assertCreated(); + $actors = []; + DB::purge(); + try { + $actors['deregister'] = $this->forkActor(fn (): array => ['status' => $this->deleteJson('/api/worker/registrations/race-worker', + [], $this->workerHeaders())->status()], pause: 'superseded'); + $this->readMessage($actors['deregister']); + fwrite($actors['deregister']['socket'], "go\n"); + $this->assertSame('superseded', $this->readMessage($actors['deregister'])['paused']); + + $before = WorkerSessionLease::query()->where('session_id', 'race-state')->firstOrFail()->getAttributes(); + foreach (['create', 'heartbeat', 'close'] as $verb) { + $this->sessionResponse($verb, null)->assertStatus(409)->assertJsonPath('reason', 'worker_registration_superseded'); + } + $this->assertSame($before, WorkerSessionLease::query()->where('session_id', 'race-state')->firstOrFail()->getAttributes()); + + fwrite($actors['deregister']['socket'], "continue\n"); + $this->assertSame(200, $this->finishActor($actors['deregister'])['status']); + unset($actors['deregister']); + } finally { + foreach ($actors as $actor) { + posix_kill($actor['pid'], SIGKILL); + pcntl_waitpid($actor['pid'], $status); + fclose($actor['socket']); + } + DB::purge(); + DB::reconnect(); + } + + $this->sessionResponse('heartbeat', null)->assertStatus(412)->assertJsonPath('reason', 'worker_not_registered'); + } + public static function creates(): array { $cases = []; @@ -187,7 +238,7 @@ public function test_a_create_admits_against_the_registration_current_when_it_co $actors['session'] = $this->forkActor(fn (): array => $this->sessionRequest('create', $sendsToken ? $first : null), pause: 'read'); $this->readMessage($actors['session']); fwrite($actors['session']['socket'], "go\n"); - $this->assertTrue($this->readMessage($actors['session'])['registration_locked']); + $this->assertNotNull($this->readMessage($actors['session'])['paused']); // The replacement registers and reacquires while the create waits // between its first read and its transaction. @@ -321,13 +372,27 @@ private function forkActor(callable $work, ?string $pause = null): array if (fgets($sockets[1]) !== "go\n") { throw new RuntimeException('Actor was not released.'); } - if ($pause !== null) { + if ($pause === 'superseded') { + // Pause right after a transaction commits the registration + // as superseded, while the row still exists. + $paused = false; + Event::listen(TransactionCommitted::class, static function () use (&$paused, $sockets, $pause): void { + if (! $paused && DB::table('workflow_worker_registrations')->where('worker_id', 'race-worker') + ->value('status') === WorkerRegistration::STATUS_SUPERSEDED) { + $paused = true; + fwrite($sockets[1], json_encode(['paused' => $pause]).PHP_EOL); + if (fgets($sockets[1]) !== "continue\n") { + throw new RuntimeException('Deregistration was not released.'); + } + } + }); + } elseif ($pause !== null) { $paused = false; DB::listen(static function (QueryExecuted $query) use (&$paused, $sockets, $pause): void { if (! $paused && str_contains($query->sql, 'workflow_worker_registrations') && ($pause === 'read' || str_contains(strtolower($query->sql), 'for update'))) { $paused = true; - fwrite($sockets[1], json_encode(['registration_locked' => true]).PHP_EOL); + fwrite($sockets[1], json_encode(['paused' => $pause]).PHP_EOL); if (fgets($sockets[1]) !== "continue\n") { throw new RuntimeException('Session request was not released.'); }