diff --git a/.github/workflows/server-parity.yml b/.github/workflows/server-parity.yml index af09c4196..aadec63cf 100644 --- a/.github/workflows/server-parity.yml +++ b/.github/workflows/server-parity.yml @@ -50,6 +50,8 @@ on: - tests/Unit/ServerParityRepairPassContractTest.mjs - tests/Unit/ServerParityClusterInfoContractTest.mjs - tests/Unit/ServerParityNamespaceDeleteContractTest.mjs + - tests/Unit/ServerParityWorkerSessionsContractTest.mjs + - tests/Unit/ServerParityWorkerSessionActivitiesContractTest.mjs - tests/Unit/ServerParityUpdateValidationContractTest.mjs - tests/Unit/ServerParityFixtureSelectionTest.mjs - tests/Unit/ServerParityMessageStreamsContractTest.mjs @@ -1713,7 +1715,7 @@ jobs: -e COMPOSER_HOME=/tmp/composer -v "$PWD:/source" -w /source "$image" \ install --working-dir=scripts/conformance/server-parity --no-interaction --no-progress docker run --rm --user=1000:1000 --entrypoint node -v "$PWD:/source:ro" -w /source \ - "$image" --test tests/Unit/ServerParityRunnerTest.mjs tests/Unit/ServerParitySdkDirectionLanesTest.mjs tests/Unit/ServerParityCooperativeContractTest.mjs tests/Unit/ServerParityChildCancellationContractTest.mjs tests/Unit/ServerParityScheduleContractTest.mjs tests/Unit/ServerParityVisibilityContractTest.mjs tests/Unit/ServerParityAdmissionContractTest.mjs tests/Unit/ServerParityRoleTokenContractTest.mjs tests/Unit/ServerParityWorkerDeregistrationContractTest.mjs tests/Unit/ServerParitySearchAttributeContractTest.mjs tests/Unit/ServerParityWorkerListingContractTest.mjs tests/Unit/ServerParityWorkerReplacementContractTest.mjs tests/Unit/ServerParityPatchDeploymentContractTest.mjs tests/Unit/ServerParityWaitingHistoryContractTest.mjs tests/Unit/ServerParityTaskRetryContractTest.mjs tests/Unit/ServerParityPublishedPatchContractTest.mjs tests/Unit/ServerParityPublishedPatchTransportTest.mjs tests/Unit/ServerParityRecordFileTest.mjs tests/Unit/ServerParityFailureLocationTest.mjs tests/Unit/ServerParityFailureExcerptTest.mjs tests/Unit/ServerParityLateWorkflowCompletionContractTest.mjs tests/Unit/ServerParityStorageTestContractTest.mjs tests/Unit/ServerParitySchedulePoliciesContractTest.mjs tests/Unit/ServerParitySignalsBeforeWorkerContractTest.mjs tests/Unit/ServerParityScheduleCatalogContractTest.mjs tests/Unit/ServerParityOperatorMetricsContractTest.mjs tests/Unit/ServerParityActivityLeaseRenewalContractTest.mjs tests/Unit/ServerParityActivityCancellationAckContractTest.mjs tests/Unit/ServerParityServiceCatalogContractTest.mjs tests/Unit/ServerParitySystemMetricsContractTest.mjs tests/Unit/ServerParityOperatorDashboardTypesContractTest.mjs tests/Unit/ServerParityPrometheusMetricsContractTest.mjs tests/Unit/ServerParityPayloadCleanupContractTest.mjs tests/Unit/ServerParityRepairPassContractTest.mjs tests/Unit/ServerParityClusterInfoContractTest.mjs tests/Unit/ServerParityNamespaceDeleteContractTest.mjs tests/Unit/ServerParityMessageStreamsContractTest.mjs tests/Unit/ServerParityWorkflowPollRequestContractTest.mjs tests/Unit/ServerParityUpdateValidationContractTest.mjs tests/Unit/ServerParityFixtureSelectionTest.mjs > parity-evidence/comparator-tests.txt + "$image" --test tests/Unit/ServerParityRunnerTest.mjs tests/Unit/ServerParitySdkDirectionLanesTest.mjs tests/Unit/ServerParityCooperativeContractTest.mjs tests/Unit/ServerParityChildCancellationContractTest.mjs tests/Unit/ServerParityScheduleContractTest.mjs tests/Unit/ServerParityVisibilityContractTest.mjs tests/Unit/ServerParityAdmissionContractTest.mjs tests/Unit/ServerParityRoleTokenContractTest.mjs tests/Unit/ServerParityWorkerDeregistrationContractTest.mjs tests/Unit/ServerParitySearchAttributeContractTest.mjs tests/Unit/ServerParityWorkerListingContractTest.mjs tests/Unit/ServerParityWorkerReplacementContractTest.mjs tests/Unit/ServerParityPatchDeploymentContractTest.mjs tests/Unit/ServerParityWaitingHistoryContractTest.mjs tests/Unit/ServerParityTaskRetryContractTest.mjs tests/Unit/ServerParityPublishedPatchContractTest.mjs tests/Unit/ServerParityPublishedPatchTransportTest.mjs tests/Unit/ServerParityRecordFileTest.mjs tests/Unit/ServerParityFailureLocationTest.mjs tests/Unit/ServerParityFailureExcerptTest.mjs tests/Unit/ServerParityLateWorkflowCompletionContractTest.mjs tests/Unit/ServerParityStorageTestContractTest.mjs tests/Unit/ServerParitySchedulePoliciesContractTest.mjs tests/Unit/ServerParitySignalsBeforeWorkerContractTest.mjs tests/Unit/ServerParityScheduleCatalogContractTest.mjs tests/Unit/ServerParityOperatorMetricsContractTest.mjs tests/Unit/ServerParityActivityLeaseRenewalContractTest.mjs tests/Unit/ServerParityActivityCancellationAckContractTest.mjs tests/Unit/ServerParityServiceCatalogContractTest.mjs tests/Unit/ServerParitySystemMetricsContractTest.mjs tests/Unit/ServerParityOperatorDashboardTypesContractTest.mjs tests/Unit/ServerParityPrometheusMetricsContractTest.mjs tests/Unit/ServerParityPayloadCleanupContractTest.mjs tests/Unit/ServerParityRepairPassContractTest.mjs tests/Unit/ServerParityClusterInfoContractTest.mjs tests/Unit/ServerParityNamespaceDeleteContractTest.mjs tests/Unit/ServerParityMessageStreamsContractTest.mjs tests/Unit/ServerParityWorkflowPollRequestContractTest.mjs tests/Unit/ServerParityUpdateValidationContractTest.mjs tests/Unit/ServerParityFixtureSelectionTest.mjs tests/Unit/ServerParityWorkerSessionsContractTest.mjs tests/Unit/ServerParityWorkerSessionActivitiesContractTest.mjs > parity-evidence/comparator-tests.txt docker run --rm --user=1000:1000 --cpus=1 --memory=256m --memory-swap=256m \ --network=none --entrypoint node -v "$PWD:/source:ro" -w /source \ "$image" --test tests/Unit/ServerParityCodecDeadlineLoadTest.mjs > parity-evidence/codec-load-tests.txt diff --git a/app/Http/Controllers/Api/WorkerSessionController.php b/app/Http/Controllers/Api/WorkerSessionController.php index b88ae8c35..7c895a0fe 100644 --- a/app/Http/Controllers/Api/WorkerSessionController.php +++ b/app/Http/Controllers/Api/WorkerSessionController.php @@ -11,6 +11,9 @@ class WorkerSessionController { + /** The token registration returned, as fenced deregistration accepts it. */ + private const REGISTRATION_TOKEN_RULES = ['nullable', 'string', 'regex:/^[a-f0-9]{32}$/D']; + public function __construct( private readonly WorkerSessionRegistry $sessions, ) {} @@ -52,7 +55,7 @@ public function create(Request $request): JsonResponse ], 422); } - $result = $this->sessions->createOrReacquire($namespace, $worker, $options); + $result = $this->sessions->createOrReacquire($namespace, $worker, $options, $validated['registration_token'] ?? null); return WorkerProtocol::json($result, (int) $result['status']); } @@ -71,7 +74,8 @@ public function heartbeat(Request $request, string $sessionId): JsonResponse $validated = $request->validate([ 'worker_id' => ['required', 'string', 'max:255'], - 'lease_seconds' => ['nullable', 'integer', 'min:1'], + 'lease_seconds' => ['nullable', 'integer', 'min:1', 'max:'.WorkerSessionRegistry::MAX_LEASE_SECONDS], + 'registration_token' => self::REGISTRATION_TOKEN_RULES, ]); $result = $this->sessions->heartbeat( @@ -79,6 +83,7 @@ public function heartbeat(Request $request, string $sessionId): JsonResponse $validated['worker_id'], $sessionId, $validated['lease_seconds'] ?? null, + $validated['registration_token'] ?? null, ); return WorkerProtocol::json($result, (int) $result['status']); @@ -98,9 +103,10 @@ public function close(Request $request, string $sessionId): JsonResponse $validated = $request->validate([ 'worker_id' => ['required', 'string', 'max:255'], + 'registration_token' => self::REGISTRATION_TOKEN_RULES, ]); - $result = $this->sessions->close($namespace, $validated['worker_id'], $sessionId); + $result = $this->sessions->close($namespace, $validated['worker_id'], $sessionId, $validated['registration_token'] ?? null); return WorkerProtocol::json($result, (int) $result['status']); } @@ -154,11 +160,12 @@ private function validateSessionRequest(Request $request, bool $workerIdRequired 'queue' => ['nullable', 'string', 'max:255'], 'requirements' => ['nullable', 'array'], 'requirements.*' => ['string', 'max:255'], - 'lease_seconds' => ['nullable', 'integer', 'min:1'], - 'ttl_seconds' => ['nullable', 'integer', 'min:1'], - 'max_concurrent_activities' => ['nullable', 'integer', 'min:1'], + 'lease_seconds' => ['nullable', 'integer', 'min:1', 'max:'.WorkerSessionRegistry::MAX_LEASE_SECONDS], + 'ttl_seconds' => ['nullable', 'integer', 'min:1', 'max:'.WorkerSessionRegistry::MAX_TTL_SECONDS], + 'max_concurrent_activities' => ['nullable', 'integer', 'min:1', 'max:'.WorkerSessionRegistry::MAX_CONCURRENT_ACTIVITIES], 'create_if_missing' => ['nullable', 'boolean'], 'allow_reacquire_after_failure' => ['nullable', 'boolean'], + 'registration_token' => self::REGISTRATION_TOKEN_RULES, ]); } diff --git a/app/Models/WorkerSessionLease.php b/app/Models/WorkerSessionLease.php index 1ad4e866c..0c0ab034d 100644 --- a/app/Models/WorkerSessionLease.php +++ b/app/Models/WorkerSessionLease.php @@ -22,6 +22,7 @@ class WorkerSessionLease extends Model 'requirements', 'status', 'lease_owner', + 'lease_owner_incarnation', 'lease_expires_at', 'ttl_expires_at', 'closed_at', diff --git a/app/Support/WorkerSessionRegistry.php b/app/Support/WorkerSessionRegistry.php index 2dce3b529..f59c95333 100644 --- a/app/Support/WorkerSessionRegistry.php +++ b/app/Support/WorkerSessionRegistry.php @@ -13,6 +13,24 @@ final class WorkerSessionRegistry { private const DEFAULT_LEASE_SECONDS = 120; + /** + * Portable maxima, the same in native (Server #740). A heartbeat renews + * the lease, so a longer lease only delays holder-loss detection; thirty + * days of TTL keeps ttl_expires_at representable in a MySQL TIMESTAMP. + */ + public const MAX_LEASE_SECONDS = 86400; + + public const MAX_TTL_SECONDS = 2592000; + + public const MAX_CONCURRENT_ACTIVITIES = 1000; + + /** + * A worker that declares this capability at registration sends its + * registration token on every explicit worker session create, heartbeat + * and close, and the server refuses those requests without it. + */ + public const REGISTRATION_TOKEN_CAPABILITY = 'worker_session_registration_token'; + private const DEFAULT_TTL_SECONDS = 1800; private const DEFAULT_MAX_CONCURRENT_ACTIVITIES = 1; @@ -63,9 +81,23 @@ public function createOrReacquire( string $namespace, WorkerRegistration $worker, array $options, + ?string $registrationToken = null, ): array { - return DB::transaction(function () use ($namespace, $worker, $options): array { - return $this->acquireLocked($namespace, $worker, $options, null); + return DB::transaction(function () use ($namespace, $worker, $options, $registrationToken): array { + // Admit against the registration as it is now, locked, not as the + // controller read it before the transaction. + $current = $this->lockedRegistration($namespace, $worker->worker_id); + if (! $current instanceof WorkerRegistration) { + return $this->notRegistered($worker->worker_id, 'creating'); + } + if ($superseded = $this->supersededRegistration($current)) { + return $superseded; + } + if ($stale = $this->staleRegistration($current, $registrationToken)) { + return $stale; + } + + return $this->acquireLocked($namespace, $current, $options, null); }); } @@ -87,9 +119,25 @@ public function admitActivity( /** * @return array */ - public function heartbeat(string $namespace, string $workerId, string $sessionId, ?int $leaseSeconds = null): array - { - return DB::transaction(function () use ($namespace, $workerId, $sessionId, $leaseSeconds): array { + public function heartbeat( + string $namespace, + string $workerId, + string $sessionId, + ?int $leaseSeconds = null, + ?string $registrationToken = null, + ): array { + return DB::transaction(function () use ($namespace, $workerId, $sessionId, $leaseSeconds, $registrationToken): array { + $current = $this->lockedRegistration($namespace, $workerId); + if (! $current instanceof WorkerRegistration) { + return $this->notRegistered($workerId, 'renewing'); + } + if ($superseded = $this->supersededRegistration($current)) { + return $superseded; + } + if ($stale = $this->staleRegistration($current, $registrationToken)) { + return $stale; + } + /** @var WorkerSessionLease|null $session */ $session = WorkerSessionLease::query() ->where('namespace', $namespace) @@ -176,9 +224,20 @@ public function heartbeatForAttempt(string $namespace, string $attemptId, string /** * @return array */ - public function close(string $namespace, string $workerId, string $sessionId): array + public function close(string $namespace, string $workerId, string $sessionId, ?string $registrationToken = null): array { - return DB::transaction(function () use ($namespace, $workerId, $sessionId): array { + return DB::transaction(function () use ($namespace, $workerId, $sessionId, $registrationToken): array { + $current = $this->lockedRegistration($namespace, $workerId); + if (! $current instanceof WorkerRegistration) { + return $this->notRegistered($workerId, 'closing'); + } + if ($superseded = $this->supersededRegistration($current)) { + return $superseded; + } + if ($stale = $this->staleRegistration($current, $registrationToken)) { + return $stale; + } + /** @var WorkerSessionLease|null $session */ $session = WorkerSessionLease::query() ->where('namespace', $namespace) @@ -374,6 +433,7 @@ private function acquireLocked( 'requirements' => $options['requirements'], 'status' => WorkerSessionLease::STATUS_ACTIVE, 'lease_owner' => $worker->worker_id, + 'lease_owner_incarnation' => $this->incarnation($worker), 'lease_expires_at' => $now->copy()->addSeconds($options['lease_seconds']), 'ttl_expires_at' => $now->copy()->addSeconds($options['ttl_seconds']), 'lease_seconds' => $options['lease_seconds'], @@ -404,7 +464,7 @@ private function acquireLocked( } } - $session = $this->renewHeldSession($session, $options, $worker->worker_id); + $session = $this->renewHeldSession($session, $options, $worker); return $this->success($session, 'reused', $activityTaskId); } @@ -441,7 +501,7 @@ private function acquireLocked( return $limit; } - $session = $this->renewHeldSession($session, $options, $worker->worker_id); + $session = $this->renewHeldSession($session, $options, $worker); return $this->success($session, 'reacquired', $activityTaskId); } @@ -452,11 +512,12 @@ private function acquireLocked( private function renewHeldSession( WorkerSessionLease $session, array $options, - string $workerId, + WorkerRegistration $worker, ): WorkerSessionLease { $now = now(); + $workerId = $worker->worker_id; - $session->forceFill(array_filter([ + $session->forceFill([...array_filter([ 'connection' => $options['connection'], 'queue' => $options['queue'], 'requirements' => $options['requirements'], @@ -471,7 +532,10 @@ private function renewHeldSession( 'create_if_missing' => $options['create_if_missing'], 'allow_reacquire_after_failure' => $options['allow_reacquire_after_failure'], 'last_heartbeat_at' => $now, - ], static fn (mixed $value): bool => $value !== null))->save(); + ], static fn (mixed $value): bool => $value !== null), + // Set even when null: the holder is now this registration. + 'lease_owner_incarnation' => $this->incarnation($worker), + ])->save(); return $session; } @@ -506,10 +570,11 @@ private function refreshStatusLocked(WorkerSessionLease $session): WorkerSession return $session; } - if ($session->lease_owner !== null && $this->workerIsStale($session->namespace, $session->lease_owner)) { + $holderLoss = $session->lease_owner === null ? null : $this->holderLoss($session); + if ($holderLoss !== null) { $session->forceFill([ 'status' => WorkerSessionLease::STATUS_ORPHANED, - 'failure_reason' => 'worker_heartbeat_stale', + 'failure_reason' => $holderLoss, ])->save(); } @@ -688,6 +753,127 @@ private function activeActivityCount( return (int) $query->count(); } + /** + * Why the holder no longer holds an active session: its registration is + * gone or silent, or the worker ID re-registered since it acquired the + * session. A replacement process must reacquire and rebuild its + * process-local state, so it never inherits the session as reused. + */ + private function holderLoss(WorkerSessionLease $session): ?string + { + if ($this->workerIsStale($session->namespace, (string) $session->lease_owner)) { + return 'worker_heartbeat_stale'; + } + + $worker = WorkerRegistration::query() + ->where('namespace', $session->namespace) + ->where('worker_id', $session->lease_owner) + ->first(); + + return $worker instanceof WorkerRegistration && $this->incarnation($worker) === $session->lease_owner_incarnation + ? null : 'worker_registration_replaced'; + } + + /** + * The digest a session keeps of its holder's registration, as the + * cooperative claim does, so the deregistration credential itself is + * never stored on the session. + */ + private function incarnation(WorkerRegistration $worker): ?string + { + return is_string($worker->registration_token) ? hash('sha256', $worker->registration_token) : null; + } + + /** + * The worker's current registration, locked until the session request + * commits. A same-ID re-registration locks this row before it rotates the + * token, so the request either finishes before the rotation or sees it; + * it never checks one incarnation and writes for the next. SQLite ignores + * FOR UPDATE, so its writer lock is taken first, before any read, as + * ExternalPayloadBackupHold does. + */ + private function lockedRegistration(string $namespace, string $workerId): ?WorkerRegistration + { + if (DB::getDriverName() === 'sqlite') { + DB::table((new WorkerRegistration)->getTable()) + ->where('namespace', $namespace) + ->where('worker_id', $workerId) + ->update(['worker_id' => $workerId]); + } + + /** @var WorkerRegistration|null $current */ + $current = WorkerRegistration::query() + ->where('namespace', $namespace) + ->where('worker_id', $workerId) + ->lockForUpdate() + ->first(); + + return $current; + } + + /** + * A session request needs the worker's current registration, with or + * without a token: once deregistration deletes it, the former process can + * no longer renew, close or recreate the session it held. + * + * @return array + */ + private function notRegistered(string $workerId, string $action): array + { + return $this->failure('worker_not_registered', sprintf('Worker must be registered before %s a worker session.', $action), 412, + ['worker_id' => $workerId]); + } + + /** + * A registration fenced as superseded, by deregistration before its row is + * deleted or by lease recovery, has lost its authority, as the worker + * heartbeat and polls already treat it. + * + * @return array|null + */ + private function supersededRegistration(WorkerRegistration $current): ?array + { + return $current->status === WorkerRegistration::STATUS_SUPERSEDED + ? $this->failure('worker_registration_superseded', 'Worker registration was fenced after its leases were recovered.', 409, [ + 'worker_id' => $current->worker_id, + 'remediation' => 'Restart and register the worker with a new worker_id.', + ]) + : null; + } + + /** + * A registration token the request sends must be the worker's current + * one. Without a token, a stale process and its same-ID replacement are + * indistinguishable once the replacement has reacquired, so a worker + * whose current registration declares REGISTRATION_TOKEN_CAPABILITY must + * send it on every explicit create, heartbeat and close. A registration + * without the capability keeps the published contract: a request without + * a token acts for the worker ID's current registration. + * + * @return array|null + */ + private function staleRegistration(WorkerRegistration $current, ?string $registrationToken): ?array + { + if ($registrationToken === null) { + return in_array(self::REGISTRATION_TOKEN_CAPABILITY, $current->capabilities ?? [], true) + ? $this->failure( + 'stale_worker_registration', + 'The worker\'s current registration requires its registration token on worker session requests. Send it, or register again and reacquire the session.', + 409, + ) + : null; + } + + return is_string($current->registration_token) + && hash_equals($current->registration_token, $registrationToken) + ? null + : $this->failure( + 'stale_worker_registration', + 'The registration token is not the worker\'s current registration. Register again and reacquire the session.', + 409, + ); + } + private function workerIsStale(string $namespace, string $workerId): bool { /** @var WorkerRegistration|null $worker */ diff --git a/app/Support/WorkflowTaskCompletionValidation.php b/app/Support/WorkflowTaskCompletionValidation.php index 591defb2c..15067ca14 100644 --- a/app/Support/WorkflowTaskCompletionValidation.php +++ b/app/Support/WorkflowTaskCompletionValidation.php @@ -71,9 +71,9 @@ public static function rules(): array 'commands.*.worker_session.queue' => ['nullable', 'string', 'max:255'], 'commands.*.worker_session.requirements' => ['nullable', 'array'], 'commands.*.worker_session.requirements.*' => ['string', 'max:255'], - 'commands.*.worker_session.lease_seconds' => ['nullable', 'integer', 'min:1'], - 'commands.*.worker_session.ttl_seconds' => ['nullable', 'integer', 'min:1'], - 'commands.*.worker_session.max_concurrent_activities' => ['nullable', 'integer', 'min:1'], + 'commands.*.worker_session.lease_seconds' => ['nullable', 'integer', 'min:1', 'max:'.WorkerSessionRegistry::MAX_LEASE_SECONDS], + 'commands.*.worker_session.ttl_seconds' => ['nullable', 'integer', 'min:1', 'max:'.WorkerSessionRegistry::MAX_TTL_SECONDS], + 'commands.*.worker_session.max_concurrent_activities' => ['nullable', 'integer', 'min:1', 'max:'.WorkerSessionRegistry::MAX_CONCURRENT_ACTIVITIES], 'commands.*.worker_session.create_if_missing' => ['nullable', 'boolean'], 'commands.*.worker_session.allow_reacquire_after_failure' => ['nullable', 'boolean'], 'commands.*.execution_timeout_seconds' => ['nullable', 'integer', 'min:1'], diff --git a/database/migrations/2026_10_24_000100_add_worker_session_holder_incarnation.php b/database/migrations/2026_10_24_000100_add_worker_session_holder_incarnation.php new file mode 100644 index 000000000..111411bf1 --- /dev/null +++ b/database/migrations/2026_10_24_000100_add_worker_session_holder_incarnation.php @@ -0,0 +1,46 @@ +char('lease_owner_incarnation', 64)->nullable()->after('lease_owner'); + }); + + // Each active session stays with the registration that holds it now, + // so upgrading does not orphan sessions whose workers keep running. + DB::table('workflow_worker_sessions') + ->where('status', 'active') + ->whereNotNull('lease_owner') + ->orderBy('id') + ->chunkById(500, function ($sessions): void { + foreach ($sessions as $session) { + $token = DB::table('workflow_worker_registrations') + ->where('namespace', $session->namespace) + ->where('worker_id', $session->lease_owner) + ->value('registration_token'); + + if (is_string($token)) { + DB::table('workflow_worker_sessions') + ->where('id', $session->id) + ->update(['lease_owner_incarnation' => hash('sha256', $token)]); + } + } + }); + } + + public function down(): void + { + Schema::table('workflow_worker_sessions', function (Blueprint $table) { + $table->dropColumn('lease_owner_incarnation'); + }); + } +}; diff --git a/docs/rust-server-port.md b/docs/rust-server-port.md index 0afd87729..d45b88ed6 100644 --- a/docs/rust-server-port.md +++ b/docs/rust-server-port.md @@ -82,13 +82,12 @@ Updated October 24, 2026. ([#737](https://github.com/durable-workflow/server/issues/737)). No expectation is edited to make native pass. -- **Parity surface.** Native serves 133 of the 175 API route methods in - published 2.11.1's route table, with message streams, update validation - and activity cancellation acknowledgement. An execution test sends each - one through the native router, and the ones still answering - `501 rust_capability_not_implemented` must match +- **Parity surface.** Native serves 138 of the 175 API route methods in + published 2.11.1's route table, with message streams, update validation, + activity cancellation acknowledgement and worker sessions. An execution + test sends each one through the native router, and the ones still + answering `501 rust_capability_not_implemented` must match [`unserved-php-routes.v2.11.1.txt`](../rust/tests/fixtures/unserved-php-routes.v2.11.1.txt). - The work in review serves 5 more: worker sessions. - **Next action.** Land the items above in order, keep porting the unserved routes fixture first, and package the native binary into the server image. diff --git a/resources/platform-protocol-specs/worker-sessions-runtime.schema.json b/resources/platform-protocol-specs/worker-sessions-runtime.schema.json index d52225008..301d56f21 100644 --- a/resources/platform-protocol-specs/worker-sessions-runtime.schema.json +++ b/resources/platform-protocol-specs/worker-sessions-runtime.schema.json @@ -223,9 +223,9 @@ "items": { "type": "string", "minLength": 1, "maxLength": 255 }, "uniqueItems": true }, - "lease_seconds": { "type": "integer", "minimum": 1, "default": 120 }, - "ttl_seconds": { "type": "integer", "minimum": 1, "default": 1800 }, - "max_concurrent_activities": { "type": "integer", "minimum": 1, "default": 1 }, + "lease_seconds": { "type": "integer", "minimum": 1, "maximum": 86400, "default": 120 }, + "ttl_seconds": { "type": "integer", "minimum": 1, "maximum": 2592000, "default": 1800 }, + "max_concurrent_activities": { "type": "integer", "minimum": 1, "maximum": 1000, "default": 1 }, "create_if_missing": { "type": "boolean", "default": true }, "allow_reacquire_after_failure": { "type": "boolean", "default": true } } @@ -238,18 +238,25 @@ "required": ["worker_id"], "additionalProperties": true, "properties": { - "worker_id": { "type": "string", "minLength": 1, "maxLength": 255 } + "worker_id": { "type": "string", "minLength": 1, "maxLength": 255 }, + "registration_token": { "$ref": "#/$defs/workerRegistrationToken" } } } ] }, + "workerRegistrationToken": { + "type": "string", + "pattern": "^[a-f0-9]{32}$", + "description": "The token worker registration returned. When sent, it must be the worker's current registration, otherwise the request is refused with stale_worker_registration and changes nothing. A worker whose current registration declared the worker_session_registration_token capability must send it on every explicit create, heartbeat and close: without it the request is refused the same way, so a replaced process can neither reuse, renew nor close its replacement's session. Without that capability a request without a token acts for the worker ID's current registration, as before the token existed. A same-ID re-registration orphans the sessions the previous registration held (worker_registration_replaced) until they are reacquired. Session-bound activity claims and their heartbeats keep their own attempt leases." + }, "workerSessionHeartbeatRequest": { "type": "object", "required": ["worker_id"], "additionalProperties": true, "properties": { "worker_id": { "type": "string", "minLength": 1, "maxLength": 255 }, - "lease_seconds": { "type": "integer", "minimum": 1 } + "lease_seconds": { "type": "integer", "minimum": 1, "maximum": 86400 }, + "registration_token": { "$ref": "#/$defs/workerRegistrationToken" } } }, "workerSessionCloseRequest": { @@ -257,7 +264,8 @@ "required": ["worker_id"], "additionalProperties": true, "properties": { - "worker_id": { "type": "string", "minLength": 1, "maxLength": 255 } + "worker_id": { "type": "string", "minLength": 1, "maxLength": 255 }, + "registration_token": { "$ref": "#/$defs/workerRegistrationToken" } } }, "workerSessionOperationResult": { @@ -289,7 +297,8 @@ "session_owned_by_another_worker", "session_activity_limit_exceeded", "session_closed", - "session_reacquire_disallowed" + "session_reacquire_disallowed", + "stale_worker_registration" ] }, "error": { "type": "string" }, @@ -425,7 +434,7 @@ }, "workerSessionFailureReason": { "type": ["string", "null"], - "enum": [null, "ttl_expired", "lease_expired", "worker_heartbeat_stale"] + "enum": [null, "ttl_expired", "lease_expired", "worker_heartbeat_stale", "worker_registration_replaced"] } } } diff --git a/rust/migrations/mariadb/0004_catalog.json b/rust/migrations/mariadb/0005_catalog.json similarity index 99% rename from rust/migrations/mariadb/0004_catalog.json rename to rust/migrations/mariadb/0005_catalog.json index def952353..57d64e99a 100644 --- a/rust/migrations/mariadb/0004_catalog.json +++ b/rust/migrations/mariadb/0005_catalog.json @@ -27,6 +27,10 @@ "dw_worker_registration_incarnations", "CREATE TABLE `dw_worker_registration_incarnations` (\n `token` char(32) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,\n `namespace` varchar(255) NOT NULL,\n `worker_id` varchar(255) NOT NULL,\n `status` varchar(32) NOT NULL,\n `recovered_workflow_task_count` bigint(20) DEFAULT NULL,\n `finished_at` datetime(6) DEFAULT NULL,\n `created_at` datetime(6) NOT NULL,\n `updated_at` datetime(6) NOT NULL,\n PRIMARY KEY (`token`),\n KEY `dw_worker_incarnation_authority` (`namespace`,`worker_id`,`status`),\n KEY `dw_worker_incarnation_retention` (`status`,`finished_at`),\n CONSTRAINT `dw_worker_incarnation_state` CHECK (`status` = 'active' and `finished_at` is null and `recovered_workflow_task_count` is null or `status` = 'superseded' and `finished_at` is not null and `recovered_workflow_task_count` is null or `status` = 'deregistered' and `finished_at` is not null and `recovered_workflow_task_count` is not null and `recovered_workflow_task_count` >= 0)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci" ], + [ + "dw_worker_session_holders", + "CREATE TABLE `dw_worker_session_holders` (\n `namespace` varchar(128) NOT NULL,\n `session_id` varchar(255) NOT NULL,\n `incarnation` char(64) COLLATE ascii_bin NOT NULL,\n `updated_at` datetime(6) NOT NULL,\n PRIMARY KEY (`namespace`,`session_id`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci" + ], [ "failed_jobs", "CREATE TABLE `failed_jobs` (\n `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT,\n `uuid` varchar(255) NOT NULL,\n `connection` text NOT NULL,\n `queue` text NOT NULL,\n `payload` longtext NOT NULL,\n `exception` longtext NOT NULL,\n `failed_at` timestamp NOT NULL DEFAULT current_timestamp(),\n PRIMARY KEY (`id`),\n UNIQUE KEY `failed_jobs_uuid_unique` (`uuid`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci" diff --git a/rust/migrations/mariadb/0005_worker_session_holders.sql b/rust/migrations/mariadb/0005_worker_session_holders.sql new file mode 100644 index 000000000..ad3089f47 --- /dev/null +++ b/rust/migrations/mariadb/0005_worker_session_holders.sql @@ -0,0 +1,10 @@ +-- Native development bootstrap version 5: the registration incarnation that +-- holds each worker session (Server #740). A same-ID re-registration rotates +-- the incarnation, which orphans the session until it is reacquired. +CREATE TABLE dw_worker_session_holders ( + namespace VARCHAR(128) NOT NULL, + session_id VARCHAR(255) NOT NULL, + incarnation CHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + updated_at DATETIME(6) NOT NULL, + PRIMARY KEY (namespace, session_id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; diff --git a/rust/migrations/mysql/0004_catalog.json b/rust/migrations/mysql/0005_catalog.json similarity index 99% rename from rust/migrations/mysql/0004_catalog.json rename to rust/migrations/mysql/0005_catalog.json index 5a4a6319e..4f67db5a6 100644 --- a/rust/migrations/mysql/0004_catalog.json +++ b/rust/migrations/mysql/0005_catalog.json @@ -31,6 +31,10 @@ "dw_worker_registration_incarnations", "CREATE TABLE `dw_worker_registration_incarnations` (\n `token` char(32) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,\n `namespace` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL,\n `worker_id` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL,\n `status` varchar(32) COLLATE utf8mb4_unicode_ci NOT NULL,\n `recovered_workflow_task_count` bigint DEFAULT NULL,\n `finished_at` datetime(6) DEFAULT NULL,\n `created_at` datetime(6) NOT NULL,\n `updated_at` datetime(6) NOT NULL,\n PRIMARY KEY (`token`),\n KEY `dw_worker_incarnation_authority` (`namespace`,`worker_id`,`status`),\n KEY `dw_worker_incarnation_retention` (`status`,`finished_at`),\n CONSTRAINT `dw_worker_incarnation_state` CHECK ((((`status` = _utf8mb4'active') and (`finished_at` is null) and (`recovered_workflow_task_count` is null)) or ((`status` = _utf8mb4'superseded') and (`finished_at` is not null) and (`recovered_workflow_task_count` is null)) or ((`status` = _utf8mb4'deregistered') and (`finished_at` is not null) and (`recovered_workflow_task_count` is not null) and (`recovered_workflow_task_count` >= 0))))\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci" ], + [ + "dw_worker_session_holders", + "CREATE TABLE `dw_worker_session_holders` (\n `namespace` varchar(128) COLLATE utf8mb4_unicode_ci NOT NULL,\n `session_id` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL,\n `incarnation` char(64) COLLATE ascii_bin NOT NULL,\n `updated_at` datetime(6) NOT NULL,\n PRIMARY KEY (`namespace`,`session_id`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci" + ], [ "failed_jobs", "CREATE TABLE `failed_jobs` (\n `id` bigint unsigned NOT NULL AUTO_INCREMENT,\n `uuid` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL,\n `connection` text COLLATE utf8mb4_unicode_ci NOT NULL,\n `queue` text COLLATE utf8mb4_unicode_ci NOT NULL,\n `payload` longtext COLLATE utf8mb4_unicode_ci NOT NULL,\n `exception` longtext COLLATE utf8mb4_unicode_ci NOT NULL,\n `failed_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,\n PRIMARY KEY (`id`),\n UNIQUE KEY `failed_jobs_uuid_unique` (`uuid`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci" diff --git a/rust/migrations/mysql/0005_worker_session_holders.sql b/rust/migrations/mysql/0005_worker_session_holders.sql new file mode 100644 index 000000000..ad3089f47 --- /dev/null +++ b/rust/migrations/mysql/0005_worker_session_holders.sql @@ -0,0 +1,10 @@ +-- Native development bootstrap version 5: the registration incarnation that +-- holds each worker session (Server #740). A same-ID re-registration rotates +-- the incarnation, which orphans the session until it is reacquired. +CREATE TABLE dw_worker_session_holders ( + namespace VARCHAR(128) NOT NULL, + session_id VARCHAR(255) NOT NULL, + incarnation CHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + updated_at DATETIME(6) NOT NULL, + PRIMARY KEY (namespace, session_id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; diff --git a/rust/migrations/postgres/0004_catalog.json b/rust/migrations/postgres/0005_catalog.json similarity index 99% rename from rust/migrations/postgres/0004_catalog.json rename to rust/migrations/postgres/0005_catalog.json index 3ad9fe814..4dd16a521 100644 --- a/rust/migrations/postgres/0004_catalog.json +++ b/rust/migrations/postgres/0005_catalog.json @@ -896,6 +896,58 @@ "generated": "" } ], + [ + "column", + "dw_worker_session_holders.1", + { + "name": "namespace", + "type": "text", + "default": null, + "identity": "", + "not_null": true, + "collation": "default", + "generated": "" + } + ], + [ + "column", + "dw_worker_session_holders.2", + { + "name": "session_id", + "type": "text", + "default": null, + "identity": "", + "not_null": true, + "collation": "default", + "generated": "" + } + ], + [ + "column", + "dw_worker_session_holders.3", + { + "name": "incarnation", + "type": "text", + "default": null, + "identity": "", + "not_null": true, + "collation": "default", + "generated": "" + } + ], + [ + "column", + "dw_worker_session_holders.4", + { + "name": "updated_at", + "type": "timestamp(6) without time zone", + "default": null, + "identity": "", + "not_null": true, + "collation": null, + "generated": "" + } + ], [ "column", "failed_jobs.1", @@ -11975,6 +12027,16 @@ "definition": "PRIMARY KEY (token)" } ], + [ + "constraint", + "dw_worker_session_holders.dw_worker_session_holders_pkey", + { + "deferred": false, + "validated": true, + "deferrable": false, + "definition": "PRIMARY KEY (namespace, session_id)" + } + ], [ "constraint", "failed_jobs.failed_jobs_pkey", @@ -13064,6 +13126,17 @@ "definition": "CREATE UNIQUE INDEX dw_worker_registration_incarnations_pkey ON public.dw_worker_registration_incarnations USING btree (token)" } ], + [ + "index", + "dw_worker_session_holders_pkey", + { + "ready": true, + "valid": true, + "unique": true, + "primary": true, + "definition": "CREATE UNIQUE INDEX dw_worker_session_holders_pkey ON public.dw_worker_session_holders USING btree (namespace, session_id)" + } + ], [ "index", "dw_workflow_tasks_poll", @@ -17265,6 +17338,21 @@ "force_row_security": false } ], + [ + "relation", + "dw_worker_session_holders", + { + "kind": "r", + "view": null, + "options": null, + "partition": false, + "persistence": "p", + "row_security": false, + "access_method": "heap", + "partition_key": null, + "force_row_security": false + } + ], [ "relation", "failed_jobs", diff --git a/rust/migrations/postgres/0005_worker_session_holders.sql b/rust/migrations/postgres/0005_worker_session_holders.sql new file mode 100644 index 000000000..8609300a6 --- /dev/null +++ b/rust/migrations/postgres/0005_worker_session_holders.sql @@ -0,0 +1,11 @@ +-- Native development bootstrap version 5: the registration incarnation that +-- holds each worker session (Server #740). A same-ID re-registration rotates +-- the incarnation, which orphans the session until it is reacquired. +CREATE TABLE dw_worker_session_holders ( + namespace text NOT NULL, + session_id text NOT NULL, + incarnation text NOT NULL, + updated_at timestamp(6) without time zone NOT NULL, + PRIMARY KEY (namespace, session_id) +); +UPDATE dw_server_schema SET version=5 WHERE engine='rust-development' AND version=4; diff --git a/rust/migrations/sqlite/0005_worker_session_holders.sql b/rust/migrations/sqlite/0005_worker_session_holders.sql new file mode 100644 index 000000000..d7e099452 --- /dev/null +++ b/rust/migrations/sqlite/0005_worker_session_holders.sql @@ -0,0 +1,11 @@ +-- Native development bootstrap version 5: the registration incarnation that +-- holds each worker session (Server #740). A same-ID re-registration rotates +-- the incarnation, which orphans the session until it is reacquired. +CREATE TABLE dw_worker_session_holders ( + namespace TEXT NOT NULL, + session_id TEXT NOT NULL, + incarnation TEXT NOT NULL, + updated_at TEXT NOT NULL, + PRIMARY KEY (namespace, session_id) +); +UPDATE dw_server_schema SET version=5 WHERE engine='rust-development' AND version=4; diff --git a/rust/src/runtime/activity_heartbeats.rs b/rust/src/runtime/activity_heartbeats.rs index 24412be8b..c7f25397c 100644 --- a/rust/src/runtime/activity_heartbeats.rs +++ b/rust/src/runtime/activity_heartbeats.rs @@ -43,7 +43,7 @@ where async fn observe_activity(&self, task_id: &str, body: Value, protocol: &str) -> Result { let mut tx = self.begin_read().await?; let (response, _) = self - .activity_observation(&mut tx, task_id, body, protocol) + .activity_observation(&mut tx, task_id, body, protocol, true) .await?; tx.commit().await?; Ok(response) @@ -55,6 +55,7 @@ where task_id: &str, body: Value, protocol: &str, + check_session: bool, ) -> Result<(Value, bool)> { reject_fields(&body, &["activity_attempt_id", "lease_owner"])?; let attempt_id = text(&body, "activity_attempt_id")?; @@ -127,6 +128,24 @@ where } else { (true, false, None) }; + // A session-bound attempt shows its session's compact form, read + // without a refresh; on status, a session that is no longer the + // holder's live lease stops an attempt that could otherwise continue. + let session = DB::optional_document_row(&activity, "activity_options")? + .and_then(|options| super::worker_sessions::CommandSession::from_snapshot(&options["worker_session"])); + let (worker_session, can_continue, reason) = match &session { + Some(session) => { + let queue = DB::optional_string(&activity, "queue")?.unwrap_or_default(); + let connection = DB::optional_string(&activity, "connection")?; + let (view, usable) = self.session_for_status(tx, session, &queue, connection.as_deref(), owner).await?; + if check_session && can_continue && !usable { + (view, false, Some("worker_session_unavailable")) + } else { + (view, can_continue, reason) + } + } + None => (Value::Null, can_continue, reason), + }; // Observation preserves the existing lease and never records progress. let mut response = json!({"task_id":task_id,"activity_attempt_id":attempt_id,"lease_owner":owner, "can_continue":can_continue,"cancel_requested":cancel_requested,"reason":reason,"heartbeat_recorded":false, @@ -135,7 +154,7 @@ where "last_heartbeat_at":DB::optional_instant(&attempt,"last_heartbeat_at")?.or(DB::optional_instant(&activity,"last_heartbeat_at")?).map(super::cooperative::timestamp), "run_status":run_status,"run_closed_reason":DB::optional_string(&run,"closed_reason")?,"run_closed_at":DB::optional_instant(&run,"closed_at")?.map(super::cooperative::timestamp), "activity_status":activity_status,"attempt_status":attempt_status,"task_status":task_status, - "deadlines":super::activity_deadlines::deadlines_view::(&activity)?,"worker_session":null,"protocol_version":protocol,"server_capabilities":super::store::capabilities()}); + "deadlines":super::activity_deadlines::deadlines_view::(&activity)?,"worker_session":worker_session,"protocol_version":protocol,"server_capabilities":super::store::capabilities()}); // A cooperative request that reached this attempt shows its receipt. if cancel_requested && let Some(receipt) = Self::activity_cancellation_receipt(tx, &run, &DB::string(&activity, "id")?, attempt_id).await? @@ -191,6 +210,7 @@ where task_id, json!({"activity_attempt_id":attempt_id,"lease_owner":text(&body,"lease_owner")?}), protocol, + false, ) .await .map_err(|error| match error { @@ -253,6 +273,25 @@ where return Err(refuse(StatusCode::CONFLICT, "stale_attempt")); } super::activity_deadlines::fence::(&activity)?; + // `heartbeatForAttempt`: a session-bound attempt renews its session + // by the command's lease first. A refused renewal records nothing. + let mut worker_session = Value::Null; + if let Some(session) = DB::optional_document_row(&activity, "activity_options")? + .and_then(|options| super::worker_sessions::CommandSession::from_snapshot(&options["worker_session"])) + { + match self.renew_session_for_attempt(&mut tx, &session, text(&body, "lease_owner")?).await? { + Ok(snapshot) => worker_session = snapshot, + Err((status, snapshot, renewal)) => { + return Err(super::RuntimeError::Protocol { status, + response: json!({"task_id":task_id,"activity_attempt_id":attempt_id,"lease_owner":body["lease_owner"], + "cancel_requested":observed["cancel_requested"],"can_continue":false,"reason":renewal["reason"], + "run_closed_reason":observed["run_closed_reason"],"run_closed_at":observed["run_closed_at"], + "heartbeat_recorded":false,"lease_expires_at":observed["lease_expires_at"], + "last_heartbeat_at":observed["last_heartbeat_at"],"error":renewal["error"],"worker_session":snapshot, + "worker_session_renewal":renewal,"protocol_version":protocol,"server_capabilities":super::store::capabilities()}) }); + } + } + } let at = now(); let expires = if Self::cancellation_request(&run)?.is_none() { at + chrono::Duration::seconds(self.activity_task_lease_seconds) @@ -313,7 +352,7 @@ where "cancel_requested":false,"can_continue":true,"reason":null,"run_closed_reason":null,"run_closed_at":null, // PHP formats each time in UTC with microseconds, as status reads do. "heartbeat_recorded":true,"lease_expires_at":super::cooperative::timestamp(expires), - "last_heartbeat_at":super::cooperative::timestamp(at),"worker_session":null, + "last_heartbeat_at":super::cooperative::timestamp(at),"worker_session":worker_session, "protocol_version":protocol,"server_capabilities":super::store::capabilities()}), ) } diff --git a/rust/src/runtime/activity_lease_renewal.rs b/rust/src/runtime/activity_lease_renewal.rs index f5d6d3cb6..442c50d8f 100644 --- a/rust/src/runtime/activity_lease_renewal.rs +++ b/rust/src/runtime/activity_lease_renewal.rs @@ -92,6 +92,28 @@ store_methods! { Some(task) => DB::optional_string(task, "lease_owner")?.as_deref() == Some(owner), None => false, }; + // `heartbeatForAttempt`: a session-bound attempt renews its session + // before its own lease; a refused renewal renews nothing. + let mut worker_session = Value::Null; + let session = match &rows.activity { + Some(activity) => DB::optional_document_row(activity, "activity_options")? + .and_then(|options| super::worker_sessions::CommandSession::from_snapshot(&options["worker_session"])), + None => None, + }; + if leased_by_owner && let Some(session) = &session { + match self.renew_session_for_attempt(&mut tx, session, owner).await? { + Ok(snapshot) => worker_session = snapshot, + Err((status, snapshot, renewal)) => { + let mut response = observed; + response["lease_renewed"] = json!(false); + response["can_continue"] = json!(false); + response["reason"] = renewal["reason"].clone(); + response["worker_session"] = snapshot; + response["worker_session_renewal"] = renewal; + return Err(worker(status, response)); + } + } + } let at = now(); let mut status = if leased_by_owner { let task = rows.task.as_ref().expect("task checked above"); @@ -118,7 +140,7 @@ store_methods! { status["lease_renewed"] = status["can_continue"].clone(); status["server_time"] = json!(micros(at)); status["task_id"] = json!(task_id); - status["worker_session"] = Value::Null; + status["worker_session"] = worker_session; let code = outcome_status(status["reason"].as_str()); if code != StatusCode::OK { return Err(worker(code, status)); diff --git a/rust/src/runtime/backend.rs b/rust/src/runtime/backend.rs index 4b8660955..8d92da79a 100644 --- a/rust/src/runtime/backend.rs +++ b/rust/src/runtime/backend.rs @@ -63,6 +63,9 @@ pub(super) trait Backend: Database { const WORKFLOW_POLL_EXPIRED: &str = "SELECT \"key\" FROM dw_query_cache WHERE \"key\" LIKE 'dw-native-workflow-poll:%' AND expiration<=$1 ORDER BY \"key\" LIMIT 32"; /// Add a cache row, or lock the existing one with a no-op update, so /// transactions over one poll request ID run one at a time. + /// A worker session's running activity attempts held by its owner, + /// other than one task (`activeActivityCount`). + const SESSION_RUNNING_ATTEMPTS: &str = "SELECT COUNT(*) FROM activity_attempts p JOIN activity_executions e ON e.id=p.activity_execution_id JOIN workflow_tasks t ON t.id=p.workflow_task_id WHERE t.namespace=$1 AND p.status='running' AND p.lease_expires_at>$2 AND p.lease_owner=$3 AND e.activity_options->'worker_session'->>'session_id'=$4 AND p.workflow_task_id<>$5"; const QUERY_CACHE_LOCK: &str = "INSERT INTO dw_query_cache(\"key\",value,expiration) VALUES ($1,$2,$3) ON CONFLICT (\"key\") DO UPDATE SET expiration=dw_query_cache.expiration"; /// `WorkflowTaskKindSelector`: create the queue's cursor at /// `update_validation` if it is missing, and lock it. @@ -165,6 +168,7 @@ SELECT id FROM clocks ORDER BY CASE WHEN opened_at IS NULL THEN 1 ELSE 0 END, const OBSERVATION_FAILURE_EVENT: &str = "SELECT id,sequence,event_type,recorded_at FROM workflow_history_events WHERE workflow_run_id=$1 AND json_extract(payload,'$.failure_id') COLLATE BINARY=$2 AND event_type IN ('ActivityFailed','ActivityTimedOut','ChildRunFailed','ChildRunCancelled','ChildRunTerminated','WorkflowFailed','WorkflowTimedOut','WorkflowCancelled','WorkflowTerminated','UpdateCompleted') ORDER BY sequence DESC LIMIT 1"; const ACTIVITY_RESULT_EVENT_SQL: &str = "SELECT sequence,event_type,payload FROM workflow_history_events WHERE workflow_run_id=$1 AND event_type IN ('ActivityScheduled','ActivityStarted','ActivityHeartbeatRecorded','ActivityRetryScheduled','ActivityCompleted','ActivityFailed','ActivityTimedOut','ActivityCancelled') AND (json_extract(payload,'$.activity_execution_id') COLLATE BINARY=$2 OR json_extract(payload,'$.activity.id') COLLATE BINARY=$3) ORDER BY sequence DESC LIMIT 1"; const QUERY_CACHE_SCAN: &str = "SELECT \"key\",value,expiration FROM dw_query_cache WHERE \"key\" LIKE 'dw-native-query:%' AND json_extract(value,'$.namespace')=$1 ORDER BY \"key\" LIMIT 65"; + const SESSION_RUNNING_ATTEMPTS: &str = "SELECT COUNT(*) FROM activity_attempts p JOIN activity_executions e ON e.id=p.activity_execution_id JOIN workflow_tasks t ON t.id=p.workflow_task_id WHERE t.namespace=$1 AND p.status='running' AND p.lease_expires_at>$2 AND p.lease_owner=$3 AND json_extract(e.activity_options,'$.worker_session.session_id')=$4 AND p.workflow_task_id<>$5"; const QUERY_CACHE_PENDING_FOR_RUN: &str = "SELECT json_extract(value,'$.query_admitted_at') AS query_admitted_at,json_extract(value,'$.query_deadline_at') AS query_deadline_at,json_extract(value,'$.receipt_expires_at') AS receipt_expires_at FROM dw_query_cache WHERE \"key\" LIKE 'dw-native-query:%' AND expiration>$2 AND json_extract(value,'$.run_id')=$1 AND json_extract(value,'$.status') IN ('pending','leased') ORDER BY \"key\" LIMIT 65"; const POLL_WAKE_QUERIES_SQL: &str = "SELECT \"key\" AS id FROM dw_query_cache WHERE \"key\" LIKE 'dw-native-query:%' AND expiration>$1 AND json_extract(value,'$.namespace')=$2 AND json_extract(value,'$.task_queue')=$3 AND json_extract(value,'$.query_deadline_at')>$4 AND (json_extract(value,'$.status')='pending' OR (json_extract(value,'$.status')='leased' AND json_extract(value,'$.lease_expires_at')<=$5)) ORDER BY \"key\" LIMIT 65"; const POLL_WAKE_TASKS_SQL: &str = r#" @@ -474,6 +478,7 @@ impl Backend for MySql { const QUERY_POLL_EXPIRED: &str = "SELECT `key` FROM dw_query_cache WHERE `key` LIKE 'dw-native-query-poll:%' AND expiration<=$1 ORDER BY `key` LIMIT 32"; const WORKFLOW_POLL_EXPIRED: &str = "SELECT `key` FROM dw_query_cache WHERE `key` LIKE 'dw-native-workflow-poll:%' AND expiration<=$1 ORDER BY `key` LIMIT 32"; const QUERY_CACHE_LOCK: &str = "INSERT INTO dw_query_cache(`key`,value,expiration) VALUES ($1,$2,$3) ON DUPLICATE KEY UPDATE expiration=expiration"; + const SESSION_RUNNING_ATTEMPTS: &str = "SELECT COUNT(*) FROM activity_attempts p JOIN activity_executions e ON e.id=p.activity_execution_id JOIN workflow_tasks t ON t.id=p.workflow_task_id WHERE t.namespace=? AND p.status='running' AND p.lease_expires_at>? AND p.lease_owner=? AND JSON_UNQUOTE(JSON_EXTRACT(e.activity_options,'$.worker_session.session_id'))=? AND p.workflow_task_id<>?"; const TASK_KIND_CURSOR_LOCK: &str = "INSERT INTO workflow_task_poll_cursors(namespace,task_queue,next_task_kind,created_at,updated_at) VALUES ($1,$2,'update_validation',$3,$4) ON DUPLICATE KEY UPDATE next_task_kind=next_task_kind"; const TIMER_TASK_SQL: &str = "SELECT id,payload FROM workflow_tasks WHERE workflow_run_id=? AND task_type='timer' AND status='ready' AND JSON_UNQUOTE(JSON_EXTRACT(payload,'$.timer_id'))=? AND BINARY namespace=BINARY ?"; const TIMER_SCHEDULE_SQL: &str = "SELECT payload FROM workflow_history_events WHERE workflow_run_id=? AND event_type='TimerScheduled' AND JSON_UNQUOTE(JSON_EXTRACT(payload,'$.timer_id'))=? ORDER BY sequence DESC LIMIT 1"; @@ -506,6 +511,7 @@ impl Backend for MySql { || sql == Self::POLL_WAKE_QUERIES_SQL || sql == Self::QUERY_CACHE_PENDING_FOR_RUN || sql == Self::QUERY_CACHE_SCAN + || sql == Self::SESSION_RUNNING_ATTEMPTS { return Cow::Borrowed(sql); } diff --git a/rust/src/runtime/completion_rules.rs b/rust/src/runtime/completion_rules.rs index dc887b57b..15f7de134 100644 --- a/rust/src/runtime/completion_rules.rs +++ b/rust/src/runtime/completion_rules.rs @@ -119,9 +119,9 @@ const COMMAND: &[(&str, &[Check])] = &[ ("worker_session.queue", &[NUL, STR, max(255)]), ("worker_session.requirements", &[NUL, ARR]), ("worker_session.requirements.*", &[STR, max(255)]), - ("worker_session.lease_seconds", &[NUL, INT, min(1)]), - ("worker_session.ttl_seconds", &[NUL, INT, min(1)]), - ("worker_session.max_concurrent_activities", &[NUL, INT, min(1)]), + ("worker_session.lease_seconds", &[NUL, INT, min(1), max(super::worker_sessions::MAX_LEASE_SECONDS)]), + ("worker_session.ttl_seconds", &[NUL, INT, min(1), max(super::worker_sessions::MAX_TTL_SECONDS)]), + ("worker_session.max_concurrent_activities", &[NUL, INT, min(1), max(super::worker_sessions::MAX_CONCURRENT_ACTIVITIES)]), ("worker_session.create_if_missing", &[NUL, BOOL]), ("worker_session.allow_reacquire_after_failure", &[NUL, BOOL]), ("execution_timeout_seconds", &[NUL, INT, min(1)]), diff --git a/rust/src/runtime/execution.rs b/rust/src/runtime/execution.rs index 5088c39c6..4b7819678 100644 --- a/rust/src/runtime/execution.rs +++ b/rust/src/runtime/execution.rs @@ -1359,6 +1359,11 @@ impl Runtime { #[cfg(test)] delegate!(poll(body: Value, kind: &'static str, protocol_version: &str) -> Result); delegate!(poll_with_pages(body: Value, kind: &'static str, protocol_version: &str, pages: super::history_pages::HistoryPages, default_wait: u64) -> Result); + delegate!(create_worker_session(body: Value, protocol: &str) -> Result); + delegate!(heartbeat_worker_session(session_id: &str, body: Value, protocol: &str) -> Result); + delegate!(close_worker_session(session_id: &str, body: Value, protocol: &str) -> Result); + delegate!(list_worker_sessions() -> Result); + delegate!(show_worker_session(session_id: &str) -> Result); delegate!(heartbeat_task(task_id: &str, body: Value) -> Result); pub(crate) async fn deliver_cancellation(&self, task_id: &str, body: Value, protocol: &str) -> Result { let codec = self.transition_codec.clone(); diff --git a/rust/src/runtime/http.rs b/rust/src/runtime/http.rs index 9bbb5882b..85c37bff6 100644 --- a/rust/src/runtime/http.rs +++ b/rust/src/runtime/http.rs @@ -207,6 +207,11 @@ pub fn router(runtime: Runtime) -> Router { ) .route("/api/worker/register", post(register)) .route("/api/worker/heartbeat", post(worker_heartbeat)) + .route("/api/worker/sessions", post(create_worker_session)) + .route("/api/worker/sessions/{session_id}", delete(close_worker_session)) + .route("/api/worker/sessions/{session_id}/heartbeat", post(heartbeat_worker_session)) + .route("/api/worker-sessions", get(list_worker_sessions)) + .route("/api/worker-sessions/{session_id}", get(show_worker_session)) .route("/api/worker/registrations/{worker_id}", delete(deregister)) .route( "/api/worker/registrations/{worker_id}/deregister", @@ -645,7 +650,7 @@ enum Rest { } /// Route families a named namespace may use, by prefix. -const NAMESPACE_FAMILIES: [(&str, Rest, Allowed); 8] = [ +const NAMESPACE_FAMILIES: [(&str, Rest, Allowed); 9] = [ // Runtime credentials are server-wide; the namespace only scopes a tenant. ("/api/runtime-credentials/", Rest::Anything, Allowed::Any), ("/api/external-payloads/v1/", Rest::Anything, READ), @@ -661,6 +666,8 @@ const NAMESPACE_FAMILIES: [(&str, Rest, Allowed); 8] = [ ("/api/workers/", Rest::OneSegment, Allowed::Only(&["GET", "DELETE"])), // Deployments are the namespace's own build-id rollouts. ("/api/deployments/", Rest::NonEmpty, READ_WRITE), + // Worker session visibility reads the namespace's own sessions. + ("/api/worker-sessions/", Rest::OneSegment, READ), ]; fn named_namespace_operation(method: &str, path: &str) -> bool { @@ -702,6 +709,7 @@ fn exact_namespace_route(path: &str) -> Option { "/api/task-queues" | "/api/deployments" | "/api/workers" + | "/api/worker-sessions" | "/api/system/operator-metrics" | "/api/system/health" | "/api/system/operator-dashboard" @@ -767,6 +775,9 @@ fn worker_route(rest: &str) -> Option { ["register" | "heartbeat"] | ["registrations", _] | ["registrations", _, "deregister"] + | ["sessions"] + | ["sessions", _] + | ["sessions", _, "heartbeat"] | ["workflow-tasks" | "activity-tasks", "poll"] | ["query-tasks", "poll"] | ["query-tasks", _, "complete" | "fail" | "status"] @@ -1598,6 +1609,61 @@ fn worker_response(mut body: Value) -> Value { body["server_capabilities"] = capabilities(); body } +/// A worker session answer: PHP's status, its body repeating it, and the +/// worker protocol envelope. +fn session_answer(answer: super::worker_sessions::Answer) -> Response { + (answer.status, Json(worker_response(answer.body))).into_response() +} + +async fn create_worker_session( + Extension(runtime): Extension, + headers: HeaderMap, + Json(body): Json, +) -> Result { + let protocol = request_protocol(&headers)?; + runtime.create_worker_session(body, protocol).await.map(session_answer).map_err(worker_refusal) +} + +async fn heartbeat_worker_session( + Extension(runtime): Extension, + Path(session_id): Path, + headers: HeaderMap, + Json(body): Json, +) -> Result { + let protocol = request_protocol(&headers)?; + runtime.heartbeat_worker_session(&session_id, body, protocol).await.map(session_answer).map_err(worker_refusal) +} + +/// The holder may name itself in the JSON body or as `?worker_id=`. +async fn close_worker_session( + Extension(runtime): Extension, + Path(session_id): Path, + Query(query): Query>, + headers: HeaderMap, + body: axum::body::Bytes, +) -> Result { + let protocol = request_protocol(&headers)?; + let mut fields = if body.is_empty() { + json!({}) + } else { + serde_json::from_slice::(&body).ok().filter(Value::is_object).unwrap_or_else(|| json!({})) + }; + if let Some(worker_id) = query.get("worker_id") + && fields.get("worker_id").is_none() + { + fields["worker_id"] = json!(worker_id); + } + runtime.close_worker_session(&session_id, fields, protocol).await.map(session_answer).map_err(worker_refusal) +} + +async fn list_worker_sessions(Extension(runtime): Extension) -> Result> { + runtime.list_worker_sessions().await.map(Json) +} + +async fn show_worker_session(Extension(runtime): Extension, Path(session_id): Path) -> Result> { + runtime.show_worker_session(&session_id).await.map(Json) +} + async fn worker_heartbeat( Extension(runtime): Extension, headers: HeaderMap, diff --git a/rust/src/runtime/mod.rs b/rust/src/runtime/mod.rs index 124916e3c..82d9c7382 100644 --- a/rust/src/runtime/mod.rs +++ b/rust/src/runtime/mod.rs @@ -145,6 +145,7 @@ mod wait_timeouts; mod worker_contracts; mod worker_incarnations; mod worker_poll_fence; +mod worker_sessions; mod workflow_poll_requests; mod worker_management; mod worker_admission; diff --git a/rust/src/runtime/mysql.rs b/rust/src/runtime/mysql.rs index 60b63da41..025f571e0 100644 --- a/rust/src/runtime/mysql.rs +++ b/rust/src/runtime/mysql.rs @@ -9,7 +9,7 @@ use sqlx::{ }; use std::{collections::BTreeMap, sync::OnceLock, time::Duration}; -pub const VERSION: i64 = 4; +pub const VERSION: i64 = 5; type Catalog = Vec<(String, String)>; pub const MARKER_SQL: &str = "CREATE TABLE dw_server_schema (engine VARCHAR(64) NOT NULL PRIMARY KEY,version BIGINT NOT NULL,bootstrap_checksum VARBINARY(48) NOT NULL) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin"; @@ -44,7 +44,9 @@ impl Flavor { concat!( include_str!("../../migrations/mysql/0003_full_schema.sql"), "\n", - include_str!("../../migrations/mysql/0004_worker_incarnations.sql") + include_str!("../../migrations/mysql/0004_worker_incarnations.sql"), + "\n", + include_str!("../../migrations/mysql/0005_worker_session_holders.sql") ), ), Self::MariaDb => ( @@ -52,14 +54,16 @@ impl Flavor { concat!( include_str!("../../migrations/mariadb/0003_full_schema.sql"), "\n", - include_str!("../../migrations/mariadb/0004_worker_incarnations.sql") + include_str!("../../migrations/mariadb/0004_worker_incarnations.sql"), + "\n", + include_str!("../../migrations/mariadb/0005_worker_session_holders.sql") ), ), }; slot.get_or_init(|| { let mut migrations = Migrator::with_migrations(vec![Migration::new( VERSION, - "full frozen PHP schema and native worker incarnations".into(), + "full frozen PHP schema, native worker incarnations and session holders".into(), MigrationType::Simple, sql.into_sql_str(), false, @@ -72,8 +76,8 @@ impl Flavor { } fn expected(self) -> Result { Ok(serde_json::from_str(&canonical(match self { - Self::MySql => include_str!("../../migrations/mysql/0004_catalog.json"), - Self::MariaDb => include_str!("../../migrations/mariadb/0004_catalog.json"), + Self::MySql => include_str!("../../migrations/mysql/0005_catalog.json"), + Self::MariaDb => include_str!("../../migrations/mariadb/0005_catalog.json"), }))?) } /// The single native migration, which a takeover records as applied. @@ -87,11 +91,19 @@ impl Flavor { let (frozen, incarnations) = match self { Self::MySql => ( include_str!("../../migrations/mysql/0003_full_schema.sql"), - include_str!("../../migrations/mysql/0004_worker_incarnations.sql"), + concat!( + include_str!("../../migrations/mysql/0004_worker_incarnations.sql"), + "\n", + include_str!("../../migrations/mysql/0005_worker_session_holders.sql") + ), ), Self::MariaDb => ( include_str!("../../migrations/mariadb/0003_full_schema.sql"), - include_str!("../../migrations/mariadb/0004_worker_incarnations.sql"), + concat!( + include_str!("../../migrations/mariadb/0004_worker_incarnations.sql"), + "\n", + include_str!("../../migrations/mariadb/0005_worker_session_holders.sql") + ), ), }; Ok(super::upgrade::native_relations( @@ -102,6 +114,10 @@ impl Flavor { .replace( "CREATE TABLE dw_worker_registration_incarnations", "CREATE TABLE IF NOT EXISTS dw_worker_registration_incarnations", + ) + .replace( + "CREATE TABLE dw_worker_session_holders", + "CREATE TABLE IF NOT EXISTS dw_worker_session_holders", )) } pub(super) fn checksum(self) -> &'static [u8] { diff --git a/rust/src/runtime/namespace_cleanup.rs b/rust/src/runtime/namespace_cleanup.rs index 3fbd980f1..5dd4eb923 100644 --- a/rust/src/runtime/namespace_cleanup.rs +++ b/rust/src/runtime/namespace_cleanup.rs @@ -148,6 +148,7 @@ const CLEANUP: &[(&str, &str, usize)] = &[ const NATIVE_CLEANUP: &[&str] = &[ "DELETE FROM dw_poll_receipts WHERE namespace=$1", "DELETE FROM dw_worker_registration_incarnations WHERE namespace=$1", + "DELETE FROM dw_worker_session_holders WHERE namespace=$1", ]; /// Registered external payloads and payload columns holding a reference. diff --git a/rust/src/runtime/operator_metrics.rs b/rust/src/runtime/operator_metrics.rs index 240a718f6..02fa06405 100644 --- a/rust/src/runtime/operator_metrics.rs +++ b/rust/src/runtime/operator_metrics.rs @@ -283,8 +283,7 @@ where /// server adds. pub(super) async fn operator_metrics(&self, config: &MetricsConfig) -> Result { let mut metrics = self.operator_metrics_snapshot(config).await?; - metrics["worker_sessions"] = - json!({"total": 0, "active": 0, "closed": 0, "expired": 0, "failed": 0, "orphaned": 0}); + metrics["worker_sessions"] = self.worker_session_metrics().await?; metrics["capacity_evidence"] = json!({"available": false, "reason": "native_capacity_evidence_unavailable"}); metrics["runtime_external_payload_cleanup"] = @@ -309,8 +308,7 @@ where metrics["capacity_evidence"] = json!({"available": false, "reason": "native_capacity_evidence_unavailable"}); } - metrics["worker_sessions"] = - json!({"total": 0, "active": 0, "closed": 0, "expired": 0, "failed": 0, "orphaned": 0}); + metrics["worker_sessions"] = self.worker_session_metrics().await?; metrics["runtime_external_payload_cleanup"] = self.external_payload_cleanup_metrics(config).await?; let filter = types.map(|types| Value::from(types.to_vec()).to_string()); let filter = filter.as_deref(); diff --git a/rust/src/runtime/postgres.rs b/rust/src/runtime/postgres.rs index 0744d1ac0..a89775873 100644 --- a/rust/src/runtime/postgres.rs +++ b/rust/src/runtime/postgres.rs @@ -13,7 +13,7 @@ use sqlx::{ use super::{Result, refuse}; -pub const VERSION: i64 = 4; +pub const VERSION: i64 = 5; pub const CATALOG_QUERY: &str = include_str!("postgres-catalog.sql"); type Catalog = Vec<(String, String, Json)>; @@ -22,12 +22,14 @@ fn migrator() -> &'static Migrator { MIGRATOR.get_or_init(|| { let mut migrations = Migrator::with_migrations(vec![Migration::new( VERSION, - "full frozen PHP PostgreSQL schema and native worker incarnations".into(), + "full frozen PHP PostgreSQL schema, native worker incarnations and session holders".into(), MigrationType::Simple, concat!( include_str!("../../migrations/postgres/0003_full_schema.sql"), "\n", - include_str!("../../migrations/postgres/0004_worker_incarnations.sql") + include_str!("../../migrations/postgres/0004_worker_incarnations.sql"), + "\n", + include_str!("../../migrations/postgres/0005_worker_session_holders.sql") ) .into_sql_str(), false, @@ -52,7 +54,11 @@ pub(super) fn native_relations_sql() -> Result { super::upgrade::native_relations( include_str!("../../migrations/postgres/0003_full_schema.sql"), "CREATE TABLE public.dw_server_schema ", - include_str!("../../migrations/postgres/0004_worker_incarnations.sql"), + concat!( + include_str!("../../migrations/postgres/0004_worker_incarnations.sql"), + "\n", + include_str!("../../migrations/postgres/0005_worker_session_holders.sql") + ), ) } @@ -85,7 +91,7 @@ pub(super) async fn php_source_catalog(connection: &mut PgConnection) -> Result< )); } let native: Catalog = - serde_json::from_str(include_str!("../../migrations/postgres/0004_catalog.json"))?; + serde_json::from_str(include_str!("../../migrations/postgres/0005_catalog.json"))?; let expected: Catalog = native .into_iter() .filter(|(kind, name, _)| { @@ -180,7 +186,7 @@ pub(super) async fn verify(connection: &mut PgConnection) -> Result<()> { // from SQL text. It retains types, precision, defaults, indexes, foreign // keys and sequence ownership while excluding live rows/sequence values. let expected: Catalog = - serde_json::from_str(include_str!("../../migrations/postgres/0004_catalog.json"))?; + serde_json::from_str(include_str!("../../migrations/postgres/0005_catalog.json"))?; let actual = catalog(connection).await?; if actual != expected { return Err(refuse( @@ -345,7 +351,7 @@ mod tests { #[test] fn compiled_catalog_preserves_long_object_identities_and_int64_limits() { let rows: Catalog = - serde_json::from_str(include_str!("../../migrations/postgres/0004_catalog.json")) + serde_json::from_str(include_str!("../../migrations/postgres/0005_catalog.json")) .unwrap(); let identities: std::collections::BTreeSet<_> = rows.iter().map(|(kind, name, _)| (kind, name)).collect(); diff --git a/rust/src/runtime/schema.rs b/rust/src/runtime/schema.rs index 9b602f319..55db13331 100644 --- a/rust/src/runtime/schema.rs +++ b/rust/src/runtime/schema.rs @@ -11,19 +11,21 @@ use tokio::sync::OnceCell; use super::{Result, refuse}; -pub(super) const VERSION: i64 = 4; +pub(super) const VERSION: i64 = 5; fn migrator() -> &'static Migrator { static MIGRATOR: OnceLock = OnceLock::new(); MIGRATOR.get_or_init(|| { Migrator::with_migrations(vec![Migration::new( VERSION, - "full frozen PHP schema and native worker incarnations".into(), + "full frozen PHP schema, native worker incarnations and session holders".into(), MigrationType::Simple, concat!( include_str!("../../migrations/sqlite/0003_full_schema.sql"), "\n", - include_str!("../../migrations/sqlite/0004_worker_incarnations.sql") + include_str!("../../migrations/sqlite/0004_worker_incarnations.sql"), + "\n", + include_str!("../../migrations/sqlite/0005_worker_session_holders.sql") ) .into_sql_str(), false, @@ -38,7 +40,11 @@ pub(super) fn native_relations_sql() -> Result { super::upgrade::native_relations( include_str!("../../migrations/sqlite/0003_full_schema.sql"), "CREATE TABLE dw_server_schema ", - include_str!("../../migrations/sqlite/0004_worker_incarnations.sql"), + concat!( + include_str!("../../migrations/sqlite/0004_worker_incarnations.sql"), + "\n", + include_str!("../../migrations/sqlite/0005_worker_session_holders.sql") + ), ) } diff --git a/rust/src/runtime/store.rs b/rust/src/runtime/store.rs index a42bf58f9..28e77fd0c 100644 --- a/rust/src/runtime/store.rs +++ b/rust/src/runtime/store.rs @@ -1950,7 +1950,46 @@ where { continue; } + // `workerCanSatisfySession`: a session-bound activity goes only to + // a worker on protocol 1.8 or newer, on the session's queue, with + // every requirement among its capabilities. + let session = match activity.as_ref() { + Some(activity) => DB::optional_document_row(activity, "activity_options")? + .and_then(|options| super::worker_sessions::CommandSession::from_snapshot(&options["worker_session"])), + None => None, + }; + let activity_queue = match activity.as_ref() { + Some(activity) => DB::optional_string(activity, "queue")?.unwrap_or_else(|| queue.to_owned()), + None => queue.to_owned(), + }; + if let Some(session) = &session { + let capabilities: Vec = DB::optional_document_row(&worker, "capabilities")? + .as_ref() + .and_then(Value::as_array) + .into_iter() + .flatten() + .filter_map(|value| value.as_str().map(str::to_owned)) + .collect(); + if !session.claimable_by(protocol_version, &DB::string(&worker, "task_queue")?, &activity_queue, &capabilities) { + continue; + } + } let task_id = DB::string(&task, "id")?; + // `admitActivity`: the session admits the claim or the claim is + // rolled back, so a held or full session leaves the task ready. + let session_view = match &session { + Some(session) => { + let connection = match activity.as_ref() { + Some(activity) => DB::optional_string(activity, "connection")?, + None => None, + }; + match self.admit_session_activity(tx, &worker, session, &activity_queue, connection.as_deref(), &task_id).await? { + Some(view) => Some(view), + None => continue, + } + } + None => None, + }; let attempt = DB::number(&task, "attempt_count")? .checked_add(1) .ok_or_else(|| refuse(StatusCode::CONFLICT, "task_attempt_exhausted"))?; @@ -2044,6 +2083,9 @@ where claim["attempt_number"] = json!(attempt); claim["activity_type"] = json!(type_name); claim["idempotency_key"] = json!(activity_id); + if let Some(view) = session_view.clone() { + claim["worker_session"] = view; + } // A standalone activity started without input has none. if let Some(arguments) = DB::optional_string(&activity, "arguments")? { claim["arguments"] = wire(&arguments); @@ -2463,7 +2505,9 @@ where "type", "activity_type", "arguments", + "connection", "queue", + "worker_session", "retry_policy", "start_to_close_timeout", "schedule_to_start_timeout", @@ -2578,6 +2622,8 @@ where ); } Self::workflow_task_guard(&mut tx, &task, &body).await?; + self.guard_worker_session_commands(&mut tx, &task, task_id, &body, protocol, commands).await?; + // PHP guards worker-session commands before validating streams. // `MessageStreamService::validateCompletion`, before anything changes. let streams = super::message_streams::Completion::of(&body); let stream_instance = Self::query("SELECT workflow_instance_id FROM workflow_runs WHERE id=$1") @@ -2729,11 +2775,16 @@ where let metadata = super::selections::metadata(command, "activity")?; super::selections::check_sequence(&metadata, sequence)?; let activity_id = id(); - let queue = command - .get("queue") - .map_or(Ok(DB::string(&run, "queue")?), |_| { - text(command, "queue").map(str::to_owned) - })?; + // A session-bound activity without its own queue or + // connection takes the session's (`applyWorkerSessionRouting`). + let session = super::worker_sessions::command_session(command); + let queue = match command.get("queue").filter(|queue| !queue.is_null()) { + Some(_) => text(command, "queue")?.to_owned(), + None => session.as_ref().and_then(|session| session.queue.clone()).map_or_else(|| DB::string(&run, "queue"), Ok)?, + }; + let connection = command["connection"].as_str().map(str::trim).filter(|connection| !connection.is_empty()).map(str::to_owned) + .or_else(|| session.as_ref().and_then(|session| session.connection.clone())); + let activity_options = session.as_ref().map(|session| json!({"worker_session": session.snapshot()})); let activity_type = text(command, "activity_type")?; let arguments = match external.get(&index) { Some(prepared) => { @@ -2758,11 +2809,12 @@ where // The preserved SQL schema defaults this counter to one. // Published authored scheduling explicitly starts at zero; // only the original worker claim consumes its first attempt. - Self::query("INSERT INTO activity_executions(id,workflow_run_id,sequence,activity_type,activity_class,status,attempt_count,payload_codec,arguments,queue,retry_policy,parallel_group_path,schedule_deadline_at,schedule_to_close_deadline_at) VALUES ($1,$2,$3,$4,$5,'pending',0,'avro',$6,$7,$8,$9,$10,$11)") + Self::query("INSERT INTO activity_executions(id,workflow_run_id,sequence,activity_type,activity_class,status,attempt_count,payload_codec,arguments,queue,retry_policy,parallel_group_path,schedule_deadline_at,schedule_to_close_deadline_at,connection,activity_options) VALUES ($1,$2,$3,$4,$5,'pending',0,'avro',$6,$7,$8,$9,$10,$11,$12,$13)") .bind(&activity_id).bind(&run_id).bind(sequence).bind(activity_type).bind(activity_type) .bind(&arguments).bind(&queue).bind(policy.as_ref().map(DB::document)) .bind(DB::optional_document(&metadata["parallel_group_path"])) - .bind(DB::optional_time(schedule_deadline)).bind(DB::optional_time(lifecycle_deadline)).execute(&mut *tx).await?; + .bind(DB::optional_time(schedule_deadline)).bind(DB::optional_time(lifecycle_deadline)) + .bind(connection.as_deref()).bind(activity_options.as_ref().map(DB::document)).execute(&mut *tx).await?; let activity = Self::query("SELECT * FROM activity_executions WHERE id=$1") .bind(&activity_id) .fetch_one(&mut *tx) @@ -4378,7 +4430,8 @@ pub(crate) fn capabilities() -> Value { let mut value = json!({"supported_workflow_task_commands": ["schedule_activity", "start_timer", "start_child_workflow", "open_condition_wait", "open_signal_wait", "record_version_marker", "record_side_effect", "complete_workflow", "fail_workflow", "continue_as_new", "complete_update", "fail_update"], "worker_deregistration_fencing":{"schema":"durable-workflow.v2.worker-deregistration.v1","supported":true,"receipt_retention_seconds":600,"endpoint":"/worker/registrations/{workerId}/deregister"}, "workflow_memo_updates": false, "cooperative_cancellation": true, "prepared_local_activities": false, - "worker_sessions": false, "sticky_execution": false, "local_activities": false, "message_streams": super::message_streams::worker_capability(), + "worker_sessions": super::worker_sessions::capability(), "worker_session_verbs": ["create", "heartbeat", "close"], + "sticky_execution": false, "local_activities": false, "message_streams": super::message_streams::worker_capability(), "update_validation_tasks": true, "synchronous_update_validation": super::update_validation::capability(), "workflow_updates": true, "query_tasks": true, "query_task_authority":true, "query_task_poll_request_idempotency": true, "activity_lease_renewal": true, "activity_cancellation_acknowledgement": true}); diff --git a/rust/src/runtime/upgrade.rs b/rust/src/runtime/upgrade.rs index 0995ed343..612e0977c 100644 --- a/rust/src/runtime/upgrade.rs +++ b/rust/src/runtime/upgrade.rs @@ -105,6 +105,7 @@ pub(super) fn native_relation(name: &str) -> bool { | "dw_poll_receipts" | "dw_query_cache" | "dw_worker_registration_incarnations" + | "dw_worker_session_holders" ) } @@ -120,6 +121,7 @@ pub(super) fn native_object(name: &str) -> bool { | "dw_worker_registration_incarnations_pkey" | "dw_worker_incarnation_authority" | "dw_worker_incarnation_retention" + | "dw_worker_session_holders_pkey" ) } diff --git a/rust/src/runtime/upgrade_tests.rs b/rust/src/runtime/upgrade_tests.rs index c55082a5a..a76db6497 100644 --- a/rust/src/runtime/upgrade_tests.rs +++ b/rust/src/runtime/upgrade_tests.rs @@ -25,12 +25,15 @@ async fn php_layout() -> (tempfile::TempDir, String) { sqlx::raw_sql(concat!( include_str!("../../migrations/sqlite/0003_full_schema.sql"), "\n", - include_str!("../../migrations/sqlite/0004_worker_incarnations.sql") + include_str!("../../migrations/sqlite/0004_worker_incarnations.sql"), + "\n", + include_str!("../../migrations/sqlite/0005_worker_session_holders.sql") )) .execute(&mut connection) .await .unwrap(); for table in [ + "dw_worker_session_holders", "dw_worker_registration_incarnations", "dw_task_completions", "dw_poll_receipts", @@ -944,7 +947,7 @@ async fn takeover_converts_a_php_297_database_that_native_then_opens() { .fetch_one(&mut connection) .await .unwrap(); - assert_eq!(marker, ("rust-development".to_owned(), 4)); + assert_eq!(marker, ("rust-development".to_owned(), super::super::schema::VERSION)); connection.close().await.unwrap(); } diff --git a/rust/src/runtime/worker_sessions.rs b/rust/src/runtime/worker_sessions.rs new file mode 100644 index 000000000..57f8bfdd3 --- /dev/null +++ b/rust/src/runtime/worker_sessions.rs @@ -0,0 +1,1212 @@ +//! Worker sessions, as published Server 2.11.1 serves them +//! (`WorkerSessionController` and `WorkerSessionRegistry`): a worker leases a +//! named session in its namespace, renews it with heartbeats, and closes it. +//! A session's lease and absolute TTL expire only when the session is read; +//! a holder whose registration is gone or silent leaves the session orphaned +//! on the next read. Session times are whole seconds, as PHP stores them. +//! +//! One published defect is not reproduced: PHP's `renewHeldSession` filters +//! its own `failure_reason => null` out, so a reused or reacquired session +//! keeps its old failure reason (Server #737). Native clears it, as that code +//! intends; the shared fixture bounds the difference explicitly. +use super::laravel_validation::{Rule, failure_messages, rule_errors}; +use super::transaction::Transaction; +use super::{Result, RuntimeError, store::now}; +use axum::http::StatusCode; +use chrono::{DateTime, Utc}; +use serde_json::{Map, Value, json}; +use std::collections::HashSet; +use std::hash::Hash; + +const DEFAULT_LEASE_SECONDS: i64 = 120; +/// Portable session maxima, the same in PHP and native (Server #740). A lease +/// is renewed by heartbeats, so a longer one only delays holder-loss +/// detection. Thirty days of TTL keeps `ttl_expires_at` representable in a +/// MySQL `TIMESTAMP` for years, and every value fits PostgreSQL's `integer`. +pub(super) const MAX_LEASE_SECONDS: i64 = 86_400; +pub(super) const MAX_TTL_SECONDS: i64 = 2_592_000; +pub(super) const MAX_CONCURRENT_ACTIVITIES: i64 = 1_000; +const DEFAULT_TTL_SECONDS: i64 = 1800; +const DEFAULT_MAX_CONCURRENT_ACTIVITIES: i64 = 1; +const MINIMUM_PROTOCOL: (u64, u64) = (1, 8); + +const SESSION_BY_ID: &str = "SELECT id,session_id,connection,queue,requirements,status,lease_owner,lease_expires_at,ttl_expires_at,closed_at,failure_reason,lease_seconds,ttl_seconds,max_concurrent_activities,CASE WHEN create_if_missing THEN 1 ELSE 0 END AS create_if_missing_flag,CASE WHEN allow_reacquire_after_failure THEN 1 ELSE 0 END AS allow_reacquire_flag,last_heartbeat_at,created_at,updated_at FROM workflow_worker_sessions WHERE namespace=$1 AND session_id=$2"; +const SESSIONS_TO_REFRESH: &str = "SELECT session_id FROM workflow_worker_sessions WHERE namespace=$1 AND (status='active' OR (status<>'closed' AND ttl_expires_at<=$2 AND (failure_reason IS NULL OR failure_reason<>'ttl_expired'))) ORDER BY id"; +const SESSIONS_LISTED: &str = "SELECT id,session_id,connection,queue,requirements,status,lease_owner,lease_expires_at,ttl_expires_at,closed_at,failure_reason,lease_seconds,ttl_seconds,max_concurrent_activities,CASE WHEN create_if_missing THEN 1 ELSE 0 END AS create_if_missing_flag,CASE WHEN allow_reacquire_after_failure THEN 1 ELSE 0 END AS allow_reacquire_flag,last_heartbeat_at,created_at,updated_at FROM workflow_worker_sessions WHERE namespace=$1 ORDER BY updated_at DESC,session_id ASC"; +const HELD_SESSIONS: &str = "SELECT COUNT(*) FROM workflow_worker_sessions WHERE namespace=$1 AND lease_owner=$2 AND status='active' AND (lease_expires_at IS NULL OR lease_expires_at>$3) AND session_id<>$4"; +const HOLDER: &str = "SELECT task_queue,status,last_heartbeat_at FROM workflow_worker_registrations WHERE namespace=$1 AND worker_id=$2"; +/// The worker's current registration token, as the poll fence reads it. +/// The registration terms a worker session is admitted against. +const SESSION_REGISTRATION: &str = "SELECT task_queue,capabilities,max_concurrent_worker_sessions,status FROM workflow_worker_registrations WHERE namespace=$1 AND worker_id=$2"; +/// The capabilities the worker's current registration declared. +const REGISTRATION_CAPABILITIES: &str = "SELECT capabilities FROM workflow_worker_registrations WHERE namespace=$1 AND worker_id=$2"; +/// `WorkerSessionRegistry::REGISTRATION_TOKEN_CAPABILITY`: a worker that +/// declares it sends its registration token on every explicit session +/// create, heartbeat and close, and requests without it are refused. +const REGISTRATION_TOKEN_CAPABILITY: &str = "worker_session_registration_token"; +const CURRENT_TOKEN: &str = "SELECT token FROM dw_worker_registration_incarnations WHERE namespace=$1 AND worker_id=$2 AND status='active' ORDER BY created_at DESC LIMIT 1"; +/// The registration incarnation that acquired or last renewed a session. +const SESSION_HOLDER: &str = "SELECT incarnation FROM dw_worker_session_holders WHERE namespace=$1 AND session_id=$2"; + +/// A session time: PHP keeps whole seconds. +fn second(at: DateTime) -> DateTime { + DateTime::from_timestamp(at.timestamp(), 0).expect("whole-second timestamp") +} + +/// `at` plus a session's seconds. Validation bounds request values, so this +/// fails only for a stored value no current server accepts; that answers an +/// error instead of panicking inside a durable transition. +fn after(at: DateTime, seconds: i64) -> Result> { + chrono::Duration::try_seconds(seconds) + .and_then(|span| at.checked_add_signed(span)) + .ok_or_else(|| RuntimeError::Protocol { status: StatusCode::UNPROCESSABLE_ENTITY, + response: json!({"error":"Worker session duration is out of range.","reason":"invalid_worker_session"}) }) +} + +fn wire(at: Option>) -> Value { + at.map_or(Value::Null, |at| json!(super::cooperative::timestamp(at))) +} + +/// `server_capabilities.worker_sessions`: the published contract +/// (`WorkerProtocolVersion::workerSessionSemantics`) and its support. +pub(super) fn capability() -> Value { + json!({"feature":"worker_sessions","contract_version":"1.0","minimum_protocol_version":"1.8","command_field":"worker_session", + "activity_options_field":"worker_session","verbs":["create","heartbeat","close"], + "lifecycle":{"creation":"lazy_create_on_first_admitted_activity_or_explicit_worker_create","renewal":"activity_heartbeat_or_explicit_session_heartbeat", + "close":"explicit_holder_close","lease_expiry":"session_expires_when_lease_is_not_renewed", + "ttl_expiry":"absolute_session_ttl_is_terminal_for_that_session_id"}, + "ownership":"single_worker_lease_owner", + "lease":{"scope":"namespace_session_id","owner":"registered_worker_id","activity_attempt_leases_remain_independent":true}, + "admission":{"queue_routing_first":true,"requires_registered_worker":true,"requires_capabilities":true,"create_if_missing_default":true, + "allow_reacquire_after_failure_default":true}, + "rollout_safety":{"minimum_protocol_version":"1.8","mixed_server_rollout_fenced_by_protocol_version":true, + "servers_below_minimum_must_reject_worker_session_commands":true,"servers_below_minimum_must_not_claim_worker_session_activity_tasks":true}, + "limits":{"max_concurrent_worker_sessions":"worker_registration","max_concurrent_activities":"session"}, + "default_max_concurrent_activities":1, + "renewal":{"activity_heartbeat_renews_session":true,"explicit_session_heartbeat":true}, + "failure_detection":["lease_expiry","registered_worker_heartbeat_staleness"], + "holder_loss":{"in_flight_activities_keep_at_least_once_attempt_semantics":true,"replacement_worker_must_reacquire_session":true, + "process_local_state_must_be_rebuilt_after_reacquire":true}, + "cancellation":{"workflow_cancellation_observed_through_activity_heartbeat":true,"session_lease_does_not_override_activity_cancel_requested":true, + "planned_shutdown_should_close_sessions":true}, + "routing":["queue","connection","requirements"], + "visibility":["active","closed","expired","failed","orphaned"], + "statuses":["active","closed","expired","failed","orphaned"], + "terminal_statuses":["closed"], + "terminal_conditions":["explicit_close","ttl_expired","allow_reacquire_after_failure_false"], + "authoring_guidance":["use_for_process_local_state_gpu_memory_or_filesystem_affinity","prefer_ordinary_queued_activities_for_independent_steps", + "prefer_one_larger_activity_for_atomic_side_effects"], + "supported":true}) +} + +/// `rejectWorkerSessionsUnavailable`: worker sessions need protocol 1.8. +pub(super) fn sessions_gate(protocol: &str) -> Result<()> { + let version = protocol.split_once('.').and_then(|(major, minor)| { + let digits = |text: &str| !text.is_empty() && text.bytes().all(|byte| byte.is_ascii_digit()); + (digits(major) && digits(minor)).then(|| Some((major.parse::().ok()?, minor.parse::().ok()?))).flatten() + }); + if version.is_some_and(|version| version >= MINIMUM_PROTOCOL) { + return Ok(()); + } + Err(RuntimeError::Protocol { + status: StatusCode::CONFLICT, + response: json!({"error":"Worker sessions require worker protocol 1.8 or newer.","reason":"worker_sessions_unavailable", + "supported_version":super::WORKER_PROTOCOL_VERSION,"requested_version":protocol,"minimum_protocol_version":"1.8", + "remediation":"Send the X-Durable-Workflow-Protocol-Version header with worker protocol 1.8 or newer and use a worker SDK that implements worker-session semantics."}), + }) +} + +/// The controller's rules, after Laravel's TrimStrings and +/// ConvertEmptyStringsToNull. +fn validate(body: &Value, create: bool) -> Result { + use Rule::{Array, Boolean, Integer, Max, Min, Nullable, Required}; + let body = super::laravel_rules::normalize(body); + let mut rules: Vec<(&str, &[Rule])> = vec![("worker_id", &[Required, Rule::String, Max(255)])]; + if create { + rules.extend([ + ("session_id", &[Required, Rule::String, Max(255)][..]), + ("connection", &[Nullable, Rule::String, Max(255)][..]), + ("queue", &[Nullable, Rule::String, Max(255)][..]), + ("requirements", &[Nullable, Array][..]), + ]); + } + rules.push(("lease_seconds", &[Nullable, Integer, Min(1), Max(MAX_LEASE_SECONDS)])); + if create { + rules.extend([ + ("ttl_seconds", &[Nullable, Integer, Min(1), Max(MAX_TTL_SECONDS)][..]), + ("max_concurrent_activities", &[Nullable, Integer, Min(1), Max(MAX_CONCURRENT_ACTIVITIES)][..]), + ("create_if_missing", &[Nullable, Boolean][..]), + ("allow_reacquire_after_failure", &[Nullable, Boolean][..]), + ]); + } + rules.push(("registration_token", REGISTRATION_TOKEN_RULES)); + let mut errors = rule_errors(&body, &rules, |_, _| None); + // `requirements.*`: each a string of at most 255 characters, reported + // after `requirements` itself. + if let Some(items) = body.get("requirements").and_then(Value::as_array) { + let at = errors.iter().position(|(field, _)| field == "requirements").map_or(errors.len(), |at| at + 1); + let item_errors: Vec<(String, Vec)> = items + .iter() + .enumerate() + .map(|(index, item)| { + let field = format!("requirements.{index}"); + let mut messages = Vec::new(); + match item { + Value::Null => {} + Value::String(text) if text.chars().count() > 255 => { + messages.push(format!("The {field} field must not be greater than 255 characters.")); + } + Value::String(_) => {} + _ => messages.push(format!("The {field} field must be a string.")), + } + (field, messages) + }) + .collect(); + errors.splice(at..at, item_errors); + } + if let Some(error) = failure_messages(errors) { + return Err(error); + } + Ok(body) +} + +/// `positiveInt`: an integer above zero, or a string of digits. A number +/// written with a fraction, even an integral one, is not. +fn positive_int(value: &Value) -> Option { + match value { + Value::Number(number) => number.as_i64().filter(|value| *value > 0), + Value::String(text) if !text.is_empty() && text.bytes().all(|byte| byte.is_ascii_digit()) => { + text.parse::().ok().filter(|value| *value > 0) + } + _ => None, + } +} + +/// `boolValue`, as Laravel's `boolean` admits. +fn bool_value(value: &Value, default: bool) -> bool { + match value { + Value::Bool(flag) => *flag, + Value::Number(number) => number.as_i64().map_or(default, |value| value == 1), + Value::String(text) => match text.as_str() { + "1" => true, + "0" => false, + _ => default, + }, + _ => default, + } +} + +/// The token registration returns (`bin2hex(random_bytes(16))`). +fn registration_token_format(text: &str) -> bool { + text.len() == 32 && text.bytes().all(|byte| matches!(byte, b'0'..=b'9' | b'a'..=b'f')) +} + +/// An optional `registration_token` on create, heartbeat and close. +const REGISTRATION_TOKEN_RULES: &[Rule] = &[Rule::Nullable, Rule::String, Rule::Pattern(registration_token_format)]; + +/// The digest a session keeps of its holder's registration, as PHP keeps +/// `hash('sha256', token)`, so the deregistration credential itself is never +/// stored with the session. +fn incarnation_digest(token: &str) -> String { + use sha2::{Digest, Sha256}; + Sha256::digest(token.as_bytes()).iter().map(|byte| format!("{byte:02x}")).collect() +} + +/// Equal tokens, compared without an early exit (`hash_equals`). +fn same_token(left: &str, right: &str) -> bool { + left.len() == right.len() && left.bytes().zip(right.bytes()).fold(0, |differs, (a, b)| differs | (a ^ b)) == 0 +} + +fn string_value(value: &Value) -> Option { + value.as_str().map(str::trim).filter(|text| !text.is_empty()).map(str::to_owned) +} + +/// Each item once, in order of first occurrence (`array_unique`). The seen +/// index is a hash set, so a request-sized list stays linear even while a +/// durable transition holds the server-wide lock. +fn first_occurrences(items: impl IntoIterator) -> Vec { + let mut seen = HashSet::new(); + items.into_iter().filter(|item| seen.insert(item.clone())).collect() +} + +/// The required items missing from `available`, in required order, looked up +/// in a set built once (`array_flip`), not by scanning `available` per item. +fn missing_from<'a, T: Hash + Eq>(required: &'a [T], available: &[T]) -> Vec<&'a T> { + let available: HashSet<&T> = available.iter().collect(); + required.iter().filter(|item| !available.contains(item)).collect() +} + +/// A `requirements` list as `normalizeOptions` and the stored snapshot read +/// it: trimmed, non-empty strings, each once. +fn requirement_list(value: &Value) -> Vec { + first_occurrences(value.as_array().into_iter().flatten().filter_map(string_value)) +} + +/// `normalizeOptions`. +struct Options { + session_id: String, + connection: Option, + queue: Option, + requirements: Vec, + lease_seconds: i64, + ttl_seconds: i64, + max_concurrent_activities: i64, + create_if_missing: bool, + allow_reacquire_after_failure: bool, +} + +impl Options { + fn from(body: &Value, fallback_queue: Option) -> Option { + let requirements = requirement_list(&body["requirements"]); + Some(Self { + session_id: string_value(&body["session_id"])?, + connection: string_value(&body["connection"]), + queue: string_value(&body["queue"]).or(fallback_queue), + requirements, + lease_seconds: positive_int(&body["lease_seconds"]).unwrap_or(DEFAULT_LEASE_SECONDS), + ttl_seconds: positive_int(&body["ttl_seconds"]).unwrap_or(DEFAULT_TTL_SECONDS), + max_concurrent_activities: positive_int(&body["max_concurrent_activities"]).unwrap_or(DEFAULT_MAX_CONCURRENT_ACTIVITIES), + create_if_missing: bool_value(&body["create_if_missing"], true), + allow_reacquire_after_failure: bool_value(&body["allow_reacquire_after_failure"], true), + }) + } +} + +/// A `schedule_activity` command's `worker_session`, as the command +/// normalizer leaves it and `WorkerSessionOptions::toSnapshot` stores it in +/// the activity's options: omitted values stay null. +pub(super) struct CommandSession { + pub(super) session_id: String, + pub(super) connection: Option, + pub(super) queue: Option, + requirements: Vec, + lease_seconds: Option, + ttl_seconds: Option, + max_concurrent_activities: Option, + create_if_missing: bool, + allow_reacquire_after_failure: bool, +} + +impl CommandSession { + pub(super) fn snapshot(&self) -> Value { + json!({"session_id":self.session_id,"connection":self.connection,"queue":self.queue,"requirements":self.requirements, + "lease_seconds":self.lease_seconds,"ttl_seconds":self.ttl_seconds,"max_concurrent_activities":self.max_concurrent_activities, + "create_if_missing":self.create_if_missing,"allow_reacquire_after_failure":self.allow_reacquire_after_failure}) + } + + /// The stored snapshot, from an activity's options. + pub(super) fn from_snapshot(value: &Value) -> Option { + let requirements = requirement_list(&value["requirements"]); + Some(Self { + session_id: string_value(&value["session_id"])?, + connection: string_value(&value["connection"]), + queue: string_value(&value["queue"]), + requirements, + lease_seconds: value["lease_seconds"].as_i64().filter(|value| *value > 0), + ttl_seconds: value["ttl_seconds"].as_i64().filter(|value| *value > 0), + max_concurrent_activities: value["max_concurrent_activities"].as_i64().filter(|value| *value > 0), + create_if_missing: value["create_if_missing"].as_bool().unwrap_or(true), + allow_reacquire_after_failure: value["allow_reacquire_after_failure"].as_bool().unwrap_or(true), + }) + } + + /// `normalizeOptions` with the activity's queue and connection as the + /// fallbacks. + fn options(&self, queue: Option<&str>, connection: Option<&str>) -> Options { + Options { + session_id: self.session_id.clone(), + connection: self.connection.clone().or_else(|| connection.map(str::to_owned)), + queue: self.queue.clone().or_else(|| queue.map(str::to_owned)), + requirements: self.requirements.clone(), + lease_seconds: self.lease_seconds.unwrap_or(DEFAULT_LEASE_SECONDS), + ttl_seconds: self.ttl_seconds.unwrap_or(DEFAULT_TTL_SECONDS), + max_concurrent_activities: self.max_concurrent_activities.unwrap_or(DEFAULT_MAX_CONCURRENT_ACTIVITIES), + create_if_missing: self.create_if_missing, + allow_reacquire_after_failure: self.allow_reacquire_after_failure, + } + } + + /// `workerCanSatisfySession`: the claiming worker's protocol, task queue + /// and capabilities. + pub(super) fn claimable_by(&self, protocol: &str, worker_queue: &str, activity_queue: &str, capabilities: &[String]) -> bool { + sessions_gate(protocol).is_ok() + && self.queue.as_deref().unwrap_or(activity_queue) == worker_queue + && missing_from(&self.requirements, capabilities).is_empty() + } + + /// The lease an activity heartbeat renews the session by. + pub(super) fn heartbeat_lease(&self) -> i64 { + self.lease_seconds.unwrap_or(DEFAULT_LEASE_SECONDS) + } +} + +/// A validated command's `worker_session`, when it names a session. +pub(super) fn command_session(command: &Value) -> Option { + let raw = command.get("worker_session").filter(|value| value.is_object())?; + CommandSession::from_snapshot(raw) +} + +/// The command normalizer's `worker_session` checks, with its messages. +fn normalizer_errors(index: usize, command: &Value) -> Vec<(String, Vec)> { + let Some(raw) = command.get("worker_session").filter(|value| !value.is_null()) else { + return Vec::new(); + }; + let at = |field: &str| format!("commands.{index}.worker_session{field}"); + let Some(raw) = raw.as_object() else { + return vec![(at(""), vec!["Workflow task command field [worker_session] must be an object when provided.".into()])]; + }; + if raw.get("session_id").and_then(Value::as_str).is_none_or(|id| id.trim().is_empty()) { + return vec![(at(".session_id"), vec!["Worker session commands require a non-empty session_id.".into()])]; + } + let mut errors = Vec::new(); + for field in ["connection", "queue"] { + if let Some(value) = raw.get(field).filter(|value| !value.is_null()) + && value.as_str().is_none_or(|text| text.trim().is_empty()) + { + errors.push((at(&format!(".{field}")), vec![format!("Worker session field [{field}] must be a non-empty string when provided.")])); + } + } + for field in ["lease_seconds", "ttl_seconds", "max_concurrent_activities"] { + if let Some(value) = raw.get(field).filter(|value| !value.is_null()) + && value.as_i64().is_none_or(|value| value < 1) + { + errors.push((at(&format!(".{field}")), vec![format!("Worker session field [{field}] must be a positive integer when provided.")])); + } + } + for field in ["create_if_missing", "allow_reacquire_after_failure"] { + if let Some(value) = raw.get(field).filter(|value| !value.is_null()) + && !value.is_boolean() + { + errors.push((at(&format!(".{field}")), vec![format!("Worker session field [{field}] must be boolean when provided.")])); + } + } + if let Some(requirements) = raw.get("requirements").filter(|value| !value.is_null()) { + match requirements.as_array() { + None => errors.push((at(".requirements"), vec!["Worker session requirements must be a list of non-empty strings.".into()])), + Some(items) => { + for (position, item) in items.iter().enumerate() { + if item.as_str().is_none_or(|text| text.trim().is_empty()) { + errors.push((at(&format!(".requirements.{position}")), vec!["Worker session requirements must be non-empty strings.".into()])); + } + } + } + } + } + errors +} + +/// A protocol version as `major.minor`. +fn version(protocol: Option<&str>) -> Option<(u64, u64)> { + let (major, minor) = protocol?.split_once('.')?; + let digits = |text: &str| !text.is_empty() && text.bytes().all(|byte| byte.is_ascii_digit()); + (digits(major) && digits(minor)).then(|| Some((major.parse().ok()?, minor.parse().ok()?))).flatten() +} + +/// A session row. +struct Session { + id: i64, + session_id: String, + connection: Option, + queue: Option, + requirements: Vec, + status: String, + lease_owner: Option, + lease_expires_at: Option>, + ttl_expires_at: Option>, + closed_at: Option>, + failure_reason: Option, + lease_seconds: i64, + ttl_seconds: i64, + max_concurrent_activities: i64, + create_if_missing: bool, + allow_reacquire_after_failure: bool, + last_heartbeat_at: Option>, + created_at: Option>, + updated_at: Option>, +} + +/// A registry answer: the HTTP status and PHP's body, which repeats it. +pub(crate) struct Answer { + pub(crate) status: StatusCode, + pub(crate) body: Value, +} + +fn success(outcome: &str, session: Value, status: StatusCode) -> Answer { + Answer { status, body: json!({"admitted":true,"outcome":outcome,"session":session,"activity_task_id":null, + "status":status.as_u16(),"reason":null}) } +} + +/// `supersededRegistration`: a registration fenced as superseded has lost +/// its authority, as PHP's worker heartbeat treats it. +fn superseded_answer(worker_id: &str) -> Answer { + failure("worker_registration_superseded", "Worker registration was fenced after its leases were recovered.", + StatusCode::CONFLICT, Map::from_iter([("worker_id".to_owned(), json!(worker_id)), + ("remediation".to_owned(), json!("Restart and register the worker with a new worker_id."))])) +} + +/// `notRegistered`: an explicit session request from a worker without a +/// current registration. +fn not_registered(worker_id: &str, action: &str) -> Answer { + failure("worker_not_registered", &format!("Worker must be registered before {action} a worker session."), + StatusCode::PRECONDITION_FAILED, Map::from_iter([("worker_id".to_owned(), json!(worker_id))])) +} + +fn failure(reason: &str, error: &str, status: StatusCode, context: Map) -> Answer { + let mut body = context; + body.insert("admitted".into(), json!(false)); + body.insert("error".into(), json!(error)); + body.insert("reason".into(), json!(reason)); + body.insert("status".into(), json!(status.as_u16())); + Answer { status, body: Value::Object(body) } +} + +fn with_session(session: Value) -> Map { + Map::from_iter([("session".to_owned(), session)]) +} + +/// The compact form a task or status carries (`activitySessionPayload`): +/// the normalized options with the row's lease, without null values. +fn compact(options: &Options, session: Option<&Session>) -> Value { + let mut fields = json!({"session_id":options.session_id,"connection":options.connection,"queue":options.queue, + "requirements":options.requirements,"lease_seconds":options.lease_seconds,"ttl_seconds":options.ttl_seconds, + "max_concurrent_activities":options.max_concurrent_activities,"create_if_missing":options.create_if_missing, + "allow_reacquire_after_failure":options.allow_reacquire_after_failure}); + if let Some(session) = session { + fields["status"] = json!(session.status); + fields["lease_owner"] = json!(session.lease_owner); + fields["lease_expires_at"] = wire(session.lease_expires_at); + fields["ttl_expires_at"] = wire(session.ttl_expires_at); + } + if let Some(map) = fields.as_object_mut() { + map.retain(|_, value| !value.is_null()); + } + fields +} + +store_methods! { + /// `POST /api/worker/sessions`. + pub(crate) async fn create_worker_session(&self, body: Value, protocol: &str) -> Result { + sessions_gate(protocol)?; + let body = validate(&body, true)?; + let worker_id = body["worker_id"].as_str().unwrap_or_default().to_owned(); + let Some(worker) = Self::query(SESSION_REGISTRATION) + .bind(self.namespace.as_ref()).bind(&worker_id).fetch_optional(&self.pool).await? + else { + return Err(RuntimeError::Protocol { status: StatusCode::PRECONDITION_FAILED, + response: json!({"error":"Worker must be registered before creating a worker session.","reason":"worker_not_registered","worker_id":worker_id}) }); + }; + let task_queue = DB::string(&worker, "task_queue")?; + if let Some(queue) = string_value(&body["queue"]).filter(|queue| *queue != task_queue) { + return Err(RuntimeError::Protocol { status: StatusCode::CONFLICT, + response: json!({"error":format!("Worker [{worker_id}] is registered for task queue [{task_queue}], not session queue [{queue}]."), + "reason":"session_queue_mismatch","worker_id":worker_id,"registered_task_queue":task_queue,"requested_session_queue":queue}) }); + } + let Some(options) = Options::from(&body, Some(task_queue.clone())) else { + return Err(RuntimeError::Protocol { status: StatusCode::UNPROCESSABLE_ENTITY, + response: json!({"error":"Worker session requires a non-empty session_id.","reason":"invalid_worker_session"}) }); + }; + let mut tx = self.begin().await?; + // Admit against the registration as it is inside the transaction, not + // as it was read above, as PHP's createOrReacquire does under its row + // lock; begin() serializes this with every re-registration. + let current = Self::query(SESSION_REGISTRATION) + .bind(self.namespace.as_ref()).bind(&worker_id).fetch_optional(&mut *tx).await?; + let answer = match current { + None => not_registered(&worker_id, "creating"), + Some(current) if Self::registration_superseded(¤t)? => superseded_answer(&worker_id), + Some(current) => match self.stale_registration(&mut tx, &worker_id, body["registration_token"].as_str()).await? { + Some(stale) => stale, + None => { + let capabilities: Vec = DB::optional_document_row(¤t, "capabilities")? + .as_ref() + .and_then(Value::as_array) + .into_iter() + .flatten() + .filter_map(|value| value.as_str().map(str::to_owned)) + .collect(); + let limit = DB::optional_number(¤t, "max_concurrent_worker_sessions")?.unwrap_or(10); + let queue = DB::string(¤t, "task_queue")?; + self.acquire(&mut tx, &worker_id, &queue, &capabilities, limit, &options, None).await? + } + }, + }; + tx.commit().await?; + Ok(answer) + } + + /// `acquireLocked`. + async fn acquire( + &self, + tx: &mut Transaction<'_, DB>, + worker_id: &str, + task_queue: &str, + capabilities: &[String], + limit: i64, + options: &Options, + activity_task: Option<&str>, + ) -> Result { + if options.queue.as_deref().is_some_and(|queue| queue != task_queue) { + return Ok(failure("session_queue_mismatch", "Worker session queue does not match the worker's task queue.", + StatusCode::CONFLICT, Map::new())); + } + // Requirements are checked before the session is looked up. + let missing = missing_from(&options.requirements, capabilities); + if !missing.is_empty() { + return Ok(failure("session_requirements_not_met", "Worker does not advertise every worker-session requirement.", + StatusCode::CONFLICT, Map::from_iter([("missing_requirements".to_owned(), json!(missing))]))); + } + let Some(mut session) = self.locked_session(tx, &options.session_id).await? else { + if !options.create_if_missing { + return Ok(failure("session_missing", "Worker session does not exist and creation is disabled.", + StatusCode::CONFLICT, Map::new())); + } + if let Some(refused) = self.session_limit(tx, worker_id, limit, &options.session_id).await? { + return Ok(refused); + } + let at = second(now()); + Self::query("INSERT INTO workflow_worker_sessions(namespace,session_id,connection,queue,requirements,status,lease_owner,lease_expires_at,ttl_expires_at,lease_seconds,ttl_seconds,max_concurrent_activities,create_if_missing,allow_reacquire_after_failure,last_heartbeat_at,created_at,updated_at) VALUES ($1,$2,$3,$4,$5,'active',$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16)") + .bind(self.namespace.as_ref()).bind(&options.session_id).bind(options.connection.as_deref()).bind(options.queue.as_deref()) + .bind(DB::document(&json!(options.requirements))).bind(worker_id) + .bind(DB::bind_time(after(at, options.lease_seconds)?)) + .bind(DB::bind_time(after(at, options.ttl_seconds)?)) + .bind(options.lease_seconds).bind(options.ttl_seconds).bind(options.max_concurrent_activities) + .bind(DB::optional_flag(Some(options.create_if_missing))).bind(DB::optional_flag(Some(options.allow_reacquire_after_failure))) + .bind(DB::bind_time(at)).bind(DB::bind_time(at)).bind(DB::bind_time(at)) + .execute(&mut **tx).await?; + self.record_holder(tx, &options.session_id, worker_id).await?; + let created = self.locked_session(tx, &options.session_id).await?.expect("the inserted session"); + return Ok(success("created", self.snapshot(tx, &created).await?, StatusCode::CREATED)); + }; + if session.status == "active" { + if session.lease_owner.as_deref() != Some(worker_id) { + let snapshot = self.snapshot(tx, &session).await?; + return Ok(failure("session_owned_by_another_worker", "Worker session is currently leased by another worker.", + StatusCode::CONFLICT, with_session(snapshot))); + } + // An activity claim also respects the session's activity limit. + if let Some(task) = activity_task { + let running = self.running_session_activities(tx, &session, task).await?; + if running >= session.max_concurrent_activities.max(1) { + let snapshot = self.snapshot(tx, &session).await?; + return Ok(failure("session_activity_limit_exceeded", "Worker session has reached max_concurrent_activities.", + StatusCode::CONFLICT, with_session(snapshot))); + } + } + self.renew(tx, &mut session, options, worker_id).await?; + return Ok(success("reused", self.snapshot(tx, &session).await?, StatusCode::OK)); + } + if session.status == "closed" { + let snapshot = self.snapshot(tx, &session).await?; + return Ok(failure("session_closed", "Worker session was closed and cannot be reacquired.", + StatusCode::CONFLICT, with_session(snapshot))); + } + if session.failure_reason.as_deref() == Some("ttl_expired") || !session.allow_reacquire_after_failure { + let snapshot = self.snapshot(tx, &session).await?; + return Ok(failure("session_reacquire_disallowed", "Worker session is terminal and cannot be reacquired.", + StatusCode::CONFLICT, with_session(snapshot))); + } + if !options.allow_reacquire_after_failure { + let snapshot = self.snapshot(tx, &session).await?; + return Ok(failure("session_reacquire_disallowed", "Worker session options disallow reacquisition after holder failure.", + StatusCode::CONFLICT, with_session(snapshot))); + } + if let Some(refused) = self.session_limit(tx, worker_id, limit, &options.session_id).await? { + return Ok(refused); + } + self.renew(tx, &mut session, options, worker_id).await?; + Ok(success("reacquired", self.snapshot(tx, &session).await?, StatusCode::OK)) + } + + /// The worker session limit, not counting this session. + async fn session_limit(&self, tx: &mut Transaction<'_, DB>, worker_id: &str, limit: i64, session_id: &str) -> Result> { + let held = Self::scalar(HELD_SESSIONS).bind(self.namespace.as_ref()).bind(worker_id).bind(DB::bind_time(now())).bind(session_id) + .fetch_one(&mut **tx).await?; + Ok((held >= limit).then(|| failure("worker_session_limit_exceeded", "Worker has reached max_concurrent_worker_sessions.", + StatusCode::CONFLICT, Map::from_iter([("max_concurrent_worker_sessions".to_owned(), json!(limit))])))) + } + + /// `renewHeldSession`. Options the request leaves out reset to their + /// defaults; the TTL deadline is never extended; an omitted connection + /// keeps the stored one. Unlike published PHP, the failure reason is + /// cleared (Server #737). + async fn renew(&self, tx: &mut Transaction<'_, DB>, session: &mut Session, options: &Options, worker_id: &str) -> Result<()> { + let at = second(now()); + if options.connection.is_some() { + session.connection.clone_from(&options.connection); + } + session.queue.clone_from(&options.queue); + session.requirements.clone_from(&options.requirements); + session.status = "active".into(); + session.lease_owner = Some(worker_id.to_owned()); + session.lease_expires_at = Some(after(at, options.lease_seconds)?); + session.closed_at = None; + session.failure_reason = None; + session.lease_seconds = options.lease_seconds; + session.ttl_seconds = options.ttl_seconds; + session.max_concurrent_activities = options.max_concurrent_activities; + session.create_if_missing = options.create_if_missing; + session.allow_reacquire_after_failure = options.allow_reacquire_after_failure; + session.last_heartbeat_at = Some(at); + self.save(tx, session).await?; + // The renewing registration now holds the session. + self.record_holder(tx, &session.session_id, worker_id).await + } + + /// `activeActivityCount`: the owner's running, unexpired attempts of + /// activities bound to this session, other than one task. + async fn running_session_activities(&self, tx: &mut Transaction<'_, DB>, session: &Session, except_task: &str) -> Result { + let Some(owner) = session.lease_owner.as_deref() else { + return Ok(0); + }; + Ok(Self::scalar(DB::SESSION_RUNNING_ATTEMPTS).bind(self.namespace.as_ref()).bind(DB::bind_time(now())).bind(owner) + .bind(session.session_id.as_str()).bind(except_task).fetch_one(&mut **tx).await?) + } + + /// `guardWorkerSessionCommandsAvailable`, then + /// `guardPortableWorkerAffinityCompletion`, then the command normalizer's + /// `worker_session` checks, in PHP's order. + pub(super) async fn guard_worker_session_commands( + &self, + tx: &mut Transaction<'_, DB>, + task: &DB::Row, + task_id: &str, + body: &Value, + protocol: Option<&str>, + commands: &[Value], + ) -> Result<()> { + if !commands.iter().any(|command| command.get("worker_session").is_some_and(|value| !value.is_null())) { + return Ok(()); + } + let attempt = body["workflow_task_attempt"].clone(); + let version = version(protocol); + if version.is_none_or(|version| version < MINIMUM_PROTOCOL) { + return Err(RuntimeError::Protocol { status: StatusCode::CONFLICT, + response: json!({"task_id":task_id,"workflow_task_attempt":attempt,"outcome":"rejected","recorded":false, + "reason":"worker_sessions_unavailable","error":"Worker-session activity commands require worker protocol 1.8 or newer.", + "requested_version":protocol,"minimum_protocol_version":"1.8", + "remediation":"Complete worker-session workflow tasks through a server node advertising worker protocol 1.8 or newer."}) }); + } + let rejection = |reason: &str| RuntimeError::Protocol { status: StatusCode::CONFLICT, + response: json!({"task_id":task_id,"workflow_task_attempt":attempt,"outcome":"rejected","recorded":false,"reason":reason, + "capability":"worker_sessions","requested_version":protocol,"minimum_protocol_version":"1.18", + "remediation":"Use cold durable replay unless the registered worker manifest explicitly supports the required capability and exact cache identity."}) }; + if version.is_none_or(|version| version < (1, 18)) { + return Err(rejection("portable_worker_affinity_protocol_floor_unavailable")); + } + let owner = DB::optional_string(task, "lease_owner")?.unwrap_or_default(); + let worker = Self::query("SELECT capabilities,capability_manifest FROM workflow_worker_registrations WHERE namespace=$1 AND worker_id=$2") + .bind(self.namespace.as_ref()).bind(&owner).fetch_optional(&mut **tx).await?; + let capable = match &worker { + Some(row) => { + let flat = DB::optional_document_row(row, "capabilities")? + .and_then(|capabilities| capabilities.as_array().map(|items| items.iter().any(|item| item == "worker_sessions"))) + .unwrap_or(false); + let manifest = DB::optional_document_row(row, "capability_manifest")? + .is_some_and(|manifest| manifest["worker_sessions"]["supported"] == json!(true)); + flat && manifest + } + None => false, + }; + if !capable { + return Err(rejection("worker_capability_not_supported")); + } + let errors: Vec<_> = commands.iter().enumerate().flat_map(|(index, command)| normalizer_errors(index, command)).collect(); + failure_messages(errors).map_or(Ok(()), Err) + } + + /// `admitActivity` inside a claim: acquire or reuse the activity's + /// session for the claiming worker, within its activity limit. Answers + /// the compact form, or `None` when the session refuses the claim. + pub(super) async fn admit_session_activity( + &self, + tx: &mut Transaction<'_, DB>, + worker: &DB::Row, + session: &CommandSession, + activity_queue: &str, + activity_connection: Option<&str>, + task_id: &str, + ) -> Result> { + let worker_id = DB::string(worker, "worker_id")?; + let task_queue = DB::string(worker, "task_queue")?; + let capabilities: Vec = DB::optional_document_row(worker, "capabilities")? + .as_ref() + .and_then(Value::as_array) + .into_iter() + .flatten() + .filter_map(|value| value.as_str().map(str::to_owned)) + .collect(); + let limit = DB::optional_number(worker, "max_concurrent_worker_sessions")?.unwrap_or(10); + let options = session.options(Some(activity_queue), activity_connection); + let answer = self.acquire(tx, &worker_id, &task_queue, &capabilities, limit, &options, Some(task_id)).await?; + if answer.body["admitted"] != json!(true) { + return Ok(None); + } + let row = self.locked_session(tx, &options.session_id).await?; + Ok(Some(compact(&options, row.as_ref()))) + } + + /// `heartbeatForAttempt`: an activity heartbeat or lease renewal renews + /// its session by the command's lease. Answers the full snapshot, or the + /// refusal's status, its snapshot (if any) and the renewal's failure. + pub(super) async fn renew_session_for_attempt( + &self, + tx: &mut Transaction<'_, DB>, + session: &CommandSession, + lease_owner: &str, + ) -> Result> { + let Some(mut row) = self.locked_session(tx, &session.session_id).await? else { + return Ok(Err((StatusCode::NOT_FOUND, Value::Null, + json!({"admitted":false,"reason":"session_not_found","error":"Worker session was not found."})))); + }; + if row.status != "active" { + let snapshot = self.snapshot(tx, &row).await?; + return Ok(Err((StatusCode::CONFLICT, snapshot, + json!({"admitted":false,"reason":"session_not_active","error":"Worker session is not active."})))); + } + if row.lease_owner.as_deref() != Some(lease_owner) { + let snapshot = self.snapshot(tx, &row).await?; + return Ok(Err((StatusCode::CONFLICT, snapshot, + json!({"admitted":false,"reason":"session_owner_mismatch","error":"Worker session lease is owned by another worker."})))); + } + let at = second(now()); + row.lease_expires_at = Some(after(at, session.heartbeat_lease().max(1))?); + row.last_heartbeat_at = Some(at); + row.failure_reason = None; + self.save(tx, &row).await?; + Ok(Ok(self.snapshot(tx, &row).await?)) + } + + /// The compact form an attempt's status carries, read without a + /// refresh, and whether the session still lets the attempt continue. + pub(super) async fn session_for_status( + &self, + tx: &mut Transaction<'_, DB>, + session: &CommandSession, + activity_queue: &str, + activity_connection: Option<&str>, + lease_owner: &str, + ) -> Result<(Value, bool)> { + let row = Self::query(SESSION_BY_ID).bind(self.namespace.as_ref()).bind(session.session_id.as_str()).fetch_optional(&mut **tx).await?; + let row = row.as_ref().map(Self::session).transpose()?; + let at = now(); + let usable = row.as_ref().is_some_and(|row| row.status == "active" && row.lease_owner.as_deref() == Some(lease_owner) + && row.lease_expires_at.is_some_and(|lease| lease > at) && row.ttl_expires_at.is_some_and(|ttl| ttl > at)); + Ok((compact(&session.options(Some(activity_queue), activity_connection), row.as_ref()), usable)) + } + + /// `metrics`: the namespace's sessions by status, after a refresh. + pub(crate) async fn worker_session_metrics(&self) -> Result { + self.refresh_namespace_sessions().await?; + let rows = Self::query("SELECT status FROM workflow_worker_sessions WHERE namespace=$1").bind(self.namespace.as_ref()).fetch_all(&self.pool).await?; + let mut metrics = json!({"total":0,"active":0,"closed":0,"expired":0,"failed":0,"orphaned":0}); + for row in rows { + let status = DB::string(&row, "status")?; + metrics["total"] = json!(metrics["total"].as_i64().unwrap_or(0) + 1); + if let Some(count) = metrics.get_mut(status.as_str()) { + *count = json!(count.as_i64().unwrap_or(0) + 1); + } + } + Ok(metrics) + } + + /// `POST /api/worker/sessions/{sessionId}/heartbeat`. + pub(crate) async fn heartbeat_worker_session(&self, session_id: &str, body: Value, protocol: &str) -> Result { + sessions_gate(protocol)?; + let body = validate(&body, false)?; + let worker_id = body["worker_id"].as_str().unwrap_or_default(); + let mut tx = self.begin().await?; + if let Some(refused) = self.session_request_fence(&mut tx, worker_id, body["registration_token"].as_str(), "renewing").await? { + tx.commit().await?; + return Ok(refused); + } + let Some(mut session) = self.locked_session(&mut tx, session_id).await? else { + tx.commit().await?; + return Ok(failure("session_not_found", "Worker session was not found.", StatusCode::NOT_FOUND, Map::new())); + }; + let answer = if session.status != "active" { + failure("session_not_active", "Worker session is not active.", StatusCode::CONFLICT, with_session(self.snapshot(&mut tx, &session).await?)) + } else if session.lease_owner.as_deref() != Some(worker_id) { + failure("session_owner_mismatch", "Worker session lease is owned by another worker.", StatusCode::CONFLICT, + with_session(self.snapshot(&mut tx, &session).await?)) + } else { + let at = second(now()); + let lease = super::laravel_validation::integer(&body["lease_seconds"]).unwrap_or(session.lease_seconds).max(1); + session.lease_expires_at = Some(after(at, lease)?); + session.last_heartbeat_at = Some(at); + session.failure_reason = None; + self.save(&mut tx, &session).await?; + success("heartbeat_recorded", self.snapshot(&mut tx, &session).await?, StatusCode::OK) + }; + tx.commit().await?; + Ok(answer) + } + + /// `DELETE /api/worker/sessions/{sessionId}`. + pub(crate) async fn close_worker_session(&self, session_id: &str, body: Value, protocol: &str) -> Result { + sessions_gate(protocol)?; + let body = validate(&body, false)?; + let worker_id = body["worker_id"].as_str().unwrap_or_default(); + let mut tx = self.begin().await?; + if let Some(refused) = self.session_request_fence(&mut tx, worker_id, body["registration_token"].as_str(), "closing").await? { + tx.commit().await?; + return Ok(refused); + } + let Some(mut session) = self.locked_session(&mut tx, session_id).await? else { + tx.commit().await?; + return Ok(failure("session_not_found", "Worker session was not found.", StatusCode::NOT_FOUND, Map::new())); + }; + let owner = session.lease_owner.as_deref() == Some(worker_id); + let answer = if session.status == "closed" && owner { + success("already_closed", self.snapshot(&mut tx, &session).await?, StatusCode::OK) + } else if !owner { + failure("session_owner_mismatch", "Worker session lease is owned by another worker.", StatusCode::CONFLICT, + with_session(self.snapshot(&mut tx, &session).await?)) + } else { + session.status = "closed".into(); + session.closed_at = Some(second(now())); + session.failure_reason = None; + self.save(&mut tx, &session).await?; + success("closed", self.snapshot(&mut tx, &session).await?, StatusCode::OK) + }; + tx.commit().await?; + Ok(answer) + } + + /// `GET /api/worker-sessions`: refreshes the namespace's sessions, then + /// lists them, newest change first, with the status counts. + pub(crate) async fn list_worker_sessions(&self) -> Result { + self.refresh_namespace_sessions().await?; + let mut tx = self.begin().await?; + let rows = Self::query(SESSIONS_LISTED).bind(self.namespace.as_ref()).fetch_all(&mut *tx).await?; + let mut sessions = Vec::new(); + let mut metrics = json!({"total":0,"active":0,"closed":0,"expired":0,"failed":0,"orphaned":0}); + for row in &rows { + let session = Self::session(row)?; + metrics["total"] = json!(metrics["total"].as_i64().unwrap_or(0) + 1); + if let Some(count) = metrics.get_mut(session.status.as_str()) { + *count = json!(count.as_i64().unwrap_or(0) + 1); + } + sessions.push(self.snapshot(&mut tx, &session).await?); + } + tx.commit().await?; + Ok(json!({"namespace":self.namespace.as_ref(),"metrics":metrics,"sessions":sessions})) + } + + /// `GET /api/worker-sessions/{sessionId}`. + pub(crate) async fn show_worker_session(&self, session_id: &str) -> Result { + self.refresh_namespace_sessions().await?; + let mut tx = self.begin().await?; + let row = Self::query(SESSION_BY_ID).bind(self.namespace.as_ref()).bind(session_id).fetch_optional(&mut *tx).await?; + let Some(row) = row else { + tx.commit().await?; + return Err(RuntimeError::Protocol { status: StatusCode::NOT_FOUND, + response: json!({"message":format!("Worker session [{session_id}] not found in namespace [{}].", self.namespace), + "reason":"worker_session_not_found"}) }); + }; + let session = Self::session(&row)?; + let snapshot = self.snapshot(&mut tx, &session).await?; + tx.commit().await?; + Ok(json!({"namespace":self.namespace.as_ref(),"session":snapshot})) + } + + /// `refreshNamespace`: each session that may have expired, in its own + /// transaction. + async fn refresh_namespace_sessions(&self) -> Result<()> { + let ids = Self::query(SESSIONS_TO_REFRESH).bind(self.namespace.as_ref()).bind(DB::bind_time(now())).fetch_all(&self.pool).await?; + for row in ids { + let mut tx = self.begin().await?; + self.locked_session(&mut tx, &DB::string(&row, "session_id")?).await?; + tx.commit().await?; + } + Ok(()) + } + + /// The session, refreshed (`refreshStatusLocked`). + async fn locked_session(&self, tx: &mut Transaction<'_, DB>, session_id: &str) -> Result> { + let Some(row) = Self::query(SESSION_BY_ID).bind(self.namespace.as_ref()).bind(session_id).fetch_optional(&mut **tx).await? else { + return Ok(None); + }; + let mut session = Self::session(&row)?; + if session.status == "closed" || session.failure_reason.as_deref() == Some("ttl_expired") { + return Ok(Some(session)); + } + let at = now(); + let change = if session.ttl_expires_at.is_some_and(|ttl| ttl <= at) { + Some(("expired", "ttl_expired")) + } else if session.status != "active" { + None + } else if session.lease_expires_at.is_some_and(|lease| lease <= at) { + Some(("expired", "lease_expired")) + } else { + self.holder_loss(tx, &session).await?.map(|reason| ("orphaned", reason)) + }; + if let Some((status, reason)) = change { + session.status = status.into(); + session.failure_reason = Some(reason.into()); + self.save(tx, &session).await?; + } + Ok(Some(session)) + } + + /// Why an active session's holder no longer holds it: its registration + /// is gone or silent, or the worker ID registered again since it acquired + /// or renewed the session. A replacement process must reacquire it and + /// rebuild its process-local state, so it never inherits it as reused. + async fn holder_loss(&self, tx: &mut Transaction<'_, DB>, session: &Session) -> Result> { + if self.holder_stale(tx, session.lease_owner.as_deref()).await? { + return Ok(Some("worker_heartbeat_stale")); + } + let current = self.current_incarnation(tx, session.lease_owner.as_deref().unwrap_or_default()).await?; + let holder = Self::query(SESSION_HOLDER).bind(self.namespace.as_ref()).bind(&session.session_id) + .fetch_optional(&mut **tx).await? + .map(|row| DB::string(&row, "incarnation")) + .transpose()?; + Ok((current != holder).then_some("worker_registration_replaced")) + } + + async fn current_token(&self, tx: &mut Transaction<'_, DB>, worker_id: &str) -> Result> { + Self::query(CURRENT_TOKEN).bind(self.namespace.as_ref()).bind(worker_id).fetch_optional(&mut **tx).await? + .map(|row| DB::string(&row, "token")) + .transpose() + } + + async fn current_incarnation(&self, tx: &mut Transaction<'_, DB>, worker_id: &str) -> Result> { + Ok(self.current_token(tx, worker_id).await?.as_deref().map(incarnation_digest)) + } + + /// Records the worker's current registration as the session's holder. A + /// registration without a token, such as one kept from a PHP takeover, + /// records none. + async fn record_holder(&self, tx: &mut Transaction<'_, DB>, session_id: &str, worker_id: &str) -> Result<()> { + Self::query("DELETE FROM dw_worker_session_holders WHERE namespace=$1 AND session_id=$2") + .bind(self.namespace.as_ref()).bind(session_id).execute(&mut **tx).await?; + if let Some(incarnation) = self.current_incarnation(tx, worker_id).await? { + Self::query("INSERT INTO dw_worker_session_holders(namespace,session_id,incarnation,updated_at) VALUES ($1,$2,$3,$4)") + .bind(self.namespace.as_ref()).bind(session_id).bind(incarnation).bind(DB::bind_time(now())) + .execute(&mut **tx).await?; + } + Ok(()) + } + + /// Whether a registration row is fenced as superseded. + fn registration_superseded(row: &DB::Row) -> Result { + Ok(DB::optional_string(row, "status")?.as_deref() == Some("superseded")) + } + + /// The fence of an explicit heartbeat or close: the worker must have a + /// current registration, with or without a token, so a process whose + /// registration was deleted can no longer renew or close the session it + /// held; then the registration token rules apply. + async fn session_request_fence(&self, tx: &mut Transaction<'_, DB>, worker_id: &str, token: Option<&str>, action: &str) + -> Result> { + let registered = Self::query(SESSION_REGISTRATION) + .bind(self.namespace.as_ref()).bind(worker_id).fetch_optional(&mut **tx).await?; + let Some(registered) = registered else { + return Ok(Some(not_registered(worker_id, action))); + }; + if Self::registration_superseded(®istered)? { + return Ok(Some(superseded_answer(worker_id))); + } + self.stale_registration(tx, worker_id, token).await + } + + /// A registration token the request sent must be the worker's current + /// one. Without a token, a stale process and its same-ID replacement are + /// indistinguishable once the replacement has reacquired, so a worker + /// whose current registration declares [`REGISTRATION_TOKEN_CAPABILITY`] + /// must send it on every explicit create, heartbeat and close. Without + /// the capability, a request without a token acts for the worker ID's + /// current registration, as published PHP's does. + async fn stale_registration(&self, tx: &mut Transaction<'_, DB>, worker_id: &str, token: Option<&str>) -> Result> { + let Some(token) = token else { + let declared = Self::query(REGISTRATION_CAPABILITIES) + .bind(self.namespace.as_ref()).bind(worker_id).fetch_optional(&mut **tx).await? + .map(|row| DB::optional_document_row(&row, "capabilities")).transpose()?.flatten() + .is_some_and(|flags| flags.as_array().is_some_and(|flags| flags.iter().any(|flag| flag == REGISTRATION_TOKEN_CAPABILITY))); + return Ok(declared.then(|| failure("stale_worker_registration", + "The worker's current registration requires its registration token on worker session requests. Send it, or register again and reacquire the session.", + StatusCode::CONFLICT, Map::new()))); + }; + let current = self.current_token(tx, worker_id).await?; + Ok((!current.is_some_and(|current| same_token(¤t, token))).then(|| failure("stale_worker_registration", + "The registration token is not the worker's current registration. Register again and reacquire the session.", + StatusCode::CONFLICT, Map::new()))) + } + + /// `holderIsStale`: no registration, no heartbeat, or a heartbeat older + /// than the stale window. + async fn holder_stale(&self, tx: &mut Transaction<'_, DB>, owner: Option<&str>) -> Result { + let Some(owner) = owner else { + return Ok(true); + }; + let row = Self::query(HOLDER).bind(self.namespace.as_ref()).bind(owner).fetch_optional(&mut **tx).await?; + let heartbeat = match row { + Some(row) => DB::optional_instant(&row, "last_heartbeat_at")?, + None => None, + }; + Ok(heartbeat.is_none_or(|heartbeat| heartbeat < now() - chrono::Duration::seconds(self.worker_stale_after_seconds.max(1)))) + } + + async fn save(&self, tx: &mut Transaction<'_, DB>, session: &Session) -> Result<()> { + Self::query("UPDATE workflow_worker_sessions SET connection=$1,queue=$2,requirements=$3,status=$4,lease_owner=$5,lease_expires_at=$6,ttl_expires_at=$7,closed_at=$8,failure_reason=$9,lease_seconds=$10,ttl_seconds=$11,max_concurrent_activities=$12,create_if_missing=$13,allow_reacquire_after_failure=$14,last_heartbeat_at=$15,updated_at=$16 WHERE id=$17") + .bind(session.connection.as_deref()).bind(session.queue.as_deref()).bind(DB::document(&json!(session.requirements))) + .bind(session.status.as_str()).bind(session.lease_owner.as_deref()) + .bind(DB::optional_time(session.lease_expires_at)).bind(DB::optional_time(session.ttl_expires_at)).bind(DB::optional_time(session.closed_at)) + .bind(session.failure_reason.as_deref()).bind(session.lease_seconds).bind(session.ttl_seconds).bind(session.max_concurrent_activities) + .bind(DB::optional_flag(Some(session.create_if_missing))).bind(DB::optional_flag(Some(session.allow_reacquire_after_failure))) + .bind(DB::optional_time(session.last_heartbeat_at)).bind(DB::bind_time(second(now()))).bind(session.id) + .execute(&mut **tx).await?; + Ok(()) + } + + fn session(row: &DB::Row) -> Result { + let requirements = DB::optional_document_row(row, "requirements")? + .as_ref() + .and_then(Value::as_array) + .into_iter() + .flatten() + .filter_map(|value| value.as_str().map(str::to_owned)) + .collect(); + Ok(Session { + id: DB::number(row, "id")?, + session_id: DB::string(row, "session_id")?, + connection: DB::optional_string(row, "connection")?, + queue: DB::optional_string(row, "queue")?, + requirements, + status: DB::string(row, "status")?, + lease_owner: DB::optional_string(row, "lease_owner")?, + lease_expires_at: DB::optional_instant(row, "lease_expires_at")?, + ttl_expires_at: DB::optional_instant(row, "ttl_expires_at")?, + closed_at: DB::optional_instant(row, "closed_at")?, + failure_reason: DB::optional_string(row, "failure_reason")?, + lease_seconds: DB::number(row, "lease_seconds")?, + ttl_seconds: DB::number(row, "ttl_seconds")?, + max_concurrent_activities: DB::number(row, "max_concurrent_activities")?, + create_if_missing: DB::number(row, "create_if_missing_flag")? == 1, + allow_reacquire_after_failure: DB::number(row, "allow_reacquire_flag")? == 1, + last_heartbeat_at: DB::optional_instant(row, "last_heartbeat_at")?, + created_at: DB::optional_instant(row, "created_at")?, + updated_at: DB::optional_instant(row, "updated_at")?, + }) + } + + /// PHP's session snapshot, with its holder read from the registration. + async fn snapshot(&self, tx: &mut Transaction<'_, DB>, session: &Session) -> Result { + let holder = match session.lease_owner.as_deref() { + None => Value::Null, + Some(owner) => { + let row = Self::query(HOLDER).bind(self.namespace.as_ref()).bind(owner).fetch_optional(&mut **tx).await?; + let (task_queue, status, heartbeat) = match &row { + Some(row) => (DB::optional_string(row, "task_queue")?, DB::optional_string(row, "status")?, DB::optional_instant(row, "last_heartbeat_at")?), + None => (None, None, None), + }; + let stale = heartbeat.is_none_or(|heartbeat| heartbeat < now() - chrono::Duration::seconds(self.worker_stale_after_seconds.max(1))); + json!({"worker_id":owner,"task_queue":task_queue,"status":status,"last_heartbeat_at":wire(heartbeat),"is_stale":stale}) + } + }; + Ok(json!({"session_id":session.session_id,"namespace":self.namespace.as_ref(),"connection":session.connection,"queue":session.queue, + "requirements":session.requirements,"status":session.status,"lease_owner":session.lease_owner,"holder":holder, + "lease_expires_at":wire(session.lease_expires_at),"ttl_expires_at":wire(session.ttl_expires_at),"closed_at":wire(session.closed_at), + "failure_reason":session.failure_reason,"lease_seconds":session.lease_seconds,"ttl_seconds":session.ttl_seconds, + "max_concurrent_activities":session.max_concurrent_activities,"create_if_missing":session.create_if_missing, + "allow_reacquire_after_failure":session.allow_reacquire_after_failure,"active_activity_count":self.running_session_activities(tx, session, "").await?, + "last_heartbeat_at":wire(session.last_heartbeat_at),"created_at":wire(session.created_at),"updated_at":wire(session.updated_at)})) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn sessions_need_protocol_one_eight() { + assert!(sessions_gate("1.8").is_ok()); + assert!(sessions_gate("1.20").is_ok()); + assert!(sessions_gate("1.10").is_ok()); + assert!(sessions_gate("1.7").is_err()); + assert!(sessions_gate("1.x").is_err()); + assert!(sessions_gate("+1.9").is_err()); + } + + #[test] + fn options_read_like_php() { + assert_eq!(positive_int(&json!("60")), Some(60)); + assert_eq!(positive_int(&json!(7)), Some(7)); + assert_eq!(positive_int(&serde_json::from_str::("7.0").unwrap()), None); + assert_eq!(positive_int(&json!("0")), None); + let options = Options::from(&json!({"session_id":"s","requirements":["gpu",null,"gpu","tpu"]}), Some("q".into())).unwrap(); + assert_eq!(options.requirements, ["gpu", "tpu"]); + assert_eq!((options.queue.as_deref(), options.lease_seconds, options.ttl_seconds), (Some("q"), 120, 1800)); + } + + /// A string whose equality checks are counted, so the tests below fail + /// on a quadratic scan rather than on a timing threshold. + #[derive(Clone)] + struct Counted(String); + + // Hash and equality both read only the string, so equal values hash alike. + impl std::hash::Hash for Counted { + fn hash(&self, state: &mut H) { + self.0.hash(state); + } + } + + thread_local! { + static COMPARISONS: std::cell::Cell = const { std::cell::Cell::new(0) }; + } + + impl Counted { + fn eq_count() -> usize { + COMPARISONS.with(std::cell::Cell::get) + } + } + + impl PartialEq for Counted { + fn eq(&self, other: &Self) -> bool { + COMPARISONS.with(|count| count.set(count.get() + 1)); + self.0 == other.0 + } + } + + impl Eq for Counted {} + + /// Distinct near-limit strings with a shared 248-character prefix, as a + /// 2 MiB request can carry about 8,000 of them. + fn near_limit(count: usize) -> Vec { + let prefix = "r".repeat(248); + (0..count).map(|index| Counted(format!("{prefix}{index:07}"))).collect() + } + + #[test] + fn requirement_lists_stay_linear_at_the_request_limit() { + let distinct = near_limit(8_000); + let mut doubled = distinct.clone(); + doubled.extend(distinct.iter().rev().cloned()); + let before = Counted::eq_count(); + let unique = first_occurrences(doubled); + assert_eq!(unique.len(), 8_000); + assert!(unique.iter().zip(&distinct).all(|(left, right)| left.0 == right.0), "first occurrence order"); + let missing = missing_from(&unique, &distinct); + assert!(missing.is_empty()); + let mut partial = distinct.clone(); + partial.truncate(4_000); + assert_eq!(missing_from(&unique, &partial).len(), 4_000); + // A Vec scan would compare about n^2/2 = 32 million times. + assert!(Counted::eq_count() - before < 64_000, "{} comparisons", Counted::eq_count() - before); + } + + #[test] + fn near_limit_requirements_normalize_in_order() { + let items: Vec = near_limit(8_000).into_iter().map(|item| json!(item.0)).collect(); + let mut body = json!({"session_id":"s"}); + body["requirements"] = Value::Array(items.iter().chain(items.iter()).cloned().collect()); + let options = Options::from(&body, None).unwrap(); + assert_eq!(options.requirements.len(), 8_000); + assert_eq!(options.requirements.first().map(String::as_str), items[0].as_str()); + let stored = CommandSession::from_snapshot(&json!({"session_id":"s","requirements":body["requirements"]})).unwrap(); + assert_eq!(stored.requirements, options.requirements); + let capabilities = options.requirements.clone(); + assert!(stored.claimable_by("1.20", "q", "q", &capabilities)); + assert!(!stored.claimable_by("1.20", "q", "q", &capabilities[1..])); + } +} diff --git a/rust/tests/execution.rs b/rust/tests/execution.rs index 95d3f3c91..d3352e967 100644 --- a/rust/tests/execution.rs +++ b/rust/tests/execution.rs @@ -304,6 +304,9 @@ mod activity_cancellation_ack; #[path = "execution/route_coverage.rs"] mod route_coverage; +#[path = "execution/worker_sessions.rs"] +mod worker_sessions; + enum TestDatabase { Sqlite(tempfile::TempDir), Postgres { diff --git a/rust/tests/execution/worker_sessions.rs b/rust/tests/execution/worker_sessions.rs new file mode 100644 index 000000000..c52bfbc6d --- /dev/null +++ b/rust/tests/execution/worker_sessions.rs @@ -0,0 +1,468 @@ +//! Worker sessions as published 2.11.1 serves them (worker-sessions fixture): +//! create, reuse, heartbeat and close on the worker plane, lease expiry and a +//! holder's loss applied on read, and the namespace's list on the control +//! plane. +use super::*; +use std::time::Duration; + +async fn worker(app: &Router, worker: &str, capabilities: Value, sessions: Option) { + let mut body = json!({"worker_id":worker,"task_queue":"sessions","runtime":"php","supported_workflow_types":["echo"], + "supported_activity_types":["act"],"capabilities":capabilities}); + if let Some(sessions) = sessions { + body["max_concurrent_worker_sessions"] = json!(sessions); + } + let (status, answer) = request(app, "POST", "/api/worker/register", body).await; + assert_eq!(status, StatusCode::CREATED, "{answer}"); +} + +async fn create(app: &Router, body: Value) -> (StatusCode, Value) { + request(app, "POST", "/api/worker/sessions", body).await +} + +#[tokio::test] +async fn worker_sessions_lease_renew_expire_and_close_like_php() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + worker(&app, "holder", json!(["gpu"]), Some(2)).await; + worker(&app, "other", json!([]), None).await; + + // Create: the ID trimmed, requirements de-duplicated, a string lease read. + let (status, created) = create(&app, json!({"worker_id":"holder","session_id":" s1 ","requirements":["gpu"," gpu "], + "lease_seconds":"60","connection":"c1"})).await; + assert_eq!(status, StatusCode::CREATED, "{created}"); + let session = &created["session"]; + assert_eq!((created["outcome"].clone(), created["status"].clone()), (json!("created"), json!(201))); + assert_eq!((session["session_id"].clone(), session["requirements"].clone(), session["lease_seconds"].clone()), + (json!("s1"), json!(["gpu"]), json!(60)), "{session}"); + assert_eq!((session["create_if_missing"].clone(), session["allow_reacquire_after_failure"].clone()), (json!(true), json!(true))); + assert!(session["created_at"].as_str().unwrap().ends_with(".000000Z"), "whole seconds: {session}"); + // Requirements are checked before the lookup. + let (status, unmet) = create(&app, json!({"worker_id":"holder","session_id":"s1","requirements":["tpu"]})).await; + assert_eq!((status, unmet["missing_requirements"].clone()), (StatusCode::CONFLICT, json!(["tpu"])), "{unmet}"); + + // Reuse resets omitted options and keeps the TTL deadline. + let (status, reused) = create(&app, json!({"worker_id":"holder","session_id":"s1","ttl_seconds":9999,"create_if_missing":false})).await; + assert_eq!((status, reused["outcome"].clone()), (StatusCode::OK, json!("reused")), "{reused}"); + assert_eq!((reused["session"]["requirements"].clone(), reused["session"]["lease_seconds"].clone(), reused["session"]["create_if_missing"].clone(), + reused["session"]["connection"].clone(), reused["session"]["ttl_expires_at"].clone()), + (json!([]), json!(120), json!(false), json!("c1"), session["ttl_expires_at"].clone())); + let (status, foreign) = create(&app, json!({"worker_id":"other","session_id":"s1"})).await; + assert_eq!((status, foreign["reason"].clone()), (StatusCode::CONFLICT, json!("session_owned_by_another_worker"))); + + // The per-worker limit counts live leases. + let (status, _) = create(&app, json!({"worker_id":"holder","session_id":"s2","lease_seconds":2})).await; + assert_eq!(status, StatusCode::CREATED); + let (status, limited) = create(&app, json!({"worker_id":"holder","session_id":"s3"})).await; + assert_eq!((status, limited["max_concurrent_worker_sessions"].clone()), (StatusCode::CONFLICT, json!(2)), "{limited}"); + + // A lease expires on read; another worker reacquires with the failure + // reason cleared (published PHP keeps it: Server #737). + tokio::time::sleep(Duration::from_millis(3200)).await; + let (status, shown) = request(&app, "GET", "/api/worker-sessions/s2", Value::Null).await; + assert_eq!((status, shown["session"]["status"].clone(), shown["session"]["failure_reason"].clone()), + (StatusCode::OK, json!("expired"), json!("lease_expired")), "{shown}"); + let (status, reacquired) = create(&app, json!({"worker_id":"other","session_id":"s2"})).await; + assert_eq!((status, reacquired["outcome"].clone(), reacquired["session"]["failure_reason"].clone()), + (StatusCode::OK, json!("reacquired"), Value::Null), "{reacquired}"); + + // Heartbeat renews from now without changing the stored lease. + let (status, beat) = request(&app, "POST", "/api/worker/sessions/s1/heartbeat", json!({"worker_id":"holder","lease_seconds":5})).await; + assert_eq!((status, beat["outcome"].clone(), beat["session"]["lease_seconds"].clone()), (StatusCode::OK, json!("heartbeat_recorded"), json!(120))); + let (status, mismatch) = request(&app, "POST", "/api/worker/sessions/s1/heartbeat", json!({"worker_id":"other"})).await; + assert_eq!((status, mismatch["reason"].clone()), (StatusCode::CONFLICT, json!("session_owner_mismatch"))); + + // Close, close again, and a closed ID stays unusable. + let (status, closed) = request(&app, "DELETE", "/api/worker/sessions/s1?worker_id=holder", Value::Null).await; + assert_eq!((status, closed["outcome"].clone()), (StatusCode::OK, json!("closed")), "{closed}"); + let (status, again) = request(&app, "DELETE", "/api/worker/sessions/s1", json!({"worker_id":"holder"})).await; + assert_eq!((status, again["outcome"].clone(), again["session"]["closed_at"].clone()), (StatusCode::OK, json!("already_closed"), closed["session"]["closed_at"].clone())); + let (status, refused) = create(&app, json!({"worker_id":"holder","session_id":"s1"})).await; + assert_eq!((status, refused["reason"].clone()), (StatusCode::CONFLICT, json!("session_closed"))); + + // A deregistered holder leaves its session orphaned on read. + let (status, _) = request(&app, "DELETE", "/api/worker/registrations/other", Value::Null).await; + assert_eq!(status, StatusCode::OK); + let (status, listed) = request(&app, "GET", "/api/worker-sessions", Value::Null).await; + assert_eq!(status, StatusCode::OK, "{listed}"); + assert_eq!(listed["metrics"], json!({"total":2,"active":0,"closed":1,"expired":0,"failed":0,"orphaned":1}), "{listed}"); + let orphan = listed["sessions"].as_array().unwrap().iter().find(|session| session["session_id"] == "s2").unwrap(); + assert_eq!((orphan["status"].clone(), orphan["holder"]["task_queue"].clone(), orphan["holder"]["is_stale"].clone()), + (json!("orphaned"), Value::Null, json!(true)), "{orphan}"); +} + +#[tokio::test] +async fn worker_sessions_refuse_old_protocols_and_bad_requests() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + worker(&app, "gate", json!([]), None).await; + let (status, old) = request_protocol(&app, "POST", "/api/worker/sessions", json!({"worker_id":"gate","session_id":"g"}), "1.7").await; + assert_eq!((status, old["reason"].clone(), old["minimum_protocol_version"].clone()), (StatusCode::CONFLICT, json!("worker_sessions_unavailable"), json!("1.8")), "{old}"); + let (status, invalid) = request_protocol(&app, "POST", "/api/worker/sessions", json!({}), "1.8").await; + assert_eq!((status, invalid["errors"].as_object().unwrap().len()), (StatusCode::UNPROCESSABLE_ENTITY, 2), "{invalid}"); + let (status, unknown) = create(&app, json!({"worker_id":"nobody","session_id":"g"})).await; + assert_eq!((status, unknown["reason"].clone()), (StatusCode::PRECONDITION_FAILED, json!("worker_not_registered"))); + let (status, missing) = request(&app, "GET", "/api/worker-sessions/none", Value::Null).await; + assert_eq!((status, missing["reason"].clone()), (StatusCode::NOT_FOUND, json!("worker_session_not_found"))); +} + +async fn session_worker(app: &Router, worker: &str, capabilities: Value) { + let manifest = json!({"local_activities":{"supported":false,"minimum_protocol_version":"1.18"}, + "sticky_execution":{"supported":false,"minimum_protocol_version":"1.18"}, + "worker_sessions":{"supported":capabilities.as_array().unwrap().iter().any(|item| item == "worker_sessions"),"minimum_protocol_version":"1.18"}}); + let (status, answer) = request(app, "POST", "/api/worker/register", json!({"worker_id":worker,"task_queue":"test","runtime":"php", + "supported_workflow_types":["echo"],"supported_activity_types":["act"],"capabilities":capabilities,"capability_manifest":manifest})).await; + assert_eq!(status, StatusCode::CREATED, "{answer}"); +} + +fn bound(session: Value) -> Value { + json!({"type":"schedule_activity","activity_type":"act","arguments":envelope(Payload::Array(vec![])),"worker_session":session}) +} + +// Session-bound activities (worker-session-activities fixture): lazily +// created on the first admitted claim, held against other workers, bounded +// by the session's activity limit, and renewed by the activity heartbeat. +#[tokio::test] +async fn session_bound_activities_are_admitted_held_limited_and_renewed() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + session_worker(&app, "author", json!(["worker_sessions"])).await; + session_worker(&app, "gpu-a", json!(["worker_sessions", "gpu"])).await; + session_worker(&app, "gpu-b", json!(["worker_sessions", "gpu"])).await; + start(&app, "session-run").await; + let task = poll(&app, "author", "workflow").await; + let session = json!({"session_id":"lazy","requirements":["gpu"],"lease_seconds":90,"max_concurrent_activities":1}); + let (status, done) = finish_task(&app, &task, json!([bound(session.clone()), bound(session)])).await; + assert_eq!(status, StatusCode::OK, "{done}"); + let (status, _) = request(&app, "GET", "/api/worker-sessions/lazy", Value::Null).await; + assert_eq!(status, StatusCode::NOT_FOUND, "no session before the first claim"); + + let first = poll(&app, "gpu-a", "activity").await; + assert_eq!((first["worker_session"]["session_id"].clone(), first["worker_session"]["lease_owner"].clone()), + (json!("lazy"), json!("gpu-a")), "{first}"); + // The session holds one activity: neither worker gets the second. + assert!(poll(&app, "gpu-a", "activity").await.is_null()); + assert!(poll(&app, "gpu-b", "activity").await.is_null()); + let (status, shown) = request(&app, "GET", "/api/worker-sessions/lazy", Value::Null).await; + assert_eq!((status, shown["session"]["active_activity_count"].clone()), (StatusCode::OK, json!(1)), "{shown}"); + + // The heartbeat renews the session by the command's lease. + let own = json!({"activity_attempt_id":first["activity_attempt_id"],"lease_owner":"gpu-a"}); + let path = format!("/api/worker/activity-tasks/{}", first["task_id"].as_str().unwrap()); + let (status, beat) = request(&app, "POST", &format!("{path}/heartbeat"), own.clone()).await; + assert_eq!(status, StatusCode::OK, "{beat}"); + let lease = |value: &Value| chrono::DateTime::parse_from_rfc3339(value.as_str().unwrap()).unwrap().timestamp(); + assert_eq!(lease(&beat["worker_session"]["lease_expires_at"]) - lease(&beat["worker_session"]["last_heartbeat_at"]), 90, "{beat}"); + + // Once the holder closes the session, the heartbeat is refused and the + // attempt still completes; the second activity is then claimable again. + let (status, _) = request(&app, "DELETE", "/api/worker/sessions/lazy", json!({"worker_id":"gpu-a"})).await; + assert_eq!(status, StatusCode::OK); + let (status, refused) = request(&app, "POST", &format!("{path}/heartbeat"), own.clone()).await; + assert_eq!((status, refused["worker_session_renewal"]["reason"].clone()), (StatusCode::CONFLICT, json!("session_not_active")), "{refused}"); + let (status, observed) = request_protocol(&app, "POST", &format!("{path}/status"), own.clone(), "1.20").await; + assert_eq!((status, observed["reason"].clone()), (StatusCode::OK, json!("worker_session_unavailable")), "{observed}"); + let mut completion = own.clone(); + completion["result"] = envelope(Payload::Null); + assert_eq!(request(&app, "POST", &format!("{path}/complete"), completion).await.0, StatusCode::OK); +} + +// Session numbers have portable maxima, the same in PHP (Server #740). One +// above a maximum, and i64::MAX, are refused with 422 before any arithmetic +// or write, on the create and heartbeat routes and in a schedule_activity +// command; the maxima themselves are accepted. +#[tokio::test] +async fn session_numbers_are_bounded_on_every_path() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + worker(&app, "bounded", json!(["gpu"]), None).await; + let maxima = [("lease_seconds", 86_400_i64), ("ttl_seconds", 2_592_000), ("max_concurrent_activities", 1_000)]; + let message = |field: &str, maximum: i64| json!(format!("The {} field must not be greater than {maximum}.", field.replace('_', " "))); + for (field, maximum) in maxima { + for value in [maximum + 1, i64::MAX] { + let mut body = json!({"worker_id":"bounded","session_id":format!("too-large-{field}")}); + body[field] = json!(value); + let (status, refused) = create(&app, body).await; + assert_eq!((status, refused["errors"][field][0].clone()), (StatusCode::UNPROCESSABLE_ENTITY, message(field, maximum)), "{refused}"); + } + } + assert_eq!(task_retry::rows(&database, "SELECT COUNT(*) FROM workflow_worker_sessions").await, 0, "refused creates record nothing"); + + let mut at_maxima = json!({"worker_id":"bounded","session_id":"at-maxima"}); + for (field, maximum) in maxima { + at_maxima[field] = json!(maximum); + } + let (status, created) = create(&app, at_maxima).await; + assert_eq!(status, StatusCode::CREATED, "{created}"); + assert_eq!((created["session"]["lease_seconds"].clone(), created["session"]["ttl_seconds"].clone(), + created["session"]["max_concurrent_activities"].clone()), (json!(86_400), json!(2_592_000), json!(1_000)), "{created}"); + let lease = || async { + request(&app, "GET", "/api/worker-sessions/at-maxima", Value::Null).await.1["session"]["lease_expires_at"].clone() + }; + let before = lease().await; + for value in [86_401, i64::MAX] { + let (status, refused) = request(&app, "POST", "/api/worker/sessions/at-maxima/heartbeat", + json!({"worker_id":"bounded","lease_seconds":value})).await; + assert_eq!((status, refused["errors"]["lease_seconds"][0].clone()), (StatusCode::UNPROCESSABLE_ENTITY, + message("lease_seconds", 86_400)), "{refused}"); + } + assert_eq!(lease().await, before, "a refused heartbeat renews nothing"); + let (status, beat) = request(&app, "POST", "/api/worker/sessions/at-maxima/heartbeat", + json!({"worker_id":"bounded","lease_seconds":86_400})).await; + assert_eq!((status, beat["outcome"].clone()), (StatusCode::OK, json!("heartbeat_recorded")), "{beat}"); + + session_worker(&app, "author", json!(["worker_sessions"])).await; + start(&app, "bounded-run").await; + let task = poll(&app, "author", "workflow").await; + for (field, maximum) in maxima { + let mut session = json!({"session_id":"command-maxima"}); + session[field] = json!(i64::MAX); + let (status, refused) = finish_task(&app, &task, json!([bound(session)])).await; + let key = format!("commands.0.worker_session.{field}"); + assert_eq!((status, refused["errors"][&key][0].clone()), (StatusCode::UNPROCESSABLE_ENTITY, + json!(format!("The {key} field must not be greater than {maximum}."))), "{refused}"); + } + assert_eq!(task_retry::rows(&database, "SELECT COUNT(*) FROM activity_executions").await, 0, "refused commands schedule nothing"); +} + +async fn register_holder(app: &Router, worker: &str, queue: &str) -> String { + register_with(app, worker, queue, json!(["worker_sessions", "gpu"])).await +} + +/// A holder that declares it sends its registration token on every +/// explicit session request, so the server requires it. +async fn register_token_holder(app: &Router, worker: &str, queue: &str) -> String { + register_with(app, worker, queue, json!(["worker_sessions", "gpu", "worker_session_registration_token"])).await +} + +async fn register_with(app: &Router, worker: &str, queue: &str, capabilities: Value) -> String { + let manifest = json!({"local_activities":{"supported":false,"minimum_protocol_version":"1.18"}, + "sticky_execution":{"supported":false,"minimum_protocol_version":"1.18"}, + "worker_sessions":{"supported":true,"minimum_protocol_version":"1.18"}}); + let (status, answer) = request(app, "POST", "/api/worker/register", json!({"worker_id":worker,"task_queue":queue,"runtime":"php", + "supported_workflow_types":["echo"],"supported_activity_types":["act"],"capabilities":capabilities, + "capability_manifest":manifest})).await; + assert_eq!(status, StatusCode::CREATED, "{answer}"); + answer["registration_token"].as_str().expect("a registration token").to_owned() +} + +// A same-ID re-registration orphans the sessions the previous registration +// held (Server #740): the replacement reacquires, never reuses, and a request +// from the replaced process, with its old token or with none, is refused +// without changing anything. A worker heartbeat keeps the registration, so +// its holder reuses. The worker declared that it sends its token. +#[tokio::test] +async fn a_same_id_replacement_reacquires_the_session_and_fences_the_old_process() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + let with_token = |mut body: Value, token: Option<&str>| { + if let Some(token) = token { + body["registration_token"] = json!(token); + } + body + }; + let create = |token: Option<&str>, options: Value| { + let mut body = json!({"worker_id":"gpu-worker","session_id":"gpu-state","requirements":["gpu"]}); + for (key, value) in options.as_object().into_iter().flatten() { + body[key] = value.clone(); + } + create(&app, with_token(body, token)) + }; + let heartbeat = |token: Option<&str>| request(&app, "POST", "/api/worker/sessions/gpu-state/heartbeat", + with_token(json!({"worker_id":"gpu-worker","lease_seconds":900}), token)); + let close = |token: Option<&str>| request(&app, "DELETE", "/api/worker/sessions/gpu-state", + with_token(json!({"worker_id":"gpu-worker"}), token)); + let shown = || request(&app, "GET", "/api/worker-sessions/gpu-state", Value::Null); + + let first = register_token_holder(&app, "gpu-worker", "sessions").await; + let (status, created) = create(Some(&first), json!({})).await; + assert_eq!((status, created["outcome"].clone()), (StatusCode::CREATED, json!("created")), "{created}"); + let (status, beat) = request(&app, "POST", "/api/worker/heartbeat", json!({"worker_id":"gpu-worker"})).await; + assert_eq!(status, StatusCode::OK, "{beat}"); + let (status, reused) = create(Some(&first), json!({})).await; + assert_eq!((status, reused["outcome"].clone()), (StatusCode::OK, json!("reused")), "{reused}"); + + let second = register_token_holder(&app, "gpu-worker", "sessions").await; + assert_ne!(first, second); + let (status, orphaned) = heartbeat(Some(&second)).await; + assert_eq!((status, orphaned["reason"].clone(), orphaned["session"]["status"].clone(), orphaned["session"]["failure_reason"].clone()), + (StatusCode::CONFLICT, json!("session_not_active"), json!("orphaned"), json!("worker_registration_replaced")), "{orphaned}"); + for refused in [heartbeat(Some(&first)).await, create(Some(&first), json!({})).await] { + assert_eq!((refused.0, refused.1["reason"].clone()), (StatusCode::CONFLICT, json!("stale_worker_registration")), "{}", refused.1); + } + let (status, reacquired) = create(Some(&second), json!({})).await; + assert_eq!((status, reacquired["outcome"].clone(), reacquired["session"]["status"].clone()), + (StatusCode::OK, json!("reacquired"), json!("active")), "{reacquired}"); + let (status, current) = heartbeat(Some(&second)).await; + assert_eq!((status, current["outcome"].clone()), (StatusCode::OK, json!("heartbeat_recorded")), "{current}"); + + // The replaced process can neither reuse, renew nor close the session, + // with its old token or with none, and nothing changes. + let before = shown().await; + assert_eq!(before.0, StatusCode::OK, "{}", before.1); + for stale in [Some(first.as_str()), None] { + for refused in [create(stale, json!({"lease_seconds":5,"max_concurrent_activities":9})).await, heartbeat(stale).await, close(stale).await] { + assert_eq!((refused.0, refused.1["reason"].clone()), (StatusCode::CONFLICT, json!("stale_worker_registration")), "{}", refused.1); + } + assert_eq!(shown().await, before, "a refused request changes nothing"); + } + let (status, invalid) = close(Some("not-a-token")).await; + assert_eq!((status, invalid["errors"]["registration_token"][0].clone()), + (StatusCode::UNPROCESSABLE_ENTITY, json!("The registration token field format is invalid.")), "{invalid}"); + let (status, closed) = close(Some(&second)).await; + assert_eq!((status, closed["outcome"].clone()), (StatusCode::OK, json!("closed")), "{closed}"); +} + +// Deregistration deletes the registration and leaves the session for +// holder-loss handling. The former process, with its token or without, can +// neither renew nor close the session, and nothing changes: a token worker +// deregistered through the fenced route, a published one through the plain one. +#[tokio::test] +async fn a_deregistered_worker_can_neither_renew_nor_close_its_session() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + for sends_token in [true, false] { + let worker = if sends_token { "gpu-token-worker" } else { "gpu-published-worker" }; + let session = format!("{worker}-state"); + let token = if sends_token { + register_token_holder(&app, worker, "sessions").await + } else { + register_holder(&app, worker, "sessions").await + }; + let mut body = json!({"worker_id":worker,"session_id":session,"requirements":["gpu"]}); + if sends_token { + body["registration_token"] = json!(token); + } + let (status, created) = create(&app, body).await; + assert_eq!(status, StatusCode::CREATED, "{created}"); + let shown_path = format!("/api/worker-sessions/{session}"); + let shown = || request(&app, "GET", &shown_path, Value::Null); + let (status, deregistered) = if sends_token { + request(&app, "POST", &format!("/api/worker/registrations/{worker}/deregister"), json!({"registration_token":token})).await + } else { + request(&app, "DELETE", &format!("/api/worker/registrations/{worker}"), json!({})).await + }; + assert_eq!(status, StatusCode::OK, "{deregistered}"); + // The session shows its holder lost on read, as PHP's does; that read + // settles it before the refused requests are compared with it. + let before = shown().await; + assert_eq!((before.0, before.1["session"]["status"].clone()), (StatusCode::OK, json!("orphaned")), "{}", before.1); + for sent in [None, Some(token.as_str())] { + let with_token = |mut body: Value| { + if let Some(sent) = sent { + body["registration_token"] = json!(sent); + } + body + }; + let (status, renewed) = request(&app, "POST", &format!("/api/worker/sessions/{session}/heartbeat"), + with_token(json!({"worker_id":worker,"lease_seconds":900}))).await; + assert_eq!((status, renewed["reason"].clone(), renewed["error"].clone()), (StatusCode::PRECONDITION_FAILED, + json!("worker_not_registered"), json!("Worker must be registered before renewing a worker session.")), "{renewed}"); + let (status, closed) = request(&app, "DELETE", &format!("/api/worker/sessions/{session}"), + with_token(json!({"worker_id":worker}))).await; + assert_eq!((status, closed["reason"].clone(), closed["error"].clone()), (StatusCode::PRECONDITION_FAILED, + json!("worker_not_registered"), json!("Worker must be registered before closing a worker session.")), "{closed}"); + assert_eq!(shown().await, before, "a refused request changes nothing"); + } + } +} + +// A registration fenced as superseded (plain deregistration commits that +// before deleting the row, and lease recovery supersedes a stale holder) can +// neither create, renew nor close a session until the worker registers again. +#[tokio::test] +async fn a_superseded_registration_can_neither_create_renew_nor_close_its_session() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + for sends_token in [true, false] { + let worker = if sends_token { "gpu-token-worker" } else { "gpu-published-worker" }; + let session = format!("{worker}-state"); + let token = if sends_token { + register_token_holder(&app, worker, "sessions").await + } else { + register_holder(&app, worker, "sessions").await + }; + let sent = sends_token.then_some(token.as_str()); + let with_token = |mut body: Value| { + if let Some(sent) = sent { + body["registration_token"] = json!(sent); + } + body + }; + let (status, created) = create(&app, with_token(json!({"worker_id":worker,"session_id":session,"requirements":["gpu"]}))).await; + assert_eq!(status, StatusCode::CREATED, "{created}"); + // Test-owned literal only. + database.execute(sqlx::AssertSqlSafe(format!( + "UPDATE workflow_worker_registrations SET status='superseded' WHERE worker_id='{worker}'"))).await; + let shown_path = format!("/api/worker-sessions/{session}"); + let shown = || request(&app, "GET", &shown_path, Value::Null); + let before = shown().await; + assert_eq!(before.0, StatusCode::OK, "{}", before.1); + for refused in [ + create(&app, with_token(json!({"worker_id":worker,"session_id":session,"requirements":["gpu"],"lease_seconds":5}))).await, + request(&app, "POST", &format!("/api/worker/sessions/{session}/heartbeat"), with_token(json!({"worker_id":worker,"lease_seconds":900}))).await, + request(&app, "DELETE", &format!("/api/worker/sessions/{session}"), with_token(json!({"worker_id":worker}))).await, + ] { + assert_eq!((refused.0, refused.1["reason"].clone(), refused.1["remediation"].clone()), (StatusCode::CONFLICT, + json!("worker_registration_superseded"), json!("Restart and register the worker with a new worker_id.")), "{}", refused.1); + } + assert_eq!(shown().await, before, "a refused request changes nothing"); + } +} + +// Published workers send no registration token. Without the token +// capability, a request without one acts for the worker ID's current +// registration, as published PHP's does; a token that is sent must be current. +#[tokio::test] +async fn without_the_token_capability_a_tokenless_request_acts_for_the_current_registration() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + let body = json!({"worker_id":"gpu-worker","session_id":"gpu-legacy","requirements":["gpu"]}); + let first = register_holder(&app, "gpu-worker", "sessions").await; + let (status, created) = create(&app, body.clone()).await; + assert_eq!((status, created["outcome"].clone()), (StatusCode::CREATED, json!("created")), "{created}"); + register_holder(&app, "gpu-worker", "sessions").await; + let heartbeat = |token: Option<&str>| { + let mut body = json!({"worker_id":"gpu-worker"}); + if let Some(token) = token { + body["registration_token"] = json!(token); + } + request(&app, "POST", "/api/worker/sessions/gpu-legacy/heartbeat", body) + }; + let (status, orphaned) = heartbeat(None).await; + assert_eq!((status, orphaned["session"]["failure_reason"].clone()), (StatusCode::CONFLICT, json!("worker_registration_replaced")), "{orphaned}"); + let (status, reacquired) = create(&app, body).await; + assert_eq!((status, reacquired["outcome"].clone()), (StatusCode::OK, json!("reacquired")), "{reacquired}"); + let (status, beat) = heartbeat(None).await; + assert_eq!((status, beat["outcome"].clone()), (StatusCode::OK, json!("heartbeat_recorded")), "{beat}"); + let (status, stale) = heartbeat(Some(&first)).await; + assert_eq!((status, stale["reason"].clone()), (StatusCode::CONFLICT, json!("stale_worker_registration")), "{stale}"); + let (status, closed) = request(&app, "DELETE", "/api/worker/sessions/gpu-legacy", json!({"worker_id":"gpu-worker"})).await; + assert_eq!((status, closed["outcome"].clone()), (StatusCode::OK, json!("closed")), "{closed}"); +} + +// A session-bound activity claim by the replacement takes the orphaned +// session through reacquisition, so the claim names the replacement's +// registration as the holder. +#[tokio::test] +async fn a_same_id_replacement_reacquires_a_session_through_an_activity_claim() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + register_holder(&app, "gpu-worker", "test").await; + let (status, created) = create(&app, json!({"worker_id":"gpu-worker","session_id":"gpu-claim","requirements":["gpu"]})).await; + assert_eq!(status, StatusCode::CREATED, "{created}"); + start(&app, "replacement-run").await; + let task = poll(&app, "gpu-worker", "workflow").await; + let (status, done) = finish_task(&app, &task, json!([bound(json!({"session_id":"gpu-claim","requirements":["gpu"]}))])).await; + assert_eq!(status, StatusCode::OK, "{done}"); + + register_holder(&app, "gpu-worker", "test").await; + let (status, shown) = request(&app, "GET", "/api/worker-sessions/gpu-claim", Value::Null).await; + assert_eq!((status, shown["session"]["status"].clone(), shown["session"]["failure_reason"].clone()), + (StatusCode::OK, json!("orphaned"), json!("worker_registration_replaced")), "{shown}"); + let claimed = poll(&app, "gpu-worker", "activity").await; + assert_eq!((claimed["worker_session"]["session_id"].clone(), claimed["worker_session"]["lease_owner"].clone()), + (json!("gpu-claim"), json!("gpu-worker")), "{claimed}"); + let (status, shown) = request(&app, "GET", "/api/worker-sessions/gpu-claim", Value::Null).await; + assert_eq!((status, shown["session"]["status"].clone(), shown["session"]["failure_reason"].clone()), + (StatusCode::OK, json!("active"), Value::Null), "{shown}"); +} diff --git a/rust/tests/fixtures/unserved-php-routes.v2.11.1.txt b/rust/tests/fixtures/unserved-php-routes.v2.11.1.txt index e62882383..63b6590ca 100644 --- a/rust/tests/fixtures/unserved-php-routes.v2.11.1.txt +++ b/rust/tests/fixtures/unserved-php-routes.v2.11.1.txt @@ -20,11 +20,6 @@ DELETE /api/service-endpoints/{endpointName}/services/{serviceName}/operations/{ POST /api/service-endpoints/{endpointName}/services/{serviceName}/operations/{operationName}/execute GET /api/service-endpoints/{endpointName}/services/{serviceName}/operations/{operationName}/service-calls/{serviceCallId} POST /api/service-endpoints/{endpointName}/services/{serviceName}/operations/{operationName}/service-calls/{serviceCallId}/cancel -GET /api/worker-sessions -GET /api/worker-sessions/{sessionId} -POST /api/worker/sessions -DELETE /api/worker/sessions/{sessionId} -POST /api/worker/sessions/{sessionId}/heartbeat POST /api/worker/workflow-tasks/{taskId}/cancellation-scopes/checkpoint POST /api/worker/workflow-tasks/{taskId}/cancellation-scopes/deliver POST /api/worker/workflow-tasks/{taskId}/cancellation-scopes/open diff --git a/scripts/conformance/server-parity.mjs b/scripts/conformance/server-parity.mjs index 9fdbd69ca..63c3f1f94 100644 --- a/scripts/conformance/server-parity.mjs +++ b/scripts/conformance/server-parity.mjs @@ -105,7 +105,7 @@ try { for (const flag of ['url', 'application-root']) { if (values[flag]) arguments_.push(`--${flag}`, values[flag]); } - const probe = captureProbe('php', arguments_, {directory: dirname(resolve(values.output)), timeout: fixture.memo_replay ? fixture.memo_replay.overall_budget_seconds * 1000 + 5000 : fixture.corrected_markers ? fixture.corrected_markers.overall_budget_seconds * 1000 + 5000 : fixture.namespace_signal_pair || fixture.selected_run_signal_pair ? 75000 : fixture.update_validation ? 90000 : fixture.worker_poll_fence ? fixture.worker_poll_fence.overall_budget_seconds * 1000 + 5000 : 45000, + const probe = captureProbe('php', arguments_, {directory: dirname(resolve(values.output)), timeout: fixture.memo_replay ? fixture.memo_replay.overall_budget_seconds * 1000 + 5000 : fixture.corrected_markers ? fixture.corrected_markers.overall_budget_seconds * 1000 + 5000 : fixture.namespace_signal_pair || fixture.selected_run_signal_pair ? 75000 : fixture.update_validation ? 90000 : fixture.worker_poll_fence ? fixture.worker_poll_fence.overall_budget_seconds * 1000 + 5000 : fixture.worker_sessions ? fixture.worker_sessions.overall_budget_seconds * 1000 + 5000 : fixture.worker_session_activities ? fixture.worker_session_activities.overall_budget_seconds * 1000 + 5000 : 45000, env: {...process.env, DW_PARITY_FAILURE_EVIDENCE_DIR: dirname(resolve(values.output))}, preserveStdout: join(sourceDirectory, name)}); const item = {fixture_id: fixture.id, fixture, outcome: 'runner-blocked'}; diff --git a/scripts/conformance/server-parity/contract.mjs b/scripts/conformance/server-parity/contract.mjs index e57210c3c..8610a18fb 100644 --- a/scripts/conformance/server-parity/contract.mjs +++ b/scripts/conformance/server-parity/contract.mjs @@ -15,6 +15,8 @@ import {checkStorageTest} from './storage-test-contract.mjs'; import {checkActivityLeaseRenewal} from './activity-lease-renewal-contract.mjs'; import {checkWorkflowPollRequest} from './workflow-poll-request-contract.mjs'; import {checkWorkerPollFence} from './worker-poll-fence-contract.mjs'; +import {checkWorkerSessions} from './worker-sessions-contract.mjs'; +import {checkWorkerSessionActivities} from './worker-session-activities-contract.mjs'; import {checkActivityCancellationAcknowledgement} from './activity-cancellation-ack-contract.mjs'; import {checkServiceCatalog} from './service-catalog-contract.mjs'; import {checkSystemMetrics} from './system-metrics-contract.mjs'; @@ -158,6 +160,8 @@ export function checkObservation(fixture, observation, workflowId, artifacts) { const activityLeaseRenewal = fixture.activity_lease_renewal ? checkActivityLeaseRenewal(fixture, observation, workflowId) : null; const workflowPollRequest = fixture.workflow_poll_request ? checkWorkflowPollRequest(fixture, observation, workflowId) : null; const workerPollFence = fixture.worker_poll_fence ? checkWorkerPollFence(fixture, observation, workflowId) : null; + const workerSessions = fixture.worker_sessions ? checkWorkerSessions(fixture, observation, workflowId) : null; + const workerSessionActivities = fixture.worker_session_activities ? checkWorkerSessionActivities(fixture, observation, workflowId) : null; const activityCancellationAcknowledgement = fixture.activity_cancellation_acknowledgement ? checkActivityCancellationAcknowledgement(fixture, observation, workflowId) : null; const serviceCatalog = fixture.service_catalog ? checkServiceCatalog(fixture, observation, workflowId) : null; @@ -719,6 +723,8 @@ export function checkObservation(fixture, observation, workflowId, artifacts) { ...(activityLeaseRenewal ? {activity_lease_renewal: activityLeaseRenewal} : {}), ...(workflowPollRequest ? {workflow_poll_request: workflowPollRequest} : {}), ...(workerPollFence ? {worker_poll_fence: workerPollFence} : {}), + ...(workerSessions ? {worker_sessions: workerSessions} : {}), + ...(workerSessionActivities ? {worker_session_activities: workerSessionActivities} : {}), ...(activityCancellationAcknowledgement ? {activity_cancellation_acknowledgement: activityCancellationAcknowledgement} : {}), ...(serviceCatalog ? {service_catalog: serviceCatalog} : {}), ...(systemMetrics ? {system_metrics: systemMetrics} : {}), diff --git a/scripts/conformance/server-parity/mysql-schema.php b/scripts/conformance/server-parity/mysql-schema.php index 20e2bca6e..1a71793d7 100644 --- a/scripts/conformance/server-parity/mysql-schema.php +++ b/scripts/conformance/server-parity/mysql-schema.php @@ -106,7 +106,7 @@ function catalog(PDO $pdo): array } $pdo->exec('ROLLBACK'); $pdo = connection($argv[3]); -$native = array_values(array_filter(catalog($pdo), static fn (array $row): bool => ! in_array($row[0], ['dw_server_schema', '_sqlx_migrations', 'dw_task_completions', 'dw_poll_receipts', 'dw_query_cache', 'dw_worker_registration_incarnations'], true))); +$native = array_values(array_filter(catalog($pdo), static fn (array $row): bool => ! in_array($row[0], ['dw_server_schema', '_sqlx_migrations', 'dw_task_completions', 'dw_poll_receipts', 'dw_query_cache', 'dw_worker_registration_incarnations', 'dw_worker_session_holders'], true))); $pdo->exec('ROLLBACK'); // Native polling uses an additional index on an existing table. Retain all // original column, index and constraint definitions in the comparison. diff --git a/scripts/conformance/server-parity/postgres-schema.php b/scripts/conformance/server-parity/postgres-schema.php index e5da9aaf5..7ea05ff49 100644 --- a/scripts/conformance/server-parity/postgres-schema.php +++ b/scripts/conformance/server-parity/postgres-schema.php @@ -70,8 +70,8 @@ function catalog(string $database): array throw new InvalidArgumentException('Compare requires two independent databases.'); } $native = catalog($argv[3]); -$tables = ['_sqlx_migrations', 'dw_server_schema', 'dw_task_completions', 'dw_poll_receipts', 'dw_query_cache', 'dw_worker_registration_incarnations']; -$indexes = ['_sqlx_migrations_pkey', 'dw_task_completions_pkey', 'dw_poll_receipts_pkey', 'dw_workflow_tasks_poll', 'dw_query_cache_pkey', 'dw_worker_registration_incarnations_pkey', 'dw_worker_incarnation_authority', 'dw_worker_incarnation_retention']; +$tables = ['_sqlx_migrations', 'dw_server_schema', 'dw_task_completions', 'dw_poll_receipts', 'dw_query_cache', 'dw_worker_registration_incarnations', 'dw_worker_session_holders']; +$indexes = ['_sqlx_migrations_pkey', 'dw_task_completions_pkey', 'dw_poll_receipts_pkey', 'dw_workflow_tasks_poll', 'dw_query_cache_pkey', 'dw_worker_registration_incarnations_pkey', 'dw_worker_incarnation_authority', 'dw_worker_incarnation_retention', 'dw_worker_session_holders_pkey']; $native = array_values(array_filter($native, static function (array $row) use ($tables, $indexes): bool { if (in_array($row[0], ['column', 'constraint'], true)) { return ! in_array(explode('.', $row[1], 2)[0], $tables, true); diff --git a/scripts/conformance/server-parity/probe.php b/scripts/conformance/server-parity/probe.php index cd96f2d22..c124890d2 100644 --- a/scripts/conformance/server-parity/probe.php +++ b/scripts/conformance/server-parity/probe.php @@ -433,6 +433,8 @@ function httpObservation(array $fixture, string $workflowId, string $namespace, $activityLeaseRenewal = isset($fixture['activity_lease_renewal']) ? finishHttpActivityLeaseRenewal($client, $fixture, $workflowId, $url) : null; $workflowPollRequest = isset($fixture['workflow_poll_request']) ? finishHttpWorkflowPollRequest($client, $fixture, $workflowId, $url) : null; $workerPollFence = isset($fixture['worker_poll_fence']) ? finishHttpWorkerPollFence($client, $fixture, $workflowId, $url) : null; + $workerSessions = isset($fixture['worker_sessions']) ? finishHttpWorkerSessions($client, $fixture, $workflowId, $url) : null; + $workerSessionActivities = isset($fixture['worker_session_activities']) ? finishHttpWorkerSessionActivities($client, $fixture, $workflowId, $url) : null; $activityCancellationAcknowledgement = isset($fixture['activity_cancellation_acknowledgement']) ? finishHttpActivityCancellationAcknowledgement($client, $fixture, $workflowId, $url) : null; $serviceCatalog = isset($fixture['service_catalog']) ? finishHttpServiceCatalog($client, $fixture, $workflowId, $url) : null; @@ -547,6 +549,8 @@ function httpObservation(array $fixture, string $workflowId, string $namespace, 'activity_lease_renewal' => $activityLeaseRenewal, 'workflow_poll_request' => $workflowPollRequest, 'worker_poll_fence' => $workerPollFence, + 'worker_sessions' => $workerSessions, + 'worker_session_activities' => $workerSessionActivities, 'activity_cancellation_acknowledgement' => $activityCancellationAcknowledgement, 'service_catalog' => $serviceCatalog, 'system_metrics' => $systemMetrics, @@ -864,6 +868,8 @@ function embeddedObservation(array $fixture, string $workflowId, string $namespa require __DIR__.'/activity-lease-renewal-probe.php'; require __DIR__.'/workflow-poll-request-probe.php'; require __DIR__.'/worker-poll-fence-probe.php'; + require __DIR__.'/worker-sessions-probe.php'; + require __DIR__.'/worker-session-activities-probe.php'; require __DIR__.'/activity-cancellation-ack-probe.php'; require __DIR__.'/service-catalog-probe.php'; require __DIR__.'/system-metrics-probe.php'; @@ -1042,6 +1048,12 @@ function embeddedObservation(array $fixture, string $workflowId, string $namespa if ($mode === 'embedded' && isset($fixture['worker_poll_fence'])) { $observation['worker_poll_fence'] = ['applicable' => false, 'reason' => 'embedded_has_no_worker_protocol']; } + if ($mode === 'embedded' && isset($fixture['worker_sessions'])) { + $observation['worker_sessions'] = ['applicable' => false, 'reason' => 'embedded_has_no_worker_protocol']; + } + if ($mode === 'embedded' && isset($fixture['worker_session_activities'])) { + $observation['worker_session_activities'] = ['applicable' => false, 'reason' => 'embedded_has_no_worker_protocol']; + } if ($mode === 'embedded' && isset($fixture['activity_cancellation_acknowledgement'])) { $observation['activity_cancellation_acknowledgement'] = ['applicable' => false, 'reason' => 'embedded_has_no_worker_protocol']; } diff --git a/scripts/conformance/server-parity/schema.php b/scripts/conformance/server-parity/schema.php index 4dd30342e..5929ce8db 100644 --- a/scripts/conformance/server-parity/schema.php +++ b/scripts/conformance/server-parity/schema.php @@ -38,7 +38,7 @@ function catalog(string $filename): array $php = catalog($argv[1]); $native = catalog($argv[2]); -$nativeOnly = ['_sqlx_migrations', 'dw_server_schema', 'dw_task_completions', 'dw_poll_receipts', 'dw_workflow_tasks_poll', 'dw_query_cache', 'dw_worker_registration_incarnations', 'dw_worker_incarnation_authority', 'dw_worker_incarnation_retention']; +$nativeOnly = ['_sqlx_migrations', 'dw_server_schema', 'dw_task_completions', 'dw_poll_receipts', 'dw_workflow_tasks_poll', 'dw_query_cache', 'dw_worker_registration_incarnations', 'dw_worker_incarnation_authority', 'dw_worker_incarnation_retention', 'dw_worker_session_holders']; $native = array_values(array_filter($native, static fn (array $row): bool => ! in_array($row['name'], $nativeOnly, true))); if ($php !== $native) { $byName = static fn (array $rows): array => array_column($rows, null, 'name'); diff --git a/scripts/conformance/server-parity/worker-session-activities-contract.mjs b/scripts/conformance/server-parity/worker-session-activities-contract.mjs new file mode 100644 index 000000000..1a9900562 --- /dev/null +++ b/scripts/conformance/server-parity/worker-session-activities-contract.mjs @@ -0,0 +1,23 @@ +import assert from 'node:assert/strict'; + +// Reviewed session-bound activities, recorded against published Server +// 2.11.1. The fixture's `expected` holds the whole normalized observation, +// with the probe's workflow prefix as `{id}`; session times are spans or +// presence flags. Every step compares exactly. +export function checkWorkerSessionActivities(fixture, observation, workflowId) { + const state = observation.worker_session_activities; + const projection = {scope: 'worker_session_activities'}; + assert.ok(state && typeof state === 'object', 'actual worker session activities observation'); + if (observation.mode === 'embedded') { + assert.deepStrictEqual(state, {applicable: false, reason: 'embedded_has_no_worker_protocol'}, + 'embedded worker session activities explicitly inapplicable'); + return projection; + } + const expected = fixture.worker_session_activities.expected; + const actual = JSON.parse(JSON.stringify(state).replaceAll(workflowId, '{id}')); + for (const [step, value] of Object.entries(expected)) { + assert.deepStrictEqual(actual[step], value, `worker session activities ${step}`); + } + assert.deepStrictEqual(Object.keys(actual).sort(), Object.keys(expected).sort(), 'worker session activities steps'); + return projection; +} diff --git a/scripts/conformance/server-parity/worker-session-activities-probe.php b/scripts/conformance/server-parity/worker-session-activities-probe.php new file mode 100644 index 000000000..dc55bf477 --- /dev/null +++ b/scripts/conformance/server-parity/worker-session-activities-probe.php @@ -0,0 +1,196 @@ +createNamespace($namespace, 'worker session activities λ', 30); + $token = getenv('DW_PARITY_TOKEN') ?: null; + $stack = HandlerStack::create(); + $stack->push(LockPressurePollRetries::middleware(), 'lock_pressure_poll_retry'); + $http = new WorkerSessionActivityHttp(['base_uri' => rtrim($url, '/'), 'http_errors' => false, + 'connect_timeout' => 1, 'timeout' => 30, 'handler' => $stack]); + $capabilityMap = null; + $raw = static function (string $method, string $path, ?array $body = null, ?string $protocol = '1.20', bool $control = false) + use ($http, $token, $namespace, &$capabilityMap): array { + $headers = array_filter(['Authorization' => $token === null ? null : 'Bearer '.$token, + 'Accept' => 'application/json', 'X-Namespace' => $namespace, + 'X-Durable-Workflow-Protocol-Version' => $control ? null : $protocol, + 'X-Durable-Workflow-Control-Plane-Version' => $control ? '2' : null]); + $response = $http->request($method, $path, ['headers' => $headers] + ($body === null ? [] : ['json' => $body])); + $decoded = json_decode((string) $response->getBody(), true); + if (is_array($decoded)) { + $capabilityMap ??= $decoded['server_capabilities'] ?? null; + unset($decoded['server_capabilities'], $decoded['control_plane'], $decoded['protocol_version']); + } + + return ['status' => $response->getStatusCode(), 'response' => $decoded]; + }; + $at = static fn (?string $value): ?int => $value === null ? null : (new DateTimeImmutable($value))->getTimestamp(); + $present = static fn (mixed $value): string => $value === null ? 'null' : 'present'; + // The full snapshot with its times as spans and presence flags. + $snapshot = static function (?array $session) use ($at, $present): ?array { + if ($session === null) { + return null; + } + $times = ['lease_expires_at', 'ttl_expires_at', 'closed_at', 'last_heartbeat_at', 'created_at', 'updated_at']; + $plain = array_diff_key($session, array_flip($times)); + if (is_array($plain['holder'] ?? null)) { + $plain['holder']['last_heartbeat_at'] = $present($plain['holder']['last_heartbeat_at']); + } + $heartbeat = $at($session['last_heartbeat_at'] ?? null); + + return $plain + ['lease_span' => isset($session['lease_expires_at']) && $heartbeat !== null ? $at($session['lease_expires_at']) - $heartbeat : null, + 'times' => array_map(static fn (string $key): string => $present($session[$key] ?? null), array_combine($times, $times))]; + }; + // The compact form a task or status carries: its times as presence flags. + $compact = static function (?array $session) use ($present): ?array { + if ($session === null) { + return null; + } + foreach (['lease_expires_at', 'ttl_expires_at'] as $key) { + if (array_key_exists($key, $session)) { + $session[$key] = $present($session[$key]); + } + } + + return $session; + }; + $ok = static function (array $recorded, int $status, string $operation): array { + if ($recorded['status'] !== $status) { + throw new RuntimeException('Worker session activities '.$operation.': '.json_encode($recorded)); + } + + return $recorded['response']; + }; + $empty = ['codec' => 'avro', 'blob' => $definition['empty_blob']]; + $queue = $workflowId.'-wsa-queue'; + $other = $workflowId.'-wsa-other-queue'; + $manifest = static fn (bool $sessions): array => ['local_activities' => ['supported' => false, 'minimum_protocol_version' => '1.18'], + 'sticky_execution' => ['supported' => false, 'minimum_protocol_version' => '1.18'], + 'worker_sessions' => ['supported' => $sessions, 'minimum_protocol_version' => '1.18']]; + $register = static function (string $worker, string $onQueue, array $capabilities) use ($raw, $ok, $definition, $manifest): void { + $ok($raw('POST', '/api/worker/register', ['worker_id' => $worker, 'task_queue' => $onQueue, 'runtime' => 'php', + 'sdk_version' => 'raw-http', 'supported_workflow_types' => [$definition['workflow_type']], + 'supported_activity_types' => [$definition['activity_type']], 'capabilities' => $capabilities, + 'capability_manifest' => $manifest(in_array('worker_sessions', $capabilities, true))]), 201, 'registration '.$worker); + }; + [$author, $plain, $gpu, $gpu2, $far] = array_map(static fn (string $name): string => $workflowId.'-wsa-'.$name, ['author', 'plain', 'gpu', 'gpu2', 'far']); + $register($author, $queue, ['worker_sessions']); + $register($plain, $queue, []); + $register($gpu, $queue, ['worker_sessions', 'gpu']); + $register($gpu2, $queue, ['worker_sessions', 'gpu']); + $register($far, $other, ['worker_sessions', 'gpu']); + $start = static fn (string $suffix): array => $ok($raw('POST', '/api/workflows', ['workflow_id' => $workflowId.'-wsa-'.$suffix, + 'workflow_type' => $definition['workflow_type'], 'task_queue' => $queue, 'input' => $empty], null, true), 201, 'start '.$suffix); + $workflowTask = static fn (string $worker): array => $raw('POST', '/api/worker/workflow-tasks/poll', + ['worker_id' => $worker, 'task_queue' => $queue, 'timeout_seconds' => 0])['response']['task'] ?? []; + $complete = static fn (array $task, array $commands, string $protocol = '1.20'): array => $raw('POST', + '/api/worker/workflow-tasks/'.($task['task_id'] ?? 'missing').'/complete', + ['lease_owner' => $task['lease_owner'] ?? null, 'workflow_task_attempt' => $task['workflow_task_attempt'] ?? null, 'commands' => $commands], $protocol); + $activity = static fn (array $session, array $extra = []): array => ['type' => 'schedule_activity', 'activity_type' => $definition['activity_type'], + 'arguments' => $empty, 'retry_policy' => ['max_attempts' => 1, 'backoff_seconds' => []], 'start_to_close_timeout' => 300, + 'worker_session' => $session] + $extra; + $refusal = static fn (array $recorded): array => ['status' => $recorded['status'], + 'reason' => $recorded['response']['reason'] ?? null, 'message' => $recorded['response']['message'] ?? $recorded['response']['error'] ?? null]; + $poll = static fn (string $worker, string $onQueue, string $protocol = '1.20'): array => $raw('POST', '/api/worker/activity-tasks/poll', + ['worker_id' => $worker, 'task_queue' => $onQueue, 'timeout_seconds' => 0], $protocol); + $leased = static fn (array $recorded): array => ['status' => $recorded['status'], 'poll_status' => $recorded['response']['poll_status'] ?? null, + 'worker_session' => $compact($recorded['response']['task']['worker_session'] ?? null), + 'has_session_key' => is_array($recorded['response']['task'] ?? null) && array_key_exists('worker_session', $recorded['response']['task'])]; + $own = static fn (array $task): array => ['activity_attempt_id' => $task['activity_attempt_id'] ?? null, 'lease_owner' => $task['lease_owner'] ?? null]; + $show = static fn (string $session): array => $raw('GET', '/api/worker-sessions/'.rawurlencode($session), null, null, true); + $state = ['applicable' => true, 'namespace' => $namespace]; + + // 17. Completion guards on one workflow task, then an accepted command. + $start('guards'); + $task = $workflowTask($author); + $lazy = $workflowId.'-wsa-lazy'; + $session = ['session_id' => $lazy, 'requirements' => ['gpu'], 'lease_seconds' => 90]; + $state['complete_protocol_1_7'] = $refusal($complete($task, [$activity($session)], '1.7')); + $state['complete_protocol_1_17'] = $refusal($complete($task, [$activity($session)], '1.17')); + $state['complete_on_timer'] = $refusal($complete($task, [['type' => 'start_timer', 'delay_seconds' => 60, 'worker_session' => $session]])); + $state['complete_without_session_id'] = $refusal($complete($task, [$activity(['queue' => $queue])])); + $state['complete_accepted'] = $complete($task, [$activity($session)])['status']; + $start('incapable'); + $incapable = $workflowTask($plain); + $state['complete_incapable'] = $refusal($complete($incapable, [$activity($session)])); + + // 18. The activity path: lazily created on the first admitted claim. + $state['show_before_claim'] = $refusal($show($lazy)); + $state['poll_without_requirement'] = $leased($poll($plain, $queue)); + $state['poll_protocol_1_7'] = $leased($poll($gpu, $queue, '1.7')); + $claimed = $poll($gpu, $queue); + $state['poll_capable'] = $leased($claimed); + $claim = $claimed['response']['task'] ?? []; + $shown = $show($lazy); + $state['show_after_claim'] = ['status' => $shown['status'], 'session' => $snapshot($shown['response']['session'] ?? null)]; + $state['poll_other_capable'] = $leased($poll($gpu2, $queue)); + $beat = $raw('POST', '/api/worker/activity-tasks/'.($claim['task_id'] ?? 'missing').'/heartbeat', $own($claim) + ['message' => 'half']); + $state['activity_heartbeat'] = ['status' => $beat['status'], 'heartbeat_recorded' => $beat['response']['heartbeat_recorded'] ?? null, + 'worker_session' => $snapshot($beat['response']['worker_session'] ?? null)]; + $status = $raw('POST', '/api/worker/activity-tasks/'.($claim['task_id'] ?? 'missing').'/status', $own($claim)); + $state['activity_status'] = ['status' => $status['status'], 'can_continue' => $status['response']['can_continue'] ?? null, + 'reason' => $status['response']['reason'] ?? null, 'worker_session' => $compact($status['response']['worker_session'] ?? null)]; + $renewed = $raw('POST', '/api/worker/activity-tasks/'.($claim['task_id'] ?? 'missing').'/renew-lease', $own($claim)); + $state['activity_renew'] = ['status' => $renewed['status'], 'lease_renewed' => $renewed['response']['lease_renewed'] ?? null, + 'worker_session' => $snapshot($renewed['response']['worker_session'] ?? null)]; + $state['session_closed'] = $raw('DELETE', '/api/worker/sessions/'.rawurlencode($lazy), ['worker_id' => $gpu])['status']; + $beat = $raw('POST', '/api/worker/activity-tasks/'.($claim['task_id'] ?? 'missing').'/heartbeat', $own($claim)); + $state['heartbeat_after_close'] = ['status' => $beat['status'], 'reason' => $beat['response']['reason'] ?? null, + 'can_continue' => $beat['response']['can_continue'] ?? null, 'heartbeat_recorded' => $beat['response']['heartbeat_recorded'] ?? null, + 'worker_session_renewal' => $beat['response']['worker_session_renewal'] ?? null, + 'worker_session_status' => $beat['response']['worker_session']['status'] ?? null]; + $status = $raw('POST', '/api/worker/activity-tasks/'.($claim['task_id'] ?? 'missing').'/status', $own($claim)); + $state['status_after_close'] = ['status' => $status['status'], 'can_continue' => $status['response']['can_continue'] ?? null, + 'reason' => $status['response']['reason'] ?? null]; + $state['complete_after_close'] = $raw('POST', '/api/worker/activity-tasks/'.($claim['task_id'] ?? 'missing').'/complete', + $own($claim) + ['result' => $empty])['status']; + + // 19. The session's activity limit: two activities on one session. + foreach ([1 => 'one', 2 => 'two'] as $limit => $name) { + $start('limit-'.$name); + $task = $workflowTask($author); + $shared = ['session_id' => $workflowId.'-wsa-shared-'.$name, 'requirements' => ['gpu'], 'max_concurrent_activities' => $limit]; + $complete($task, [$activity($shared), $activity($shared)]); + $first = $poll($gpu2, $queue); + $second = $poll($gpu2, $queue); + $state['limit_'.$name] = [$leased($first)['poll_status'], $leased($second)['poll_status']]; + foreach ([$first, $second] as $polled) { + $leasedTask = $polled['response']['task'] ?? null; + if (is_array($leasedTask)) { + $raw('POST', '/api/worker/activity-tasks/'.$leasedTask['task_id'].'/complete', $own($leasedTask) + ['result' => $empty]); + } + } + } + + // 20. A command queue other than the session's: nobody can claim it. + $start('queue'); + $task = $workflowTask($author); + $complete($task, [$activity(['session_id' => $workflowId.'-wsa-elsewhere', 'queue' => $queue], ['queue' => $other])]); + $state['queue_mismatch_polls'] = [$leased($poll($gpu, $queue))['poll_status'], $leased($poll($far, $other))['poll_status']]; + + // 21. Operator metrics count the namespace's sessions. + $metrics = $raw('GET', '/api/system/operator-metrics', null, null, true); + $state['operator_metrics'] = ['status' => $metrics['status'], + 'worker_sessions' => $metrics['response']['operator_metrics']['worker_sessions'] ?? null]; + + // The capability map, once. + $state['capabilities'] = ['worker_sessions' => $capabilityMap['worker_sessions'] ?? null, + 'worker_session_verbs' => $capabilityMap['worker_session_verbs'] ?? null]; + + return $state; +} diff --git a/scripts/conformance/server-parity/worker-sessions-contract.mjs b/scripts/conformance/server-parity/worker-sessions-contract.mjs new file mode 100644 index 000000000..2a0c858b4 --- /dev/null +++ b/scripts/conformance/server-parity/worker-sessions-contract.mjs @@ -0,0 +1,38 @@ +import assert from 'node:assert/strict'; + +// Reviewed worker sessions (WorkerSessionController, WorkerSessionRegistry), +// recorded against published Server 2.11.1. The fixture's `expected` holds +// the whole normalized observation, with the probe's workflow prefix as +// `{id}`; session times are spans and presence flags, since PHP keeps them in +// whole seconds, and the list is compared by session ID because ties within +// a second order by ID. Every step compares exactly. +// +// Published defect boundary: a reacquired session keeps its previous +// failure_reason, because renewHeldSession filters its own `failure_reason => +// null` out (fixture `published_defects`). At those paths the published value +// or the intended null is accepted; nothing else is relaxed. +export function checkWorkerSessions(fixture, observation, workflowId) { + const state = observation.worker_sessions; + const projection = {scope: 'worker_sessions'}; + assert.ok(state && typeof state === 'object', 'actual worker sessions observation'); + if (observation.mode === 'embedded') { + assert.deepStrictEqual(state, {applicable: false, reason: 'embedded_has_no_worker_protocol'}, + 'embedded worker sessions explicitly inapplicable'); + return projection; + } + const definition = fixture.worker_sessions; + const actual = JSON.parse(JSON.stringify(state).replaceAll(workflowId, '{id}')); + const defect = definition.published_defects.reacquired_failure_reason; + const tolerate = (session, published, label) => { + assert.ok(session && typeof session === 'object', `${label}: a session`); + if (session.failure_reason === published) session.failure_reason = null; + }; + tolerate(actual.reacquired?.response?.session, defect.published[0], 'reacquired'); + tolerate(actual.orphan_reacquired?.response?.session, defect.published[1], 'orphan reacquired'); + tolerate((actual.list_sessions ?? []).find(session => session.session_id === '{id}-wss-held'), defect.published[2], 'listed held session'); + for (const [step, expected] of Object.entries(definition.expected)) { + assert.deepStrictEqual(actual[step], expected, `worker sessions ${step}`); + } + assert.deepStrictEqual(Object.keys(actual).sort(), Object.keys(definition.expected).sort(), 'worker sessions steps'); + return projection; +} diff --git a/scripts/conformance/server-parity/worker-sessions-probe.php b/scripts/conformance/server-parity/worker-sessions-probe.php new file mode 100644 index 000000000..5b093c7be --- /dev/null +++ b/scripts/conformance/server-parity/worker-sessions-probe.php @@ -0,0 +1,214 @@ +createNamespace($namespace, 'worker sessions λ', 30); + $token = getenv('DW_PARITY_TOKEN') ?: null; + $stack = HandlerStack::create(); + $stack->push(LockPressurePollRetries::middleware(), 'lock_pressure_poll_retry'); + $http = new WorkerSessionHttp(['base_uri' => rtrim($url, '/'), 'http_errors' => false, + 'connect_timeout' => 1, 'timeout' => 30, 'handler' => $stack]); + // A request on the worker plane (protocol header, null for none) or the + // control plane, in this namespace unless another is named. + $raw = static function (string $method, string $path, ?array $body = null, ?string $protocol = '1.20', + bool $control = false, ?string $as = null, ?string $json = null) use ($http, $token, $namespace): array { + $headers = array_filter(['Authorization' => $token === null ? null : 'Bearer '.$token, + 'Accept' => 'application/json', 'X-Namespace' => $as ?? $namespace, + 'X-Durable-Workflow-Protocol-Version' => $control ? null : $protocol, + 'X-Durable-Workflow-Control-Plane-Version' => $control ? '2' : null, + 'Content-Type' => $json === null ? null : 'application/json']); + $payload = $json !== null ? ['body' => $json] : ($body === null ? [] : ['json' => $body]); + $response = $http->request($method, $path, ['headers' => $headers] + $payload); + $decoded = json_decode((string) $response->getBody(), true); + if (is_array($decoded)) { + unset($decoded['server_capabilities'], $decoded['control_plane']); + } + + return ['status' => $response->getStatusCode(), 'response' => $decoded]; + }; + $at = static fn (?string $value): ?int => $value === null ? null : (new DateTimeImmutable($value))->getTimestamp(); + // A session snapshot with its times as spans and presence flags. + $snapshot = static function (array $session) use ($at): array { + $times = ['lease_expires_at', 'ttl_expires_at', 'closed_at', 'last_heartbeat_at', 'created_at', 'updated_at']; + $plain = array_diff_key($session, array_flip($times)); + if (is_array($plain['holder'] ?? null)) { + $plain['holder']['last_heartbeat_at'] = $plain['holder']['last_heartbeat_at'] === null ? null : 'present'; + } + $heartbeat = $at($session['last_heartbeat_at'] ?? null); + + return $plain + [ + 'lease_span' => isset($session['lease_expires_at']) && $heartbeat !== null ? $at($session['lease_expires_at']) - $heartbeat : null, + 'ttl_span' => isset($session['ttl_expires_at'], $session['created_at']) ? $at($session['ttl_expires_at']) - $at($session['created_at']) : null, + 'closed' => ($session['closed_at'] ?? null) !== null, + 'times' => array_map(static fn (string $key): string => ($session[$key] ?? null) === null ? 'null' : 'present', array_combine($times, $times)), + ]; + }; + $answer = static function (array $recorded) use ($snapshot): array { + $body = $recorded['response']; + if (is_array($body)) { + if (is_array($body['session'] ?? null)) { + $body['session'] = $snapshot($body['session']); + } + if (is_array($body['sessions'] ?? null)) { + $body['sessions'] = array_map($snapshot, $body['sessions']); + } + } + + return ['status' => $recorded['status'], 'response' => $body]; + }; + $ok = static function (array $recorded, int $status, string $operation): array { + if ($recorded['status'] !== $status) { + throw new RuntimeException('Worker sessions '.$operation.': '.json_encode($recorded)); + } + + return $recorded['response']; + }; + $id = static fn (string $name): string => $workflowId.'-'.$name; + $register = static function (string $worker, string $queue, array $capabilities = [], ?int $sessions = null) use ($raw, $ok, $definition): void { + $ok($raw('POST', '/api/worker/register', ['worker_id' => $worker, 'task_queue' => $queue, 'runtime' => 'php', + 'sdk_version' => 'raw-http', 'supported_workflow_types' => [$definition['workflow_type']], + 'supported_activity_types' => [$definition['activity_type']], 'capabilities' => $capabilities, + 'capability_manifest' => array_fill_keys(['local_activities', 'worker_sessions', 'sticky_execution'], + ['supported' => false, 'minimum_protocol_version' => '1.18', 'reason' => 'raw_http'])] + + ($sessions === null ? [] : ['max_concurrent_worker_sessions' => $sessions])), 201, 'registration '.$worker); + }; + $beat = static function (string ...$workers) use ($raw): void { + foreach ($workers as $worker) { + $raw('POST', '/api/worker/heartbeat', ['worker_id' => $worker]); + } + }; + $create = static fn (array $body, ?string $protocol = '1.20'): array => $raw('POST', '/api/worker/sessions', $body, $protocol); + $heartbeat = static fn (string $session, array $body): array => $raw('POST', '/api/worker/sessions/'.rawurlencode($session).'/heartbeat', $body); + $close = static fn (string $session, array $body = [], string $query = ''): array => $raw('DELETE', + '/api/worker/sessions/'.rawurlencode($session).$query, $body === [] ? null : $body); + $show = static fn (string $session): array => $raw('GET', '/api/worker-sessions/'.rawurlencode($session), null, null, true); + + [$w1, $w2, $w3] = [$id('wss-w1'), $id('wss-w2'), $id('wss-w3')]; + [$q1, $q2] = [$id('wss-q1'), $id('wss-q2')]; + $register($w1, $q1, ['gpu'], 2); + $register($w2, $q1); + $register($w3, $q2); + [$s1, $s2, $s3] = [$id('wss-s1'), $id('wss-s2'), $id('wss-s3')]; + $state = ['applicable' => true, 'namespace' => $namespace, 'workers' => [$w1, $w2, $w3], 'queues' => [$q1, $q2]]; + + // 1. Gates. + $state['gate_no_protocol'] = $answer($create(['worker_id' => $w1, 'session_id' => $s1], null)); + $state['gate_protocol_1_7'] = $answer($create(['worker_id' => $w1, 'session_id' => $s1], '1.7')); + $state['gate_protocol_1_21'] = $answer($create(['worker_id' => $w1, 'session_id' => $s1], '1.21')); + $state['gate_protocol_1_8_empty'] = $answer($create([], '1.8')); + $state['gate_unknown_namespace'] = $answer($raw('POST', '/api/worker/sessions', ['worker_id' => $w1, 'session_id' => $s1], '1.20', false, 'no-such-namespace-wss')); + $state['gate_list_without_control_plane'] = $answer($raw('GET', '/api/worker-sessions', null, '1.20')); + $state['gate_list_unknown_namespace'] = $answer($raw('GET', '/api/worker-sessions', null, null, true, 'no-such-namespace-wss')); + + // 2. Validation. + $state['invalid_empty'] = $answer($create([])); + $state['invalid_types'] = $answer($create(['worker_id' => 5, 'session_id' => ' ', 'requirements' => 'x', 'lease_seconds' => 0, + 'ttl_seconds' => 'a', 'max_concurrent_activities' => 1.5, 'create_if_missing' => 'yes', 'allow_reacquire_after_failure' => 2, + 'queue' => [], 'connection' => 7])); + $state['invalid_heartbeat'] = $answer($heartbeat($s1, ['lease_seconds' => 0])); + $state['invalid_close'] = $answer($close($s1)); + + // 3. Registration and routing. + $state['unregistered'] = $answer($create(['worker_id' => $id('wss-nobody'), 'session_id' => $s1])); + $state['queue_mismatch'] = $answer($create(['worker_id' => $w1, 'session_id' => $s1, 'queue' => $q2])); + $state['requirements_missing'] = $answer($create(['worker_id' => $w1, 'session_id' => $s1, 'requirements' => ['gpu', 'tpu', ' tpu ', 'fpga']])); + $state['missing_without_create'] = $answer($create(['worker_id' => $w1, 'session_id' => $s1, 'create_if_missing' => false])); + + // 4-6. Create, a float lease, reuse. + $state['created'] = $answer($create(['worker_id' => $w1, 'session_id' => ' '.$s1.' ', 'requirements' => ['gpu', ' gpu '], + 'lease_seconds' => '60', 'ttl_seconds' => 600, 'connection' => 'c1'])); + // A float lease passes validation as an integer; sent as JSON 7.0, since + // json_encode writes 7.0 as 7. + $state['float_lease'] = $answer($raw('POST', '/api/worker/sessions', [], '1.20', false, null, + json_encode(['worker_id' => $w1, 'session_id' => $s2, 'lease_seconds' => 7.0], JSON_PRESERVE_ZERO_FRACTION))); + $state['reused'] = $answer($create(['worker_id' => $w1, 'session_id' => $s1, 'ttl_seconds' => 9999, 'lease_seconds' => 30, + 'max_concurrent_activities' => 3, 'create_if_missing' => false])); + + // 7. Another worker. + $state['foreign_create'] = $answer($create(['worker_id' => $w2, 'session_id' => $s1])); + $state['foreign_heartbeat'] = $answer($heartbeat($s1, ['worker_id' => $w2])); + $state['foreign_close'] = $answer($close($s1, ['worker_id' => $w2])); + + // 8. Heartbeat. + $state['heartbeat_missing'] = $answer($heartbeat($id('wss-none'), ['worker_id' => $w1])); + $state['heartbeat'] = $answer($heartbeat($s1, ['worker_id' => $w1, 'lease_seconds' => 5])); + + // 9. The limit: S1 and S2 are held, so a third is refused. + $state['limit'] = $answer($create(['worker_id' => $w1, 'session_id' => $s3])); + + // 10. Lease expiry, on read. S1's lease is 5 seconds from the heartbeat. + $beat($w1, $w2, $w3); + $short = $id('wss-short'); + $state['short_created'] = $answer($create(['worker_id' => $w2, 'session_id' => $short, 'lease_seconds' => 2])); + usleep(3200000); + $state['short_expired'] = $answer($show($short)); + + // 11. Reacquire by another worker, with no options. + $state['reacquired'] = $answer($create(['worker_id' => $w3, 'session_id' => $short])); + $state['reacquired_heartbeat'] = $answer($heartbeat($short, ['worker_id' => $w3])); + + // 12. Close. + $state['close_by_old_owner'] = $answer($close($short, ['worker_id' => $w2])); + $state['closed'] = $answer($close($short, [], '?worker_id='.rawurlencode($w3))); + $state['closed_again'] = $answer($close($short, ['worker_id' => $w3])); + $state['closed_by_other'] = $answer($close($short, ['worker_id' => $w2])); + $state['closed_heartbeat'] = $answer($heartbeat($short, ['worker_id' => $w3])); + $state['closed_create'] = $answer($create(['worker_id' => $w3, 'session_id' => $short])); + $state['closed_create_unmet'] = $answer($create(['worker_id' => $w3, 'session_id' => $short, 'requirements' => ['gpu']])); + + // 13. Reacquire refused, by the stored option and by the request's. + $beat($w1, $w2, $w3); + [$terminal, $optional] = [$id('wss-terminal'), $id('wss-optional')]; + $create(['worker_id' => $w2, 'session_id' => $terminal, 'lease_seconds' => 2, 'allow_reacquire_after_failure' => false]); + $create(['worker_id' => $w2, 'session_id' => $optional, 'lease_seconds' => 2]); + usleep(3200000); + $state['reacquire_terminal'] = $answer($create(['worker_id' => $w3, 'session_id' => $terminal])); + $state['reacquire_disallowed'] = $answer($create(['worker_id' => $w3, 'session_id' => $optional, 'allow_reacquire_after_failure' => false])); + + // 14. TTL expiry is terminal; the owner can still close. + $beat($w1, $w2, $w3); + $ttl = $id('wss-ttl'); + $create(['worker_id' => $w2, 'session_id' => $ttl, 'ttl_seconds' => 2, 'lease_seconds' => 60]); + usleep(3200000); + $state['ttl_heartbeat'] = $answer($heartbeat($ttl, ['worker_id' => $w2])); + $state['ttl_create'] = $answer($create(['worker_id' => $w2, 'session_id' => $ttl])); + $state['ttl_close'] = $answer($close($ttl, ['worker_id' => $w2])); + + // 15. A holder deregistered: orphaned on read, then reacquired. + $beat($w1, $w2, $w3); + $held = $id('wss-held'); + $state['held_created'] = $answer($create(['worker_id' => $w3, 'session_id' => $held])); + $state['holder_deregistered'] = $raw('DELETE', '/api/worker/registrations/'.rawurlencode($w3))['status']; + $state['held_orphaned'] = $answer($show($held)); + $state['deregistered_create'] = $answer($create(['worker_id' => $w3, 'session_id' => $held])); + $state['orphan_reacquired'] = $answer($create(['worker_id' => $w2, 'session_id' => $held])); + $state['orphan_queue_mismatch'] = $answer($create(['worker_id' => $w2, 'session_id' => $held, 'queue' => $q2])); + + // Visibility: one session, and the namespace's list and metrics. + $state['show_missing'] = $answer($show($id('wss-none'))); + $list = $answer($raw('GET', '/api/worker-sessions', null, null, true)); + $state['list_metrics'] = $list['response']['metrics'] ?? null; + $state['list_namespace'] = $list['response']['namespace'] ?? null; + // Order ties within a second break by session ID; record membership. + $sessions = $list['response']['sessions'] ?? []; + usort($sessions, static fn (array $a, array $b): int => strcmp($a['session_id'], $b['session_id'])); + $state['list_sessions'] = $sessions; + + return $state; +} diff --git a/tests/Feature/WorkerSessionHolderMigrationTest.php b/tests/Feature/WorkerSessionHolderMigrationTest.php new file mode 100644 index 000000000..ccd9f1196 --- /dev/null +++ b/tests/Feature/WorkerSessionHolderMigrationTest.php @@ -0,0 +1,69 @@ +down(); + + $token = bin2hex(random_bytes(16)); + $this->seedWorker('holder', $token); + $this->seedWorker('tokenless', null); + $this->seedSession('held', 'holder', WorkerSessionLease::STATUS_ACTIVE); + $this->seedSession('closed', 'holder', WorkerSessionLease::STATUS_CLOSED); + $this->seedSession('tokenless-held', 'tokenless', WorkerSessionLease::STATUS_ACTIVE); + $this->seedSession('unregistered', 'gone', WorkerSessionLease::STATUS_ACTIVE); + + $migration->up(); + + $holders = WorkerSessionLease::query()->pluck('lease_owner_incarnation', 'session_id')->all(); + $this->assertSame([ + 'held' => hash('sha256', $token), + 'closed' => null, + 'tokenless-held' => null, + 'unregistered' => null, + ], $holders); + } + + private function seedWorker(string $workerId, ?string $token): void + { + WorkerRegistration::query()->create([ + 'worker_id' => $workerId, + 'namespace' => 'default', + 'task_queue' => 'gpu-activities', + 'runtime' => 'php', + 'last_heartbeat_at' => now(), + 'status' => 'active', + ])->forceFill(['registration_token' => $token])->save(); + } + + private function seedSession(string $sessionId, string $owner, string $status): void + { + WorkerSessionLease::query()->create([ + 'namespace' => 'default', + 'session_id' => $sessionId, + 'status' => $status, + 'lease_owner' => $owner, + 'lease_expires_at' => now()->addMinutes(2), + 'ttl_expires_at' => now()->addMinutes(30), + 'lease_seconds' => 120, + 'ttl_seconds' => 1800, + 'max_concurrent_activities' => 1, + 'create_if_missing' => true, + 'allow_reacquire_after_failure' => true, + 'last_heartbeat_at' => now(), + ]); + } +} diff --git a/tests/Feature/WorkerSessionProtocolTest.php b/tests/Feature/WorkerSessionProtocolTest.php index 80525a7db..242993d0d 100644 --- a/tests/Feature/WorkerSessionProtocolTest.php +++ b/tests/Feature/WorkerSessionProtocolTest.php @@ -9,6 +9,7 @@ use App\Support\ActivityTaskPollRequestStore; use App\Support\ServerPollingCache; use App\Support\WorkerProtocol; +use App\Support\WorkerSessionRegistry; use Illuminate\Foundation\Testing\RefreshDatabase; use Illuminate\Support\Facades\Queue; use Mockery\MockInterface; @@ -748,17 +749,21 @@ public function test_activity_heartbeat_reports_worker_session_renewal_failure() ->assertJsonPath('task.worker_session.session_id', 'gpu-heartbeat-renewal') ->assertJsonPath('task.worker_session.lease_owner', 'gpu-heartbeat-renewal-worker'); - $this->registerWorkerThroughProtocol( + $otherToken = $this->registerWorkerThroughProtocol( workerId: 'gpu-heartbeat-renewal-other-worker', taskQueue: 'gpu-activities', supportedActivityTypes: ['tests.external-greeting-activity'], capabilities: ['gpu:nvidia-l4'], ); + // Hand the session to the other worker's current registration. WorkerSessionLease::query() ->where('namespace', 'default') ->where('session_id', 'gpu-heartbeat-renewal') - ->update(['lease_owner' => 'gpu-heartbeat-renewal-other-worker']); + ->update([ + 'lease_owner' => 'gpu-heartbeat-renewal-other-worker', + 'lease_owner_incarnation' => hash('sha256', (string) $otherToken), + ]); $taskId = (string) $activityPoll->json('task.task_id'); $attemptId = (string) $activityPoll->json('task.activity_attempt_id'); @@ -1020,6 +1025,305 @@ public function test_absolute_ttl_remains_terminal_after_a_lease_was_already_mar * @param list $supportedActivityTypes * @param list $capabilities */ + public function test_session_numbers_have_portable_maxima_on_every_path(): void + { + Queue::fake(); + $this->registerWorkerThroughProtocol( + workerId: 'bounded-worker', + taskQueue: 'gpu-activities', + supportedWorkflowTypes: ['tests.external-greeting-workflow'], + capabilities: ['gpu:nvidia-l4'], + ); + $maxima = [ + 'lease_seconds' => WorkerSessionRegistry::MAX_LEASE_SECONDS, + 'ttl_seconds' => WorkerSessionRegistry::MAX_TTL_SECONDS, + 'max_concurrent_activities' => WorkerSessionRegistry::MAX_CONCURRENT_ACTIVITIES, + ]; + $this->assertSame([86400, 2592000, 1000], array_values($maxima)); + + foreach ($maxima as $field => $maximum) { + foreach ([$maximum + 1, PHP_INT_MAX] as $value) { + $this->postJson('/api/worker/sessions', [ + 'worker_id' => 'bounded-worker', 'session_id' => "too-large-{$field}", $field => $value, + ], $this->workerHeaders()) + ->assertStatus(422) + ->assertJsonPath("errors.{$field}.0", sprintf( + 'The %s field must not be greater than %d.', str_replace('_', ' ', $field), $maximum, + )); + } + } + $this->assertSame(0, WorkerSessionLease::query()->count(), 'refused creates record nothing'); + + $this->postJson('/api/worker/sessions', ['worker_id' => 'bounded-worker', 'session_id' => 'at-maxima', ...$maxima], + $this->workerHeaders()) + ->assertCreated() + ->assertJsonPath('session.lease_seconds', 86400) + ->assertJsonPath('session.ttl_seconds', 2592000) + ->assertJsonPath('session.max_concurrent_activities', 1000); + $leaseExpiresAt = WorkerSessionLease::query()->where('session_id', 'at-maxima')->value('lease_expires_at'); + + foreach ([86401, PHP_INT_MAX] as $value) { + $this->postJson('/api/worker/sessions/at-maxima/heartbeat', ['worker_id' => 'bounded-worker', 'lease_seconds' => $value], + $this->workerHeaders()) + ->assertStatus(422) + ->assertJsonPath('errors.lease_seconds.0', 'The lease seconds field must not be greater than 86400.'); + } + $this->assertEquals($leaseExpiresAt, WorkerSessionLease::query()->where('session_id', 'at-maxima')->value('lease_expires_at'), + 'a refused heartbeat renews nothing'); + $this->postJson('/api/worker/sessions/at-maxima/heartbeat', ['worker_id' => 'bounded-worker', 'lease_seconds' => 86400], + $this->workerHeaders()) + ->assertOk() + ->assertJsonPath('outcome', 'heartbeat_recorded'); + + $this->configureWorkflowTypes(['tests.external-greeting-workflow' => ExternalGreetingWorkflow::class]); + $this->postJson('/api/workflows', [ + 'workflow_id' => 'wf-worker-session-maxima', 'workflow_type' => 'tests.external-greeting-workflow', + 'task_queue' => 'gpu-activities', 'input' => ['Ada'], + ], $this->apiHeaders())->assertCreated(); + $poll = $this->postJson('/api/worker/workflow-tasks/poll', ['worker_id' => 'bounded-worker', 'task_queue' => 'gpu-activities'], + $this->workerHeaders()); + $poll->assertOk(); + foreach ($maxima as $field => $maximum) { + $errors = $this->postJson(sprintf('/api/worker/workflow-tasks/%s/complete', $poll->json('task.task_id')), [ + 'lease_owner' => $poll->json('task.lease_owner'), + 'workflow_task_attempt' => $poll->json('task.workflow_task_attempt'), + 'commands' => [[ + 'type' => 'schedule_activity', + 'activity_type' => 'tests.external-greeting-activity', + 'arguments' => Serializer::serializeWithCodec((string) config('workflows.serializer'), ['Ada']), + 'worker_session' => ['session_id' => 'command-maxima', $field => PHP_INT_MAX], + ]], + ], $this->workerHeaders()) + ->assertStatus(422) + ->json('errors'); + $this->assertSame( + sprintf('The commands.0.worker_session.%s field must not be greater than %d.', $field, $maximum), + $errors["commands.0.worker_session.{$field}"][0] ?? null, + json_encode($errors), + ); + } + $this->assertSame(0, ActivityExecution::query()->count(), 'refused commands schedule nothing'); + } + + public function test_a_same_id_replacement_reacquires_the_session_and_fences_the_old_process(): void + { + // The worker declares that it sends its registration token on every + // explicit session request, so the server requires it. + $register = fn (): ?string => $this->registerWorkerThroughProtocol( + workerId: 'gpu-worker', + taskQueue: 'gpu-activities', + supportedActivityTypes: ['tests.external-greeting-activity'], + capabilities: ['gpu:nvidia-l4', WorkerSessionRegistry::REGISTRATION_TOKEN_CAPABILITY], + ); + $holder = static fn (): ?string => WorkerSessionLease::query()->where('session_id', 'gpu-state')->value('lease_owner_incarnation'); + $row = static fn (): array => WorkerSessionLease::query()->where('session_id', 'gpu-state')->firstOrFail()->getAttributes(); + $create = fn (?string $token = null, array $options = []) => $this->postJson('/api/worker/sessions', array_filter([ + 'worker_id' => 'gpu-worker', 'session_id' => 'gpu-state', 'requirements' => ['gpu:nvidia-l4'], 'registration_token' => $token, + ...$options, + ]), $this->workerHeaders()); + $heartbeat = fn (?string $token = null) => $this->postJson('/api/worker/sessions/gpu-state/heartbeat', + array_filter(['worker_id' => 'gpu-worker', 'registration_token' => $token, 'lease_seconds' => 900]), $this->workerHeaders()); + $close = fn (?string $token = null) => $this->deleteJson('/api/worker/sessions/gpu-state', + array_filter(['worker_id' => 'gpu-worker', 'registration_token' => $token]), $this->workerHeaders()); + + $first = $register(); + $this->assertIsString($first); + $create($first)->assertCreated()->assertJsonPath('outcome', 'created'); + $this->assertSame(hash('sha256', $first), $holder(), 'the session keeps a digest, never the token'); + + // A worker heartbeat keeps the registration, so the holder reuses it. + $this->postJson('/api/worker/heartbeat', ['worker_id' => 'gpu-worker'], $this->workerHeaders())->assertOk(); + $create($first)->assertOk()->assertJsonPath('outcome', 'reused'); + + // A same-ID re-registration rotates the token: the session is orphaned. + $second = $register(); + $this->assertIsString($second); + $this->assertNotSame($first, $second); + $heartbeat($second) + ->assertStatus(409) + ->assertJsonPath('reason', 'session_not_active') + ->assertJsonPath('session.status', 'orphaned') + ->assertJsonPath('session.failure_reason', 'worker_registration_replaced'); + $heartbeat($first)->assertStatus(409)->assertJsonPath('reason', 'stale_worker_registration'); + $create($first)->assertStatus(409)->assertJsonPath('reason', 'stale_worker_registration'); + $this->assertSame(hash('sha256', $first), $holder(), 'a stale token changes nothing'); + + // The replacement reacquires, never reuses, and becomes the holder. + $create($second)->assertOk()->assertJsonPath('outcome', 'reacquired')->assertJsonPath('session.status', 'active'); + $this->assertSame(hash('sha256', $second), $holder()); + $heartbeat($second)->assertOk()->assertJsonPath('outcome', 'heartbeat_recorded'); + + // The stale process, with its old token or with none, can neither + // reuse, renew nor close the replacement's session, and changes nothing. + $before = $row(); + foreach ([$first, null] as $stale) { + $create($stale, ['lease_seconds' => 5, 'max_concurrent_activities' => 9]) + ->assertStatus(409)->assertJsonPath('reason', 'stale_worker_registration'); + $heartbeat($stale)->assertStatus(409)->assertJsonPath('reason', 'stale_worker_registration'); + $close($stale)->assertStatus(409)->assertJsonPath('reason', 'stale_worker_registration'); + $this->assertSame($before, $row(), 'a refused request changes nothing'); + } + $close($second)->assertOk()->assertJsonPath('outcome', 'closed'); + } + + public function test_a_deregistered_worker_can_neither_renew_nor_close_its_session(): void + { + // Deregistration deletes the registration and leaves the session for + // holder-loss handling. The former process, with its token or without, + // is refused and changes nothing: a token worker deregistered through + // the fenced route, a published worker through the plain one. + foreach ([true, false] as $sendsToken) { + $workerId = $sendsToken ? 'gpu-token-worker' : 'gpu-published-worker'; + $sessionId = $workerId.'-state'; + $token = $this->registerWorkerThroughProtocol( + workerId: $workerId, + taskQueue: 'gpu-activities', + supportedActivityTypes: ['tests.external-greeting-activity'], + capabilities: ['gpu:nvidia-l4', ...($sendsToken ? [WorkerSessionRegistry::REGISTRATION_TOKEN_CAPABILITY] : [])], + ); + $this->assertIsString($token); + $this->postJson('/api/worker/sessions', array_filter(['worker_id' => $workerId, 'session_id' => $sessionId, + 'requirements' => ['gpu:nvidia-l4'], 'registration_token' => $sendsToken ? $token : null]), $this->workerHeaders()) + ->assertCreated(); + $row = static fn (): array => WorkerSessionLease::query()->where('session_id', $sessionId)->firstOrFail()->getAttributes(); + $before = $row(); + if ($sendsToken) { + $this->postJson("/api/worker/registrations/{$workerId}/deregister", ['registration_token' => $token], + $this->workerHeaders())->assertOk(); + } else { + $this->deleteJson("/api/worker/registrations/{$workerId}", [], $this->workerHeaders())->assertOk(); + } + foreach ([null, $token] as $sent) { + $this->postJson("/api/worker/sessions/{$sessionId}/heartbeat", + array_filter(['worker_id' => $workerId, 'registration_token' => $sent, 'lease_seconds' => 900]), $this->workerHeaders()) + ->assertStatus(412)->assertJsonPath('reason', 'worker_not_registered') + ->assertJsonPath('error', 'Worker must be registered before renewing a worker session.'); + $this->deleteJson("/api/worker/sessions/{$sessionId}", + array_filter(['worker_id' => $workerId, 'registration_token' => $sent]), $this->workerHeaders()) + ->assertStatus(412)->assertJsonPath('reason', 'worker_not_registered') + ->assertJsonPath('error', 'Worker must be registered before closing a worker session.'); + $this->assertSame($before, $row(), 'a refused request changes nothing'); + } + } + } + + public function test_a_superseded_registration_can_neither_create_renew_nor_close_its_session(): void + { + // Plain deregistration commits the registration as superseded before it + // deletes the row, and lease recovery supersedes a stale holder. Until + // the worker registers again, its session requests are refused. + foreach ([true, false] as $sendsToken) { + $workerId = $sendsToken ? 'gpu-token-worker' : 'gpu-published-worker'; + $sessionId = $workerId.'-state'; + $token = $this->registerWorkerThroughProtocol( + workerId: $workerId, + taskQueue: 'gpu-activities', + supportedActivityTypes: ['tests.external-greeting-activity'], + capabilities: ['gpu:nvidia-l4', ...($sendsToken ? [WorkerSessionRegistry::REGISTRATION_TOKEN_CAPABILITY] : [])], + ); + $sent = $sendsToken ? $token : null; + $this->postJson('/api/worker/sessions', array_filter(['worker_id' => $workerId, 'session_id' => $sessionId, + 'requirements' => ['gpu:nvidia-l4'], 'registration_token' => $sent]), $this->workerHeaders())->assertCreated(); + WorkerRegistration::query()->where('worker_id', $workerId)->update(['status' => WorkerRegistration::STATUS_SUPERSEDED]); + $row = static fn (): array => WorkerSessionLease::query()->where('session_id', $sessionId)->firstOrFail()->getAttributes(); + $before = $row(); + foreach ([ + $this->postJson('/api/worker/sessions', array_filter(['worker_id' => $workerId, 'session_id' => $sessionId, + 'requirements' => ['gpu:nvidia-l4'], 'registration_token' => $sent, 'lease_seconds' => 5]), $this->workerHeaders()), + $this->postJson("/api/worker/sessions/{$sessionId}/heartbeat", + array_filter(['worker_id' => $workerId, 'registration_token' => $sent, 'lease_seconds' => 900]), $this->workerHeaders()), + $this->deleteJson("/api/worker/sessions/{$sessionId}", + array_filter(['worker_id' => $workerId, 'registration_token' => $sent]), $this->workerHeaders()), + ] as $refused) { + $refused->assertStatus(409)->assertJsonPath('reason', 'worker_registration_superseded') + ->assertJsonPath('remediation', 'Restart and register the worker with a new worker_id.'); + } + $this->assertSame($before, $row(), 'a refused request changes nothing'); + } + } + + public function test_without_the_token_capability_a_tokenless_request_acts_for_the_current_registration(): void + { + // Published workers do not send a registration token. For them the + // published contract holds: a request without one acts for the worker + // ID's current registration, while a token that is sent must be current. + $register = fn (): ?string => $this->registerWorkerThroughProtocol( + workerId: 'gpu-worker', + taskQueue: 'gpu-activities', + supportedActivityTypes: ['tests.external-greeting-activity'], + capabilities: ['gpu:nvidia-l4'], + ); + $create = fn (?string $token = null) => $this->postJson('/api/worker/sessions', array_filter([ + 'worker_id' => 'gpu-worker', 'session_id' => 'gpu-legacy', 'requirements' => ['gpu:nvidia-l4'], 'registration_token' => $token, + ]), $this->workerHeaders()); + $heartbeat = fn (?string $token = null) => $this->postJson('/api/worker/sessions/gpu-legacy/heartbeat', + array_filter(['worker_id' => 'gpu-worker', 'registration_token' => $token]), $this->workerHeaders()); + + $first = $register(); + $create()->assertCreated()->assertJsonPath('outcome', 'created'); + $second = $register(); + $heartbeat()->assertStatus(409)->assertJsonPath('session.failure_reason', 'worker_registration_replaced'); + $create()->assertOk()->assertJsonPath('outcome', 'reacquired'); + $this->assertSame(hash('sha256', (string) $second), + WorkerSessionLease::query()->where('session_id', 'gpu-legacy')->value('lease_owner_incarnation')); + $heartbeat()->assertOk()->assertJsonPath('outcome', 'heartbeat_recorded'); + $heartbeat($first)->assertStatus(409)->assertJsonPath('reason', 'stale_worker_registration'); + $this->deleteJson('/api/worker/sessions/gpu-legacy', ['worker_id' => 'gpu-worker'], $this->workerHeaders()) + ->assertOk()->assertJsonPath('outcome', 'closed'); + } + + public function test_a_same_id_replacement_reacquires_a_session_through_an_activity_claim(): void + { + Queue::fake(); + $this->configureWorkflowTypes(['tests.external-greeting-workflow' => ExternalGreetingWorkflow::class]); + $register = fn (): ?string => $this->registerWorkerThroughProtocol( + workerId: 'gpu-worker', + taskQueue: 'gpu-activities', + supportedWorkflowTypes: ['tests.external-greeting-workflow'], + supportedActivityTypes: ['tests.external-greeting-activity'], + capabilities: ['gpu:nvidia-l4'], + ); + $first = $register(); + $this->postJson('/api/worker/sessions', ['worker_id' => 'gpu-worker', 'session_id' => 'gpu-claim', 'requirements' => ['gpu:nvidia-l4']], + $this->workerHeaders())->assertCreated(); + $this->postJson('/api/workflows', [ + 'workflow_id' => 'wf-worker-session-replacement', 'workflow_type' => 'tests.external-greeting-workflow', + 'task_queue' => 'gpu-activities', 'input' => ['Ada'], + ], $this->apiHeaders())->assertCreated(); + $poll = $this->postJson('/api/worker/workflow-tasks/poll', ['worker_id' => 'gpu-worker', 'task_queue' => 'gpu-activities'], + $this->workerHeaders()); + $this->postJson(sprintf('/api/worker/workflow-tasks/%s/complete', $poll->json('task.task_id')), [ + 'lease_owner' => $poll->json('task.lease_owner'), + 'workflow_task_attempt' => $poll->json('task.workflow_task_attempt'), + 'commands' => [[ + 'type' => 'schedule_activity', + 'activity_type' => 'tests.external-greeting-activity', + 'arguments' => Serializer::serializeWithCodec((string) config('workflows.serializer'), ['Ada']), + 'worker_session' => ['session_id' => 'gpu-claim', 'requirements' => ['gpu:nvidia-l4']], + ]], + ], $this->workerHeaders())->assertOk(); + + $second = $register(); + $this->assertNotSame($first, $second); + $this->getJson('/api/worker-sessions/gpu-claim', $this->apiHeaders()) + ->assertOk() + ->assertJsonPath('session.status', 'orphaned') + ->assertJsonPath('session.failure_reason', 'worker_registration_replaced'); + $this->postJson('/api/worker/activity-tasks/poll', ['worker_id' => 'gpu-worker', 'task_queue' => 'gpu-activities'], + $this->workerHeaders()) + ->assertOk() + ->assertJsonPath('poll_status', 'leased') + ->assertJsonPath('task.worker_session.session_id', 'gpu-claim') + ->assertJsonPath('task.worker_session.lease_owner', 'gpu-worker'); + $this->assertSame(hash('sha256', (string) $second), + WorkerSessionLease::query()->where('session_id', 'gpu-claim')->value('lease_owner_incarnation')); + // Published PHP keeps the old failure reason on a reacquired session + // (Server #737), so only the status is asserted here. + $this->getJson('/api/worker-sessions/gpu-claim', $this->apiHeaders()) + ->assertOk() + ->assertJsonPath('session.status', 'active'); + } + private function registerWorkerThroughProtocol( string $workerId, string $taskQueue, @@ -1028,7 +1332,7 @@ private function registerWorkerThroughProtocol( array $capabilities = [], int $maxConcurrentWorkerSessions = 10, ?string $protocolVersion = null, - ): void { + ): ?string { $requestedProtocol = $protocolVersion ?? WorkerProtocol::VERSION; $workerSupportsSessions = version_compare( $requestedProtocol, @@ -1061,10 +1365,11 @@ private function registerWorkerThroughProtocol( ); } - $this->postJson('/api/worker/register', $registration, $protocolVersion === null + return $this->postJson('/api/worker/register', $registration, $protocolVersion === null ? $this->workerHeaders() : $this->workerProtocolHeaders($protocolVersion)) - ->assertCreated(); + ->assertCreated() + ->json('registration_token'); } /** diff --git a/tests/Feature/WorkerSessionRegistrationRaceTest.php b/tests/Feature/WorkerSessionRegistrationRaceTest.php new file mode 100644 index 000000000..595995527 --- /dev/null +++ b/tests/Feature/WorkerSessionRegistrationRaceTest.php @@ -0,0 +1,455 @@ +initializeDatabase($driver); + $first = $this->register(); + $this->postJson('/api/worker/sessions', ['worker_id' => 'race-worker', 'session_id' => 'race-state', + 'requirements' => ['gpu'], 'registration_token' => $first], $this->workerHeaders())->assertCreated(); + $actors = []; + DB::purge(); + try { + $actors['session'] = $this->forkActor(fn (): array => $this->sessionRequest($verb, $first), pause: 'lock'); + $this->readMessage($actors['session']); + fwrite($actors['session']['socket'], "go\n"); + $this->assertNotNull($this->readMessage($actors['session'])['paused']); + + $actors['register'] = $this->forkActor(fn (): array => ['token' => $this->register()]); + $replacement = $this->readMessage($actors['register']); + fwrite($actors['register']['socket'], "go\n"); + // The re-registration waits on the row the session request holds. + $this->awaitLockWait($replacement['connection_id']); + + fwrite($actors['session']['socket'], "continue\n"); + $session = $this->finishActor($actors['session']); + unset($actors['session']); + $second = $this->finishActor($actors['register'])['token']; + unset($actors['register']); + } finally { + foreach ($actors as $actor) { + posix_kill($actor['pid'], SIGKILL); + pcntl_waitpid($actor['pid'], $status); + fclose($actor['socket']); + } + DB::purge(); + DB::reconnect(); + } + + // The request ran wholly before the rotation, for the incarnation + // that was current then. + $this->assertSame(200, $session['status'], json_encode($session)); + $this->assertSame(match ($verb) { + 'create' => 'reused', 'heartbeat' => 'heartbeat_recorded', 'close' => 'closed' + }, + $session['outcome'], json_encode($session)); + $this->assertIsString($second); + $this->assertNotSame($first, $second); + $this->assertSame(hash('sha256', $first), + WorkerSessionLease::query()->where('session_id', 'race-state')->value('lease_owner_incarnation')); + + // After the rotation the old process is refused and changes nothing. + $before = WorkerSessionLease::query()->where('session_id', 'race-state')->firstOrFail()->getAttributes(); + $this->sessionResponse($verb, $first)->assertStatus(409)->assertJsonPath('reason', 'stale_worker_registration'); + $this->assertSame($before, WorkerSessionLease::query()->where('session_id', 'race-state')->firstOrFail()->getAttributes()); + } + + public static function deregistrations(): array + { + $cases = []; + foreach (['mysql', 'pgsql'] as $driver) { + foreach (['heartbeat', 'close'] as $verb) { + $cases[$driver.' '.$verb] = [$driver, $verb]; + } + } + + return $cases; + } + + /** + * A published tokenless heartbeat or close and the worker's deregistration + * serialize on the registration row: the request either commits while the + * registration exists, as here, or sees none and is refused, as it is + * afterwards, without changing the session. + */ + #[DataProvider('deregistrations')] + public function test_a_session_request_and_a_deregistration_serialize(string $driver, string $verb): void + { + $this->initializeDatabase($driver); + $this->register(sendsToken: false); + $this->sessionResponse('create', null)->assertCreated(); + $actors = []; + DB::purge(); + try { + $actors['session'] = $this->forkActor(fn (): array => $this->sessionRequest($verb, null), pause: 'lock'); + $this->readMessage($actors['session']); + fwrite($actors['session']['socket'], "go\n"); + $this->assertNotNull($this->readMessage($actors['session'])['paused']); + + $actors['deregister'] = $this->forkActor(fn (): array => ['status' => $this->deleteJson('/api/worker/registrations/race-worker', + [], $this->workerHeaders())->status()]); + $deregistration = $this->readMessage($actors['deregister']); + fwrite($actors['deregister']['socket'], "go\n"); + // The deregistration waits on the row the session request holds. + $this->awaitLockWait($deregistration['connection_id']); + + fwrite($actors['session']['socket'], "continue\n"); + $session = $this->finishActor($actors['session']); + unset($actors['session']); + $this->assertSame(200, $this->finishActor($actors['deregister'])['status']); + unset($actors['deregister']); + } finally { + foreach ($actors as $actor) { + posix_kill($actor['pid'], SIGKILL); + pcntl_waitpid($actor['pid'], $status); + fclose($actor['socket']); + } + DB::purge(); + DB::reconnect(); + } + + $this->assertSame([200, $verb === 'heartbeat' ? 'heartbeat_recorded' : 'closed'], [$session['status'], $session['outcome']], + json_encode($session)); + $before = WorkerSessionLease::query()->where('session_id', 'race-state')->firstOrFail()->getAttributes(); + $this->sessionResponse($verb, null)->assertStatus(412)->assertJsonPath('reason', 'worker_not_registered'); + $this->assertSame($before, WorkerSessionLease::query()->where('session_id', 'race-state')->firstOrFail()->getAttributes()); + } + + public static function drivers(): array + { + return ['mysql' => ['mysql'], 'pgsql' => ['pgsql']]; + } + + /** + * Plain deregistration commits the registration as superseded, recovers + * its leases and only then deletes the row. Inside that window a + * published tokenless create, heartbeat or close is refused and changes + * nothing; once the row is gone it is refused as unregistered. + */ + #[DataProvider('drivers')] + public function test_session_requests_inside_a_deregistrations_superseded_window_are_refused(string $driver): void + { + $this->initializeDatabase($driver); + $this->register(sendsToken: false); + $this->sessionResponse('create', null)->assertCreated(); + $actors = []; + DB::purge(); + try { + $actors['deregister'] = $this->forkActor(fn (): array => ['status' => $this->deleteJson('/api/worker/registrations/race-worker', + [], $this->workerHeaders())->status()], pause: 'superseded'); + $this->readMessage($actors['deregister']); + fwrite($actors['deregister']['socket'], "go\n"); + $this->assertSame('superseded', $this->readMessage($actors['deregister'])['paused']); + + $before = WorkerSessionLease::query()->where('session_id', 'race-state')->firstOrFail()->getAttributes(); + foreach (['create', 'heartbeat', 'close'] as $verb) { + $this->sessionResponse($verb, null)->assertStatus(409)->assertJsonPath('reason', 'worker_registration_superseded'); + } + $this->assertSame($before, WorkerSessionLease::query()->where('session_id', 'race-state')->firstOrFail()->getAttributes()); + + fwrite($actors['deregister']['socket'], "continue\n"); + $this->assertSame(200, $this->finishActor($actors['deregister'])['status']); + unset($actors['deregister']); + } finally { + foreach ($actors as $actor) { + posix_kill($actor['pid'], SIGKILL); + pcntl_waitpid($actor['pid'], $status); + fclose($actor['socket']); + } + DB::purge(); + DB::reconnect(); + } + + $this->sessionResponse('heartbeat', null)->assertStatus(412)->assertJsonPath('reason', 'worker_not_registered'); + } + + public static function creates(): array + { + $cases = []; + foreach (['mysql', 'pgsql'] as $driver) { + $cases[$driver.' token worker'] = [$driver, true]; + $cases[$driver.' published worker'] = [$driver, false]; + } + + return $cases; + } + + /** + * A create read the worker's registration before a same-ID replacement + * registered and reacquired the session, and continues only afterwards. + * It is admitted against the registration as it is when it commits: a + * worker that sends its token is refused with its old token and changes + * nothing, and a published tokenless create reuses the replacement's + * session. Neither restores the old incarnation. + */ + #[DataProvider('creates')] + public function test_a_create_admits_against_the_registration_current_when_it_commits(string $driver, bool $sendsToken): void + { + $this->initializeDatabase($driver); + $first = $this->register($sendsToken); + $this->sessionResponse('create', $sendsToken ? $first : null)->assertCreated(); + $actors = []; + DB::purge(); + try { + $actors['session'] = $this->forkActor(fn (): array => $this->sessionRequest('create', $sendsToken ? $first : null), pause: 'read'); + $this->readMessage($actors['session']); + fwrite($actors['session']['socket'], "go\n"); + $this->assertNotNull($this->readMessage($actors['session'])['paused']); + + // The replacement registers and reacquires while the create waits + // between its first read and its transaction. + $second = $this->register($sendsToken); + $this->sessionResponse('heartbeat', $sendsToken ? $second : null)->assertStatus(409) + ->assertJsonPath('session.failure_reason', 'worker_registration_replaced'); + $this->sessionResponse('create', $sendsToken ? $second : null)->assertOk()->assertJsonPath('outcome', 'reacquired'); + $before = WorkerSessionLease::query()->where('session_id', 'race-state')->firstOrFail()->getAttributes(); + + fwrite($actors['session']['socket'], "continue\n"); + $session = $this->finishActor($actors['session']); + unset($actors['session']); + } finally { + foreach ($actors as $actor) { + posix_kill($actor['pid'], SIGKILL); + pcntl_waitpid($actor['pid'], $status); + fclose($actor['socket']); + } + DB::purge(); + DB::reconnect(); + } + + $this->assertNotSame($first, $second); + $after = WorkerSessionLease::query()->where('session_id', 'race-state')->firstOrFail()->getAttributes(); + if ($sendsToken) { + $this->assertSame([409, 'stale_worker_registration'], [$session['status'], $session['reason']], json_encode($session)); + $this->assertSame($before, $after, 'the stale create changes nothing'); + } else { + $this->assertSame([200, 'reused'], [$session['status'], $session['outcome']], json_encode($session)); + } + $this->assertSame(hash('sha256', $second), $after['lease_owner_incarnation'], 'the session keeps the replacement as its holder'); + } + + private function initializeDatabase(string $driver): void + { + $prefix = $driver === 'pgsql' ? 'DW_TEST_COMPLETION_PGSQL_' : 'DW_TEST_COMPLETION_MYSQL_'; + $host = getenv($prefix.'HOST'); + if (! $host) { + $this->markTestSkipped('Set '.$prefix.'HOST to a disposable database test server.'); + } + if (! function_exists('pcntl_fork') || ! function_exists('posix_kill') || ! function_exists('stream_socket_pair')) { + $this->markTestSkipped('Process control and local sockets are required.'); + } + $port = getenv($prefix.'PORT') ?: ($driver === 'pgsql' ? '5432' : '3306'); + $user = getenv($prefix.'USER') ?: ($driver === 'pgsql' ? 'postgres' : 'root'); + $password = getenv($prefix.'PASSWORD') ?: ''; + $this->databaseAdmin = [$driver.':host='.$host.';port='.$port.($driver === 'pgsql' ? ';dbname=postgres' : ''), $user, $password]; + $this->databaseName = 'dw_session_race_'.bin2hex(random_bytes(6)); + $admin = new PDO(...$this->databaseAdmin); + $admin->exec('CREATE DATABASE '.$this->databaseName); + unset($admin); + config(['database.default' => $driver, 'database.connections.'.$driver.'.host' => $host, + 'database.connections.'.$driver.'.port' => $port, 'database.connections.'.$driver.'.database' => $this->databaseName, + 'database.connections.'.$driver.'.username' => $user, 'database.connections.'.$driver.'.password' => $password, + 'database.connections.'.$driver.'.url' => null]); + DB::purge($driver); + $this->artisan('migrate:fresh', ['--force' => true])->assertExitCode(0); + $this->createNamespace('default'); + } + + protected function tearDown(): void + { + if ($this->databaseAdmin !== null) { + DB::disconnect(); + $admin = new PDO(...$this->databaseAdmin); + $admin->exec('DROP DATABASE '.$this->databaseName); + } + parent::tearDown(); + } + + /** Registers race-worker, by default declaring that it sends its token; returns the token. */ + private function register(bool $sendsToken = true): string + { + return (string) $this->postJson('/api/worker/register', [ + 'worker_id' => 'race-worker', + 'task_queue' => 'race-queue', + 'runtime' => 'php', + 'supported_activity_types' => ['tests.race-activity'], + 'capabilities' => ['gpu', 'worker_sessions', ...($sendsToken ? [WorkerSessionRegistry::REGISTRATION_TOKEN_CAPABILITY] : [])], + 'capability_manifest' => array_replace($this->portableWorkerAffinityRefusalManifest(), [ + 'worker_sessions' => ['supported' => true, + 'minimum_protocol_version' => WorkerProtocol::PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION, + 'implementation' => 'worker_session_registration_race_test'], + ]), + ], $this->workerHeaders())->assertCreated()->json('registration_token'); + } + + private function sessionResponse(string $verb, ?string $token) + { + return match ($verb) { + 'create' => $this->postJson('/api/worker/sessions', array_filter(['worker_id' => 'race-worker', 'session_id' => 'race-state', + 'requirements' => ['gpu'], 'registration_token' => $token]), $this->workerHeaders()), + 'heartbeat' => $this->postJson('/api/worker/sessions/race-state/heartbeat', + ['worker_id' => 'race-worker', 'registration_token' => $token, 'lease_seconds' => 900], $this->workerHeaders()), + 'close' => $this->deleteJson('/api/worker/sessions/race-state', + ['worker_id' => 'race-worker', 'registration_token' => $token], $this->workerHeaders()), + }; + } + + /** @return array */ + private function sessionRequest(string $verb, ?string $token): array + { + $response = $this->sessionResponse($verb, $token); + + return ['status' => $response->status(), 'outcome' => $response->json('outcome'), 'reason' => $response->json('reason')]; + } + + /** + * Runs $work in a child process on its own database connection. The child + * reports its connection id, waits for "go", and with $pause stops until + * "continue" right after it first reads the worker's registration ('read') + * or locks it ('lock'). + * + * @return array{pid: int, socket: resource} + */ + private function forkActor(callable $work, ?string $pause = null): array + { + $sockets = stream_socket_pair(STREAM_PF_UNIX, STREAM_SOCK_STREAM, STREAM_IPPROTO_IP); + $this->assertIsArray($sockets); + $pid = pcntl_fork(); + $this->assertNotSame(-1, $pid); + if ($pid === 0) { + fclose($sockets[0]); + stream_set_timeout($sockets[1], 15); + $ok = false; + try { + DB::reconnect(); + $idQuery = DB::connection()->getDriverName() === 'mysql' + ? 'SELECT CONNECTION_ID() AS id' : 'SELECT pg_backend_pid() AS id'; + fwrite($sockets[1], json_encode(['connection_id' => (int) DB::selectOne($idQuery)->id]).PHP_EOL); + if (fgets($sockets[1]) !== "go\n") { + throw new RuntimeException('Actor was not released.'); + } + if ($pause === 'superseded') { + // Pause right after a transaction commits the registration + // as superseded, while the row still exists. + $paused = false; + Event::listen(TransactionCommitted::class, static function () use (&$paused, $sockets, $pause): void { + if (! $paused && DB::table('workflow_worker_registrations')->where('worker_id', 'race-worker') + ->value('status') === WorkerRegistration::STATUS_SUPERSEDED) { + $paused = true; + fwrite($sockets[1], json_encode(['paused' => $pause]).PHP_EOL); + if (fgets($sockets[1]) !== "continue\n") { + throw new RuntimeException('Deregistration was not released.'); + } + } + }); + } elseif ($pause !== null) { + $paused = false; + DB::listen(static function (QueryExecuted $query) use (&$paused, $sockets, $pause): void { + if (! $paused && str_contains($query->sql, 'workflow_worker_registrations') + && ($pause === 'read' || str_contains(strtolower($query->sql), 'for update'))) { + $paused = true; + fwrite($sockets[1], json_encode(['paused' => $pause]).PHP_EOL); + if (fgets($sockets[1]) !== "continue\n") { + throw new RuntimeException('Session request was not released.'); + } + } + }); + } + fwrite($sockets[1], json_encode(['result' => $work()], JSON_THROW_ON_ERROR).PHP_EOL); + $ok = true; + } catch (Throwable $error) { + fwrite($sockets[1], json_encode(['error' => $error::class.': '.$error->getMessage()]).PHP_EOL); + } finally { + DB::disconnect(); + fclose($sockets[1]); + } + exit($ok ? 0 : 1); + } + fclose($sockets[1]); + stream_set_timeout($sockets[0], 15); + + return ['pid' => $pid, 'socket' => $sockets[0]]; + } + + private function awaitLockWait(int $connectionId): void + { + $query = DB::connection()->getDriverName() === 'mysql' + ? 'SELECT COUNT(*) AS waiting FROM performance_schema.data_lock_waits w JOIN performance_schema.threads t ON t.THREAD_ID = w.REQUESTING_THREAD_ID WHERE t.PROCESSLIST_ID = ?' + : 'SELECT COUNT(*) AS waiting FROM pg_locks WHERE pid = ? AND NOT granted'; + $until = microtime(true) + 5; + do { + if ((int) DB::selectOne($query, [$connectionId])->waiting > 0) { + $this->addToAssertionCount(1); + + return; + } + usleep(10_000); + } while (microtime(true) < $until); + $this->fail('The re-registration did not wait on the session request\'s registration lock, connection '.$connectionId.'.'); + } + + private function readMessage(array $actor): array + { + $line = fgets($actor['socket']); + $this->assertIsString($line, 'Actor did not respond.'); + $message = json_decode($line, true, flags: JSON_THROW_ON_ERROR); + $this->assertArrayNotHasKey('error', $message, $message['error'] ?? ''); + + return $message; + } + + private function finishActor(array $actor): array + { + $message = $this->readMessage($actor); + pcntl_waitpid($actor['pid'], $status); + $this->assertTrue(pcntl_wifexited($status)); + $this->assertSame(0, pcntl_wexitstatus($status)); + fclose($actor['socket']); + + return $message['result']; + } +} diff --git a/tests/Fixtures/ServerParity/worker-session-activities.json b/tests/Fixtures/ServerParity/worker-session-activities.json new file mode 100644 index 000000000..fa0aa7df2 --- /dev/null +++ b/tests/Fixtures/ServerParity/worker-session-activities.json @@ -0,0 +1,372 @@ +{ + "schema": "durable-workflow.server-parity-fixture/v1", + "id": "worker-session-activities", + "review_state": "reviewed", + "workflow_type": "parity.v1.echo", + "input": {"message": "worker session activities λ", "count": 9007199254740993, "optional": null}, + "typed_value": {"type": "map", "value": { + "message": {"type": "string", "value": "worker session activities λ"}, + "count": {"type": "int64", "value": "9007199254740993"}, + "optional": {"type": "null", "value": null} + }}, + "activity": false, + "expected_events": ["StartAccepted", "WorkflowStarted", "WorkflowCompleted"], + "worker_session_activities": { + "workflow_type": "parity.v1.session_activities", + "activity_type": "parity.v1.session_activity", + "empty_blob": "wwHioz3/VYAiNwwA", + "overall_budget_seconds": 120, + "expected": { + "applicable": true, + "namespace": "{id}-wsa", + "complete_protocol_1_7": { + "status": 409, + "reason": "worker_sessions_unavailable", + "message": "Worker-session activity commands require worker protocol 1.8 or newer." + }, + "complete_protocol_1_17": { + "status": 409, + "reason": "portable_worker_affinity_protocol_floor_unavailable", + "message": null + }, + "complete_on_timer": { + "status": 422, + "reason": "validation_failed", + "message": "worker_session is only supported for schedule_activity commands." + }, + "complete_without_session_id": { + "status": 422, + "reason": "validation_failed", + "message": "Worker session commands require a non-empty session_id." + }, + "complete_accepted": 200, + "complete_incapable": { + "status": 409, + "reason": "worker_capability_not_supported", + "message": null + }, + "show_before_claim": { + "status": 404, + "reason": "worker_session_not_found", + "message": "Worker session [{id}-wsa-lazy] not found in namespace [{id}-wsa]." + }, + "poll_without_requirement": { + "status": 200, + "poll_status": "empty", + "worker_session": null, + "has_session_key": false + }, + "poll_protocol_1_7": { + "status": 200, + "poll_status": "empty", + "worker_session": null, + "has_session_key": false + }, + "poll_capable": { + "status": 200, + "poll_status": "leased", + "worker_session": { + "session_id": "{id}-wsa-lazy", + "queue": "{id}-wsa-queue", + "requirements": [ + "gpu" + ], + "lease_seconds": 90, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "status": "active", + "lease_owner": "{id}-wsa-gpu", + "lease_expires_at": "present", + "ttl_expires_at": "present" + }, + "has_session_key": true + }, + "show_after_claim": { + "status": 200, + "session": { + "session_id": "{id}-wsa-lazy", + "namespace": "{id}-wsa", + "connection": null, + "queue": "{id}-wsa-queue", + "requirements": [ + "gpu" + ], + "status": "active", + "lease_owner": "{id}-wsa-gpu", + "holder": { + "worker_id": "{id}-wsa-gpu", + "task_queue": "{id}-wsa-queue", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 90, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 1, + "lease_span": 90, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + } + }, + "poll_other_capable": { + "status": 200, + "poll_status": "empty", + "worker_session": null, + "has_session_key": false + }, + "activity_heartbeat": { + "status": 200, + "heartbeat_recorded": true, + "worker_session": { + "session_id": "{id}-wsa-lazy", + "namespace": "{id}-wsa", + "connection": null, + "queue": "{id}-wsa-queue", + "requirements": [ + "gpu" + ], + "status": "active", + "lease_owner": "{id}-wsa-gpu", + "holder": { + "worker_id": "{id}-wsa-gpu", + "task_queue": "{id}-wsa-queue", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 90, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 1, + "lease_span": 90, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + } + }, + "activity_status": { + "status": 200, + "can_continue": true, + "reason": null, + "worker_session": { + "session_id": "{id}-wsa-lazy", + "queue": "{id}-wsa-queue", + "requirements": [ + "gpu" + ], + "lease_seconds": 90, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "status": "active", + "lease_owner": "{id}-wsa-gpu", + "lease_expires_at": "present", + "ttl_expires_at": "present" + } + }, + "activity_renew": { + "status": 200, + "lease_renewed": true, + "worker_session": { + "session_id": "{id}-wsa-lazy", + "namespace": "{id}-wsa", + "connection": null, + "queue": "{id}-wsa-queue", + "requirements": [ + "gpu" + ], + "status": "active", + "lease_owner": "{id}-wsa-gpu", + "holder": { + "worker_id": "{id}-wsa-gpu", + "task_queue": "{id}-wsa-queue", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 90, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 1, + "lease_span": 90, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + } + }, + "session_closed": 200, + "heartbeat_after_close": { + "status": 409, + "reason": "session_not_active", + "can_continue": false, + "heartbeat_recorded": false, + "worker_session_renewal": { + "admitted": false, + "reason": "session_not_active", + "error": "Worker session is not active." + }, + "worker_session_status": "closed" + }, + "status_after_close": { + "status": 200, + "can_continue": false, + "reason": "worker_session_unavailable" + }, + "complete_after_close": 200, + "limit_one": [ + "leased", + "empty" + ], + "limit_two": [ + "leased", + "leased" + ], + "queue_mismatch_polls": [ + "empty", + "empty" + ], + "operator_metrics": { + "status": 200, + "worker_sessions": { + "total": 3, + "active": 2, + "closed": 1, + "expired": 0, + "failed": 0, + "orphaned": 0 + } + }, + "capabilities": { + "worker_sessions": { + "feature": "worker_sessions", + "contract_version": "1.0", + "minimum_protocol_version": "1.8", + "command_field": "worker_session", + "activity_options_field": "worker_session", + "verbs": [ + "create", + "heartbeat", + "close" + ], + "lifecycle": { + "creation": "lazy_create_on_first_admitted_activity_or_explicit_worker_create", + "renewal": "activity_heartbeat_or_explicit_session_heartbeat", + "close": "explicit_holder_close", + "lease_expiry": "session_expires_when_lease_is_not_renewed", + "ttl_expiry": "absolute_session_ttl_is_terminal_for_that_session_id" + }, + "ownership": "single_worker_lease_owner", + "lease": { + "scope": "namespace_session_id", + "owner": "registered_worker_id", + "activity_attempt_leases_remain_independent": true + }, + "admission": { + "queue_routing_first": true, + "requires_registered_worker": true, + "requires_capabilities": true, + "create_if_missing_default": true, + "allow_reacquire_after_failure_default": true + }, + "rollout_safety": { + "minimum_protocol_version": "1.8", + "mixed_server_rollout_fenced_by_protocol_version": true, + "servers_below_minimum_must_reject_worker_session_commands": true, + "servers_below_minimum_must_not_claim_worker_session_activity_tasks": true + }, + "limits": { + "max_concurrent_worker_sessions": "worker_registration", + "max_concurrent_activities": "session" + }, + "default_max_concurrent_activities": 1, + "renewal": { + "activity_heartbeat_renews_session": true, + "explicit_session_heartbeat": true + }, + "failure_detection": [ + "lease_expiry", + "registered_worker_heartbeat_staleness" + ], + "holder_loss": { + "in_flight_activities_keep_at_least_once_attempt_semantics": true, + "replacement_worker_must_reacquire_session": true, + "process_local_state_must_be_rebuilt_after_reacquire": true + }, + "cancellation": { + "workflow_cancellation_observed_through_activity_heartbeat": true, + "session_lease_does_not_override_activity_cancel_requested": true, + "planned_shutdown_should_close_sessions": true + }, + "routing": [ + "queue", + "connection", + "requirements" + ], + "visibility": [ + "active", + "closed", + "expired", + "failed", + "orphaned" + ], + "statuses": [ + "active", + "closed", + "expired", + "failed", + "orphaned" + ], + "terminal_statuses": [ + "closed" + ], + "terminal_conditions": [ + "explicit_close", + "ttl_expired", + "allow_reacquire_after_failure_false" + ], + "authoring_guidance": [ + "use_for_process_local_state_gpu_memory_or_filesystem_affinity", + "prefer_ordinary_queued_activities_for_independent_steps", + "prefer_one_larger_activity_for_atomic_side_effects" + ], + "supported": true + }, + "worker_session_verbs": [ + "create", + "heartbeat", + "close" + ] + } + } + }, + "contract": "Session-bound activities as published Server 2.11.1 serves them: schedule_activity with worker_session is refused below protocol 1.8 (worker_sessions_unavailable) and below 1.18 (portable_worker_affinity_protocol_floor_unavailable), on other commands and without a session_id (422), and from a worker without the worker_sessions capability (409 worker_capability_not_supported); an accepted command creates no session until its first admitted claim. A claim skips workers below 1.8, on another queue or missing a requirement, and a session held by one worker; the task carries the compact session. The activity heartbeat and renew-lease renew the session by the command's lease and answer the full snapshot; status answers the compact form. After the holder closes the session, the heartbeat is refused 409 session_not_active with worker_session_renewal, status answers worker_session_unavailable, and completion still succeeds. A session's max_concurrent_activities bounds its running activities; a command queue other than the session queue is unclaimable. Operator metrics count the namespace's sessions, and the capability map serves worker_sessions as an object with its verbs. Session times are compared as spans or presence. Embedded applications have no worker protocol." +} diff --git a/tests/Fixtures/ServerParity/worker-sessions.json b/tests/Fixtures/ServerParity/worker-sessions.json new file mode 100644 index 000000000..f55d91c45 --- /dev/null +++ b/tests/Fixtures/ServerParity/worker-sessions.json @@ -0,0 +1,1735 @@ +{ + "schema": "durable-workflow.server-parity-fixture/v1", + "id": "worker-sessions", + "review_state": "reviewed", + "workflow_type": "parity.v1.echo", + "input": {"message": "worker sessions λ", "count": 9007199254740993, "optional": null}, + "typed_value": {"type": "map", "value": { + "message": {"type": "string", "value": "worker sessions λ"}, + "count": {"type": "int64", "value": "9007199254740993"}, + "optional": {"type": "null", "value": null} + }}, + "activity": false, + "expected_events": ["StartAccepted", "WorkflowStarted", "WorkflowCompleted"], + "worker_sessions": { + "workflow_type": "parity.v1.sessions", + "activity_type": "parity.v1.session_activity", + "empty_blob": "wwHioz3/VYAiNwwA", + "overall_budget_seconds": 120, + "published_defects": { + "reacquired_failure_reason": { + "paths": [ + "reacquired.response.session.failure_reason", + "orphan_reacquired.response.session.failure_reason", + "list_sessions[{id}-wss-held].failure_reason" + ], + "published": [ + "lease_expired", + "worker_heartbeat_stale", + "worker_heartbeat_stale" + ], + "reason": "WorkerSessionRegistry::renewHeldSession writes failure_reason => null and closed_at => null through array_filter(... !== null), which drops both; a reacquired session stays active with the old failure_reason until a heartbeat or close clears it. Native clears it, as the code intends." + } + }, + "expected": { + "applicable": true, + "namespace": "{id}-wss", + "workers": [ + "{id}-wss-w1", + "{id}-wss-w2", + "{id}-wss-w3" + ], + "queues": [ + "{id}-wss-q1", + "{id}-wss-q2" + ], + "gate_no_protocol": { + "status": 400, + "response": { + "error": "Missing worker protocol version header.", + "reason": "missing_protocol_version", + "supported_version": "1.20", + "requested_version": null, + "remediation": "Send the X-Durable-Workflow-Protocol-Version: 1.20 header on worker protocol requests.", + "protocol_version": "1.20" + } + }, + "gate_protocol_1_7": { + "status": 409, + "response": { + "error": "Worker sessions require worker protocol 1.8 or newer.", + "reason": "worker_sessions_unavailable", + "supported_version": "1.20", + "requested_version": "1.7", + "minimum_protocol_version": "1.8", + "remediation": "Send the X-Durable-Workflow-Protocol-Version header with worker protocol 1.8 or newer and use a worker SDK that implements worker-session semantics.", + "protocol_version": "1.20" + } + }, + "gate_protocol_1_21": { + "status": 400, + "response": { + "error": "Unsupported worker protocol version.", + "reason": "unsupported_protocol_version", + "supported_version": "1.20", + "requested_version": "1.21", + "remediation": "Worker requested protocol version 1.21; this server supports 1.20. Workers may target any 1.x version with x ≤ 20. Upgrade the worker to a release that targets a compatible version, or connect to a server that matches.", + "protocol_version": "1.20" + } + }, + "gate_protocol_1_8_empty": { + "status": 422, + "response": { + "message": "The worker id field is required. (and 1 more error)", + "reason": "validation_failed", + "errors": { + "worker_id": [ + "The worker id field is required." + ], + "session_id": [ + "The session id field is required." + ] + }, + "validation_errors": { + "worker_id": [ + "The worker id field is required." + ], + "session_id": [ + "The session id field is required." + ] + }, + "protocol_version": "1.20" + } + }, + "gate_unknown_namespace": { + "status": 404, + "response": { + "message": "Namespace 'no-such-namespace-wss' does not exist.", + "reason": "namespace_not_found", + "namespace": "no-such-namespace-wss", + "remediation": "Register the namespace via POST /api/namespaces, or send an X-Namespace header naming an existing namespace.", + "protocol_version": "1.20" + } + }, + "gate_list_without_control_plane": { + "status": 400, + "response": { + "message": "Missing control-plane version header.", + "reason": "missing_control_plane_version", + "supported_version": "2", + "requested_version": null, + "remediation": "Send the X-Durable-Workflow-Control-Plane-Version: 2 header on control-plane requests." + } + }, + "gate_list_unknown_namespace": { + "status": 404, + "response": { + "message": "Namespace 'no-such-namespace-wss' does not exist.", + "reason": "namespace_not_found", + "namespace": "no-such-namespace-wss", + "remediation": "Register the namespace via POST /api/namespaces, or send an X-Namespace header naming an existing namespace." + } + }, + "invalid_empty": { + "status": 422, + "response": { + "message": "The worker id field is required. (and 1 more error)", + "reason": "validation_failed", + "errors": { + "worker_id": [ + "The worker id field is required." + ], + "session_id": [ + "The session id field is required." + ] + }, + "validation_errors": { + "worker_id": [ + "The worker id field is required." + ], + "session_id": [ + "The session id field is required." + ] + }, + "protocol_version": "1.20" + } + }, + "invalid_types": { + "status": 422, + "response": { + "message": "The worker id field must be a string. (and 9 more errors)", + "reason": "validation_failed", + "errors": { + "worker_id": [ + "The worker id field must be a string." + ], + "session_id": [ + "The session id field is required." + ], + "connection": [ + "The connection field must be a string." + ], + "queue": [ + "The queue field must be a string." + ], + "requirements": [ + "The requirements field must be an array." + ], + "lease_seconds": [ + "The lease seconds field must be at least 1." + ], + "ttl_seconds": [ + "The ttl seconds field must be an integer." + ], + "max_concurrent_activities": [ + "The max concurrent activities field must be an integer." + ], + "create_if_missing": [ + "The create if missing field must be true or false." + ], + "allow_reacquire_after_failure": [ + "The allow reacquire after failure field must be true or false." + ] + }, + "validation_errors": { + "worker_id": [ + "The worker id field must be a string." + ], + "session_id": [ + "The session id field is required." + ], + "connection": [ + "The connection field must be a string." + ], + "queue": [ + "The queue field must be a string." + ], + "requirements": [ + "The requirements field must be an array." + ], + "lease_seconds": [ + "The lease seconds field must be at least 1." + ], + "ttl_seconds": [ + "The ttl seconds field must be an integer." + ], + "max_concurrent_activities": [ + "The max concurrent activities field must be an integer." + ], + "create_if_missing": [ + "The create if missing field must be true or false." + ], + "allow_reacquire_after_failure": [ + "The allow reacquire after failure field must be true or false." + ] + }, + "protocol_version": "1.20" + } + }, + "invalid_heartbeat": { + "status": 422, + "response": { + "message": "The worker id field is required. (and 1 more error)", + "reason": "validation_failed", + "errors": { + "worker_id": [ + "The worker id field is required." + ], + "lease_seconds": [ + "The lease seconds field must be at least 1." + ] + }, + "validation_errors": { + "worker_id": [ + "The worker id field is required." + ], + "lease_seconds": [ + "The lease seconds field must be at least 1." + ] + }, + "protocol_version": "1.20" + } + }, + "invalid_close": { + "status": 422, + "response": { + "message": "The worker id field is required.", + "reason": "validation_failed", + "errors": { + "worker_id": [ + "The worker id field is required." + ] + }, + "validation_errors": { + "worker_id": [ + "The worker id field is required." + ] + }, + "protocol_version": "1.20" + } + }, + "unregistered": { + "status": 412, + "response": { + "error": "Worker must be registered before creating a worker session.", + "reason": "worker_not_registered", + "worker_id": "{id}-wss-nobody", + "protocol_version": "1.20" + } + }, + "queue_mismatch": { + "status": 409, + "response": { + "error": "Worker [{id}-wss-w1] is registered for task queue [{id}-wss-q1], not session queue [{id}-wss-q2].", + "reason": "session_queue_mismatch", + "worker_id": "{id}-wss-w1", + "registered_task_queue": "{id}-wss-q1", + "requested_session_queue": "{id}-wss-q2", + "protocol_version": "1.20" + } + }, + "requirements_missing": { + "status": 409, + "response": { + "missing_requirements": [ + "tpu", + "fpga" + ], + "admitted": false, + "error": "Worker does not advertise every worker-session requirement.", + "reason": "session_requirements_not_met", + "status": 409, + "protocol_version": "1.20" + } + }, + "missing_without_create": { + "status": 409, + "response": { + "admitted": false, + "error": "Worker session does not exist and creation is disabled.", + "reason": "session_missing", + "status": 409, + "protocol_version": "1.20" + } + }, + "created": { + "status": 201, + "response": { + "admitted": true, + "outcome": "created", + "session": { + "session_id": "{id}-wss-s1", + "namespace": "{id}-wss", + "connection": "c1", + "queue": "{id}-wss-q1", + "requirements": [ + "gpu" + ], + "status": "active", + "lease_owner": "{id}-wss-w1", + "holder": { + "worker_id": "{id}-wss-w1", + "task_queue": "{id}-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 60, + "ttl_seconds": 600, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 60, + "ttl_span": 600, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "activity_task_id": null, + "status": 201, + "reason": null, + "protocol_version": "1.20" + } + }, + "float_lease": { + "status": 201, + "response": { + "admitted": true, + "outcome": "created", + "session": { + "session_id": "{id}-wss-s2", + "namespace": "{id}-wss", + "connection": null, + "queue": "{id}-wss-q1", + "requirements": [], + "status": "active", + "lease_owner": "{id}-wss-w1", + "holder": { + "worker_id": "{id}-wss-w1", + "task_queue": "{id}-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 120, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 120, + "ttl_span": 1800, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "activity_task_id": null, + "status": 201, + "reason": null, + "protocol_version": "1.20" + } + }, + "reused": { + "status": 200, + "response": { + "admitted": true, + "outcome": "reused", + "session": { + "session_id": "{id}-wss-s1", + "namespace": "{id}-wss", + "connection": "c1", + "queue": "{id}-wss-q1", + "requirements": [], + "status": "active", + "lease_owner": "{id}-wss-w1", + "holder": { + "worker_id": "{id}-wss-w1", + "task_queue": "{id}-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 30, + "ttl_seconds": 9999, + "max_concurrent_activities": 3, + "create_if_missing": false, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 30, + "ttl_span": 600, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "activity_task_id": null, + "status": 200, + "reason": null, + "protocol_version": "1.20" + } + }, + "foreign_create": { + "status": 409, + "response": { + "session": { + "session_id": "{id}-wss-s1", + "namespace": "{id}-wss", + "connection": "c1", + "queue": "{id}-wss-q1", + "requirements": [], + "status": "active", + "lease_owner": "{id}-wss-w1", + "holder": { + "worker_id": "{id}-wss-w1", + "task_queue": "{id}-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 30, + "ttl_seconds": 9999, + "max_concurrent_activities": 3, + "create_if_missing": false, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 30, + "ttl_span": 600, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "admitted": false, + "error": "Worker session is currently leased by another worker.", + "reason": "session_owned_by_another_worker", + "status": 409, + "protocol_version": "1.20" + } + }, + "foreign_heartbeat": { + "status": 409, + "response": { + "session": { + "session_id": "{id}-wss-s1", + "namespace": "{id}-wss", + "connection": "c1", + "queue": "{id}-wss-q1", + "requirements": [], + "status": "active", + "lease_owner": "{id}-wss-w1", + "holder": { + "worker_id": "{id}-wss-w1", + "task_queue": "{id}-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 30, + "ttl_seconds": 9999, + "max_concurrent_activities": 3, + "create_if_missing": false, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 30, + "ttl_span": 600, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "admitted": false, + "error": "Worker session lease is owned by another worker.", + "reason": "session_owner_mismatch", + "status": 409, + "protocol_version": "1.20" + } + }, + "foreign_close": { + "status": 409, + "response": { + "session": { + "session_id": "{id}-wss-s1", + "namespace": "{id}-wss", + "connection": "c1", + "queue": "{id}-wss-q1", + "requirements": [], + "status": "active", + "lease_owner": "{id}-wss-w1", + "holder": { + "worker_id": "{id}-wss-w1", + "task_queue": "{id}-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 30, + "ttl_seconds": 9999, + "max_concurrent_activities": 3, + "create_if_missing": false, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 30, + "ttl_span": 600, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "admitted": false, + "error": "Worker session lease is owned by another worker.", + "reason": "session_owner_mismatch", + "status": 409, + "protocol_version": "1.20" + } + }, + "heartbeat_missing": { + "status": 404, + "response": { + "admitted": false, + "error": "Worker session was not found.", + "reason": "session_not_found", + "status": 404, + "protocol_version": "1.20" + } + }, + "heartbeat": { + "status": 200, + "response": { + "admitted": true, + "outcome": "heartbeat_recorded", + "session": { + "session_id": "{id}-wss-s1", + "namespace": "{id}-wss", + "connection": "c1", + "queue": "{id}-wss-q1", + "requirements": [], + "status": "active", + "lease_owner": "{id}-wss-w1", + "holder": { + "worker_id": "{id}-wss-w1", + "task_queue": "{id}-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 30, + "ttl_seconds": 9999, + "max_concurrent_activities": 3, + "create_if_missing": false, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 5, + "ttl_span": 600, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "activity_task_id": null, + "status": 200, + "reason": null, + "protocol_version": "1.20" + } + }, + "limit": { + "status": 409, + "response": { + "max_concurrent_worker_sessions": 2, + "admitted": false, + "error": "Worker has reached max_concurrent_worker_sessions.", + "reason": "worker_session_limit_exceeded", + "status": 409, + "protocol_version": "1.20" + } + }, + "short_created": { + "status": 201, + "response": { + "admitted": true, + "outcome": "created", + "session": { + "session_id": "{id}-wss-short", + "namespace": "{id}-wss", + "connection": null, + "queue": "{id}-wss-q1", + "requirements": [], + "status": "active", + "lease_owner": "{id}-wss-w2", + "holder": { + "worker_id": "{id}-wss-w2", + "task_queue": "{id}-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 2, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 2, + "ttl_span": 1800, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "activity_task_id": null, + "status": 201, + "reason": null, + "protocol_version": "1.20" + } + }, + "short_expired": { + "status": 200, + "response": { + "namespace": "{id}-wss", + "session": { + "session_id": "{id}-wss-short", + "namespace": "{id}-wss", + "connection": null, + "queue": "{id}-wss-q1", + "requirements": [], + "status": "expired", + "lease_owner": "{id}-wss-w2", + "holder": { + "worker_id": "{id}-wss-w2", + "task_queue": "{id}-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": "lease_expired", + "lease_seconds": 2, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 2, + "ttl_span": 1800, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + } + } + }, + "reacquired": { + "status": 200, + "response": { + "admitted": true, + "outcome": "reacquired", + "session": { + "session_id": "{id}-wss-short", + "namespace": "{id}-wss", + "connection": null, + "queue": "{id}-wss-q2", + "requirements": [], + "status": "active", + "lease_owner": "{id}-wss-w3", + "holder": { + "worker_id": "{id}-wss-w3", + "task_queue": "{id}-wss-q2", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 120, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 120, + "ttl_span": 1800, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "activity_task_id": null, + "status": 200, + "reason": null, + "protocol_version": "1.20" + } + }, + "reacquired_heartbeat": { + "status": 200, + "response": { + "admitted": true, + "outcome": "heartbeat_recorded", + "session": { + "session_id": "{id}-wss-short", + "namespace": "{id}-wss", + "connection": null, + "queue": "{id}-wss-q2", + "requirements": [], + "status": "active", + "lease_owner": "{id}-wss-w3", + "holder": { + "worker_id": "{id}-wss-w3", + "task_queue": "{id}-wss-q2", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 120, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 120, + "ttl_span": 1800, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "activity_task_id": null, + "status": 200, + "reason": null, + "protocol_version": "1.20" + } + }, + "close_by_old_owner": { + "status": 409, + "response": { + "session": { + "session_id": "{id}-wss-short", + "namespace": "{id}-wss", + "connection": null, + "queue": "{id}-wss-q2", + "requirements": [], + "status": "active", + "lease_owner": "{id}-wss-w3", + "holder": { + "worker_id": "{id}-wss-w3", + "task_queue": "{id}-wss-q2", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 120, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 120, + "ttl_span": 1800, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "admitted": false, + "error": "Worker session lease is owned by another worker.", + "reason": "session_owner_mismatch", + "status": 409, + "protocol_version": "1.20" + } + }, + "closed": { + "status": 200, + "response": { + "admitted": true, + "outcome": "closed", + "session": { + "session_id": "{id}-wss-short", + "namespace": "{id}-wss", + "connection": null, + "queue": "{id}-wss-q2", + "requirements": [], + "status": "closed", + "lease_owner": "{id}-wss-w3", + "holder": { + "worker_id": "{id}-wss-w3", + "task_queue": "{id}-wss-q2", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 120, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 120, + "ttl_span": 1800, + "closed": true, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "present", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "activity_task_id": null, + "status": 200, + "reason": null, + "protocol_version": "1.20" + } + }, + "closed_again": { + "status": 200, + "response": { + "admitted": true, + "outcome": "already_closed", + "session": { + "session_id": "{id}-wss-short", + "namespace": "{id}-wss", + "connection": null, + "queue": "{id}-wss-q2", + "requirements": [], + "status": "closed", + "lease_owner": "{id}-wss-w3", + "holder": { + "worker_id": "{id}-wss-w3", + "task_queue": "{id}-wss-q2", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 120, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 120, + "ttl_span": 1800, + "closed": true, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "present", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "activity_task_id": null, + "status": 200, + "reason": null, + "protocol_version": "1.20" + } + }, + "closed_by_other": { + "status": 409, + "response": { + "session": { + "session_id": "{id}-wss-short", + "namespace": "{id}-wss", + "connection": null, + "queue": "{id}-wss-q2", + "requirements": [], + "status": "closed", + "lease_owner": "{id}-wss-w3", + "holder": { + "worker_id": "{id}-wss-w3", + "task_queue": "{id}-wss-q2", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 120, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 120, + "ttl_span": 1800, + "closed": true, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "present", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "admitted": false, + "error": "Worker session lease is owned by another worker.", + "reason": "session_owner_mismatch", + "status": 409, + "protocol_version": "1.20" + } + }, + "closed_heartbeat": { + "status": 409, + "response": { + "session": { + "session_id": "{id}-wss-short", + "namespace": "{id}-wss", + "connection": null, + "queue": "{id}-wss-q2", + "requirements": [], + "status": "closed", + "lease_owner": "{id}-wss-w3", + "holder": { + "worker_id": "{id}-wss-w3", + "task_queue": "{id}-wss-q2", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 120, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 120, + "ttl_span": 1800, + "closed": true, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "present", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "admitted": false, + "error": "Worker session is not active.", + "reason": "session_not_active", + "status": 409, + "protocol_version": "1.20" + } + }, + "closed_create": { + "status": 409, + "response": { + "session": { + "session_id": "{id}-wss-short", + "namespace": "{id}-wss", + "connection": null, + "queue": "{id}-wss-q2", + "requirements": [], + "status": "closed", + "lease_owner": "{id}-wss-w3", + "holder": { + "worker_id": "{id}-wss-w3", + "task_queue": "{id}-wss-q2", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 120, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 120, + "ttl_span": 1800, + "closed": true, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "present", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "admitted": false, + "error": "Worker session was closed and cannot be reacquired.", + "reason": "session_closed", + "status": 409, + "protocol_version": "1.20" + } + }, + "closed_create_unmet": { + "status": 409, + "response": { + "missing_requirements": [ + "gpu" + ], + "admitted": false, + "error": "Worker does not advertise every worker-session requirement.", + "reason": "session_requirements_not_met", + "status": 409, + "protocol_version": "1.20" + } + }, + "reacquire_terminal": { + "status": 409, + "response": { + "session": { + "session_id": "{id}-wss-terminal", + "namespace": "{id}-wss", + "connection": null, + "queue": "{id}-wss-q1", + "requirements": [], + "status": "expired", + "lease_owner": "{id}-wss-w2", + "holder": { + "worker_id": "{id}-wss-w2", + "task_queue": "{id}-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": "lease_expired", + "lease_seconds": 2, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": false, + "active_activity_count": 0, + "lease_span": 2, + "ttl_span": 1800, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "admitted": false, + "error": "Worker session is terminal and cannot be reacquired.", + "reason": "session_reacquire_disallowed", + "status": 409, + "protocol_version": "1.20" + } + }, + "reacquire_disallowed": { + "status": 409, + "response": { + "session": { + "session_id": "{id}-wss-optional", + "namespace": "{id}-wss", + "connection": null, + "queue": "{id}-wss-q1", + "requirements": [], + "status": "expired", + "lease_owner": "{id}-wss-w2", + "holder": { + "worker_id": "{id}-wss-w2", + "task_queue": "{id}-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": "lease_expired", + "lease_seconds": 2, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 2, + "ttl_span": 1800, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "admitted": false, + "error": "Worker session options disallow reacquisition after holder failure.", + "reason": "session_reacquire_disallowed", + "status": 409, + "protocol_version": "1.20" + } + }, + "ttl_heartbeat": { + "status": 409, + "response": { + "session": { + "session_id": "{id}-wss-ttl", + "namespace": "{id}-wss", + "connection": null, + "queue": "{id}-wss-q1", + "requirements": [], + "status": "expired", + "lease_owner": "{id}-wss-w2", + "holder": { + "worker_id": "{id}-wss-w2", + "task_queue": "{id}-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": "ttl_expired", + "lease_seconds": 60, + "ttl_seconds": 2, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 60, + "ttl_span": 2, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "admitted": false, + "error": "Worker session is not active.", + "reason": "session_not_active", + "status": 409, + "protocol_version": "1.20" + } + }, + "ttl_create": { + "status": 409, + "response": { + "session": { + "session_id": "{id}-wss-ttl", + "namespace": "{id}-wss", + "connection": null, + "queue": "{id}-wss-q1", + "requirements": [], + "status": "expired", + "lease_owner": "{id}-wss-w2", + "holder": { + "worker_id": "{id}-wss-w2", + "task_queue": "{id}-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": "ttl_expired", + "lease_seconds": 60, + "ttl_seconds": 2, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 60, + "ttl_span": 2, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "admitted": false, + "error": "Worker session is terminal and cannot be reacquired.", + "reason": "session_reacquire_disallowed", + "status": 409, + "protocol_version": "1.20" + } + }, + "ttl_close": { + "status": 200, + "response": { + "admitted": true, + "outcome": "closed", + "session": { + "session_id": "{id}-wss-ttl", + "namespace": "{id}-wss", + "connection": null, + "queue": "{id}-wss-q1", + "requirements": [], + "status": "closed", + "lease_owner": "{id}-wss-w2", + "holder": { + "worker_id": "{id}-wss-w2", + "task_queue": "{id}-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 60, + "ttl_seconds": 2, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 60, + "ttl_span": 2, + "closed": true, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "present", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "activity_task_id": null, + "status": 200, + "reason": null, + "protocol_version": "1.20" + } + }, + "held_created": { + "status": 201, + "response": { + "admitted": true, + "outcome": "created", + "session": { + "session_id": "{id}-wss-held", + "namespace": "{id}-wss", + "connection": null, + "queue": "{id}-wss-q2", + "requirements": [], + "status": "active", + "lease_owner": "{id}-wss-w3", + "holder": { + "worker_id": "{id}-wss-w3", + "task_queue": "{id}-wss-q2", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 120, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 120, + "ttl_span": 1800, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "activity_task_id": null, + "status": 201, + "reason": null, + "protocol_version": "1.20" + } + }, + "holder_deregistered": 200, + "held_orphaned": { + "status": 200, + "response": { + "namespace": "{id}-wss", + "session": { + "session_id": "{id}-wss-held", + "namespace": "{id}-wss", + "connection": null, + "queue": "{id}-wss-q2", + "requirements": [], + "status": "orphaned", + "lease_owner": "{id}-wss-w3", + "holder": { + "worker_id": "{id}-wss-w3", + "task_queue": null, + "status": null, + "last_heartbeat_at": null, + "is_stale": true + }, + "failure_reason": "worker_heartbeat_stale", + "lease_seconds": 120, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 120, + "ttl_span": 1800, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + } + } + }, + "deregistered_create": { + "status": 412, + "response": { + "error": "Worker must be registered before creating a worker session.", + "reason": "worker_not_registered", + "worker_id": "{id}-wss-w3", + "protocol_version": "1.20" + } + }, + "orphan_reacquired": { + "status": 200, + "response": { + "admitted": true, + "outcome": "reacquired", + "session": { + "session_id": "{id}-wss-held", + "namespace": "{id}-wss", + "connection": null, + "queue": "{id}-wss-q1", + "requirements": [], + "status": "active", + "lease_owner": "{id}-wss-w2", + "holder": { + "worker_id": "{id}-wss-w2", + "task_queue": "{id}-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 120, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 120, + "ttl_span": 1800, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "activity_task_id": null, + "status": 200, + "reason": null, + "protocol_version": "1.20" + } + }, + "orphan_queue_mismatch": { + "status": 409, + "response": { + "error": "Worker [{id}-wss-w2] is registered for task queue [{id}-wss-q1], not session queue [{id}-wss-q2].", + "reason": "session_queue_mismatch", + "worker_id": "{id}-wss-w2", + "registered_task_queue": "{id}-wss-q1", + "requested_session_queue": "{id}-wss-q2", + "protocol_version": "1.20" + } + }, + "show_missing": { + "status": 404, + "response": { + "message": "Worker session [{id}-wss-none] not found in namespace [{id}-wss].", + "reason": "worker_session_not_found" + } + }, + "list_metrics": { + "total": 7, + "active": 2, + "closed": 2, + "expired": 3, + "failed": 0, + "orphaned": 0 + }, + "list_namespace": "{id}-wss", + "list_sessions": [ + { + "session_id": "{id}-wss-held", + "namespace": "{id}-wss", + "connection": null, + "queue": "{id}-wss-q1", + "requirements": [], + "status": "active", + "lease_owner": "{id}-wss-w2", + "holder": { + "worker_id": "{id}-wss-w2", + "task_queue": "{id}-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 120, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 120, + "ttl_span": 1800, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + { + "session_id": "{id}-wss-optional", + "namespace": "{id}-wss", + "connection": null, + "queue": "{id}-wss-q1", + "requirements": [], + "status": "expired", + "lease_owner": "{id}-wss-w2", + "holder": { + "worker_id": "{id}-wss-w2", + "task_queue": "{id}-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": "lease_expired", + "lease_seconds": 2, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 2, + "ttl_span": 1800, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + { + "session_id": "{id}-wss-s1", + "namespace": "{id}-wss", + "connection": "c1", + "queue": "{id}-wss-q1", + "requirements": [], + "status": "expired", + "lease_owner": "{id}-wss-w1", + "holder": { + "worker_id": "{id}-wss-w1", + "task_queue": "{id}-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": "lease_expired", + "lease_seconds": 30, + "ttl_seconds": 9999, + "max_concurrent_activities": 3, + "create_if_missing": false, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 5, + "ttl_span": 600, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + { + "session_id": "{id}-wss-s2", + "namespace": "{id}-wss", + "connection": null, + "queue": "{id}-wss-q1", + "requirements": [], + "status": "active", + "lease_owner": "{id}-wss-w1", + "holder": { + "worker_id": "{id}-wss-w1", + "task_queue": "{id}-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 120, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 120, + "ttl_span": 1800, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + { + "session_id": "{id}-wss-short", + "namespace": "{id}-wss", + "connection": null, + "queue": "{id}-wss-q2", + "requirements": [], + "status": "closed", + "lease_owner": "{id}-wss-w3", + "holder": { + "worker_id": "{id}-wss-w3", + "task_queue": null, + "status": null, + "last_heartbeat_at": null, + "is_stale": true + }, + "failure_reason": null, + "lease_seconds": 120, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 120, + "ttl_span": 1800, + "closed": true, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "present", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + { + "session_id": "{id}-wss-terminal", + "namespace": "{id}-wss", + "connection": null, + "queue": "{id}-wss-q1", + "requirements": [], + "status": "expired", + "lease_owner": "{id}-wss-w2", + "holder": { + "worker_id": "{id}-wss-w2", + "task_queue": "{id}-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": "lease_expired", + "lease_seconds": 2, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": false, + "active_activity_count": 0, + "lease_span": 2, + "ttl_span": 1800, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + { + "session_id": "{id}-wss-ttl", + "namespace": "{id}-wss", + "connection": null, + "queue": "{id}-wss-q1", + "requirements": [], + "status": "closed", + "lease_owner": "{id}-wss-w2", + "holder": { + "worker_id": "{id}-wss-w2", + "task_queue": "{id}-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 60, + "ttl_seconds": 2, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 60, + "ttl_span": 2, + "closed": true, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "present", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + } + ] + } + }, + "contract": "Worker sessions over the worker protocol and their visibility on the control plane, as published Server 2.11.1 serves them: the protocol gates (no header 400, below 1.8 409 worker_sessions_unavailable, above 1.20 400), validation, registration (412) and queue (409) refusals, requirements checked before the session lookup, create (201), reuse and reacquire (200, options the request omits reset to defaults, the TTL deadline never extended, the queue defaulting to the worker's own), heartbeat (lease renewed from now, stored lease unchanged), the per-worker session limit, lease and TTL expiry applied when the session is read, close and already_closed, the reacquire refusals, a deregistered holder orphaned on read, and the namespace list and metrics. Session times are whole seconds, so times are compared as spans. One published defect is bounded: a reacquired session keeps its old failure_reason (renewHeldSession filters out its own null clears); the published value or the intended null is accepted. Embedded applications have no worker protocol." +} diff --git a/tests/Fixtures/ServerParityExpected/worker-session-activities-observation.json b/tests/Fixtures/ServerParityExpected/worker-session-activities-observation.json new file mode 100644 index 000000000..7cfdb5c46 --- /dev/null +++ b/tests/Fixtures/ServerParityExpected/worker-session-activities-observation.json @@ -0,0 +1,351 @@ +{ + "applicable": true, + "namespace": "pr1-worker-session-activities-wsa", + "complete_protocol_1_7": { + "status": 409, + "reason": "worker_sessions_unavailable", + "message": "Worker-session activity commands require worker protocol 1.8 or newer." + }, + "complete_protocol_1_17": { + "status": 409, + "reason": "portable_worker_affinity_protocol_floor_unavailable", + "message": null + }, + "complete_on_timer": { + "status": 422, + "reason": "validation_failed", + "message": "worker_session is only supported for schedule_activity commands." + }, + "complete_without_session_id": { + "status": 422, + "reason": "validation_failed", + "message": "Worker session commands require a non-empty session_id." + }, + "complete_accepted": 200, + "complete_incapable": { + "status": 409, + "reason": "worker_capability_not_supported", + "message": null + }, + "show_before_claim": { + "status": 404, + "reason": "worker_session_not_found", + "message": "Worker session [pr1-worker-session-activities-wsa-lazy] not found in namespace [pr1-worker-session-activities-wsa]." + }, + "poll_without_requirement": { + "status": 200, + "poll_status": "empty", + "worker_session": null, + "has_session_key": false + }, + "poll_protocol_1_7": { + "status": 200, + "poll_status": "empty", + "worker_session": null, + "has_session_key": false + }, + "poll_capable": { + "status": 200, + "poll_status": "leased", + "worker_session": { + "session_id": "pr1-worker-session-activities-wsa-lazy", + "queue": "pr1-worker-session-activities-wsa-queue", + "requirements": [ + "gpu" + ], + "lease_seconds": 90, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "status": "active", + "lease_owner": "pr1-worker-session-activities-wsa-gpu", + "lease_expires_at": "present", + "ttl_expires_at": "present" + }, + "has_session_key": true + }, + "show_after_claim": { + "status": 200, + "session": { + "session_id": "pr1-worker-session-activities-wsa-lazy", + "namespace": "pr1-worker-session-activities-wsa", + "connection": null, + "queue": "pr1-worker-session-activities-wsa-queue", + "requirements": [ + "gpu" + ], + "status": "active", + "lease_owner": "pr1-worker-session-activities-wsa-gpu", + "holder": { + "worker_id": "pr1-worker-session-activities-wsa-gpu", + "task_queue": "pr1-worker-session-activities-wsa-queue", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 90, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 1, + "lease_span": 90, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + } + }, + "poll_other_capable": { + "status": 200, + "poll_status": "empty", + "worker_session": null, + "has_session_key": false + }, + "activity_heartbeat": { + "status": 200, + "heartbeat_recorded": true, + "worker_session": { + "session_id": "pr1-worker-session-activities-wsa-lazy", + "namespace": "pr1-worker-session-activities-wsa", + "connection": null, + "queue": "pr1-worker-session-activities-wsa-queue", + "requirements": [ + "gpu" + ], + "status": "active", + "lease_owner": "pr1-worker-session-activities-wsa-gpu", + "holder": { + "worker_id": "pr1-worker-session-activities-wsa-gpu", + "task_queue": "pr1-worker-session-activities-wsa-queue", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 90, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 1, + "lease_span": 90, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + } + }, + "activity_status": { + "status": 200, + "can_continue": true, + "reason": null, + "worker_session": { + "session_id": "pr1-worker-session-activities-wsa-lazy", + "queue": "pr1-worker-session-activities-wsa-queue", + "requirements": [ + "gpu" + ], + "lease_seconds": 90, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "status": "active", + "lease_owner": "pr1-worker-session-activities-wsa-gpu", + "lease_expires_at": "present", + "ttl_expires_at": "present" + } + }, + "activity_renew": { + "status": 200, + "lease_renewed": true, + "worker_session": { + "session_id": "pr1-worker-session-activities-wsa-lazy", + "namespace": "pr1-worker-session-activities-wsa", + "connection": null, + "queue": "pr1-worker-session-activities-wsa-queue", + "requirements": [ + "gpu" + ], + "status": "active", + "lease_owner": "pr1-worker-session-activities-wsa-gpu", + "holder": { + "worker_id": "pr1-worker-session-activities-wsa-gpu", + "task_queue": "pr1-worker-session-activities-wsa-queue", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 90, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 1, + "lease_span": 90, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + } + }, + "session_closed": 200, + "heartbeat_after_close": { + "status": 409, + "reason": "session_not_active", + "can_continue": false, + "heartbeat_recorded": false, + "worker_session_renewal": { + "admitted": false, + "reason": "session_not_active", + "error": "Worker session is not active." + }, + "worker_session_status": "closed" + }, + "status_after_close": { + "status": 200, + "can_continue": false, + "reason": "worker_session_unavailable" + }, + "complete_after_close": 200, + "limit_one": [ + "leased", + "empty" + ], + "limit_two": [ + "leased", + "leased" + ], + "queue_mismatch_polls": [ + "empty", + "empty" + ], + "operator_metrics": { + "status": 200, + "worker_sessions": { + "total": 3, + "active": 2, + "closed": 1, + "expired": 0, + "failed": 0, + "orphaned": 0 + } + }, + "capabilities": { + "worker_sessions": { + "feature": "worker_sessions", + "contract_version": "1.0", + "minimum_protocol_version": "1.8", + "command_field": "worker_session", + "activity_options_field": "worker_session", + "verbs": [ + "create", + "heartbeat", + "close" + ], + "lifecycle": { + "creation": "lazy_create_on_first_admitted_activity_or_explicit_worker_create", + "renewal": "activity_heartbeat_or_explicit_session_heartbeat", + "close": "explicit_holder_close", + "lease_expiry": "session_expires_when_lease_is_not_renewed", + "ttl_expiry": "absolute_session_ttl_is_terminal_for_that_session_id" + }, + "ownership": "single_worker_lease_owner", + "lease": { + "scope": "namespace_session_id", + "owner": "registered_worker_id", + "activity_attempt_leases_remain_independent": true + }, + "admission": { + "queue_routing_first": true, + "requires_registered_worker": true, + "requires_capabilities": true, + "create_if_missing_default": true, + "allow_reacquire_after_failure_default": true + }, + "rollout_safety": { + "minimum_protocol_version": "1.8", + "mixed_server_rollout_fenced_by_protocol_version": true, + "servers_below_minimum_must_reject_worker_session_commands": true, + "servers_below_minimum_must_not_claim_worker_session_activity_tasks": true + }, + "limits": { + "max_concurrent_worker_sessions": "worker_registration", + "max_concurrent_activities": "session" + }, + "default_max_concurrent_activities": 1, + "renewal": { + "activity_heartbeat_renews_session": true, + "explicit_session_heartbeat": true + }, + "failure_detection": [ + "lease_expiry", + "registered_worker_heartbeat_staleness" + ], + "holder_loss": { + "in_flight_activities_keep_at_least_once_attempt_semantics": true, + "replacement_worker_must_reacquire_session": true, + "process_local_state_must_be_rebuilt_after_reacquire": true + }, + "cancellation": { + "workflow_cancellation_observed_through_activity_heartbeat": true, + "session_lease_does_not_override_activity_cancel_requested": true, + "planned_shutdown_should_close_sessions": true + }, + "routing": [ + "queue", + "connection", + "requirements" + ], + "visibility": [ + "active", + "closed", + "expired", + "failed", + "orphaned" + ], + "statuses": [ + "active", + "closed", + "expired", + "failed", + "orphaned" + ], + "terminal_statuses": [ + "closed" + ], + "terminal_conditions": [ + "explicit_close", + "ttl_expired", + "allow_reacquire_after_failure_false" + ], + "authoring_guidance": [ + "use_for_process_local_state_gpu_memory_or_filesystem_affinity", + "prefer_ordinary_queued_activities_for_independent_steps", + "prefer_one_larger_activity_for_atomic_side_effects" + ], + "supported": true + }, + "worker_session_verbs": [ + "create", + "heartbeat", + "close" + ] + } +} diff --git a/tests/Fixtures/ServerParityExpected/worker-sessions-observation.json b/tests/Fixtures/ServerParityExpected/worker-sessions-observation.json new file mode 100644 index 000000000..e7b44937c --- /dev/null +++ b/tests/Fixtures/ServerParityExpected/worker-sessions-observation.json @@ -0,0 +1,1699 @@ +{ + "applicable": true, + "namespace": "pr1-worker-sessions-wss", + "workers": [ + "pr1-worker-sessions-wss-w1", + "pr1-worker-sessions-wss-w2", + "pr1-worker-sessions-wss-w3" + ], + "queues": [ + "pr1-worker-sessions-wss-q1", + "pr1-worker-sessions-wss-q2" + ], + "gate_no_protocol": { + "status": 400, + "response": { + "error": "Missing worker protocol version header.", + "reason": "missing_protocol_version", + "supported_version": "1.20", + "requested_version": null, + "remediation": "Send the X-Durable-Workflow-Protocol-Version: 1.20 header on worker protocol requests.", + "protocol_version": "1.20" + } + }, + "gate_protocol_1_7": { + "status": 409, + "response": { + "error": "Worker sessions require worker protocol 1.8 or newer.", + "reason": "worker_sessions_unavailable", + "supported_version": "1.20", + "requested_version": "1.7", + "minimum_protocol_version": "1.8", + "remediation": "Send the X-Durable-Workflow-Protocol-Version header with worker protocol 1.8 or newer and use a worker SDK that implements worker-session semantics.", + "protocol_version": "1.20" + } + }, + "gate_protocol_1_21": { + "status": 400, + "response": { + "error": "Unsupported worker protocol version.", + "reason": "unsupported_protocol_version", + "supported_version": "1.20", + "requested_version": "1.21", + "remediation": "Worker requested protocol version 1.21; this server supports 1.20. Workers may target any 1.x version with x ≤ 20. Upgrade the worker to a release that targets a compatible version, or connect to a server that matches.", + "protocol_version": "1.20" + } + }, + "gate_protocol_1_8_empty": { + "status": 422, + "response": { + "message": "The worker id field is required. (and 1 more error)", + "reason": "validation_failed", + "errors": { + "worker_id": [ + "The worker id field is required." + ], + "session_id": [ + "The session id field is required." + ] + }, + "validation_errors": { + "worker_id": [ + "The worker id field is required." + ], + "session_id": [ + "The session id field is required." + ] + }, + "protocol_version": "1.20" + } + }, + "gate_unknown_namespace": { + "status": 404, + "response": { + "message": "Namespace 'no-such-namespace-wss' does not exist.", + "reason": "namespace_not_found", + "namespace": "no-such-namespace-wss", + "remediation": "Register the namespace via POST /api/namespaces, or send an X-Namespace header naming an existing namespace.", + "protocol_version": "1.20" + } + }, + "gate_list_without_control_plane": { + "status": 400, + "response": { + "message": "Missing control-plane version header.", + "reason": "missing_control_plane_version", + "supported_version": "2", + "requested_version": null, + "remediation": "Send the X-Durable-Workflow-Control-Plane-Version: 2 header on control-plane requests." + } + }, + "gate_list_unknown_namespace": { + "status": 404, + "response": { + "message": "Namespace 'no-such-namespace-wss' does not exist.", + "reason": "namespace_not_found", + "namespace": "no-such-namespace-wss", + "remediation": "Register the namespace via POST /api/namespaces, or send an X-Namespace header naming an existing namespace." + } + }, + "invalid_empty": { + "status": 422, + "response": { + "message": "The worker id field is required. (and 1 more error)", + "reason": "validation_failed", + "errors": { + "worker_id": [ + "The worker id field is required." + ], + "session_id": [ + "The session id field is required." + ] + }, + "validation_errors": { + "worker_id": [ + "The worker id field is required." + ], + "session_id": [ + "The session id field is required." + ] + }, + "protocol_version": "1.20" + } + }, + "invalid_types": { + "status": 422, + "response": { + "message": "The worker id field must be a string. (and 9 more errors)", + "reason": "validation_failed", + "errors": { + "worker_id": [ + "The worker id field must be a string." + ], + "session_id": [ + "The session id field is required." + ], + "connection": [ + "The connection field must be a string." + ], + "queue": [ + "The queue field must be a string." + ], + "requirements": [ + "The requirements field must be an array." + ], + "lease_seconds": [ + "The lease seconds field must be at least 1." + ], + "ttl_seconds": [ + "The ttl seconds field must be an integer." + ], + "max_concurrent_activities": [ + "The max concurrent activities field must be an integer." + ], + "create_if_missing": [ + "The create if missing field must be true or false." + ], + "allow_reacquire_after_failure": [ + "The allow reacquire after failure field must be true or false." + ] + }, + "validation_errors": { + "worker_id": [ + "The worker id field must be a string." + ], + "session_id": [ + "The session id field is required." + ], + "connection": [ + "The connection field must be a string." + ], + "queue": [ + "The queue field must be a string." + ], + "requirements": [ + "The requirements field must be an array." + ], + "lease_seconds": [ + "The lease seconds field must be at least 1." + ], + "ttl_seconds": [ + "The ttl seconds field must be an integer." + ], + "max_concurrent_activities": [ + "The max concurrent activities field must be an integer." + ], + "create_if_missing": [ + "The create if missing field must be true or false." + ], + "allow_reacquire_after_failure": [ + "The allow reacquire after failure field must be true or false." + ] + }, + "protocol_version": "1.20" + } + }, + "invalid_heartbeat": { + "status": 422, + "response": { + "message": "The worker id field is required. (and 1 more error)", + "reason": "validation_failed", + "errors": { + "worker_id": [ + "The worker id field is required." + ], + "lease_seconds": [ + "The lease seconds field must be at least 1." + ] + }, + "validation_errors": { + "worker_id": [ + "The worker id field is required." + ], + "lease_seconds": [ + "The lease seconds field must be at least 1." + ] + }, + "protocol_version": "1.20" + } + }, + "invalid_close": { + "status": 422, + "response": { + "message": "The worker id field is required.", + "reason": "validation_failed", + "errors": { + "worker_id": [ + "The worker id field is required." + ] + }, + "validation_errors": { + "worker_id": [ + "The worker id field is required." + ] + }, + "protocol_version": "1.20" + } + }, + "unregistered": { + "status": 412, + "response": { + "error": "Worker must be registered before creating a worker session.", + "reason": "worker_not_registered", + "worker_id": "pr1-worker-sessions-wss-nobody", + "protocol_version": "1.20" + } + }, + "queue_mismatch": { + "status": 409, + "response": { + "error": "Worker [pr1-worker-sessions-wss-w1] is registered for task queue [pr1-worker-sessions-wss-q1], not session queue [pr1-worker-sessions-wss-q2].", + "reason": "session_queue_mismatch", + "worker_id": "pr1-worker-sessions-wss-w1", + "registered_task_queue": "pr1-worker-sessions-wss-q1", + "requested_session_queue": "pr1-worker-sessions-wss-q2", + "protocol_version": "1.20" + } + }, + "requirements_missing": { + "status": 409, + "response": { + "missing_requirements": [ + "tpu", + "fpga" + ], + "admitted": false, + "error": "Worker does not advertise every worker-session requirement.", + "reason": "session_requirements_not_met", + "status": 409, + "protocol_version": "1.20" + } + }, + "missing_without_create": { + "status": 409, + "response": { + "admitted": false, + "error": "Worker session does not exist and creation is disabled.", + "reason": "session_missing", + "status": 409, + "protocol_version": "1.20" + } + }, + "created": { + "status": 201, + "response": { + "admitted": true, + "outcome": "created", + "session": { + "session_id": "pr1-worker-sessions-wss-s1", + "namespace": "pr1-worker-sessions-wss", + "connection": "c1", + "queue": "pr1-worker-sessions-wss-q1", + "requirements": [ + "gpu" + ], + "status": "active", + "lease_owner": "pr1-worker-sessions-wss-w1", + "holder": { + "worker_id": "pr1-worker-sessions-wss-w1", + "task_queue": "pr1-worker-sessions-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 60, + "ttl_seconds": 600, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 60, + "ttl_span": 600, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "activity_task_id": null, + "status": 201, + "reason": null, + "protocol_version": "1.20" + } + }, + "float_lease": { + "status": 201, + "response": { + "admitted": true, + "outcome": "created", + "session": { + "session_id": "pr1-worker-sessions-wss-s2", + "namespace": "pr1-worker-sessions-wss", + "connection": null, + "queue": "pr1-worker-sessions-wss-q1", + "requirements": [], + "status": "active", + "lease_owner": "pr1-worker-sessions-wss-w1", + "holder": { + "worker_id": "pr1-worker-sessions-wss-w1", + "task_queue": "pr1-worker-sessions-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 120, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 120, + "ttl_span": 1800, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "activity_task_id": null, + "status": 201, + "reason": null, + "protocol_version": "1.20" + } + }, + "reused": { + "status": 200, + "response": { + "admitted": true, + "outcome": "reused", + "session": { + "session_id": "pr1-worker-sessions-wss-s1", + "namespace": "pr1-worker-sessions-wss", + "connection": "c1", + "queue": "pr1-worker-sessions-wss-q1", + "requirements": [], + "status": "active", + "lease_owner": "pr1-worker-sessions-wss-w1", + "holder": { + "worker_id": "pr1-worker-sessions-wss-w1", + "task_queue": "pr1-worker-sessions-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 30, + "ttl_seconds": 9999, + "max_concurrent_activities": 3, + "create_if_missing": false, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 30, + "ttl_span": 600, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "activity_task_id": null, + "status": 200, + "reason": null, + "protocol_version": "1.20" + } + }, + "foreign_create": { + "status": 409, + "response": { + "session": { + "session_id": "pr1-worker-sessions-wss-s1", + "namespace": "pr1-worker-sessions-wss", + "connection": "c1", + "queue": "pr1-worker-sessions-wss-q1", + "requirements": [], + "status": "active", + "lease_owner": "pr1-worker-sessions-wss-w1", + "holder": { + "worker_id": "pr1-worker-sessions-wss-w1", + "task_queue": "pr1-worker-sessions-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 30, + "ttl_seconds": 9999, + "max_concurrent_activities": 3, + "create_if_missing": false, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 30, + "ttl_span": 600, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "admitted": false, + "error": "Worker session is currently leased by another worker.", + "reason": "session_owned_by_another_worker", + "status": 409, + "protocol_version": "1.20" + } + }, + "foreign_heartbeat": { + "status": 409, + "response": { + "session": { + "session_id": "pr1-worker-sessions-wss-s1", + "namespace": "pr1-worker-sessions-wss", + "connection": "c1", + "queue": "pr1-worker-sessions-wss-q1", + "requirements": [], + "status": "active", + "lease_owner": "pr1-worker-sessions-wss-w1", + "holder": { + "worker_id": "pr1-worker-sessions-wss-w1", + "task_queue": "pr1-worker-sessions-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 30, + "ttl_seconds": 9999, + "max_concurrent_activities": 3, + "create_if_missing": false, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 30, + "ttl_span": 600, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "admitted": false, + "error": "Worker session lease is owned by another worker.", + "reason": "session_owner_mismatch", + "status": 409, + "protocol_version": "1.20" + } + }, + "foreign_close": { + "status": 409, + "response": { + "session": { + "session_id": "pr1-worker-sessions-wss-s1", + "namespace": "pr1-worker-sessions-wss", + "connection": "c1", + "queue": "pr1-worker-sessions-wss-q1", + "requirements": [], + "status": "active", + "lease_owner": "pr1-worker-sessions-wss-w1", + "holder": { + "worker_id": "pr1-worker-sessions-wss-w1", + "task_queue": "pr1-worker-sessions-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 30, + "ttl_seconds": 9999, + "max_concurrent_activities": 3, + "create_if_missing": false, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 30, + "ttl_span": 600, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "admitted": false, + "error": "Worker session lease is owned by another worker.", + "reason": "session_owner_mismatch", + "status": 409, + "protocol_version": "1.20" + } + }, + "heartbeat_missing": { + "status": 404, + "response": { + "admitted": false, + "error": "Worker session was not found.", + "reason": "session_not_found", + "status": 404, + "protocol_version": "1.20" + } + }, + "heartbeat": { + "status": 200, + "response": { + "admitted": true, + "outcome": "heartbeat_recorded", + "session": { + "session_id": "pr1-worker-sessions-wss-s1", + "namespace": "pr1-worker-sessions-wss", + "connection": "c1", + "queue": "pr1-worker-sessions-wss-q1", + "requirements": [], + "status": "active", + "lease_owner": "pr1-worker-sessions-wss-w1", + "holder": { + "worker_id": "pr1-worker-sessions-wss-w1", + "task_queue": "pr1-worker-sessions-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 30, + "ttl_seconds": 9999, + "max_concurrent_activities": 3, + "create_if_missing": false, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 5, + "ttl_span": 600, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "activity_task_id": null, + "status": 200, + "reason": null, + "protocol_version": "1.20" + } + }, + "limit": { + "status": 409, + "response": { + "max_concurrent_worker_sessions": 2, + "admitted": false, + "error": "Worker has reached max_concurrent_worker_sessions.", + "reason": "worker_session_limit_exceeded", + "status": 409, + "protocol_version": "1.20" + } + }, + "short_created": { + "status": 201, + "response": { + "admitted": true, + "outcome": "created", + "session": { + "session_id": "pr1-worker-sessions-wss-short", + "namespace": "pr1-worker-sessions-wss", + "connection": null, + "queue": "pr1-worker-sessions-wss-q1", + "requirements": [], + "status": "active", + "lease_owner": "pr1-worker-sessions-wss-w2", + "holder": { + "worker_id": "pr1-worker-sessions-wss-w2", + "task_queue": "pr1-worker-sessions-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 2, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 2, + "ttl_span": 1800, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "activity_task_id": null, + "status": 201, + "reason": null, + "protocol_version": "1.20" + } + }, + "short_expired": { + "status": 200, + "response": { + "namespace": "pr1-worker-sessions-wss", + "session": { + "session_id": "pr1-worker-sessions-wss-short", + "namespace": "pr1-worker-sessions-wss", + "connection": null, + "queue": "pr1-worker-sessions-wss-q1", + "requirements": [], + "status": "expired", + "lease_owner": "pr1-worker-sessions-wss-w2", + "holder": { + "worker_id": "pr1-worker-sessions-wss-w2", + "task_queue": "pr1-worker-sessions-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": "lease_expired", + "lease_seconds": 2, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 2, + "ttl_span": 1800, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + } + } + }, + "reacquired": { + "status": 200, + "response": { + "admitted": true, + "outcome": "reacquired", + "session": { + "session_id": "pr1-worker-sessions-wss-short", + "namespace": "pr1-worker-sessions-wss", + "connection": null, + "queue": "pr1-worker-sessions-wss-q2", + "requirements": [], + "status": "active", + "lease_owner": "pr1-worker-sessions-wss-w3", + "holder": { + "worker_id": "pr1-worker-sessions-wss-w3", + "task_queue": "pr1-worker-sessions-wss-q2", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": "lease_expired", + "lease_seconds": 120, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 120, + "ttl_span": 1800, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "activity_task_id": null, + "status": 200, + "reason": null, + "protocol_version": "1.20" + } + }, + "reacquired_heartbeat": { + "status": 200, + "response": { + "admitted": true, + "outcome": "heartbeat_recorded", + "session": { + "session_id": "pr1-worker-sessions-wss-short", + "namespace": "pr1-worker-sessions-wss", + "connection": null, + "queue": "pr1-worker-sessions-wss-q2", + "requirements": [], + "status": "active", + "lease_owner": "pr1-worker-sessions-wss-w3", + "holder": { + "worker_id": "pr1-worker-sessions-wss-w3", + "task_queue": "pr1-worker-sessions-wss-q2", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 120, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 120, + "ttl_span": 1800, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "activity_task_id": null, + "status": 200, + "reason": null, + "protocol_version": "1.20" + } + }, + "close_by_old_owner": { + "status": 409, + "response": { + "session": { + "session_id": "pr1-worker-sessions-wss-short", + "namespace": "pr1-worker-sessions-wss", + "connection": null, + "queue": "pr1-worker-sessions-wss-q2", + "requirements": [], + "status": "active", + "lease_owner": "pr1-worker-sessions-wss-w3", + "holder": { + "worker_id": "pr1-worker-sessions-wss-w3", + "task_queue": "pr1-worker-sessions-wss-q2", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 120, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 120, + "ttl_span": 1800, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "admitted": false, + "error": "Worker session lease is owned by another worker.", + "reason": "session_owner_mismatch", + "status": 409, + "protocol_version": "1.20" + } + }, + "closed": { + "status": 200, + "response": { + "admitted": true, + "outcome": "closed", + "session": { + "session_id": "pr1-worker-sessions-wss-short", + "namespace": "pr1-worker-sessions-wss", + "connection": null, + "queue": "pr1-worker-sessions-wss-q2", + "requirements": [], + "status": "closed", + "lease_owner": "pr1-worker-sessions-wss-w3", + "holder": { + "worker_id": "pr1-worker-sessions-wss-w3", + "task_queue": "pr1-worker-sessions-wss-q2", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 120, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 120, + "ttl_span": 1800, + "closed": true, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "present", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "activity_task_id": null, + "status": 200, + "reason": null, + "protocol_version": "1.20" + } + }, + "closed_again": { + "status": 200, + "response": { + "admitted": true, + "outcome": "already_closed", + "session": { + "session_id": "pr1-worker-sessions-wss-short", + "namespace": "pr1-worker-sessions-wss", + "connection": null, + "queue": "pr1-worker-sessions-wss-q2", + "requirements": [], + "status": "closed", + "lease_owner": "pr1-worker-sessions-wss-w3", + "holder": { + "worker_id": "pr1-worker-sessions-wss-w3", + "task_queue": "pr1-worker-sessions-wss-q2", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 120, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 120, + "ttl_span": 1800, + "closed": true, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "present", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "activity_task_id": null, + "status": 200, + "reason": null, + "protocol_version": "1.20" + } + }, + "closed_by_other": { + "status": 409, + "response": { + "session": { + "session_id": "pr1-worker-sessions-wss-short", + "namespace": "pr1-worker-sessions-wss", + "connection": null, + "queue": "pr1-worker-sessions-wss-q2", + "requirements": [], + "status": "closed", + "lease_owner": "pr1-worker-sessions-wss-w3", + "holder": { + "worker_id": "pr1-worker-sessions-wss-w3", + "task_queue": "pr1-worker-sessions-wss-q2", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 120, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 120, + "ttl_span": 1800, + "closed": true, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "present", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "admitted": false, + "error": "Worker session lease is owned by another worker.", + "reason": "session_owner_mismatch", + "status": 409, + "protocol_version": "1.20" + } + }, + "closed_heartbeat": { + "status": 409, + "response": { + "session": { + "session_id": "pr1-worker-sessions-wss-short", + "namespace": "pr1-worker-sessions-wss", + "connection": null, + "queue": "pr1-worker-sessions-wss-q2", + "requirements": [], + "status": "closed", + "lease_owner": "pr1-worker-sessions-wss-w3", + "holder": { + "worker_id": "pr1-worker-sessions-wss-w3", + "task_queue": "pr1-worker-sessions-wss-q2", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 120, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 120, + "ttl_span": 1800, + "closed": true, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "present", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "admitted": false, + "error": "Worker session is not active.", + "reason": "session_not_active", + "status": 409, + "protocol_version": "1.20" + } + }, + "closed_create": { + "status": 409, + "response": { + "session": { + "session_id": "pr1-worker-sessions-wss-short", + "namespace": "pr1-worker-sessions-wss", + "connection": null, + "queue": "pr1-worker-sessions-wss-q2", + "requirements": [], + "status": "closed", + "lease_owner": "pr1-worker-sessions-wss-w3", + "holder": { + "worker_id": "pr1-worker-sessions-wss-w3", + "task_queue": "pr1-worker-sessions-wss-q2", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 120, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 120, + "ttl_span": 1800, + "closed": true, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "present", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "admitted": false, + "error": "Worker session was closed and cannot be reacquired.", + "reason": "session_closed", + "status": 409, + "protocol_version": "1.20" + } + }, + "closed_create_unmet": { + "status": 409, + "response": { + "missing_requirements": [ + "gpu" + ], + "admitted": false, + "error": "Worker does not advertise every worker-session requirement.", + "reason": "session_requirements_not_met", + "status": 409, + "protocol_version": "1.20" + } + }, + "reacquire_terminal": { + "status": 409, + "response": { + "session": { + "session_id": "pr1-worker-sessions-wss-terminal", + "namespace": "pr1-worker-sessions-wss", + "connection": null, + "queue": "pr1-worker-sessions-wss-q1", + "requirements": [], + "status": "expired", + "lease_owner": "pr1-worker-sessions-wss-w2", + "holder": { + "worker_id": "pr1-worker-sessions-wss-w2", + "task_queue": "pr1-worker-sessions-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": "lease_expired", + "lease_seconds": 2, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": false, + "active_activity_count": 0, + "lease_span": 2, + "ttl_span": 1800, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "admitted": false, + "error": "Worker session is terminal and cannot be reacquired.", + "reason": "session_reacquire_disallowed", + "status": 409, + "protocol_version": "1.20" + } + }, + "reacquire_disallowed": { + "status": 409, + "response": { + "session": { + "session_id": "pr1-worker-sessions-wss-optional", + "namespace": "pr1-worker-sessions-wss", + "connection": null, + "queue": "pr1-worker-sessions-wss-q1", + "requirements": [], + "status": "expired", + "lease_owner": "pr1-worker-sessions-wss-w2", + "holder": { + "worker_id": "pr1-worker-sessions-wss-w2", + "task_queue": "pr1-worker-sessions-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": "lease_expired", + "lease_seconds": 2, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 2, + "ttl_span": 1800, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "admitted": false, + "error": "Worker session options disallow reacquisition after holder failure.", + "reason": "session_reacquire_disallowed", + "status": 409, + "protocol_version": "1.20" + } + }, + "ttl_heartbeat": { + "status": 409, + "response": { + "session": { + "session_id": "pr1-worker-sessions-wss-ttl", + "namespace": "pr1-worker-sessions-wss", + "connection": null, + "queue": "pr1-worker-sessions-wss-q1", + "requirements": [], + "status": "expired", + "lease_owner": "pr1-worker-sessions-wss-w2", + "holder": { + "worker_id": "pr1-worker-sessions-wss-w2", + "task_queue": "pr1-worker-sessions-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": "ttl_expired", + "lease_seconds": 60, + "ttl_seconds": 2, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 60, + "ttl_span": 2, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "admitted": false, + "error": "Worker session is not active.", + "reason": "session_not_active", + "status": 409, + "protocol_version": "1.20" + } + }, + "ttl_create": { + "status": 409, + "response": { + "session": { + "session_id": "pr1-worker-sessions-wss-ttl", + "namespace": "pr1-worker-sessions-wss", + "connection": null, + "queue": "pr1-worker-sessions-wss-q1", + "requirements": [], + "status": "expired", + "lease_owner": "pr1-worker-sessions-wss-w2", + "holder": { + "worker_id": "pr1-worker-sessions-wss-w2", + "task_queue": "pr1-worker-sessions-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": "ttl_expired", + "lease_seconds": 60, + "ttl_seconds": 2, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 60, + "ttl_span": 2, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "admitted": false, + "error": "Worker session is terminal and cannot be reacquired.", + "reason": "session_reacquire_disallowed", + "status": 409, + "protocol_version": "1.20" + } + }, + "ttl_close": { + "status": 200, + "response": { + "admitted": true, + "outcome": "closed", + "session": { + "session_id": "pr1-worker-sessions-wss-ttl", + "namespace": "pr1-worker-sessions-wss", + "connection": null, + "queue": "pr1-worker-sessions-wss-q1", + "requirements": [], + "status": "closed", + "lease_owner": "pr1-worker-sessions-wss-w2", + "holder": { + "worker_id": "pr1-worker-sessions-wss-w2", + "task_queue": "pr1-worker-sessions-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 60, + "ttl_seconds": 2, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 60, + "ttl_span": 2, + "closed": true, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "present", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "activity_task_id": null, + "status": 200, + "reason": null, + "protocol_version": "1.20" + } + }, + "held_created": { + "status": 201, + "response": { + "admitted": true, + "outcome": "created", + "session": { + "session_id": "pr1-worker-sessions-wss-held", + "namespace": "pr1-worker-sessions-wss", + "connection": null, + "queue": "pr1-worker-sessions-wss-q2", + "requirements": [], + "status": "active", + "lease_owner": "pr1-worker-sessions-wss-w3", + "holder": { + "worker_id": "pr1-worker-sessions-wss-w3", + "task_queue": "pr1-worker-sessions-wss-q2", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 120, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 120, + "ttl_span": 1800, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "activity_task_id": null, + "status": 201, + "reason": null, + "protocol_version": "1.20" + } + }, + "holder_deregistered": 200, + "held_orphaned": { + "status": 200, + "response": { + "namespace": "pr1-worker-sessions-wss", + "session": { + "session_id": "pr1-worker-sessions-wss-held", + "namespace": "pr1-worker-sessions-wss", + "connection": null, + "queue": "pr1-worker-sessions-wss-q2", + "requirements": [], + "status": "orphaned", + "lease_owner": "pr1-worker-sessions-wss-w3", + "holder": { + "worker_id": "pr1-worker-sessions-wss-w3", + "task_queue": null, + "status": null, + "last_heartbeat_at": null, + "is_stale": true + }, + "failure_reason": "worker_heartbeat_stale", + "lease_seconds": 120, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 120, + "ttl_span": 1800, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + } + } + }, + "deregistered_create": { + "status": 412, + "response": { + "error": "Worker must be registered before creating a worker session.", + "reason": "worker_not_registered", + "worker_id": "pr1-worker-sessions-wss-w3", + "protocol_version": "1.20" + } + }, + "orphan_reacquired": { + "status": 200, + "response": { + "admitted": true, + "outcome": "reacquired", + "session": { + "session_id": "pr1-worker-sessions-wss-held", + "namespace": "pr1-worker-sessions-wss", + "connection": null, + "queue": "pr1-worker-sessions-wss-q1", + "requirements": [], + "status": "active", + "lease_owner": "pr1-worker-sessions-wss-w2", + "holder": { + "worker_id": "pr1-worker-sessions-wss-w2", + "task_queue": "pr1-worker-sessions-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": "worker_heartbeat_stale", + "lease_seconds": 120, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 120, + "ttl_span": 1800, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + "activity_task_id": null, + "status": 200, + "reason": null, + "protocol_version": "1.20" + } + }, + "orphan_queue_mismatch": { + "status": 409, + "response": { + "error": "Worker [pr1-worker-sessions-wss-w2] is registered for task queue [pr1-worker-sessions-wss-q1], not session queue [pr1-worker-sessions-wss-q2].", + "reason": "session_queue_mismatch", + "worker_id": "pr1-worker-sessions-wss-w2", + "registered_task_queue": "pr1-worker-sessions-wss-q1", + "requested_session_queue": "pr1-worker-sessions-wss-q2", + "protocol_version": "1.20" + } + }, + "show_missing": { + "status": 404, + "response": { + "message": "Worker session [pr1-worker-sessions-wss-none] not found in namespace [pr1-worker-sessions-wss].", + "reason": "worker_session_not_found" + } + }, + "list_metrics": { + "total": 7, + "active": 2, + "closed": 2, + "expired": 3, + "failed": 0, + "orphaned": 0 + }, + "list_namespace": "pr1-worker-sessions-wss", + "list_sessions": [ + { + "session_id": "pr1-worker-sessions-wss-held", + "namespace": "pr1-worker-sessions-wss", + "connection": null, + "queue": "pr1-worker-sessions-wss-q1", + "requirements": [], + "status": "active", + "lease_owner": "pr1-worker-sessions-wss-w2", + "holder": { + "worker_id": "pr1-worker-sessions-wss-w2", + "task_queue": "pr1-worker-sessions-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": "worker_heartbeat_stale", + "lease_seconds": 120, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 120, + "ttl_span": 1800, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + { + "session_id": "pr1-worker-sessions-wss-optional", + "namespace": "pr1-worker-sessions-wss", + "connection": null, + "queue": "pr1-worker-sessions-wss-q1", + "requirements": [], + "status": "expired", + "lease_owner": "pr1-worker-sessions-wss-w2", + "holder": { + "worker_id": "pr1-worker-sessions-wss-w2", + "task_queue": "pr1-worker-sessions-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": "lease_expired", + "lease_seconds": 2, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 2, + "ttl_span": 1800, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + { + "session_id": "pr1-worker-sessions-wss-s1", + "namespace": "pr1-worker-sessions-wss", + "connection": "c1", + "queue": "pr1-worker-sessions-wss-q1", + "requirements": [], + "status": "expired", + "lease_owner": "pr1-worker-sessions-wss-w1", + "holder": { + "worker_id": "pr1-worker-sessions-wss-w1", + "task_queue": "pr1-worker-sessions-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": "lease_expired", + "lease_seconds": 30, + "ttl_seconds": 9999, + "max_concurrent_activities": 3, + "create_if_missing": false, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 5, + "ttl_span": 600, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + { + "session_id": "pr1-worker-sessions-wss-s2", + "namespace": "pr1-worker-sessions-wss", + "connection": null, + "queue": "pr1-worker-sessions-wss-q1", + "requirements": [], + "status": "active", + "lease_owner": "pr1-worker-sessions-wss-w1", + "holder": { + "worker_id": "pr1-worker-sessions-wss-w1", + "task_queue": "pr1-worker-sessions-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 120, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 120, + "ttl_span": 1800, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + { + "session_id": "pr1-worker-sessions-wss-short", + "namespace": "pr1-worker-sessions-wss", + "connection": null, + "queue": "pr1-worker-sessions-wss-q2", + "requirements": [], + "status": "closed", + "lease_owner": "pr1-worker-sessions-wss-w3", + "holder": { + "worker_id": "pr1-worker-sessions-wss-w3", + "task_queue": null, + "status": null, + "last_heartbeat_at": null, + "is_stale": true + }, + "failure_reason": null, + "lease_seconds": 120, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 120, + "ttl_span": 1800, + "closed": true, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "present", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + { + "session_id": "pr1-worker-sessions-wss-terminal", + "namespace": "pr1-worker-sessions-wss", + "connection": null, + "queue": "pr1-worker-sessions-wss-q1", + "requirements": [], + "status": "expired", + "lease_owner": "pr1-worker-sessions-wss-w2", + "holder": { + "worker_id": "pr1-worker-sessions-wss-w2", + "task_queue": "pr1-worker-sessions-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": "lease_expired", + "lease_seconds": 2, + "ttl_seconds": 1800, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": false, + "active_activity_count": 0, + "lease_span": 2, + "ttl_span": 1800, + "closed": false, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "null", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + }, + { + "session_id": "pr1-worker-sessions-wss-ttl", + "namespace": "pr1-worker-sessions-wss", + "connection": null, + "queue": "pr1-worker-sessions-wss-q1", + "requirements": [], + "status": "closed", + "lease_owner": "pr1-worker-sessions-wss-w2", + "holder": { + "worker_id": "pr1-worker-sessions-wss-w2", + "task_queue": "pr1-worker-sessions-wss-q1", + "status": "active", + "last_heartbeat_at": "present", + "is_stale": false + }, + "failure_reason": null, + "lease_seconds": 60, + "ttl_seconds": 2, + "max_concurrent_activities": 1, + "create_if_missing": true, + "allow_reacquire_after_failure": true, + "active_activity_count": 0, + "lease_span": 60, + "ttl_span": 2, + "closed": true, + "times": { + "lease_expires_at": "present", + "ttl_expires_at": "present", + "closed_at": "present", + "last_heartbeat_at": "present", + "created_at": "present", + "updated_at": "present" + } + } + ] +} diff --git a/tests/Unit/ServerParityWorkerSessionActivitiesContractTest.mjs b/tests/Unit/ServerParityWorkerSessionActivitiesContractTest.mjs new file mode 100644 index 000000000..049a2ebcd --- /dev/null +++ b/tests/Unit/ServerParityWorkerSessionActivitiesContractTest.mjs @@ -0,0 +1,61 @@ +import assert from 'node:assert/strict'; +import {readFileSync} from 'node:fs'; +import test from 'node:test'; +import {checkWorkerSessionActivities} from '../../scripts/conformance/server-parity/worker-session-activities-contract.mjs'; + +const fixture = JSON.parse(readFileSync(new URL('../Fixtures/ServerParity/worker-session-activities.json', import.meta.url))); +// The observation published Server 2.11.1 returned for this fixture. +const recorded = JSON.parse(readFileSync( + new URL('../Fixtures/ServerParityExpected/worker-session-activities-observation.json', import.meta.url))); +const id = 'pr1-worker-session-activities'; +const observation = (change = () => {}) => { + const state = structuredClone(recorded); + change(state); + return {mode: 'http', worker_session_activities: state}; +}; +const check = state => checkWorkerSessionActivities(fixture, state, id); + +test('the published 2.11.1 recording satisfies the contract', () => { + assert.deepStrictEqual(check(observation()), {scope: 'worker_session_activities'}); +}); + +test('embedded applications are explicitly inapplicable', () => { + assert.deepStrictEqual(check({mode: 'embedded', worker_session_activities: + {applicable: false, reason: 'embedded_has_no_worker_protocol'}}), {scope: 'worker_session_activities'}); +}); + +for (const [label, change, message] of [ + // Native before this fixture: worker_session refused as an unknown field, + // sessions advertised as a bare false, and metrics fixed at zero. + ['worker_session refused as an unknown command field', state => { + state.complete_accepted = 422; + }, /worker session activities complete_accepted/], + ['a claim that ignores the session requirements', state => { + state.poll_without_requirement.poll_status = 'leased'; + }, /worker session activities poll_without_requirement/], + ['a task without its session', state => { + state.poll_capable.worker_session = null; + state.poll_capable.has_session_key = false; + }, /worker session activities poll_capable/], + ['a heartbeat that does not renew the session', state => { + state.activity_heartbeat.worker_session = null; + }, /worker session activities activity_heartbeat/], + ['a heartbeat recorded after the session closed', state => { + state.heartbeat_after_close.status = 200; + state.heartbeat_after_close.heartbeat_recorded = true; + }, /worker session activities heartbeat_after_close/], + ['a session limit that is not enforced', state => { + state.limit_one = ['leased', 'leased']; + }, /worker session activities limit_one/], + ['sessions advertised as a bare flag', state => { + state.capabilities.worker_sessions = false; + state.capabilities.worker_session_verbs = null; + }, /worker session activities capabilities/], + ['metrics fixed at zero', state => { + state.operator_metrics.worker_sessions = {total: 0, active: 0, closed: 0, expired: 0, failed: 0, orphaned: 0}; + }, /worker session activities operator_metrics/], +]) { + test(`the contract refuses ${label}`, () => { + assert.throws(() => check(observation(change)), message); + }); +} diff --git a/tests/Unit/ServerParityWorkerSessionsContractTest.mjs b/tests/Unit/ServerParityWorkerSessionsContractTest.mjs new file mode 100644 index 000000000..615b7f7b8 --- /dev/null +++ b/tests/Unit/ServerParityWorkerSessionsContractTest.mjs @@ -0,0 +1,72 @@ +import assert from 'node:assert/strict'; +import {readFileSync} from 'node:fs'; +import test from 'node:test'; +import {checkWorkerSessions} from '../../scripts/conformance/server-parity/worker-sessions-contract.mjs'; + +const fixture = JSON.parse(readFileSync(new URL('../Fixtures/ServerParity/worker-sessions.json', import.meta.url))); +// The observation published Server 2.11.1 returned for this fixture. +const recorded = JSON.parse(readFileSync( + new URL('../Fixtures/ServerParityExpected/worker-sessions-observation.json', import.meta.url))); +const id = 'pr1-worker-sessions'; +const observation = (change = () => {}) => { + const state = structuredClone(recorded); + change(state); + return {mode: 'http', worker_sessions: state}; +}; +const check = state => checkWorkerSessions(fixture, state, id); + +test('the published 2.11.1 recording satisfies the contract', () => { + assert.deepStrictEqual(check(observation()), {scope: 'worker_sessions'}); +}); + +test('the intended clearing of failure_reason on reacquire is accepted too', () => { + assert.deepStrictEqual(check(observation(state => { + state.reacquired.response.session.failure_reason = null; + state.orphan_reacquired.response.session.failure_reason = null; + state.list_sessions.find(session => session.session_id.endsWith('-wss-held')).failure_reason = null; + })), {scope: 'worker_sessions'}); +}); + +test('embedded applications are explicitly inapplicable', () => { + assert.deepStrictEqual(check({mode: 'embedded', worker_sessions: + {applicable: false, reason: 'embedded_has_no_worker_protocol'}}), {scope: 'worker_sessions'}); +}); + +for (const [label, change, message] of [ + // Native answered 501 for every session route before this fixture. + ['an unimplemented create', state => { + state.created = {status: 501, response: {reason: 'rust_capability_not_implemented'}}; + }, /worker sessions created/], + ['a protocol gate that admits 1.7', state => { + state.gate_protocol_1_7 = structuredClone(state.gate_protocol_1_8_empty); + }, /worker sessions gate_protocol_1_7/], + ['requirements checked after the lookup', state => { + state.closed_create_unmet = structuredClone(state.closed_create); + }, /worker sessions closed_create_unmet/], + ['a heartbeat that rewrites the stored lease', state => { + state.heartbeat.response.session.lease_seconds = 5; + }, /worker sessions heartbeat/], + ['a reuse that extends the TTL deadline', state => { + state.reused.response.session.ttl_span = 9999; + }, /worker sessions reused/], + ['lease expiry not applied on read', state => { + state.short_expired.response.session.status = 'active'; + state.short_expired.response.session.failure_reason = null; + }, /worker sessions short_expired/], + ['an orphan not detected on read', state => { + state.held_orphaned.response.session.status = 'active'; + }, /worker sessions held_orphaned/], + ['a closed session that can be reacquired', state => { + state.closed_create = {status: 200, response: {outcome: 'reacquired'}}; + }, /worker sessions closed_create/], + ['a different failure_reason after reacquire', state => { + state.reacquired.response.session.failure_reason = 'ttl_expired'; + }, /worker sessions reacquired/], + ['a missing step', state => { + delete state.ttl_close; + }, /worker sessions ttl_close/], +]) { + test(`the contract refuses ${label}`, () => { + assert.throws(() => check(observation(change)), message); + }); +}