From d93bd666a4f769c8e6463fcf52f9450451939d7a Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Wed, 21 Oct 2026 01:05:11 +0000 Subject: [PATCH 01/22] Reproduce missing runtime-only activity lease transport --- tests/Feature/ActivityLeaseRenewalTest.php | 148 +++++++++++++++++++++ 1 file changed, 148 insertions(+) create mode 100644 tests/Feature/ActivityLeaseRenewalTest.php diff --git a/tests/Feature/ActivityLeaseRenewalTest.php b/tests/Feature/ActivityLeaseRenewalTest.php new file mode 100644 index 000000000..664dff6ef --- /dev/null +++ b/tests/Feature/ActivityLeaseRenewalTest.php @@ -0,0 +1,148 @@ +createNamespace('default'); + $this->createNamespace('other'); + config(['workflows.v2.activity_task_lease_seconds' => 5]); + $this->travelTo(Carbon::parse('2026-10-21T01:00:00.123456Z')); + } + + public function test_runtime_renewal_keeps_one_attempt_without_application_progress_or_history(): void + { + $task = $this->claimActivity(); + $execution = ActivityExecution::query()->findOrFail($task['activity_execution_id']); + $before = $execution->getAttributes(); + $history = WorkflowHistoryEvent::query()->where('workflow_run_id', $task['run_id'])->get()->toArray(); + $this->assertSame(now()->toJSON(), $task['server_time']); + + for ($renewal = 0; $renewal < 3; ++$renewal) { + $this->travel(4)->seconds(); + $this->postJson($this->renewalPath($task), $this->ownership($task), $this->workerHeaders()) + ->assertOk() + ->assertJsonPath('lease_renewed', true) + ->assertJsonPath('can_continue', true) + ->assertJsonPath('heartbeat_recorded', false) + ->assertJsonPath('workflow_task_id', $task['task_id']) + ->assertJsonPath('activity_attempt_id', $task['activity_attempt_id']) + ->assertJsonPath('attempt_number', 1) + ->assertJsonPath('lease_owner', $task['lease_owner']) + ->assertJsonPath('server_time', now()->toJSON()) + ->assertJsonPath('lease_expires_at', now()->addSeconds(5)->toJSON()) + ->assertJsonPath('server_capabilities.activity_lease_renewal', true); + } + + $this->assertSame($before, $execution->fresh()->getAttributes()); + $this->assertSame($history, WorkflowHistoryEvent::query()->where('workflow_run_id', $task['run_id'])->get()->toArray()); + $this->assertSame(1, ActivityAttempt::query()->where('activity_execution_id', $execution->id)->count()); + } + + public function test_renewal_refuses_wrong_owner_and_namespace_without_extending_the_lease(): void + { + $task = $this->claimActivity(); + $before = ActivityAttempt::query()->findOrFail($task['activity_attempt_id'])->getAttributes(); + $this->postJson($this->renewalPath($task), [ + ...$this->ownership($task), 'lease_owner' => 'another-worker', + ], $this->workerHeaders())->assertConflict()->assertJsonPath('reason', 'lease_owner_mismatch'); + $this->postJson($this->renewalPath($task), $this->ownership($task), $this->workerHeaders('other')) + ->assertNotFound()->assertJsonPath('reason', 'task_not_found'); + $this->assertSame($before, ActivityAttempt::query()->findOrFail($task['activity_attempt_id'])->getAttributes()); + } + + public function test_an_expired_lease_cannot_be_revived(): void + { + $task = $this->claimActivity(); + $this->travel(5)->seconds(); + $this->postJson($this->renewalPath($task), $this->ownership($task), $this->workerHeaders()) + ->assertConflict() + ->assertJsonPath('lease_renewed', false) + ->assertJsonPath('reason', 'lease_expired'); + } + + public static function businessTimeouts(): array + { + return [ + 'heartbeat' => ['heartbeat_timeout', 'heartbeat_timeout'], + 'start to close' => ['start_to_close_timeout', 'start_to_close_timeout'], + 'schedule to close' => ['schedule_to_close_timeout', 'schedule_to_close_timeout'], + ]; + } + + #[DataProvider('businessTimeouts')] + public function test_runtime_renewal_never_extends_business_timeout(string $field, string $reason): void + { + $task = $this->claimActivity([$field => 3]); + $execution = ActivityExecution::query()->findOrFail($task['activity_execution_id']); + $before = $execution->getAttributes(); + $this->travel(3)->seconds(); + $this->postJson($this->renewalPath($task), $this->ownership($task), $this->workerHeaders()) + ->assertConflict()->assertJsonPath('lease_renewed', false)->assertJsonPath('reason', $reason); + $this->assertSame($before, $execution->fresh()->getAttributes()); + } + + public function test_custom_bridge_without_renewal_is_not_advertised_or_called(): void + { + $this->app->instance(ActivityTaskBridge::class, \Mockery::mock(ActivityTaskBridge::class)); + $this->postJson('/api/worker/activity-tasks/unknown/renew-lease', [ + 'activity_attempt_id' => 'unknown', 'lease_owner' => 'worker', + ], $this->workerHeaders()) + ->assertUnprocessable() + ->assertJsonPath('lease_renewed', false) + ->assertJsonPath('reason', 'activity_lease_renewal_unavailable') + ->assertJsonPath('server_capabilities.activity_lease_renewal', false); + } + + private function claimActivity(array $policy = []): array + { + $workflow = WorkflowStub::make(ExternalGreetingWorkflow::class, 'renewal-workflow'); + $start = $workflow->start('Ada'); + NamespaceWorkflowScope::bind('default', $workflow->id(), ExternalGreetingWorkflow::class); + $this->runReadyWorkflowTask($start->runId()); + $execution = ActivityExecution::query()->where('workflow_run_id', $start->runId())->firstOrFail(); + $execution->forceFill([ + 'retry_policy' => [...$execution->retry_policy, ...$policy], + 'schedule_to_close_deadline_at' => isset($policy['schedule_to_close_timeout']) + ? now()->addSeconds($policy['schedule_to_close_timeout']) : null, + ])->save(); + $this->registerWorker('renewal-worker', 'external-activities'); + + return $this->postJson('/api/worker/activity-tasks/poll', [ + 'worker_id' => 'renewal-worker', 'task_queue' => 'external-activities', + ], $this->workerHeaders())->assertOk()->json('task'); + } + + private function ownership(array $task): array + { + return ['activity_attempt_id' => $task['activity_attempt_id'], 'lease_owner' => $task['lease_owner']]; + } + + private function renewalPath(array $task): string + { + return '/api/worker/activity-tasks/'.$task['task_id'].'/renew-lease'; + } +} From db1601943b5ce836bc260857e36f754db7055d1c Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Wed, 21 Oct 2026 01:15:20 +0000 Subject: [PATCH 02/22] Describe the existing leased completion upload environment setting --- config/dw-contract.php | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/config/dw-contract.php b/config/dw-contract.php index 2e8abf2f3..5a075c3f5 100644 --- a/config/dw-contract.php +++ b/config/dw-contract.php @@ -598,6 +598,12 @@ 'since' => '2.0.0', 'legacy' => 'WORKFLOW_SERVER_EXTERNAL_PAYLOAD_MAX_BYTES', ], + 'DW_EXTERNAL_PAYLOAD_COMPLETION_MAX_BYTES' => [ + 'description' => 'Optional byte allowance for leased completion uploads; unset uses DW_EXTERNAL_PAYLOAD_MAX_BYTES.', + 'default' => null, + 'since' => '2.3.5', + 'legacy' => 'WORKFLOW_SERVER_EXTERNAL_PAYLOAD_COMPLETION_MAX_BYTES', + ], 'DW_EXTERNAL_PAYLOAD_REQUEST_TIMEOUT' => [ 'description' => 'Client-facing request-timeout budget advertised for external-payload upload and fetch.', 'default' => '30', From cf465b0a6b9926d41555a8d212869d87c43fe53f Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Wed, 21 Oct 2026 01:28:37 +0000 Subject: [PATCH 03/22] Expose fenced runtime-only activity lease renewal --- .../Api/ActivityTaskController.php | 77 +++++++++++++++++++ app/Providers/AppServiceProvider.php | 2 + .../ActivityTaskLeaseConfiguration.php | 31 ++++++++ app/Support/WorkerProtocol.php | 3 + config/server.php | 2 +- routes/api.php | 1 + tests/Feature/ActivityLeaseRenewalTest.php | 46 +++++++++-- .../ActivityTaskLeaseConfigurationTest.php | 36 +++++++++ 8 files changed, 192 insertions(+), 6 deletions(-) create mode 100644 app/Support/ActivityTaskLeaseConfiguration.php create mode 100644 tests/Unit/ActivityTaskLeaseConfigurationTest.php diff --git a/app/Http/Controllers/Api/ActivityTaskController.php b/app/Http/Controllers/Api/ActivityTaskController.php index c1a9a89da..226f44eb2 100644 --- a/app/Http/Controllers/Api/ActivityTaskController.php +++ b/app/Http/Controllers/Api/ActivityTaskController.php @@ -31,6 +31,7 @@ use Illuminate\Validation\Rule; use Illuminate\Validation\ValidationException; use InvalidArgumentException; +use Workflow\V2\Contracts\ActivityLeaseRenewal; use Workflow\V2\Contracts\ActivityTaskBridge as ActivityTaskBridgeContract; use Workflow\V2\Enums\HistoryEventType; use Workflow\V2\Exceptions\ExternalPayloadIntegrityException; @@ -217,6 +218,7 @@ public function poll(Request $request): JsonResponse ), 'external_executor' => $externalExecutor, ], static fn (mixed $v): bool => $v !== null), + 'server_time' => now()->toJSON(), 'poll_status' => is_string($poll['poll_status'] ?? null) ? $poll['poll_status'] : ($claim === null ? 'empty' : 'leased'), @@ -692,6 +694,81 @@ private function activityCancellationReceipt( ]; } + public function renewLease(Request $request, string $taskId): JsonResponse + { + if ($response = WorkerProtocol::rejectUnsupported($request)) { + return $response; + } + + $bridge = app(ActivityTaskBridgeContract::class); + if (! $bridge instanceof ActivityLeaseRenewal) { + return WorkerProtocol::json([ + 'lease_renewed' => false, + 'can_continue' => false, + 'reason' => 'activity_lease_renewal_unavailable', + 'error' => 'The configured activity bridge does not support runtime lease renewal.', + ], 422); + } + + $validated = $request->validate([ + 'activity_attempt_id' => ['required', 'string'], + 'lease_owner' => ['required', 'string'], + ]); + $namespace = $request->attributes->get('namespace'); + $observed = null; + try { + if ($response = $this->guardAttemptOwnership( + $namespace, + $taskId, + $validated['activity_attempt_id'], + $validated['lease_owner'], + $observed, + )) { + return $response; + } + + if (($observed['can_continue'] ?? false) !== true) { + return WorkerProtocol::json([ + ...$observed, 'lease_renewed' => false, + ], $this->outcomeStatus($observed['reason'] ?? null)); + } + + $workerSession = $this->storageMutations->run( + fn (): ?array => $this->workerSessions->heartbeatForAttempt( + $namespace, + $validated['activity_attempt_id'], + $validated['lease_owner'], + ), + ); + if ($workerSession !== null && ($workerSession['admitted'] ?? false) !== true) { + return WorkerProtocol::json([ + ...$observed, + 'lease_renewed' => false, + 'can_continue' => false, + 'reason' => $workerSession['reason'] ?? 'worker_session_renewal_failed', + 'worker_session' => $this->workerSessionSnapshotFromRenewal($workerSession), + 'worker_session_renewal' => $this->workerSessionRenewalStatusPayload($workerSession), + ], $this->workerSessionRenewalHttpStatus($workerSession)); + } + + $status = $this->storageMutations->run( + static fn (): array => $bridge->renewLease($validated['activity_attempt_id'], $validated['lease_owner']), + ); + } catch (\Throwable $exception) { + if (! BackendLockPressure::is($exception)) { + throw $exception; + } + + return BackendLockPressure::workerOperationResponse($request, false); + } + + return WorkerProtocol::json([ + ...$status, + 'task_id' => $taskId, + 'worker_session' => $workerSession['session'] ?? null, + ], $this->outcomeStatus($status['reason'] ?? null)); + } + /** * Heartbeat an in-progress activity task. * diff --git a/app/Providers/AppServiceProvider.php b/app/Providers/AppServiceProvider.php index 7b628941f..22a6c1aec 100644 --- a/app/Providers/AppServiceProvider.php +++ b/app/Providers/AppServiceProvider.php @@ -19,6 +19,7 @@ use App\Observers\WorkflowHistoryEventObserver; use App\Observers\WorkflowTaskObserver; use App\Observers\WorkflowUpdateValidationObserver; +use App\Support\ActivityTaskLeaseConfiguration; use App\Support\ExternalWorkflowUpdateAdmission; use App\Support\NamespaceExternalPayloadStorage; use App\Support\RemoteScheduleStarter; @@ -200,6 +201,7 @@ public function boot(): void WorkflowPackageApiFloor::assert(); WorkflowTaskLeaseConfiguration::apply(); + ActivityTaskLeaseConfiguration::apply(); WorkflowMemoRollingCompatibility::register(); config([ diff --git a/app/Support/ActivityTaskLeaseConfiguration.php b/app/Support/ActivityTaskLeaseConfiguration.php new file mode 100644 index 000000000..d721ce76e --- /dev/null +++ b/app/Support/ActivityTaskLeaseConfiguration.php @@ -0,0 +1,31 @@ + ['min_range' => 1, 'max_range' => 86_400], + ]) : false; + if ($seconds === false) { + throw new RuntimeException('DW_ACTIVITY_TASK_TIMEOUT must be an integer between 1 and 86400 seconds.'); + } + + config(['server.lease.activity_task_timeout' => $seconds]); + if (defined(ActivityLease::class.'::CONFIG_KEY') && method_exists(ActivityLease::class, 'seconds')) { + config([ActivityLease::CONFIG_KEY => $seconds]); + if (ActivityLease::seconds() !== $seconds) { + throw new RuntimeException('The configured activity-task lease does not match the Workflow runtime.'); + } + } + + return $seconds; + } +} diff --git a/app/Support/WorkerProtocol.php b/app/Support/WorkerProtocol.php index 871ef0fcf..eda8ded41 100644 --- a/app/Support/WorkerProtocol.php +++ b/app/Support/WorkerProtocol.php @@ -6,6 +6,8 @@ use App\Models\WorkerRegistration; use Illuminate\Http\JsonResponse; use Illuminate\Http\Request; +use Workflow\V2\Contracts\ActivityLeaseRenewal; +use Workflow\V2\Contracts\ActivityTaskBridge; use Workflow\V2\Support\WorkerHistoryPayloadContract; use Workflow\V2\Support\WorkerProtocolVersion; @@ -554,6 +556,7 @@ public static function serverCapabilities(): array ], 'activity_retry_policy' => true, 'activity_timeouts' => true, + 'activity_lease_renewal' => app(ActivityTaskBridge::class) instanceof ActivityLeaseRenewal, 'cooperative_cancellation' => CooperativeCancellationPolicy::serverSupported(), 'nexus_cooperative_cancellation' => NexusCooperativeCancellationPolicy::serverSupported(), 'nexus_cooperative_cancellation_binding_kinds' => NexusCooperativeCancellationPolicy::serverSupported() ? ['workflow_run'] : [], diff --git a/config/server.php b/config/server.php index 8b6be75e9..b3cd22ae5 100644 --- a/config/server.php +++ b/config/server.php @@ -604,7 +604,7 @@ 'lease' => [ 'workflow_task_timeout' => (int) EnvAuditor::env('DW_WORKFLOW_TASK_TIMEOUT', 'WORKFLOW_TASK_TIMEOUT', 60), - 'activity_task_timeout' => (int) EnvAuditor::env('DW_ACTIVITY_TASK_TIMEOUT', 'ACTIVITY_TASK_TIMEOUT', 300), + 'activity_task_timeout' => EnvAuditor::env('DW_ACTIVITY_TASK_TIMEOUT', 'ACTIVITY_TASK_TIMEOUT', 300), ], /* diff --git a/routes/api.php b/routes/api.php index dd6b8d2dd..ec39b2e87 100644 --- a/routes/api.php +++ b/routes/api.php @@ -276,6 +276,7 @@ Route::post('/activity-tasks/{taskId}/complete', [ActivityTaskController::class, 'complete']); Route::post('/activity-tasks/{taskId}/fail', [ActivityTaskController::class, 'fail']); Route::post('/activity-tasks/{taskId}/heartbeat', [ActivityTaskController::class, 'heartbeat']); + Route::post('/activity-tasks/{taskId}/renew-lease', [ActivityTaskController::class, 'renewLease']); }); // ── Workers (Management) ────────────────────────────────────────── diff --git a/tests/Feature/ActivityLeaseRenewalTest.php b/tests/Feature/ActivityLeaseRenewalTest.php index 664dff6ef..2922dcdae 100644 --- a/tests/Feature/ActivityLeaseRenewalTest.php +++ b/tests/Feature/ActivityLeaseRenewalTest.php @@ -39,9 +39,8 @@ public function test_runtime_renewal_keeps_one_attempt_without_application_progr $execution = ActivityExecution::query()->findOrFail($task['activity_execution_id']); $before = $execution->getAttributes(); $history = WorkflowHistoryEvent::query()->where('workflow_run_id', $task['run_id'])->get()->toArray(); - $this->assertSame(now()->toJSON(), $task['server_time']); - for ($renewal = 0; $renewal < 3; ++$renewal) { + for ($renewal = 0; $renewal < 3; $renewal++) { $this->travel(4)->seconds(); $this->postJson($this->renewalPath($task), $this->ownership($task), $this->workerHeaders()) ->assertOk() @@ -84,6 +83,42 @@ public function test_an_expired_lease_cannot_be_revived(): void ->assertJsonPath('reason', 'lease_expired'); } + public function test_only_worker_credentials_can_renew_an_original_activity(): void + { + $task = $this->claimActivity(); + config([ + 'server.auth.driver' => 'token', + 'server.auth.token' => null, + 'server.auth.role_tokens' => ['worker' => 'worker-token', 'operator' => 'operator-token'], + ]); + $this->postJson($this->renewalPath($task), $this->ownership($task), $this->workerHeaders()) + ->assertUnauthorized(); + $this->postJson($this->renewalPath($task), $this->ownership($task), [ + ...$this->workerHeaders(), 'Authorization' => 'Bearer operator-token', + ])->assertForbidden(); + $this->postJson($this->renewalPath($task), $this->ownership($task), [ + ...$this->workerHeaders(), 'Authorization' => 'Bearer worker-token', + ])->assertOk()->assertJsonPath('lease_renewed', true); + } + + public function test_a_lost_worker_session_does_not_extend_the_activity_lease(): void + { + $task = $this->claimActivity(session: [ + 'session_id' => 'renewal-session', + 'lease_seconds' => 2, + 'ttl_seconds' => 30, + 'queue' => 'external-activities', + ]); + $before = ActivityAttempt::query()->findOrFail($task['activity_attempt_id'])->lease_expires_at->toJSON(); + $this->travel(2)->seconds(); + $this->postJson($this->renewalPath($task), $this->ownership($task), $this->workerHeaders()) + ->assertConflict() + ->assertJsonPath('lease_renewed', false) + ->assertJsonPath('can_continue', false) + ->assertJsonPath('reason', 'session_not_active'); + $this->assertSame($before, ActivityAttempt::query()->findOrFail($task['activity_attempt_id'])->lease_expires_at->toJSON()); + } + public static function businessTimeouts(): array { return [ @@ -117,7 +152,7 @@ public function test_custom_bridge_without_renewal_is_not_advertised_or_called() ->assertJsonPath('server_capabilities.activity_lease_renewal', false); } - private function claimActivity(array $policy = []): array + private function claimActivity(array $policy = [], ?array $session = null): array { $workflow = WorkflowStub::make(ExternalGreetingWorkflow::class, 'renewal-workflow'); $start = $workflow->start('Ada'); @@ -126,14 +161,15 @@ private function claimActivity(array $policy = []): array $execution = ActivityExecution::query()->where('workflow_run_id', $start->runId())->firstOrFail(); $execution->forceFill([ 'retry_policy' => [...$execution->retry_policy, ...$policy], + 'activity_options' => [...($execution->activity_options ?? []), 'worker_session' => $session], 'schedule_to_close_deadline_at' => isset($policy['schedule_to_close_timeout']) ? now()->addSeconds($policy['schedule_to_close_timeout']) : null, ])->save(); - $this->registerWorker('renewal-worker', 'external-activities'); + $this->registerWorker('renewal-worker', 'external-activities', capabilities: ['worker_sessions']); return $this->postJson('/api/worker/activity-tasks/poll', [ 'worker_id' => 'renewal-worker', 'task_queue' => 'external-activities', - ], $this->workerHeaders())->assertOk()->json('task'); + ], $this->workerHeaders())->assertOk()->assertJsonPath('server_time', now()->toJSON())->json('task'); } private function ownership(array $task): array diff --git a/tests/Unit/ActivityTaskLeaseConfigurationTest.php b/tests/Unit/ActivityTaskLeaseConfigurationTest.php new file mode 100644 index 000000000..8da31178d --- /dev/null +++ b/tests/Unit/ActivityTaskLeaseConfigurationTest.php @@ -0,0 +1,36 @@ + '17']); + $this->assertSame(17, ActivityTaskLeaseConfiguration::apply()); + $this->assertSame(17, config('server.lease.activity_task_timeout')); + if (method_exists(ActivityLease::class, 'seconds')) { + $this->assertSame(17, ActivityLease::seconds()); + } + } + + public static function invalidLeases(): array + { + return [[0], [-1], [86_401], ['invalid'], ['1.5'], [1.5], [true], [null]]; + } + + #[DataProvider('invalidLeases')] + public function test_invalid_lease_is_refused_instead_of_silently_replaced(mixed $value): void + { + config(['server.lease.activity_task_timeout' => $value]); + $this->expectException(RuntimeException::class); + $this->expectExceptionMessage('DW_ACTIVITY_TASK_TIMEOUT'); + ActivityTaskLeaseConfiguration::apply(); + } +} From 98dd6331cf84bb9f8eccbad6dbc645ab599f4689 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Wed, 21 Oct 2026 03:06:16 +0000 Subject: [PATCH 04/22] Reproduce missing workflow deadline fences on activity delivery --- tests/Feature/ActivityAttemptStatusTest.php | 28 +++++++++++++++++ tests/Feature/ActivityLeaseRenewalTest.php | 35 ++++++++++++++++++++- 2 files changed, 62 insertions(+), 1 deletion(-) diff --git a/tests/Feature/ActivityAttemptStatusTest.php b/tests/Feature/ActivityAttemptStatusTest.php index 8fab65105..ad1b3c275 100644 --- a/tests/Feature/ActivityAttemptStatusTest.php +++ b/tests/Feature/ActivityAttemptStatusTest.php @@ -208,6 +208,34 @@ public function test_live_outcome_still_resolves_payload_and_records(): void ->assertOk()->assertJsonPath('recorded', true); } + public static function workflowDeadlines(): array + { + return [ + 'run' => ['run_deadline_at', 'run_timeout'], + 'execution' => ['execution_deadline_at', 'execution_timeout'], + ]; + } + + #[DataProvider('workflowDeadlines')] + public function test_workflow_deadline_refuses_outcome_before_external_payload_resolution(string $field, string $reason): void + { + $task = $this->lease(); + WorkflowRun::query()->findOrFail($task['run_id'])->forceFill([$field => now()])->save(); + $before = $this->snapshot($task); + $this->mock(NamespaceExternalPayloadStorage::class, static function (MockInterface $mock): void { + $mock->shouldNotReceive('driverFor'); + }); + $this->flushControllers(); + + foreach (['complete' => ['result' => null], 'fail' => ['failure' => ['message' => 'late failure']]] as $action => $payload) { + $this->withHeaders($this->headers())->postJson($this->path($task, $action), $this->fence($task) + $payload) + ->assertConflict()->assertJsonPath('recorded', false)->assertJsonPath('reason', $reason); + } + + $this->observe($task)->assertOk()->assertJsonPath('can_continue', false)->assertJsonPath('reason', $reason); + $this->assertSame($before, $this->snapshot($task)); + } + #[DataProvider('expiredOutcomeProvider')] public function test_outcome_crossing_expiry_during_payload_resolution_cannot_record(string $action, string $protocol): void { diff --git a/tests/Feature/ActivityLeaseRenewalTest.php b/tests/Feature/ActivityLeaseRenewalTest.php index 2922dcdae..8ba7775f2 100644 --- a/tests/Feature/ActivityLeaseRenewalTest.php +++ b/tests/Feature/ActivityLeaseRenewalTest.php @@ -16,6 +16,7 @@ use Workflow\V2\Models\ActivityAttempt; use Workflow\V2\Models\ActivityExecution; use Workflow\V2\Models\WorkflowHistoryEvent; +use Workflow\V2\Models\WorkflowRun; use Workflow\V2\WorkflowStub; class ActivityLeaseRenewalTest extends TestCase @@ -61,6 +62,35 @@ public function test_runtime_renewal_keeps_one_attempt_without_application_progr $this->assertSame(1, ActivityAttempt::query()->where('activity_execution_id', $execution->id)->count()); } + public function test_poll_and_observation_preserve_original_workflow_deadlines_after_renewal(): void + { + $runDeadline = now()->addSeconds(30); + $executionDeadline = now()->addSeconds(60); + $task = $this->claimActivity(runDeadlines: [ + 'run_deadline_at' => $runDeadline, + 'execution_deadline_at' => $executionDeadline, + ]); + $this->assertSame($runDeadline->toJSON(), $task['deadlines']['run'] ?? null); + $this->assertSame($executionDeadline->toJSON(), $task['deadlines']['execution'] ?? null); + $runBefore = WorkflowRun::query()->findOrFail($task['run_id'])->getAttributes(); + + $this->travel(4)->seconds(); + $this->postJson($this->renewalPath($task), $this->ownership($task), $this->workerHeaders()) + ->assertOk()->assertJsonPath('lease_renewed', true); + $this->postJson('/api/worker/activity-tasks/'.$task['task_id'].'/status', $this->ownership($task), $this->workerHeaders()) + ->assertOk()->assertJsonPath('can_continue', true) + ->assertJsonPath('deadlines.run', $runDeadline->toJSON()) + ->assertJsonPath('deadlines.execution', $executionDeadline->toJSON()); + $this->assertSame($runBefore, WorkflowRun::query()->findOrFail($task['run_id'])->getAttributes()); + } + + public function test_absent_workflow_deadlines_are_not_invented(): void + { + $task = $this->claimActivity(); + $this->assertArrayNotHasKey('run', $task['deadlines'] ?? []); + $this->assertArrayNotHasKey('execution', $task['deadlines'] ?? []); + } + public function test_renewal_refuses_wrong_owner_and_namespace_without_extending_the_lease(): void { $task = $this->claimActivity(); @@ -152,12 +182,15 @@ public function test_custom_bridge_without_renewal_is_not_advertised_or_called() ->assertJsonPath('server_capabilities.activity_lease_renewal', false); } - private function claimActivity(array $policy = [], ?array $session = null): array + private function claimActivity(array $policy = [], ?array $session = null, array $runDeadlines = []): array { $workflow = WorkflowStub::make(ExternalGreetingWorkflow::class, 'renewal-workflow'); $start = $workflow->start('Ada'); NamespaceWorkflowScope::bind('default', $workflow->id(), ExternalGreetingWorkflow::class); $this->runReadyWorkflowTask($start->runId()); + if ($runDeadlines !== []) { + WorkflowRun::query()->findOrFail($start->runId())->forceFill($runDeadlines)->save(); + } $execution = ActivityExecution::query()->where('workflow_run_id', $start->runId())->firstOrFail(); $execution->forceFill([ 'retry_policy' => [...$execution->retry_policy, ...$policy], From b003cd816e2097a37f43935c4c257870482117f0 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Wed, 21 Oct 2026 03:12:05 +0000 Subject: [PATCH 05/22] Cover expiry during renewal and external activity result resolution --- tests/Feature/ActivityAttemptStatusTest.php | 21 +++++++++++++++++++++ tests/Feature/ActivityLeaseRenewalTest.php | 20 ++++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/tests/Feature/ActivityAttemptStatusTest.php b/tests/Feature/ActivityAttemptStatusTest.php index ad1b3c275..74d25b6e5 100644 --- a/tests/Feature/ActivityAttemptStatusTest.php +++ b/tests/Feature/ActivityAttemptStatusTest.php @@ -236,6 +236,27 @@ public function test_workflow_deadline_refuses_outcome_before_external_payload_r $this->assertSame($before, $this->snapshot($task)); } + #[DataProvider('workflowDeadlines')] + public function test_workflow_deadline_crossing_during_payload_resolution_cannot_record(string $field, string $reason): void + { + $task = $this->lease(); + $deadline = now()->addSecond(); + WorkflowRun::query()->findOrFail($task['run_id'])->forceFill([$field => $deadline])->save(); + $before = $this->snapshot($task); + $this->mock(NamespaceExternalPayloadStorage::class, function (MockInterface $mock) use ($deadline): void { + $mock->shouldReceive('driverFor')->once()->andReturnUsing(function () use ($deadline): mixed { + $this->travelTo($deadline); + + return null; + }); + }); + $this->flushControllers(); + + $this->withHeaders($this->headers())->postJson($this->path($task, 'complete'), $this->fence($task) + ['result' => null]) + ->assertConflict()->assertJsonPath('recorded', false)->assertJsonPath('reason', $reason); + $this->assertSame($before, $this->snapshot($task)); + } + #[DataProvider('expiredOutcomeProvider')] public function test_outcome_crossing_expiry_during_payload_resolution_cannot_record(string $action, string $protocol): void { diff --git a/tests/Feature/ActivityLeaseRenewalTest.php b/tests/Feature/ActivityLeaseRenewalTest.php index 8ba7775f2..285552e2b 100644 --- a/tests/Feature/ActivityLeaseRenewalTest.php +++ b/tests/Feature/ActivityLeaseRenewalTest.php @@ -91,6 +91,26 @@ public function test_absent_workflow_deadlines_are_not_invented(): void $this->assertArrayNotHasKey('execution', $task['deadlines'] ?? []); } + public static function workflowDeadlineFields(): array + { + return [ + 'run' => ['run_deadline_at', 'run_timeout'], + 'execution' => ['execution_deadline_at', 'execution_timeout'], + ]; + } + + #[DataProvider('workflowDeadlineFields')] + public function test_renewal_cannot_pass_an_original_workflow_deadline(string $field, string $reason): void + { + $task = $this->claimActivity(runDeadlines: [$field => now()->addSeconds(2)]); + $attemptBefore = ActivityAttempt::query()->findOrFail($task['activity_attempt_id'])->getAttributes(); + $this->travel(2)->seconds(); + $this->postJson($this->renewalPath($task), $this->ownership($task), $this->workerHeaders()) + ->assertConflict()->assertJsonPath('lease_renewed', false) + ->assertJsonPath('can_continue', false)->assertJsonPath('reason', $reason); + $this->assertSame($attemptBefore, ActivityAttempt::query()->findOrFail($task['activity_attempt_id'])->getAttributes()); + } + public function test_renewal_refuses_wrong_owner_and_namespace_without_extending_the_lease(): void { $task = $this->claimActivity(); From 38678857226fabef145ddfb5d40278dd5b51a225 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Wed, 21 Oct 2026 03:18:52 +0000 Subject: [PATCH 06/22] Project original workflow deadlines and refuse expired activity work --- .../Api/ActivityTaskController.php | 44 +++++++++++++------ docs/server-reference.md | 32 +++++++++++++- docs/worker-activity-observation.md | 17 +++++++ 3 files changed, 77 insertions(+), 16 deletions(-) diff --git a/app/Http/Controllers/Api/ActivityTaskController.php b/app/Http/Controllers/Api/ActivityTaskController.php index 226f44eb2..f06525b01 100644 --- a/app/Http/Controllers/Api/ActivityTaskController.php +++ b/app/Http/Controllers/Api/ActivityTaskController.php @@ -494,12 +494,10 @@ public function status(Request $request, string $taskId): JsonResponse $status['can_continue'] = false; $status['reason'] = 'lease_expired'; } - foreach (['heartbeat', 'start_to_close', 'schedule_to_close'] as $kind) { - if ($status['can_continue'] === true && isset($deadlines[$kind]) - && ! CarbonImmutable::parse($deadlines[$kind])->isAfter(now())) { - $status['can_continue'] = false; - $status['reason'] = $kind.'_timeout'; - } + $expiredReason = $this->expiredDeadlineReason($deadlines); + if ($status['can_continue'] === true && $expiredReason !== null) { + $status['can_continue'] = false; + $status['reason'] = $expiredReason; } if ($status['can_continue'] === true && $workerSession !== null) { $sessionLease = $workerSession['lease_expires_at'] ?? null; @@ -733,6 +731,15 @@ public function renewLease(Request $request, string $taskId): JsonResponse ], $this->outcomeStatus($observed['reason'] ?? null)); } + $expiredReason = $this->expiredDeadlineReason( + $this->executionDeadlines($observed['activity_execution_id'] ?? null) ?? [], + ); + if ($expiredReason !== null) { + return WorkerProtocol::json([ + ...$observed, 'lease_renewed' => false, 'can_continue' => false, 'reason' => $expiredReason, + ], 409); + } + $workerSession = $this->storageMutations->run( fn (): ?array => $this->workerSessions->heartbeatForAttempt( $namespace, @@ -1106,18 +1113,13 @@ private function guardOutcomeAuthority(string $taskId, string $attemptId, array $reason = 'lease_expired'; } else { $deadlines = $this->executionDeadlines($status['activity_execution_id'] ?? null) ?? []; - foreach (['heartbeat', 'start_to_close', 'schedule_to_close'] as $kind) { - if (isset($deadlines[$kind]) && ! CarbonImmutable::parse($deadlines[$kind])->isAfter(now())) { - $reason = $kind.'_timeout'; - break; - } - } + $reason = $this->expiredDeadlineReason($deadlines); } } // Closed, cancelled and reclaimed attempts retain their existing handling, // including the committed completion receipt and cancellation diagnostics. - if (! in_array($reason, ['lease_expired', 'heartbeat_timeout', 'start_to_close_timeout', 'schedule_to_close_timeout'], true)) { + if (! in_array($reason, ['lease_expired', 'heartbeat_timeout', 'start_to_close_timeout', 'schedule_to_close_timeout', 'run_timeout', 'execution_timeout'], true)) { return null; } @@ -1259,7 +1261,7 @@ private function executionDeadlines(?string $executionId): ?array } /** @var ActivityExecution|null $execution */ - $execution = ActivityExecution::query()->find($executionId); + $execution = ActivityExecution::query()->with('run')->find($executionId); if (! $execution) { return null; @@ -1270,11 +1272,25 @@ private function executionDeadlines(?string $executionId): ?array 'start_to_close' => $this->deadlineTimestamp($execution->close_deadline_at), 'schedule_to_close' => $this->deadlineTimestamp($execution->schedule_to_close_deadline_at), 'heartbeat' => $this->deadlineTimestamp($execution->heartbeat_deadline_at), + 'run' => $this->deadlineTimestamp($execution->run?->run_deadline_at), + 'execution' => $this->deadlineTimestamp($execution->run?->execution_deadline_at), ], static fn (mixed $v): bool => $v !== null); return $deadlines !== [] ? $deadlines : null; } + /** @param array $deadlines */ + private function expiredDeadlineReason(array $deadlines): ?string + { + foreach (['heartbeat', 'start_to_close', 'schedule_to_close', 'run', 'execution'] as $kind) { + if (isset($deadlines[$kind]) && ! CarbonImmutable::parse($deadlines[$kind])->isAfter(now())) { + return $kind.'_timeout'; + } + } + + return null; + } + private function deadlineTimestamp(?CarbonInterface $deadline): ?string { return $deadline?->copy() diff --git a/docs/server-reference.md b/docs/server-reference.md index 91365dce5..d4ce80589 100644 --- a/docs/server-reference.md +++ b/docs/server-reference.md @@ -1110,13 +1110,41 @@ workflow ID is bound remain rejected as `instance_not_found`. Activity task polling returns a leased attempt identity. Clients must echo both `activity_attempt_id` and `lease_owner` on activity `complete`, `fail`, and `heartbeat` calls. When the activity execution has timeout deadlines configured, -the poll response includes a `deadlines` object with ISO-8601 timestamps for -`schedule_to_start`, `start_to_close`, `schedule_to_close`, and/or `heartbeat`. +the poll response includes a `deadlines` object with UTC timestamps and six +fractional digits for `schedule_to_start`, `start_to_close`, `schedule_to_close`, +and/or `heartbeat`. Configured workflow deadlines are included as `run` and +`execution`, copied from the original durable run. Missing deadlines are omitted; +polling, observation and runtime renewal do not invent or restart them. Workers should use these deadlines to self-cancel before the server enforces the timeout. The server runs `activity:timeout-enforce` periodically to expire activities that exceed their deadlines. Heartbeats accept `message`, `current`, `total`, `unit`, and `details` fields; the server normalizes them to the package heartbeat-progress contract before recording the heartbeat. + +The optional `activity_lease_renewal` capability is available only with an +installed bridge implementing runtime renewal. When discovery explicitly +returns `true`, a worker can send its original `activity_attempt_id` and +`lease_owner` to `POST /api/worker/activity-tasks/{id}/renew-lease` using the +same namespace and worker credentials. A successful response contains +`lease_renewed: true`, `can_continue: true`, the original task/run/execution/ +attempt/owner identity, `lease_expires_at`, `server_time`, and +`heartbeat_recorded: false`. Validate that identity and charge the request's +elapsed time against the returned grant before executing or publishing work. +An HTTP success alone is not execution authority. A lost required worker +session, expired lease or elapsed business deadline refuses renewal; this +operation cannot revive a claim. Missing capability retains the existing +lease/heartbeat behavior and requires no new protocol floor. + +Runtime renewal records no application progress or durable history. Only an +actual acknowledged application heartbeat may advance its originally recorded +heartbeat interval. Workers must preserve the original run, execution and +activity deadlines, keep renewal active during argument loading and result +publication, and recheck authority before every terminal HTTP retry. A late +outcome is refused before resolving an external result. The package also needs +to validate authority under its mutation locks; a preflight response is not a +reservation. The automatic-worker and published long-activity acceptance for +this optional candidate remain tracked in Workflow #878. + When a run-level cancel or terminate command stops a leased activity task, heartbeat, complete, and fail responses include `run_closed_reason` and `run_closed_at` alongside `cancel_requested: true`. diff --git a/docs/worker-activity-observation.md b/docs/worker-activity-observation.md index f29c3ba71..ece6b06ff 100644 --- a/docs/worker-activity-observation.md +++ b/docs/worker-activity-observation.md @@ -34,3 +34,20 @@ Stopping an activity process cannot undo external effects. Safe retries still require idempotency or reconciliation. This endpoint supplies the observation contract for connected SDK qualification and does not by itself prove callback supervision, graceful shutdown or cold replacement in any SDK. + +## Original workflow deadlines and optional renewal + +The `deadlines` map on activity polls and observations includes `run` and +`execution` when the durable run has those original bounds. Timestamps are UTC +with six fractional digits. Missing values are omitted; an observation or +runtime renewal cannot restart the workflow clock. The Server refuses an +already elapsed bound before resolving external completion/failure payloads. + +Runtime maintenance is separate from this read-only status route. The optional +`activity_lease_renewal` capability enables +`POST /api/worker/activity-tasks/{taskId}/renew-lease`, with the original +`activity_attempt_id`, `lease_owner`, namespace and worker credentials. It is +not an application heartbeat and does not record history or reset progress +timeouts. Only an actual stored application heartbeat can advance its recorded +progress interval. See the [Server reference](server-reference.md#worker-protocol) +and Workflow #878 for the source-candidate and published-consumer boundaries. From 9cf82f665b377ce278caf547afbcbfb6109d5a18 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Wed, 21 Oct 2026 05:41:08 +0000 Subject: [PATCH 07/22] refactor: make activity lease renewal fences explicit --- .../Api/ActivityTaskController.php | 113 ++++++++++-------- .../ActivityTaskLeaseConfiguration.php | 5 +- ...ctivityTaskLeaseConfigurationException.php | 7 ++ .../ActivityTaskLeaseConfigurationTest.php | 4 +- 4 files changed, 74 insertions(+), 55 deletions(-) create mode 100644 app/Support/ActivityTaskLeaseConfigurationException.php diff --git a/app/Http/Controllers/Api/ActivityTaskController.php b/app/Http/Controllers/Api/ActivityTaskController.php index f06525b01..918ddb576 100644 --- a/app/Http/Controllers/Api/ActivityTaskController.php +++ b/app/Http/Controllers/Api/ActivityTaskController.php @@ -708,67 +708,53 @@ public function renewLease(Request $request, string $taskId): JsonResponse ], 422); } + try { + $response = $this->renewOwnedActivityLease($request, $taskId, $bridge); + } catch (\Throwable $exception) { + if (! BackendLockPressure::is($exception)) { + throw $exception; + } + + $response = BackendLockPressure::workerOperationResponse($request, false); + } + + return $response; + } + + private function renewOwnedActivityLease(Request $request, string $taskId, ActivityLeaseRenewal $bridge): JsonResponse + { $validated = $request->validate([ 'activity_attempt_id' => ['required', 'string'], 'lease_owner' => ['required', 'string'], ]); $namespace = $request->attributes->get('namespace'); $observed = null; - try { - if ($response = $this->guardAttemptOwnership( + if ($response = $this->leaseRenewalRefusal($namespace, $taskId, $validated, $observed)) { + return $response; + } + + $workerSession = $this->storageMutations->run( + fn (): ?array => $this->workerSessions->heartbeatForAttempt( $namespace, - $taskId, $validated['activity_attempt_id'], $validated['lease_owner'], - $observed, - )) { - return $response; - } - - if (($observed['can_continue'] ?? false) !== true) { - return WorkerProtocol::json([ - ...$observed, 'lease_renewed' => false, - ], $this->outcomeStatus($observed['reason'] ?? null)); - } - - $expiredReason = $this->expiredDeadlineReason( - $this->executionDeadlines($observed['activity_execution_id'] ?? null) ?? [], - ); - if ($expiredReason !== null) { - return WorkerProtocol::json([ - ...$observed, 'lease_renewed' => false, 'can_continue' => false, 'reason' => $expiredReason, - ], 409); - } - - $workerSession = $this->storageMutations->run( - fn (): ?array => $this->workerSessions->heartbeatForAttempt( - $namespace, - $validated['activity_attempt_id'], - $validated['lease_owner'], - ), - ); - if ($workerSession !== null && ($workerSession['admitted'] ?? false) !== true) { - return WorkerProtocol::json([ - ...$observed, - 'lease_renewed' => false, - 'can_continue' => false, - 'reason' => $workerSession['reason'] ?? 'worker_session_renewal_failed', - 'worker_session' => $this->workerSessionSnapshotFromRenewal($workerSession), - 'worker_session_renewal' => $this->workerSessionRenewalStatusPayload($workerSession), - ], $this->workerSessionRenewalHttpStatus($workerSession)); - } - - $status = $this->storageMutations->run( - static fn (): array => $bridge->renewLease($validated['activity_attempt_id'], $validated['lease_owner']), - ); - } catch (\Throwable $exception) { - if (! BackendLockPressure::is($exception)) { - throw $exception; - } - - return BackendLockPressure::workerOperationResponse($request, false); + ), + ); + if ($workerSession !== null && ($workerSession['admitted'] ?? false) !== true) { + return WorkerProtocol::json([ + ...$observed, + 'lease_renewed' => false, + 'can_continue' => false, + 'reason' => $workerSession['reason'] ?? 'worker_session_renewal_failed', + 'worker_session' => $this->workerSessionSnapshotFromRenewal($workerSession), + 'worker_session_renewal' => $this->workerSessionRenewalStatusPayload($workerSession), + ], $this->workerSessionRenewalHttpStatus($workerSession)); } + $status = $this->storageMutations->run( + static fn (): array => $bridge->renewLease($validated['activity_attempt_id'], $validated['lease_owner']), + ); + return WorkerProtocol::json([ ...$status, 'task_id' => $taskId, @@ -776,6 +762,33 @@ public function renewLease(Request $request, string $taskId): JsonResponse ], $this->outcomeStatus($status['reason'] ?? null)); } + private function leaseRenewalRefusal(string $namespace, string $taskId, array $validated, ?array &$observed): ?JsonResponse + { + if ($response = $this->guardAttemptOwnership( + $namespace, + $taskId, + $validated['activity_attempt_id'], + $validated['lease_owner'], + $observed, + )) { + return $response; + } + + if (($observed['can_continue'] ?? false) !== true) { + return WorkerProtocol::json([ + ...$observed, 'lease_renewed' => false, + ], $this->outcomeStatus($observed['reason'] ?? null)); + } + + $expiredReason = $this->expiredDeadlineReason( + $this->executionDeadlines($observed['activity_execution_id'] ?? null) ?? [], + ); + + return $expiredReason === null ? null : WorkerProtocol::json([ + ...$observed, 'lease_renewed' => false, 'can_continue' => false, 'reason' => $expiredReason, + ], 409); + } + /** * Heartbeat an in-progress activity task. * diff --git a/app/Support/ActivityTaskLeaseConfiguration.php b/app/Support/ActivityTaskLeaseConfiguration.php index d721ce76e..06b513b2d 100644 --- a/app/Support/ActivityTaskLeaseConfiguration.php +++ b/app/Support/ActivityTaskLeaseConfiguration.php @@ -2,7 +2,6 @@ namespace App\Support; -use RuntimeException; use Workflow\V2\Support\ActivityLease; final class ActivityTaskLeaseConfiguration @@ -15,14 +14,14 @@ public static function apply(): int 'options' => ['min_range' => 1, 'max_range' => 86_400], ]) : false; if ($seconds === false) { - throw new RuntimeException('DW_ACTIVITY_TASK_TIMEOUT must be an integer between 1 and 86400 seconds.'); + throw new ActivityTaskLeaseConfigurationException('DW_ACTIVITY_TASK_TIMEOUT must be an integer between 1 and 86400 seconds.'); } config(['server.lease.activity_task_timeout' => $seconds]); if (defined(ActivityLease::class.'::CONFIG_KEY') && method_exists(ActivityLease::class, 'seconds')) { config([ActivityLease::CONFIG_KEY => $seconds]); if (ActivityLease::seconds() !== $seconds) { - throw new RuntimeException('The configured activity-task lease does not match the Workflow runtime.'); + throw new ActivityTaskLeaseConfigurationException('The configured activity-task lease does not match the Workflow runtime.'); } } diff --git a/app/Support/ActivityTaskLeaseConfigurationException.php b/app/Support/ActivityTaskLeaseConfigurationException.php new file mode 100644 index 000000000..53a2531ac --- /dev/null +++ b/app/Support/ActivityTaskLeaseConfigurationException.php @@ -0,0 +1,7 @@ + $value]); - $this->expectException(RuntimeException::class); + $this->expectException(ActivityTaskLeaseConfigurationException::class); $this->expectExceptionMessage('DW_ACTIVITY_TASK_TIMEOUT'); ActivityTaskLeaseConfiguration::apply(); } From 53759639c0c57c60196d16ebd45353963433635d Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Wed, 21 Oct 2026 05:52:27 +0000 Subject: [PATCH 08/22] test: reproduce activity leases retained after worker deregistration --- .../ActivityWorkerDeregistrationTest.php | 143 ++++++++++++++++++ 1 file changed, 143 insertions(+) create mode 100644 tests/Feature/ActivityWorkerDeregistrationTest.php diff --git a/tests/Feature/ActivityWorkerDeregistrationTest.php b/tests/Feature/ActivityWorkerDeregistrationTest.php new file mode 100644 index 000000000..9dc952121 --- /dev/null +++ b/tests/Feature/ActivityWorkerDeregistrationTest.php @@ -0,0 +1,143 @@ +createNamespace('default'); + $this->travelTo(Carbon::parse('2026-10-21T05:00:00.123456Z')); + } + + public static function deregistrationRoutes(): array + { + return ['fenced' => [true], 'legacy' => [false]]; + } + + #[DataProvider('deregistrationRoutes')] + public function test_deregistration_settles_the_original_attempt_with_its_recorded_retry_policy(bool $fenced): void + { + [$task, $token] = $this->leaseActivity(2); + $originalExpiry = WorkflowTask::query()->findOrFail($task['task_id'])->lease_expires_at->toJSON(); + $closedAt = now(); + $this->deregister($token, $fenced)->assertOk(); + + $attempt = ActivityAttempt::query()->findOrFail($task['activity_attempt_id']); + $this->assertSame(ActivityAttemptStatus::Expired, $attempt->status); + $this->assertTrue($attempt->closed_at->equalTo($closedAt)); + $this->assertSame(TaskStatus::Cancelled, WorkflowTask::query()->findOrFail($task['task_id'])->status); + $execution = ActivityExecution::query()->findOrFail($task['activity_execution_id']); + $this->assertSame(ActivityStatus::Pending, $execution->status); + $this->assertSame(1, $execution->attempt_count); + $retry = WorkflowTask::query()->where('workflow_run_id', $task['run_id']) + ->where('task_type', TaskType::Activity->value)->where('status', TaskStatus::Ready->value)->sole(); + $this->assertTrue($retry->available_at->equalTo($closedAt->copy()->addSeconds(7))); + $event = WorkflowHistoryEvent::query()->where('workflow_run_id', $task['run_id']) + ->where('event_type', HistoryEventType::ActivityRetryScheduled->value)->sole(); + $this->assertSame($task['activity_attempt_id'], $event->payload['activity_attempt_id']); + $this->assertSame('worker_deregistered', $event->payload['failure_reason']); + $this->assertSame($originalExpiry, $event->payload['lease_expires_at']); + $this->assertSame($closedAt->toJSON(), $event->payload['lease_revoked_at']); + $this->assertSame(1, $event->payload['retry_after_attempt']); + + $historyCount = WorkflowHistoryEvent::query()->count(); + $this->postJson('/api/worker/activity-tasks/'.$task['task_id'].'/complete', [ + 'activity_attempt_id' => $task['activity_attempt_id'], 'lease_owner' => $task['lease_owner'], + 'result' => 'late original result', + ], $this->workerHeaders())->assertConflict()->assertJsonPath('recorded', false); + $this->assertSame($historyCount, WorkflowHistoryEvent::query()->count()); + if ($fenced) { + $this->deregister($token, true)->assertOk(); + $this->assertSame($historyCount, WorkflowHistoryEvent::query()->count()); + } + } + + #[DataProvider('deregistrationRoutes')] + public function test_deregistration_cannot_reset_an_exhausted_activity_retry_budget(bool $fenced): void + { + [$task, $token] = $this->leaseActivity(1); + $this->deregister($token, $fenced)->assertOk(); + $this->assertSame(ActivityStatus::Failed, ActivityExecution::query()->findOrFail($task['activity_execution_id'])->status); + $this->assertSame(ActivityAttemptStatus::Expired, ActivityAttempt::query()->findOrFail($task['activity_attempt_id'])->status); + $this->assertSame(0, WorkflowTask::query()->where('workflow_run_id', $task['run_id']) + ->where('task_type', TaskType::Activity->value)->where('status', TaskStatus::Ready->value)->count()); + $this->assertSame(1, WorkflowHistoryEvent::query()->where('workflow_run_id', $task['run_id']) + ->where('event_type', HistoryEventType::ActivityFailed->value)->count()); + } + + public function test_activity_settlement_failure_rolls_back_the_fenced_registration_receipt(): void + { + if (DB::connection()->getDriverName() !== 'sqlite') { + $this->markTestSkipped('SQLite trigger fault injection.'); + } + [$task, $token] = $this->leaseActivity(2); + $before = WorkflowTask::query()->findOrFail($task['task_id'])->getRawOriginal(); + $historyCount = WorkflowHistoryEvent::query()->count(); + DB::unprepared("CREATE TRIGGER reject_activity_shutdown BEFORE INSERT ON workflow_history_events WHEN NEW.event_type = 'ActivityRetryScheduled' BEGIN SELECT RAISE(ABORT, 'activity shutdown rejected'); END"); + try { + $this->deregister($token, true)->assertStatus(500); + } finally { + DB::unprepared('DROP TRIGGER reject_activity_shutdown'); + } + $this->assertSame($before, WorkflowTask::query()->findOrFail($task['task_id'])->getRawOriginal()); + $this->assertSame($historyCount, WorkflowHistoryEvent::query()->count()); + $this->assertDatabaseHas('workflow_worker_registrations', ['worker_id' => 'activity-owner', 'status' => 'active']); + $this->assertSame(WorkerRegistrationIncarnation::ACTIVE, WorkerRegistrationIncarnation::query()->findOrFail($token)->status); + } + + private function leaseActivity(int $maxAttempts): array + { + $workflow = WorkflowStub::make(ExternalGreetingWorkflow::class, 'deregister-original-activity'); + $start = $workflow->start('Ada'); + $this->runReadyWorkflowTask($start->runId()); + $execution = ActivityExecution::query()->where('workflow_run_id', $start->runId())->sole(); + $execution->forceFill(['retry_policy' => [ + ...$execution->retry_policy, 'max_attempts' => $maxAttempts, 'backoff' => [7], + ]])->save(); + $registration = $this->postJson('/api/worker/register', [ + 'worker_id' => 'activity-owner', 'task_queue' => 'external-activities', 'runtime' => 'php', + 'supported_workflow_types' => [], 'supported_activity_types' => ['tests.external-greeting-activity'], + 'capabilities' => [], 'capability_manifest' => $this->portableWorkerAffinityRefusalManifest(), + ], $this->workerHeaders())->assertCreated(); + $task = $this->postJson('/api/worker/activity-tasks/poll', [ + 'worker_id' => 'activity-owner', 'task_queue' => 'external-activities', + ], $this->workerHeaders())->assertOk()->json('task'); + $this->assertIsArray($task); + + return [$task, $registration->json('registration_token')]; + } + + private function deregister(string $token, bool $fenced): TestResponse + { + return $fenced + ? $this->postJson('/api/worker/registrations/activity-owner/deregister', ['registration_token' => $token], $this->workerHeaders()) + : $this->deleteJson('/api/worker/registrations/activity-owner', [], $this->workerHeaders()); + } +} From c7765d2e75acab0d2571ccc421a21342ad93aab5 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Wed, 21 Oct 2026 06:06:20 +0000 Subject: [PATCH 09/22] fix: revoke activity claims at worker deregistration --- .../Api/WorkerDeregistrationController.php | 15 ++- app/Support/ActivityTaskLeaseRecovery.php | 62 ++++++++++++ app/Support/BackendLockPressure.php | 4 +- app/Support/FencedWorkerDeregistration.php | 9 +- app/Support/WorkerRegistrationDeregistrar.php | 3 + .../ActivityWorkerDeregistrationTest.php | 99 +++++++++++++++++-- 6 files changed, 178 insertions(+), 14 deletions(-) create mode 100644 app/Support/ActivityTaskLeaseRecovery.php diff --git a/app/Http/Controllers/Api/WorkerDeregistrationController.php b/app/Http/Controllers/Api/WorkerDeregistrationController.php index 2e841dbcd..c69539cf5 100644 --- a/app/Http/Controllers/Api/WorkerDeregistrationController.php +++ b/app/Http/Controllers/Api/WorkerDeregistrationController.php @@ -22,8 +22,21 @@ public function destroy(Request $request, string $workerId): JsonResponse return $response; } + return $this->completeLegacyDeregistration($request, $workerId); + } + + private function completeLegacyDeregistration(Request $request, string $workerId): JsonResponse + { $namespace = (string) $request->attributes->get('namespace'); - $recoveredWorkflowTaskCount = $this->registrar->deregister($request, $namespace, $workerId); + try { + $recoveredWorkflowTaskCount = $this->registrar->deregister($request, $namespace, $workerId); + } catch (\Throwable $exception) { + if (! BackendLockPressure::is($exception)) { + throw $exception; + } + + return BackendLockPressure::workerOperationResponse($request, false); + } if ($recoveredWorkflowTaskCount === null) { return WorkerProtocol::json([ diff --git a/app/Support/ActivityTaskLeaseRecovery.php b/app/Support/ActivityTaskLeaseRecovery.php new file mode 100644 index 000000000..ac42fa5fe --- /dev/null +++ b/app/Support/ActivityTaskLeaseRecovery.php @@ -0,0 +1,62 @@ +getConnectionName() ?? config('database.default'); + foreach ([new WorkflowTask, new WorkflowRun, new ActivityExecution, new ActivityAttempt] as $model) { + if (($model->getConnectionName() ?? config('database.default')) !== $connection) { + return false; + } + } + + return true; + } + + /** The caller retains the stopped registration's row/incarnation fence until commit. */ + public function recoverForStoppedWorker(string $namespace, string $workerId): void + { + if (! self::supported()) { + return; + } + if (! self::sharesRegistrationConnection()) { + throw new HttpResponseException(WorkerProtocol::json([ + 'reason' => 'activity_lease_recovery_transaction_boundary_unavailable', + 'worker_id' => $workerId, + 'retryable' => false, + ], 412)); + } + + // Do not lock tasks ahead of attempts/executions. Core rechecks the + // original task, owner, generation and namespace under canonical locks. + $taskIds = WorkflowTask::query()->where('namespace', $namespace) + ->where('task_type', TaskType::Activity->value) + ->where('status', TaskStatus::Leased->value) + ->where('lease_owner', $workerId)->pluck('id'); + foreach ($taskIds as $taskId) { + $outcome = ActivityTimeoutEnforcer::revokeWorkerLease($taskId, $namespace, $workerId); + if ($outcome['next_task'] instanceof WorkflowTask) { + TaskDispatcher::dispatch($outcome['next_task'], boundedWorkflowTaskProjection: true); + } + } + } +} diff --git a/app/Support/BackendLockPressure.php b/app/Support/BackendLockPressure.php index 272633cae..a5059e01b 100644 --- a/app/Support/BackendLockPressure.php +++ b/app/Support/BackendLockPressure.php @@ -240,11 +240,11 @@ public static function isSqliteBackend(): bool private static function hasConcurrencyErrorCode(array $errorInfo): bool { foreach ($errorInfo as $part) { - if (in_array($part, [5, 6, 1205, 1213, '40001', '40P01', '55P03'], true)) { + if (in_array($part, [5, 6, 1205, 1213, 3572, '40001', '40P01', '55P03'], true)) { return true; } - if (is_string($part) && in_array((int) $part, [5, 6, 1205, 1213], true)) { + if (is_string($part) && in_array((int) $part, [5, 6, 1205, 1213, 3572], true)) { return true; } } diff --git a/app/Support/FencedWorkerDeregistration.php b/app/Support/FencedWorkerDeregistration.php index 680d6629d..69e5093b4 100644 --- a/app/Support/FencedWorkerDeregistration.php +++ b/app/Support/FencedWorkerDeregistration.php @@ -14,14 +14,18 @@ final class FencedWorkerDeregistration { - public function __construct(private readonly WorkflowTaskLeaseRecovery $recovery) {} + public function __construct( + private readonly WorkflowTaskLeaseRecovery $recovery, + private readonly ActivityTaskLeaseRecovery $activityRecovery, + ) {} public static function available(): bool { // Independent connection handles cannot commit one receipt covering // both registration deletion and workflow-task recovery. return ((new WorkerRegistration)->getConnectionName() ?? config('database.default')) - === ((new WorkflowTask)->getConnectionName() ?? config('database.default')); + === ((new WorkflowTask)->getConnectionName() ?? config('database.default')) + && (! ActivityTaskLeaseRecovery::supported() || ActivityTaskLeaseRecovery::sharesRegistrationConnection()); } /** @return array{status: int, body: array} */ @@ -65,6 +69,7 @@ public function complete(Request $request, string $namespace, string $workerId, } $worker->forceFill(['status' => WorkerRegistration::STATUS_SUPERSEDED])->save(); + $this->activityRecovery->recoverForStoppedWorker($namespace, $workerId); $tasks = WorkflowTask::query() ->where('namespace', $namespace) ->where('task_type', TaskType::Workflow->value) diff --git a/app/Support/WorkerRegistrationDeregistrar.php b/app/Support/WorkerRegistrationDeregistrar.php index 7cca5d478..af2663ce0 100644 --- a/app/Support/WorkerRegistrationDeregistrar.php +++ b/app/Support/WorkerRegistrationDeregistrar.php @@ -15,6 +15,7 @@ final class WorkerRegistrationDeregistrar { public function __construct( private readonly WorkflowTaskLeaseRecovery $workflowTaskLeaseRecovery, + private readonly ActivityTaskLeaseRecovery $activityTaskLeaseRecovery, ) {} public function deregister(Request $request, string $namespace, string $workerId): ?int @@ -39,6 +40,8 @@ public function deregister(Request $request, string $namespace, string $workerId ->forceFill(['status' => WorkerRegistration::STATUS_SUPERSEDED]) ->save(); + $this->activityTaskLeaseRecovery->recoverForStoppedWorker($namespace, $workerId); + return WorkflowTask::query() ->where('namespace', $namespace) ->where('task_type', TaskType::Workflow->value) diff --git a/tests/Feature/ActivityWorkerDeregistrationTest.php b/tests/Feature/ActivityWorkerDeregistrationTest.php index 9dc952121..93401e700 100644 --- a/tests/Feature/ActivityWorkerDeregistrationTest.php +++ b/tests/Feature/ActivityWorkerDeregistrationTest.php @@ -3,7 +3,7 @@ namespace Tests\Feature; use App\Models\WorkerRegistrationIncarnation; -use Illuminate\Foundation\Testing\RefreshDatabase; +use Illuminate\Foundation\Testing\DatabaseTruncation; use Illuminate\Support\Carbon; use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Queue; @@ -21,11 +21,12 @@ use Workflow\V2\Models\ActivityExecution; use Workflow\V2\Models\WorkflowHistoryEvent; use Workflow\V2\Models\WorkflowTask; +use Workflow\V2\Support\ActivityTimeoutEnforcer; use Workflow\V2\WorkflowStub; class ActivityWorkerDeregistrationTest extends TestCase { - use RefreshDatabase; + use DatabaseTruncation; use ServerTestHelpers; protected function setUp(): void @@ -112,6 +113,81 @@ public function test_activity_settlement_failure_rolls_back_the_fenced_registrat $this->assertSame(WorkerRegistrationIncarnation::ACTIVE, WorkerRegistrationIncarnation::query()->findOrFail($token)->status); } + public function test_completed_receipt_cannot_revoke_a_replacement_incarnation(): void + { + [$task, $token] = $this->leaseActivity(2); + $this->deregister($token, true)->assertOk(); + $this->travel(7)->seconds(); + $replacementToken = $this->registerActivityWorker(); + $this->assertNotSame($token, $replacementToken); + $replacement = $this->postJson('/api/worker/activity-tasks/poll', [ + 'worker_id' => 'activity-owner', 'task_queue' => 'external-activities', + ], $this->workerHeaders())->assertOk()->json('task'); + $this->assertIsArray($replacement); + $this->assertNotSame($task['activity_attempt_id'], $replacement['activity_attempt_id']); + $before = ActivityAttempt::query()->findOrFail($replacement['activity_attempt_id'])->getRawOriginal(); + $historyCount = WorkflowHistoryEvent::query()->count(); + $this->deregister($token, true)->assertOk(); + $this->assertSame($before, ActivityAttempt::query()->findOrFail($replacement['activity_attempt_id'])->getRawOriginal()); + $this->assertSame($historyCount, WorkflowHistoryEvent::query()->count()); + $this->assertDatabaseHas('workflow_worker_registrations', ['registration_token' => $replacementToken, 'status' => 'active']); + } + + public function test_revocation_cannot_replace_an_already_elapsed_business_timeout(): void + { + [$task, $token] = $this->leaseActivity(1); + $execution = ActivityExecution::query()->findOrFail($task['activity_execution_id']); + $execution->forceFill([ + 'heartbeat_deadline_at' => now(), + 'retry_policy' => [...$execution->retry_policy, 'heartbeat_timeout' => 3], + ])->save(); + $this->deregister($token, true)->assertOk(); + $event = WorkflowHistoryEvent::query()->where('workflow_run_id', $task['run_id']) + ->where('event_type', HistoryEventType::ActivityTimedOut->value)->sole(); + $this->assertSame('heartbeat', $event->payload['timeout_kind']); + $this->assertArrayNotHasKey('lease_revoked_at', $event->payload); + } + + public function test_revocation_refuses_foreign_namespace_and_owner_without_mutation(): void + { + [$task] = $this->leaseActivity(2); + $before = WorkflowTask::query()->findOrFail($task['task_id'])->getRawOriginal(); + $historyCount = WorkflowHistoryEvent::query()->count(); + foreach ([['other', 'activity-owner'], ['default', 'another-worker']] as [$namespace, $owner]) { + $outcome = ActivityTimeoutEnforcer::revokeWorkerLease($task['task_id'], $namespace, $owner); + $this->assertFalse($outcome['enforced']); + $this->assertSame('lease_owner_mismatch', $outcome['reason']); + } + $this->assertSame($before, WorkflowTask::query()->findOrFail($task['task_id'])->getRawOriginal()); + $this->assertSame($historyCount, WorkflowHistoryEvent::query()->count()); + } + + #[DataProvider('deregistrationRoutes')] + public function test_contended_attempt_refuses_without_committing_the_registration(bool $fenced): void + { + if (! in_array(DB::connection()->getDriverName(), ['mysql', 'pgsql'], true)) { + $this->markTestSkipped('Two actual row-locking database connections are required.'); + } + [$task, $token] = $this->leaseActivity(2); + $before = WorkflowTask::query()->findOrFail($task['task_id'])->getRawOriginal(); + config(['database.connections.activity_blocker' => DB::connection()->getConfig()]); + $blocker = DB::connection('activity_blocker'); + $blocker->beginTransaction(); + try { + $this->assertNotNull($blocker->table((new ActivityAttempt)->getTable()) + ->where('id', $task['activity_attempt_id'])->lockForUpdate()->first()); + $started = hrtime(true); + $this->deregister($token, $fenced)->assertStatus(503)->assertJsonPath('reason', 'backend_lock_pressure'); + $this->assertLessThan(2.0, (hrtime(true) - $started) / 1e9); + } finally { + $blocker->rollBack(); + DB::purge('activity_blocker'); + } + $this->assertSame($before, WorkflowTask::query()->findOrFail($task['task_id'])->getRawOriginal()); + $this->assertDatabaseHas('workflow_worker_registrations', ['worker_id' => 'activity-owner', 'status' => 'active']); + $this->assertSame(WorkerRegistrationIncarnation::ACTIVE, WorkerRegistrationIncarnation::query()->findOrFail($token)->status); + } + private function leaseActivity(int $maxAttempts): array { $workflow = WorkflowStub::make(ExternalGreetingWorkflow::class, 'deregister-original-activity'); @@ -119,19 +195,24 @@ private function leaseActivity(int $maxAttempts): array $this->runReadyWorkflowTask($start->runId()); $execution = ActivityExecution::query()->where('workflow_run_id', $start->runId())->sole(); $execution->forceFill(['retry_policy' => [ - ...$execution->retry_policy, 'max_attempts' => $maxAttempts, 'backoff' => [7], + ...$execution->retry_policy, 'max_attempts' => $maxAttempts, 'backoff_seconds' => [7], ]])->save(); - $registration = $this->postJson('/api/worker/register', [ - 'worker_id' => 'activity-owner', 'task_queue' => 'external-activities', 'runtime' => 'php', - 'supported_workflow_types' => [], 'supported_activity_types' => ['tests.external-greeting-activity'], - 'capabilities' => [], 'capability_manifest' => $this->portableWorkerAffinityRefusalManifest(), - ], $this->workerHeaders())->assertCreated(); + $token = $this->registerActivityWorker(); $task = $this->postJson('/api/worker/activity-tasks/poll', [ 'worker_id' => 'activity-owner', 'task_queue' => 'external-activities', ], $this->workerHeaders())->assertOk()->json('task'); $this->assertIsArray($task); - return [$task, $registration->json('registration_token')]; + return [$task, $token]; + } + + private function registerActivityWorker(): string + { + return $this->postJson('/api/worker/register', [ + 'worker_id' => 'activity-owner', 'task_queue' => 'external-activities', 'runtime' => 'php', + 'supported_workflow_types' => [], 'supported_activity_types' => ['tests.external-greeting-activity'], + 'capabilities' => [], 'capability_manifest' => $this->portableWorkerAffinityRefusalManifest(), + ], $this->workerHeaders())->assertCreated()->json('registration_token'); } private function deregister(string $token, bool $fenced): TestResponse From 18cf8f44a2e42020ac8e6118933fbf875128a32f Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Wed, 21 Oct 2026 17:46:02 +0000 Subject: [PATCH 10/22] prep: adopt published Core 2.10.1 for guarded activity lease maintenance --- composer.json | 2 +- composer.lock | 16 ++++++++-------- .../ReleaseImagePublishWorkflowContractTest.php | 10 +++++----- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/composer.json b/composer.json index ec9ad5514..3655bad88 100644 --- a/composer.json +++ b/composer.json @@ -6,7 +6,7 @@ "require": { "php": "^8.2", "apache/avro": "^1.12", - "durable-workflow/workflow": "2.9.0", + "durable-workflow/workflow": "2.10.1", "laravel/framework": "^13.30", "laravel/tinker": "^3.0", "league/flysystem-aws-s3-v3": "^3.35.3" diff --git a/composer.lock b/composer.lock index 8e9e72613..574fd2c36 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "050062197e287c4c7ddc1ccd2605d0e5", + "content-hash": "20a53b8d696f48dbec19c185c444f437", "packages": [ { "name": "apache/avro", @@ -655,16 +655,16 @@ }, { "name": "durable-workflow/workflow", - "version": "2.9.0", + "version": "2.10.1", "source": { "type": "git", "url": "https://github.com/durable-workflow/workflow.git", - "reference": "2bc1ba0b57df34b74de02427d2225bd8e4dc6322" + "reference": "1a5ec33f83a68021a163bc78d06e990c1eedc5da" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/durable-workflow/workflow/zipball/2bc1ba0b57df34b74de02427d2225bd8e4dc6322", - "reference": "2bc1ba0b57df34b74de02427d2225bd8e4dc6322", + "url": "https://api.github.com/repos/durable-workflow/workflow/zipball/1a5ec33f83a68021a163bc78d06e990c1eedc5da", + "reference": "1a5ec33f83a68021a163bc78d06e990c1eedc5da", "shasum": "" }, "require": { @@ -705,7 +705,7 @@ "dev-main": "2.0.x-dev" }, "durable-workflow": { - "product-train": "2.9.0", + "product-train": "2.10.1", "laravel-embedded-upgrade-contract": "resources/laravel-embedded-upgrade-contract.json", "laravel-dependency-security-policy": "resources/laravel-dependency-security-policy.json" } @@ -732,9 +732,9 @@ "description": "Embedded durable workflow runtime and orchestration engine for Laravel applications.", "support": { "issues": "https://github.com/durable-workflow/workflow/issues", - "source": "https://github.com/durable-workflow/workflow/tree/2.9.0" + "source": "https://github.com/durable-workflow/workflow/tree/2.10.1" }, - "time": "2026-10-21T02:33:14+00:00" + "time": "2026-10-21T16:30:55+00:00" }, { "name": "egulias/email-validator", diff --git a/tests/Unit/ReleaseImagePublishWorkflowContractTest.php b/tests/Unit/ReleaseImagePublishWorkflowContractTest.php index 6a061bd4b..312d90007 100644 --- a/tests/Unit/ReleaseImagePublishWorkflowContractTest.php +++ b/tests/Unit/ReleaseImagePublishWorkflowContractTest.php @@ -310,7 +310,7 @@ public function test_standalone_apache_and_cli_processes_share_compiled_applicat $dockerfile = $this->read('Dockerfile'); $phpConfig = $this->read('docker/php-custom.ini'); - $this->assertStringContainsString('FROM php:8.3-apache AS base', $dockerfile); + $this->assertMatchesRegularExpression('/^FROM php@sha256:[a-f0-9]{64} AS base$/m', $dockerfile); $this->assertStringContainsString('docker-php-ext-install opcache', $dockerfile); $this->assertStringContainsString('opcache.enable = 1', $phpConfig); $this->assertStringContainsString('opcache.enable_cli = 1', $phpConfig); @@ -369,12 +369,12 @@ public function test_dockerfile_installs_node_for_published_conformance_handoffs $dockerfile = $this->read('Dockerfile'); $activitiesRunner = $this->read('scripts/conformance/activities-published-artifacts.sh'); - $this->assertStringContainsString('FROM php:8.3-apache AS base', $dockerfile); + $this->assertMatchesRegularExpression('/^FROM php@sha256:[a-f0-9]{64} AS base$/m', $dockerfile); $this->assertStringContainsString('nodejs', $dockerfile); $this->assertStringContainsString('if ! require_command node; then', $activitiesRunner); $this->assertStringContainsString('required command not found: node', $activitiesRunner); - $baseOffset = strpos($dockerfile, 'FROM php:8.3-apache AS base'); + $baseOffset = strpos($dockerfile, " AS base\n"); $nodeOffset = strpos($dockerfile, 'nodejs'); $productionOffset = strpos($dockerfile, 'FROM base AS production'); @@ -390,7 +390,7 @@ public function test_dockerfile_installs_python_for_focused_activity_sdk_cells() $dockerfile = $this->read('Dockerfile'); $activitiesRunner = $this->read('scripts/conformance/activities-published-artifacts.sh'); - $this->assertStringContainsString('FROM php:8.3-apache AS base', $dockerfile); + $this->assertMatchesRegularExpression('/^FROM php@sha256:[a-f0-9]{64} AS base$/m', $dockerfile); $this->assertStringContainsString('python3', $dockerfile); $this->assertStringContainsString('python3-venv', $dockerfile); $this->assertStringContainsString('prepare_focused_python_sdk', $activitiesRunner); @@ -399,7 +399,7 @@ public function test_dockerfile_installs_python_for_focused_activity_sdk_cells() $this->assertStringContainsString('run_python_activity_executor', $activitiesRunner); $this->assertStringContainsString('activity_host_evidence missing passing ${requiredMode}/${runtime} cell', $activitiesRunner); - $baseOffset = strpos($dockerfile, 'FROM php:8.3-apache AS base'); + $baseOffset = strpos($dockerfile, " AS base\n"); $pythonOffset = strpos($dockerfile, 'python3'); $productionOffset = strpos($dockerfile, 'FROM base AS production'); From fbfaec814f904378aecd8004c038153710bcca47 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Wed, 21 Oct 2026 17:58:37 +0000 Subject: [PATCH 11/22] Isolate deregistration migrations from transactional feature suites --- tests/Feature/ActivityWorkerDeregistrationTest.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/Feature/ActivityWorkerDeregistrationTest.php b/tests/Feature/ActivityWorkerDeregistrationTest.php index 93401e700..d1617f45d 100644 --- a/tests/Feature/ActivityWorkerDeregistrationTest.php +++ b/tests/Feature/ActivityWorkerDeregistrationTest.php @@ -3,7 +3,7 @@ namespace Tests\Feature; use App\Models\WorkerRegistrationIncarnation; -use Illuminate\Foundation\Testing\DatabaseTruncation; +use Illuminate\Foundation\Testing\DatabaseMigrations; use Illuminate\Support\Carbon; use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Queue; @@ -26,7 +26,7 @@ class ActivityWorkerDeregistrationTest extends TestCase { - use DatabaseTruncation; + use DatabaseMigrations; use ServerTestHelpers; protected function setUp(): void From 7a282014eeb89e8328d3bbf764620736d5baf398 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Wed, 21 Oct 2026 18:05:13 +0000 Subject: [PATCH 12/22] Document and validate runtime-only activity lease renewal --- .../worker-protocol-api.openapi.yaml | 55 ++++++++++++++++++- tests/Feature/ActivityLeaseRenewalTest.php | 5 +- 2 files changed, 58 insertions(+), 2 deletions(-) diff --git a/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml b/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml index b10e932ac..022f00c17 100644 --- a/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml +++ b/resources/platform-protocol-specs/worker-protocol-api.openapi.yaml @@ -1,7 +1,7 @@ openapi: 3.1.0 info: title: durable-workflow.v2.worker-protocol-api - version: "55" + version: "56" summary: Durable Workflow worker-plane HTTP+JSON API description: > Normative OpenAPI specification for the worker-plane HTTP+JSON API: @@ -1212,6 +1212,38 @@ paths: responses: "200": { $ref: "#/components/responses/WorkerEnvelope" } "409": { $ref: "#/components/responses/WorkerError" } + /worker/activity-tasks/{taskId}/renew-lease: + post: + operationId: renewActivityTaskLease + tags: [activity-tasks] + description: > + Renew runtime ownership of the original activity attempt using its + namespace, task, attempt and lease-owner fence. Requires affirmative + activity_lease_renewal discovery from the actual bound activity bridge. + This adds no protocol-version floor. It renews neither application + heartbeat progress nor any workflow, activity or required-session TTL + deadline, and appends no history event. Expired, replaced or closed + attempts cannot regain authority. A successful renewal is not authority + to publish a later result without independently checking the same fence. + parameters: + - $ref: "#/components/parameters/WorkerProtocolVersionHeader" + - $ref: "#/components/parameters/TaskIdPath" + requestBody: + required: true + content: + application/json: + schema: { $ref: "#/components/schemas/ActivityTaskStatusRequest" } + responses: + "200": + description: The original runtime lease was renewed without recording application progress. + content: + application/json: + schema: { $ref: "#/components/schemas/ActivityTaskLeaseRenewalResponse" } + "400": { $ref: "#/components/responses/WorkerError" } + "404": { $ref: "#/components/responses/WorkerError" } + "409": { $ref: "#/components/responses/WorkerError" } + "422": { $ref: "#/components/responses/WorkerError" } + "503": { $ref: "#/components/responses/WorkerServiceUnavailable" } /worker/activity-tasks/{taskId}/complete: post: operationId: completeActivityTask @@ -1692,6 +1724,9 @@ components: activity_cancellation_acknowledgement: type: boolean description: Original remote callback-stop receipts require an installed runtime primitive and worker protocol 1.20. False for earlier protocols and runtimes without that primitive. + activity_lease_renewal: + type: boolean + description: Runtime-only activity lease renewal is supported by the actual bound optional ActivityLeaseRenewal bridge. Independent of application heartbeat progress; false for bridges without that role. prepared_local_activities: type: boolean description: Preparation, prefix checkpoint, supervisor control, application heartbeat, outcome, recovery and joined-stop receipts. Requires protocol 1.20 and the actual bound PreparedLocalActivityTaskBridge role including heartbeat support. False for earlier protocols or an absent backend role. @@ -2897,6 +2932,22 @@ components: properties: activity_attempt_id: { type: string, minLength: 1 } lease_owner: { type: string, minLength: 1 } + ActivityTaskLeaseRenewalResponse: + allOf: + - $ref: "#/components/schemas/WorkerEnvelope" + - type: object + required: [task_id, workflow_task_id, activity_attempt_id, lease_owner, lease_renewed, can_continue, heartbeat_recorded, server_time, lease_expires_at] + properties: + task_id: { type: string, minLength: 1 } + workflow_task_id: { type: string, minLength: 1 } + activity_attempt_id: { type: string, minLength: 1 } + lease_owner: { type: string, minLength: 1 } + lease_renewed: { type: boolean, const: true } + can_continue: { type: boolean, const: true } + heartbeat_recorded: { type: boolean, const: false } + server_time: { type: string, format: date-time } + lease_expires_at: { type: string, format: date-time } + last_heartbeat_at: { type: [string, "null"], format: date-time } ActivityCancellationAcknowledgementRequest: type: object additionalProperties: true @@ -2968,6 +3019,8 @@ components: start_to_close: { type: string, format: date-time } schedule_to_close: { type: string, format: date-time } heartbeat: { type: string, format: date-time } + run: { type: string, format: date-time } + execution: { type: string, format: date-time } worker_session: type: [object, "null"] additionalProperties: true diff --git a/tests/Feature/ActivityLeaseRenewalTest.php b/tests/Feature/ActivityLeaseRenewalTest.php index 285552e2b..a19f0d251 100644 --- a/tests/Feature/ActivityLeaseRenewalTest.php +++ b/tests/Feature/ActivityLeaseRenewalTest.php @@ -11,6 +11,7 @@ use PHPUnit\Framework\Attributes\DataProvider; use Tests\Feature\Concerns\ServerTestHelpers; use Tests\Fixtures\ExternalGreetingWorkflow; +use Tests\Support\OpenApiSchema; use Tests\TestCase; use Workflow\V2\Contracts\ActivityTaskBridge; use Workflow\V2\Models\ActivityAttempt; @@ -43,7 +44,7 @@ public function test_runtime_renewal_keeps_one_attempt_without_application_progr for ($renewal = 0; $renewal < 3; $renewal++) { $this->travel(4)->seconds(); - $this->postJson($this->renewalPath($task), $this->ownership($task), $this->workerHeaders()) + $response = $this->postJson($this->renewalPath($task), $this->ownership($task), $this->workerHeaders()) ->assertOk() ->assertJsonPath('lease_renewed', true) ->assertJsonPath('can_continue', true) @@ -55,6 +56,8 @@ public function test_runtime_renewal_keeps_one_attempt_without_application_progr ->assertJsonPath('server_time', now()->toJSON()) ->assertJsonPath('lease_expires_at', now()->addSeconds(5)->toJSON()) ->assertJsonPath('server_capabilities.activity_lease_renewal', true); + OpenApiSchema::fromFile(resource_path('platform-protocol-specs/worker-protocol-api.openapi.yaml')) + ->assertReferenceMatches('#/components/schemas/ActivityTaskLeaseRenewalResponse', json_decode($response->getContent(), flags: JSON_THROW_ON_ERROR)); } $this->assertSame($before, $execution->fresh()->getAttributes()); From 8a04f9125e26c9e63551af10737a355a471a8288 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Wed, 21 Oct 2026 19:36:36 +0000 Subject: [PATCH 13/22] chore: prepare Server 2.11 and Helm 0.3 lease release --- composer.json | 2 +- composer.lock | 2 +- docker-compose.dedicated-matching.yml | 4 ++-- docker-compose.memo-rolling.yml | 4 ++-- docker-compose.published.yml | 4 ++-- docker-compose.small-cluster.yml | 2 +- docker-compose.yml | 8 ++++---- k8s/README.md | 10 +++++----- k8s/helm/durable-workflow/Chart.yaml | 10 +++++----- k8s/helm/durable-workflow/README.md | 2 +- .../ci/existing-secrets-values.yaml | 2 +- .../ci/ingress-and-hpa-values.yaml | 2 +- .../durable-workflow/ci/inline-secrets-values.yaml | 2 +- k8s/helm/durable-workflow/docs/UPGRADING.md | 14 ++++++++++++++ k8s/helm/durable-workflow/templates/_helpers.tpl | 2 +- k8s/helm/durable-workflow/values.yaml | 2 +- k8s/helm/examples/values-dev.yaml | 2 +- .../examples/values-external-secrets-operator.yaml | 2 +- .../values-production-existing-secrets.yaml | 2 +- k8s/migration-job.yaml | 2 +- k8s/scheduler-cronjob.yaml | 2 +- k8s/secret.yaml | 2 +- k8s/server-deployment.yaml | 2 +- k8s/worker-deployment.yaml | 2 +- resources/release/source-release.json | 4 ++-- scripts/k8s-kind-smoke.sh | 2 +- 26 files changed, 54 insertions(+), 40 deletions(-) diff --git a/composer.json b/composer.json index 3655bad88..5cf974531 100644 --- a/composer.json +++ b/composer.json @@ -48,7 +48,7 @@ }, "extra": { "durable-workflow": { - "product-train": "2.10.1" + "product-train": "2.11.0" }, "laravel": { "dont-discover": [] diff --git a/composer.lock b/composer.lock index 574fd2c36..e1d88a414 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "20a53b8d696f48dbec19c185c444f437", + "content-hash": "f9149b34c915f7af598a4d603ba10e75", "packages": [ { "name": "apache/avro", diff --git a/docker-compose.dedicated-matching.yml b/docker-compose.dedicated-matching.yml index 1ec3d1349..02cdc157f 100644 --- a/docker-compose.dedicated-matching.yml +++ b/docker-compose.dedicated-matching.yml @@ -32,13 +32,13 @@ name: durable-workflow-server # daemon reports `shape: dedicated`. # Generated by scripts/ci/sync-source-release.mjs. Do not edit the fallback. -x-server-image: &server-image ${DW_SERVER_IMAGE:-durableworkflow/server:${DW_SERVER_TAG:-2.10.1}} +x-server-image: &server-image ${DW_SERVER_IMAGE:-durableworkflow/server:${DW_SERVER_TAG:-2.11.0}} x-server-environment: &server-environment APP_NAME: "Durable Workflow Server" APP_ENV: ${APP_ENV:-local} DW_SERVER_KEY: ${DW_SERVER_KEY:-} - APP_VERSION: ${APP_VERSION:-${DW_SERVER_TAG:-2.10.1}} + APP_VERSION: ${APP_VERSION:-${DW_SERVER_TAG:-2.11.0}} APP_DEBUG: ${APP_DEBUG:-false} DB_CONNECTION: mysql DB_HOST: mysql diff --git a/docker-compose.memo-rolling.yml b/docker-compose.memo-rolling.yml index 83147fc2b..b69523c8d 100644 --- a/docker-compose.memo-rolling.yml +++ b/docker-compose.memo-rolling.yml @@ -49,14 +49,14 @@ services: command: ["server-bootstrap"] environment: <<: *runtime-environment - APP_VERSION: ${APP_VERSION:-2.10.1} + APP_VERSION: ${APP_VERSION:-2.11.0} successor: image: ${DW_MEMO_SUCCESSOR_IMAGE:-durable-workflow/server-memo-rolling:local} ports: !override [] environment: <<: *runtime-environment - APP_VERSION: ${APP_VERSION:-2.10.1} + APP_VERSION: ${APP_VERSION:-2.11.0} DW_SERVER_ID: memo-successor DW_SERVER_TOPOLOGY_SHAPE: standalone_server DW_SERVER_PROCESS_CLASS: server_http_node diff --git a/docker-compose.published.yml b/docker-compose.published.yml index cf3bed2b7..8ebfde010 100644 --- a/docker-compose.published.yml +++ b/docker-compose.published.yml @@ -1,13 +1,13 @@ name: durable-workflow-server # Generated by scripts/ci/sync-source-release.mjs. Do not edit the fallback. -x-server-image: &server-image ${DW_SERVER_IMAGE:-durableworkflow/server:${DW_SERVER_TAG:-2.10.1}} +x-server-image: &server-image ${DW_SERVER_IMAGE:-durableworkflow/server:${DW_SERVER_TAG:-2.11.0}} x-server-environment: &server-environment APP_NAME: "Durable Workflow Server" APP_ENV: ${APP_ENV:-local} DW_SERVER_KEY: ${DW_SERVER_KEY:-} - APP_VERSION: ${APP_VERSION:-${DW_SERVER_TAG:-2.10.1}} + APP_VERSION: ${APP_VERSION:-${DW_SERVER_TAG:-2.11.0}} APP_DEBUG: ${APP_DEBUG:-false} LOG_CHANNEL: ${LOG_CHANNEL:-stderr} LOG_LEVEL: ${LOG_LEVEL:-info} diff --git a/docker-compose.small-cluster.yml b/docker-compose.small-cluster.yml index b416bcf96..d3de1183c 100644 --- a/docker-compose.small-cluster.yml +++ b/docker-compose.small-cluster.yml @@ -12,7 +12,7 @@ x-server-build: &server-build x-server-environment: &server-environment APP_NAME: "Durable Workflow Server" APP_ENV: testing - APP_VERSION: ${APP_VERSION:-2.10.1} + APP_VERSION: ${APP_VERSION:-2.11.0} APP_DEBUG: "false" DW_SERVER_KEY: ${DW_SERVER_KEY:-base64:5Zt4nUhlCm3DD0nLXZJQdHiwPfb56yGo9gNV/g3jYbY=} DB_CONNECTION: ${DW_SMALL_CLUSTER_DB:-mysql} diff --git a/docker-compose.yml b/docker-compose.yml index c7d5d7b5a..3f40ae4ec 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,7 +15,7 @@ services: DW_SERVER_KEY: "${DW_SERVER_KEY:-}" DW_SERVER_TOPOLOGY_SHAPE: standalone_server DW_SERVER_PROCESS_CLASS: server_http_node - APP_VERSION: "${APP_VERSION:-2.10.1}" + APP_VERSION: "${APP_VERSION:-2.11.0}" APP_DEBUG: "false" DB_CONNECTION: mysql DB_HOST: mysql @@ -62,7 +62,7 @@ services: APP_NAME: "Durable Workflow Server" APP_ENV: local DW_SERVER_KEY: "${DW_SERVER_KEY:-}" - APP_VERSION: "${APP_VERSION:-2.10.1}" + APP_VERSION: "${APP_VERSION:-2.11.0}" APP_DEBUG: "false" DB_CONNECTION: mysql DB_HOST: mysql @@ -124,7 +124,7 @@ services: DW_SERVER_KEY: "${DW_SERVER_KEY:-}" DW_SERVER_TOPOLOGY_SHAPE: standalone_server DW_SERVER_PROCESS_CLASS: worker_node - APP_VERSION: "${APP_VERSION:-2.10.1}" + APP_VERSION: "${APP_VERSION:-2.11.0}" DB_CONNECTION: mysql DB_HOST: mysql DB_PORT: 3306 @@ -178,7 +178,7 @@ services: DW_SERVER_KEY: "${DW_SERVER_KEY:-}" DW_SERVER_TOPOLOGY_SHAPE: standalone_server DW_SERVER_PROCESS_CLASS: scheduler_node - APP_VERSION: "${APP_VERSION:-2.10.1}" + APP_VERSION: "${APP_VERSION:-2.11.0}" DB_CONNECTION: mysql DB_HOST: mysql DB_PORT: 3306 diff --git a/k8s/README.md b/k8s/README.md index 1f64202c7..24bca9a97 100644 --- a/k8s/README.md +++ b/k8s/README.md @@ -13,7 +13,7 @@ The checked-in manifests are synchronized with the repository's stable source release and pin its Docker Hub tag: ```text -durableworkflow/server:2.10.1 +durableworkflow/server:2.11.0 ``` Before production use, patch every workload image to the exact published tag or @@ -21,15 +21,15 @@ digest you intend to run: ```bash kubectl set image -n durable-workflow deploy/durable-workflow-server \ - server=durableworkflow/server:2.10.1 + server=durableworkflow/server:2.11.0 kubectl set image -n durable-workflow deploy/durable-workflow-worker \ - worker=durableworkflow/server:2.10.1 + worker=durableworkflow/server:2.11.0 kubectl set image -n durable-workflow cronjob/durable-workflow-scheduler \ - scheduler=durableworkflow/server:2.10.1 + scheduler=durableworkflow/server:2.11.0 ``` GitHub Container Registry publishes the same release line at -`ghcr.io/durable-workflow/server:2.10.1`. Digest pinning is preferred for strict +`ghcr.io/durable-workflow/server:2.11.0`. Digest pinning is preferred for strict change control. The manifests expect you to provide: diff --git a/k8s/helm/durable-workflow/Chart.yaml b/k8s/helm/durable-workflow/Chart.yaml index ee733b24b..ccac317ee 100644 --- a/k8s/helm/durable-workflow/Chart.yaml +++ b/k8s/helm/durable-workflow/Chart.yaml @@ -5,11 +5,11 @@ type: application # The chart's own semver version. Bumped on every chart release; treated as # independent of the server image version (appVersion). Breaking-change rules # for this version live in docs/UPGRADING.md alongside the chart. -version: 0.2.3 +version: 0.3.0 # The immutable Durable Workflow Server identity this chart release packages. # The onboarding default in values.yaml and appVersion are generated from the # checked-in source release record. -appVersion: "2.10.1" +appVersion: "2.11.0" kubeVersion: ">=1.27.0-0" home: https://durable-workflow.github.io/docs/2.0/deployment sources: @@ -30,12 +30,12 @@ annotations: # exact commit that most recently changed the packaged chart. org.opencontainers.image.source: https://github.com/durable-workflow/server dev.durable-workflow.source-revision: "unreleased" - dev.durable-workflow.image-reference: "docker.io/durableworkflow/server:2.10.1" + dev.durable-workflow.image-reference: "docker.io/durableworkflow/server:2.11.0" artifacthub.io/license: MIT artifacthub.io/category: integration-delivery # Free-form changelog for the current chart release shown by Artifact Hub. artifacthub.io/changes: | - kind: changed - description: Package the Server release with optional PHP observability context and nullable carrier migrations. + description: Package Server 2.11 with capability-gated activity lease maintenance and configurable lease duration. - kind: added - description: Document application-owned API configuration and capability discovery without adding a required dependency. + description: Document compatible worker rollout and the distinction between leases and application deadlines. diff --git a/k8s/helm/durable-workflow/README.md b/k8s/helm/durable-workflow/README.md index 4883a86fa..1fcd7e887 100644 --- a/k8s/helm/durable-workflow/README.md +++ b/k8s/helm/durable-workflow/README.md @@ -63,7 +63,7 @@ helm install durable-workflow ./k8s/helm/durable-workflow \ ```yaml image: - tag: "2.10.1" + tag: "2.11.0" # Pin a digest in production: # digest: "sha256:abc123..." # memoPayloadStorage: "raw-json-v1" # Required for a digest or custom image. diff --git a/k8s/helm/durable-workflow/ci/existing-secrets-values.yaml b/k8s/helm/durable-workflow/ci/existing-secrets-values.yaml index 3d1e2d812..9c0134877 100644 --- a/k8s/helm/durable-workflow/ci/existing-secrets-values.yaml +++ b/k8s/helm/durable-workflow/ci/existing-secrets-values.yaml @@ -1,7 +1,7 @@ # CI fixture: GitOps / externally-managed-secret path. The chart consumes # existing Secrets and renders no Secret resources of its own. image: - tag: "2.10.1" + tag: "2.11.0" externalDatabase: connection: pgsql diff --git a/k8s/helm/durable-workflow/ci/ingress-and-hpa-values.yaml b/k8s/helm/durable-workflow/ci/ingress-and-hpa-values.yaml index ab05a5d9f..09ddab75b 100644 --- a/k8s/helm/durable-workflow/ci/ingress-and-hpa-values.yaml +++ b/k8s/helm/durable-workflow/ci/ingress-and-hpa-values.yaml @@ -1,6 +1,6 @@ # CI fixture: ingress + autoscaling enabled. Exercises optional templates. image: - tag: "2.10.1" + tag: "2.11.0" externalDatabase: connection: mysql diff --git a/k8s/helm/durable-workflow/ci/inline-secrets-values.yaml b/k8s/helm/durable-workflow/ci/inline-secrets-values.yaml index 5a32320fd..9995bd14f 100644 --- a/k8s/helm/durable-workflow/ci/inline-secrets-values.yaml +++ b/k8s/helm/durable-workflow/ci/inline-secrets-values.yaml @@ -2,7 +2,7 @@ # chart's render path is exercised end-to-end. Real deployments should use # existingSecret instead. image: - tag: "2.10.1" + tag: "2.11.0" externalDatabase: connection: mysql diff --git a/k8s/helm/durable-workflow/docs/UPGRADING.md b/k8s/helm/durable-workflow/docs/UPGRADING.md index 3967e939c..10a9c9d1a 100644 --- a/k8s/helm/durable-workflow/docs/UPGRADING.md +++ b/k8s/helm/durable-workflow/docs/UPGRADING.md @@ -67,6 +67,20 @@ chart MINOR bump and an entry below. ## Per-version migration notes +### Chart 0.3.0 / Server 2.11 + +This chart advances the default Server minor version. Existing values retain +their shape. Server adds the optional `activity_lease_renewal` capability and +reads `DW_ACTIVITY_TASK_TIMEOUT` as the activity-task lease duration, in seconds +from 1 to 86400, with the existing 300-second default. + +Lease maintenance requires a compatible worker that discovers the capability. +It does not replace application heartbeats or extend an activity's authored +timeout, execution deadline or workflow deadline. See +[activity observation and lease maintenance](../../../../docs/worker-activity-observation.md) +before changing worker shutdown or lease settings. Complete the ordinary +database backup, bootstrap and worker rollout checks above. + ### Chart 0.2.2 / Server 2.10 This chart changes the default application image without removing or renaming diff --git a/k8s/helm/durable-workflow/templates/_helpers.tpl b/k8s/helm/durable-workflow/templates/_helpers.tpl index df6f4a5b7..93ed3a41b 100644 --- a/k8s/helm/durable-workflow/templates/_helpers.tpl +++ b/k8s/helm/durable-workflow/templates/_helpers.tpl @@ -88,7 +88,7 @@ resolved by an explicit capability declaration or an existing workload marker. {{- define "durable-workflow.memoPayloadStorageForImage" -}} {{- $image := toString . -}} {{- $normalized := regexReplaceAll "^index\\.docker\\.io/" $image "docker.io/" -}} -{{- if eq $normalized "docker.io/durableworkflow/server:2.10.1" -}} +{{- if eq $normalized "docker.io/durableworkflow/server:2.11.0" -}} dual-v1 {{- else if regexMatch "^docker\\.io/durableworkflow/server:2\\.0\\.0-rc\\.[0-9]+$" $normalized -}} {{- $releaseCandidate := atoi (regexFind "[0-9]+$" $normalized) -}} diff --git a/k8s/helm/durable-workflow/values.yaml b/k8s/helm/durable-workflow/values.yaml index d521dd292..490fde641 100644 --- a/k8s/helm/durable-workflow/values.yaml +++ b/k8s/helm/durable-workflow/values.yaml @@ -21,7 +21,7 @@ image: registry: docker.io repository: durableworkflow/server # Generated by scripts/ci/sync-source-release.mjs. Do not edit this default. - tag: "2.10.1" + tag: "2.11.0" # Optional digest pin. When set, takes precedence over tag for change control. # Example: "sha256:abc123..." digest: "" diff --git a/k8s/helm/examples/values-dev.yaml b/k8s/helm/examples/values-dev.yaml index 9ba84bb69..34d70165f 100644 --- a/k8s/helm/examples/values-dev.yaml +++ b/k8s/helm/examples/values-dev.yaml @@ -3,7 +3,7 @@ # shape in production. image: - tag: "2.10.1" + tag: "2.11.0" externalDatabase: connection: mysql diff --git a/k8s/helm/examples/values-external-secrets-operator.yaml b/k8s/helm/examples/values-external-secrets-operator.yaml index bc6d93e1b..868d8b4be 100644 --- a/k8s/helm/examples/values-external-secrets-operator.yaml +++ b/k8s/helm/examples/values-external-secrets-operator.yaml @@ -5,7 +5,7 @@ # concern. image: - tag: "2.10.1" + tag: "2.11.0" externalDatabase: connection: pgsql diff --git a/k8s/helm/examples/values-production-existing-secrets.yaml b/k8s/helm/examples/values-production-existing-secrets.yaml index 1bcccace7..1848a3933 100644 --- a/k8s/helm/examples/values-production-existing-secrets.yaml +++ b/k8s/helm/examples/values-production-existing-secrets.yaml @@ -10,7 +10,7 @@ image: repository: durable-workflow/server # Pin a digest in production for change-control auditability. digest: "" # e.g. "sha256:abc123..." - tag: "2.10.1" + tag: "2.11.0" externalDatabase: connection: pgsql diff --git a/k8s/migration-job.yaml b/k8s/migration-job.yaml index f1d87ebdf..882d703aa 100644 --- a/k8s/migration-job.yaml +++ b/k8s/migration-job.yaml @@ -13,7 +13,7 @@ spec: restartPolicy: OnFailure containers: - name: migrate - image: durableworkflow/server:2.10.1 + image: durableworkflow/server:2.11.0 command: ["server-entrypoint"] args: ["server-bootstrap"] envFrom: diff --git a/k8s/scheduler-cronjob.yaml b/k8s/scheduler-cronjob.yaml index fed3121ca..49a5573ed 100644 --- a/k8s/scheduler-cronjob.yaml +++ b/k8s/scheduler-cronjob.yaml @@ -24,7 +24,7 @@ spec: restartPolicy: Never containers: - name: scheduler - image: durableworkflow/server:2.10.1 + image: durableworkflow/server:2.11.0 command: ["server-entrypoint"] args: - sh diff --git a/k8s/secret.yaml b/k8s/secret.yaml index 5236c860a..3ef21b6a2 100644 --- a/k8s/secret.yaml +++ b/k8s/secret.yaml @@ -12,7 +12,7 @@ metadata: app.kubernetes.io/name: durable-workflow data: APP_NAME: "Durable Workflow Server" - APP_VERSION: "2.10.1" + APP_VERSION: "2.11.0" APP_ENV: production APP_DEBUG: "false" DB_CONNECTION: mysql diff --git a/k8s/server-deployment.yaml b/k8s/server-deployment.yaml index 38ca279b0..dff9553e1 100644 --- a/k8s/server-deployment.yaml +++ b/k8s/server-deployment.yaml @@ -23,7 +23,7 @@ spec: spec: containers: - name: server - image: durableworkflow/server:2.10.1 + image: durableworkflow/server:2.11.0 ports: - containerPort: 8080 name: http diff --git a/k8s/worker-deployment.yaml b/k8s/worker-deployment.yaml index 39e90f133..256545dad 100644 --- a/k8s/worker-deployment.yaml +++ b/k8s/worker-deployment.yaml @@ -19,7 +19,7 @@ spec: spec: containers: - name: worker - image: durableworkflow/server:2.10.1 + image: durableworkflow/server:2.11.0 command: ["server-entrypoint"] args: ["php", "artisan", "queue:work", "--sleep=1", "--tries=3", "--max-time=3600"] envFrom: diff --git a/resources/release/source-release.json b/resources/release/source-release.json index 6d20c9916..d07ffa248 100644 --- a/resources/release/source-release.json +++ b/resources/release/source-release.json @@ -1,9 +1,9 @@ { "schema": "durable-workflow.server.source-release/v1", "server": { - "version": "2.10.1" + "version": "2.11.0" }, "helm_chart": { - "version": "0.2.3" + "version": "0.3.0" } } diff --git a/scripts/k8s-kind-smoke.sh b/scripts/k8s-kind-smoke.sh index 5e4e2ab07..b8156665d 100755 --- a/scripts/k8s-kind-smoke.sh +++ b/scripts/k8s-kind-smoke.sh @@ -7,7 +7,7 @@ cluster="${K8S_SMOKE_CLUSTER:-durable-workflow-server-smoke}" image="${K8S_SMOKE_IMAGE:-durableworkflow/server:k8s-smoke}" # Generated by scripts/ci/sync-source-release.mjs so the smoke replaces the # same default shipped by the public manifests. -manifest_image="durableworkflow/server:2.10.1" +manifest_image="durableworkflow/server:2.11.0" kind_node_image="${K8S_SMOKE_KIND_NODE_IMAGE:-kindest/node:v1.29.4}" artifact_dir="${K8S_SMOKE_ARTIFACT_DIR:-/tmp/durable-workflow-k8s-kind-smoke-artifacts}" rendered_dir="${artifact_dir}/rendered-manifests" From 7a73dd29b344c714283b04174cbba3a0503077f3 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Wed, 21 Oct 2026 19:38:20 +0000 Subject: [PATCH 14/22] docs: explain previously ignored activity lease overrides --- k8s/helm/durable-workflow/docs/UPGRADING.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/k8s/helm/durable-workflow/docs/UPGRADING.md b/k8s/helm/durable-workflow/docs/UPGRADING.md index 10a9c9d1a..d4f89bd7f 100644 --- a/k8s/helm/durable-workflow/docs/UPGRADING.md +++ b/k8s/helm/durable-workflow/docs/UPGRADING.md @@ -73,6 +73,8 @@ This chart advances the default Server minor version. Existing values retain their shape. Server adds the optional `activity_lease_renewal` capability and reads `DW_ACTIVITY_TASK_TIMEOUT` as the activity-task lease duration, in seconds from 1 to 86400, with the existing 300-second default. +Review any existing override before upgrading: values previously ignored by +the PHP Server now take effect. Lease maintenance requires a compatible worker that discovers the capability. It does not replace application heartbeats or extend an activity's authored From cf51b36dc93e5af1779be3868db72b9df9baff89 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Wed, 21 Oct 2026 19:44:15 +0000 Subject: [PATCH 15/22] docs: acknowledge the historical chart version exception --- k8s/helm/durable-workflow/docs/UPGRADING.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/k8s/helm/durable-workflow/docs/UPGRADING.md b/k8s/helm/durable-workflow/docs/UPGRADING.md index d4f89bd7f..21de7431f 100644 --- a/k8s/helm/durable-workflow/docs/UPGRADING.md +++ b/k8s/helm/durable-workflow/docs/UPGRADING.md @@ -85,6 +85,10 @@ database backup, bootstrap and worker rollout checks above. ### Chart 0.2.2 / Server 2.10 +This historical release crossed a Server minor version under a chart patch +number, contrary to the policy above. Its published version stays immutable. +Chart 0.3.0 resumes the stated minor-version policy for the next Server minor. + This chart changes the default application image without removing or renaming chart values. Server bootstrap adds Core's nullable context columns to runs, commands and tasks, including installations with tracing disabled. Back up the From 3c26df1f9d2c35511195afd87c99a9a8ab153b14 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Wed, 21 Oct 2026 20:27:42 +0000 Subject: [PATCH 16/22] test: preserve installed PHP activity lease acceptance inputs --- .../PhpActivityLeaseConsumer/README.md | 64 + .../PhpActivityLeaseConsumer/client.php | 31 + .../PhpActivityLeaseConsumer/composer.json | 9 + .../PhpActivityLeaseConsumer/composer.lock | 1035 +++++++++++++++++ .../PhpActivityLeaseConsumer/helpers.php | 12 + .../PhpActivityLeaseConsumer/snapshot.php | 26 + .../PhpActivityLeaseConsumer/verify-long.php | 47 + .../PhpActivityLeaseConsumer/verify-loss.php | 77 ++ .../PhpActivityLeaseConsumer/worker.php | 66 ++ 9 files changed, 1367 insertions(+) create mode 100644 tests/Fixtures/PhpActivityLeaseConsumer/README.md create mode 100644 tests/Fixtures/PhpActivityLeaseConsumer/client.php create mode 100644 tests/Fixtures/PhpActivityLeaseConsumer/composer.json create mode 100644 tests/Fixtures/PhpActivityLeaseConsumer/composer.lock create mode 100644 tests/Fixtures/PhpActivityLeaseConsumer/helpers.php create mode 100644 tests/Fixtures/PhpActivityLeaseConsumer/snapshot.php create mode 100644 tests/Fixtures/PhpActivityLeaseConsumer/verify-long.php create mode 100644 tests/Fixtures/PhpActivityLeaseConsumer/verify-loss.php create mode 100644 tests/Fixtures/PhpActivityLeaseConsumer/worker.php diff --git a/tests/Fixtures/PhpActivityLeaseConsumer/README.md b/tests/Fixtures/PhpActivityLeaseConsumer/README.md new file mode 100644 index 000000000..c21216900 --- /dev/null +++ b/tests/Fixtures/PhpActivityLeaseConsumer/README.md @@ -0,0 +1,64 @@ +# Installed PHP SDK activity-lease controls + +These are the small PHP inputs used for [Workflow #878](https://github.com/durable-workflow/workflow/issues/878). +The recorded [source-candidate result](https://github.com/durable-workflow/workflow/issues/878#issuecomment-6325709459) +distinguishes local Server qualification from published-image acceptance. +They are explicit, slow integration controls and do not run in the unit suite. + +Use a disposable Server/database that advertises `activity_lease_renewal`. +Check actual discovery before admission. Do not infer that capability from +the Server version, and do not run these controls against a production database. +The fixture uses the test-only credential `synthetic`. + +## Package and mounts + +Run `composer install` from this directory using the committed lock. It installs +the published PHP SDK without a path repository or source overlay. The Worker +checks the loaded SDK version and reference. + +Mount this directory at `/consumer` and a fresh writable evidence directory at +`/state`, using the container's unprivileged user. Set `SERVER_URL`, `NAMESPACE` +and a unique `WORKER_ID`. Run `php /consumer/worker.php` in a separate process. +Wait for `worker.registered` in `/state/worker-events.jsonl` before starting a +client. Give every new original its own evidence directory and `WORKFLOW_ID`. + +## Six-minute callback + +Keep the runtime's default 300-second activity-task lease. Run +`php /consumer/client.php long`. The callback blocks for 360 seconds without +an authored activity timeout or application heartbeat; original run/execution +limits are 420 seconds. The result must retain the exact int64/null/Unicode +values, one invocation and one attempt. Stop the Worker with SIGTERM after +completion and inspect its actual exit status. + +## Callback-owner loss + +Use an explicitly configured five-second lease and a named namespace. Namespace +setup uses control-plane version 2. Start `client.php loss` with a fresh workflow +ID and synchronize the fault with the first invocation record. Its result wait +is 30 seconds, with the same 420-second business limits and declared two attempts. + +Keep the container alive while killing only the callback-owner PID from +`/state/worker.pid`. `helpers.php` identifies the independent maintainer. Within +two seconds it must be gone, and the original task's expiry must stop moving. +Start a replacement Worker with a different ID. After the lease expires, run +the ordinary, instance-scoped recovery command: + +```sh +php artisan workflow:v2:repair-pass --instance-id="$WORKFLOW_ID" --json +``` + +The retry must keep the original activity, two-attempt policy and exact typed +result. History must record `lease_expired`; no application heartbeat is added. +This short configured-lease case does not measure default worker-loss latency. + +## Durable audit + +`snapshot.php` opens the default Server SQLite database read-only and records +bounded state plus hashes of history rows. Capture a prefix before the fault +and a final snapshot after stopping the API. `verify-long.php` and +`verify-loss.php` check the evidence names used in the recorded result when +mounted at `/evidence`, including helper exit, fixed expiry, original/retry +binding, heartbeat absence and unchanged history. Preserve every failed +original separately; a later original does not rescue it. Retire all task +containers, databases, images, installed packages and raw evidence afterward. diff --git a/tests/Fixtures/PhpActivityLeaseConsumer/client.php b/tests/Fixtures/PhpActivityLeaseConsumer/client.php new file mode 100644 index 000000000..61419d7cd --- /dev/null +++ b/tests/Fixtures/PhpActivityLeaseConsumer/client.php @@ -0,0 +1,31 @@ +startWorkflow('qualification.runtime-lease', $workflowId, + 'lease-qualification', [360, $mode], executionTimeoutSeconds: 420, runTimeoutSeconds: 420); +file_put_contents('/state/started.json', json_encode([ + 'workflow_id' => $handle->workflowId, 'run_id' => $handle->selectedRunId, + 'mode' => $mode, 'monotonic_ns' => $started, +], JSON_THROW_ON_ERROR)); +$result = $handle->result($mode === 'long' ? 420 : 30); +if ($result !== [9007199254740993, null, 'λ']) { + throw new RuntimeException('The published SDK returned a different typed result.'); +} +$elapsed = (hrtime(true) - $started) / 1e9; +if ($mode === 'long' && $elapsed < 360) { + throw new RuntimeException('The real-duration control finished before six minutes.'); +} +echo json_encode(['mode' => $mode, 'elapsed_seconds' => $elapsed, + 'run_id' => $handle->selectedRunId, 'result' => $result], JSON_THROW_ON_ERROR | JSON_UNESCAPED_UNICODE)."\n"; diff --git a/tests/Fixtures/PhpActivityLeaseConsumer/composer.json b/tests/Fixtures/PhpActivityLeaseConsumer/composer.json new file mode 100644 index 000000000..e72a8a153 --- /dev/null +++ b/tests/Fixtures/PhpActivityLeaseConsumer/composer.json @@ -0,0 +1,9 @@ +{ + "name": "qualification/server-activity-lease", + "require": { + "durable-workflow/sdk": "3.2.0" + }, + "config": { + "allow-plugins": false + } +} diff --git a/tests/Fixtures/PhpActivityLeaseConsumer/composer.lock b/tests/Fixtures/PhpActivityLeaseConsumer/composer.lock new file mode 100644 index 000000000..858663986 --- /dev/null +++ b/tests/Fixtures/PhpActivityLeaseConsumer/composer.lock @@ -0,0 +1,1035 @@ +{ + "_readme": [ + "This file locks the dependencies of your project to a known state", + "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", + "This file is @generated automatically" + ], + "content-hash": "683f517f1349bbc497bd6e9d049a8651", + "packages": [ + { + "name": "apache/avro", + "version": "1.12.2", + "source": { + "type": "git", + "url": "https://github.com/apache/avro.git", + "reference": "8fa2067f70e3012cb3fd9a8839cd97e8c7cc1772" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/apache/avro/zipball/8fa2067f70e3012cb3fd9a8839cd97e8c7cc1772", + "reference": "8fa2067f70e3012cb3fd9a8839cd97e8c7cc1772", + "shasum": "" + }, + "require": { + "php": "^7.1 || ^8.0" + }, + "require-dev": { + "ext-curl": "*", + "ext-json": "*", + "ext-pcntl": "*", + "ext-xml": "*", + "php-mock/php-mock-phpunit": "^2.10", + "phpunit/phpunit": "^7.5 || ^8.5 || ^9.5", + "squizlabs/php_codesniffer": "3.*" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "1.0.x-dev" + } + }, + "autoload": { + "psr-4": { + "Apache\\Avro\\": "lang/php/lib/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "Apache-2.0" + ], + "authors": [ + { + "name": "Apache Avro Developers", + "email": "dev@avro.apache.org", + "homepage": "http://avro.apache.org" + } + ], + "description": "Apache Avro™ is a data serialization system.", + "homepage": "http://avro.apache.org", + "keywords": [ + "avro", + "data", + "serialization" + ], + "support": { + "email": "dev@avro.apache.org", + "issues": "https://issues.apache.org/jira/browse/AVRO", + "source": "https://github.com/apache/avro/tree/release-1.12.2" + }, + "time": "2026-08-07T12:56:37+00:00" + }, + { + "name": "durable-workflow/sdk", + "version": "3.2.0", + "source": { + "type": "git", + "url": "https://github.com/durable-workflow/sdk-php.git", + "reference": "dfaae75a94b1d855379863799ce2c9a53f900f97" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/durable-workflow/sdk-php/zipball/dfaae75a94b1d855379863799ce2c9a53f900f97", + "reference": "dfaae75a94b1d855379863799ce2c9a53f900f97", + "shasum": "" + }, + "require": { + "apache/avro": "^1.12", + "composer-runtime-api": "^2.0", + "ext-json": "*", + "guzzlehttp/guzzle": "^7.8|^8.0", + "php": "^8.1", + "psr/container": "^1.1|^2.0", + "psr/http-client": "^1.0", + "psr/http-factory": "^1.0", + "psr/http-message": "^1.1|^2.0", + "psr/log": "^1.1|^2.0|^3.0" + }, + "conflict": { + "open-telemetry/api": "<1.10.1 || >=2.0" + }, + "require-dev": { + "monolog/monolog": "^2.9|^3.0", + "open-telemetry/api": "1.10.1", + "open-telemetry/sdk": "^1.15", + "phpdocumentor/phpdocumentor": "^3.7", + "phpstan/phpstan": "^2.0", + "phpunit/phpunit": "^10.5|^11.0" + }, + "suggest": { + "ext-pcntl": "Required by framework console workers for graceful SIGINT and SIGTERM handling.", + "laravel/framework": "Enables the auto-discovered Laravel 9 through 13 service-mode bridge.", + "open-telemetry/api": "Optional worker tracing with the application's provider; supports API 1.10.1 and later in the 1.x line.", + "symfony/console": "Required with symfony/framework-bundle to run the Symfony 6.4, 7, and 8 worker command.", + "symfony/framework-bundle": "Enables the Symfony 6.4, 7, and 8 service-mode bundle." + }, + "type": "library", + "extra": { + "laravel": { + "aliases": { + "DurableWorkflow": "DurableWorkflow\\Bridge\\Laravel\\Facades\\DurableWorkflow" + }, + "providers": [ + "DurableWorkflow\\Bridge\\Laravel\\DurableWorkflowServiceProvider" + ] + }, + "durable-workflow": { + "frameworks": { + "laravel": "^9.0|^10.0|^11.0|^12.0|^13.0", + "symfony": "^6.4|^7.0|^8.0" + }, + "product-train": "3.2.0", + "payload-codecs": [ + "avro" + ], + "message-streams": true, + "version-markers": true, + "workflow-streams": { + "contract": "durable-workflow.v2.workflow-streams.contract@1", + "delivery": "at-least-once", + "operations": [ + "list", + "describe", + "subscribe", + "append", + "close", + "error" + ], + "authoring-command": "record_side_effect.workflow_stream", + "external-payload-references": "opaque-reference" + }, + "durable-selection": true, + "control-plane-version": "2", + "version-marker-command": "record_version_marker", + "version-marker-helpers": [ + "patched", + "deprecatePatch" + ], + "worker-protocol-version": "1.20", + "supported-server-versions": "2.5.0", + "version-marker-history-event": "VersionMarkerRecorded", + "message-streams-minimum-worker-protocol-version": "1.15", + "durable-selection-minimum-worker-protocol-version": "1.19" + } + }, + "autoload": { + "psr-4": { + "DurableWorkflow\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Durable Workflow Contributors" + } + ], + "description": "PHP client and worker SDK for Durable Workflow Cloud and self-hosted Server", + "homepage": "https://php.durable-workflow.com/", + "keywords": [ + "cloud", + "durable", + "laravel", + "orchestration", + "php", + "sdk", + "symfony", + "worker", + "workflow" + ], + "support": { + "docs": "https://php.durable-workflow.com/", + "issues": "https://github.com/durable-workflow/sdk-php/issues", + "source": "https://github.com/durable-workflow/sdk-php" + }, + "time": "2026-10-21T16:52:26+00:00" + }, + { + "name": "guzzlehttp/guzzle", + "version": "8.3.0", + "source": { + "type": "git", + "url": "https://github.com/guzzle/guzzle.git", + "reference": "b920e5fcaf81d761b8fabe600ca90999beaefdb7" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/guzzle/guzzle/zipball/b920e5fcaf81d761b8fabe600ca90999beaefdb7", + "reference": "b920e5fcaf81d761b8fabe600ca90999beaefdb7", + "shasum": "" + }, + "require": { + "ext-json": "*", + "guzzlehttp/promises": "^3.1", + "guzzlehttp/psr7": "^3.2", + "php": "^7.4 || ^8.0", + "psr/http-client": "^1.0", + "psr/http-factory": "^1.0", + "symfony/polyfill-php80": "^1.25", + "symfony/polyfill-php82": "^1.27" + }, + "provide": { + "psr/http-client-implementation": "1.0" + }, + "require-dev": { + "bamarni/composer-bin-plugin": "^1.8.2", + "ext-curl": "*", + "guzzle/client-integration-tests": "4.0.1", + "guzzlehttp/test-server": "^1.1", + "php-http/message-factory": "^1.1", + "phpunit/phpunit": "^9.6.34", + "psr/log": "^1.1 || ^2.0 || ^3.0" + }, + "suggest": { + "ext-curl": "Required for CURL handler support", + "ext-intl": "Required for Internationalized Domain Name (IDN) support", + "psr/log": "Required for using the Log middleware" + }, + "type": "library", + "extra": { + "bamarni-bin": { + "bin-links": true, + "forward-command": false + } + }, + "autoload": { + "psr-4": { + "GuzzleHttp\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Graham Campbell", + "email": "hello@gjcampbell.co.uk", + "homepage": "https://github.com/GrahamCampbell" + }, + { + "name": "Michael Dowling", + "email": "mtdowling@gmail.com", + "homepage": "https://github.com/mtdowling" + }, + { + "name": "Jeremy Lindblom", + "email": "jeremeamia@gmail.com", + "homepage": "https://github.com/jeremeamia" + }, + { + "name": "George Mponos", + "email": "gmponos@gmail.com", + "homepage": "https://github.com/gmponos" + }, + { + "name": "Tobias Nyholm", + "email": "tobias.nyholm@gmail.com", + "homepage": "https://github.com/Nyholm" + }, + { + "name": "Márk Sági-Kazár", + "email": "mark.sagikazar@gmail.com", + "homepage": "https://github.com/sagikazarmark" + }, + { + "name": "Tobias Schultze", + "email": "webmaster@tubo-world.de", + "homepage": "https://github.com/Tobion" + } + ], + "description": "Guzzle is a PHP HTTP client library", + "keywords": [ + "client", + "curl", + "framework", + "http", + "http client", + "psr-18", + "psr-7", + "rest", + "web service" + ], + "support": { + "issues": "https://github.com/guzzle/guzzle/issues", + "source": "https://github.com/guzzle/guzzle/tree/8.3.0" + }, + "funding": [ + { + "url": "https://github.com/GrahamCampbell", + "type": "github" + }, + { + "url": "https://github.com/Nyholm", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/guzzlehttp/guzzle", + "type": "tidelift" + } + ], + "time": "2026-10-18T13:38:43+00:00" + }, + { + "name": "guzzlehttp/promises", + "version": "3.1.0", + "source": { + "type": "git", + "url": "https://github.com/guzzle/promises.git", + "reference": "094d0ec418eb3b1411ea271a180ec95cae6ae933" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/guzzle/promises/zipball/094d0ec418eb3b1411ea271a180ec95cae6ae933", + "reference": "094d0ec418eb3b1411ea271a180ec95cae6ae933", + "shasum": "" + }, + "require": { + "php": "^7.4 || ^8.0", + "symfony/deprecation-contracts": "^2.5 || ^3.0" + }, + "require-dev": { + "bamarni/composer-bin-plugin": "^1.8.2", + "phpunit/phpunit": "^9.6.34" + }, + "type": "library", + "extra": { + "bamarni-bin": { + "bin-links": true, + "forward-command": false + } + }, + "autoload": { + "psr-4": { + "GuzzleHttp\\Promise\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Graham Campbell", + "email": "hello@gjcampbell.co.uk", + "homepage": "https://github.com/GrahamCampbell" + }, + { + "name": "Michael Dowling", + "email": "mtdowling@gmail.com", + "homepage": "https://github.com/mtdowling" + }, + { + "name": "Tobias Nyholm", + "email": "tobias.nyholm@gmail.com", + "homepage": "https://github.com/Nyholm" + }, + { + "name": "Tobias Schultze", + "email": "webmaster@tubo-world.de", + "homepage": "https://github.com/Tobion" + } + ], + "description": "Guzzle promises library", + "keywords": [ + "promise" + ], + "support": { + "issues": "https://github.com/guzzle/promises/issues", + "source": "https://github.com/guzzle/promises/tree/3.1.0" + }, + "funding": [ + { + "url": "https://github.com/GrahamCampbell", + "type": "github" + }, + { + "url": "https://github.com/Nyholm", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/guzzlehttp/promises", + "type": "tidelift" + } + ], + "time": "2026-10-18T12:48:16+00:00" + }, + { + "name": "guzzlehttp/psr7", + "version": "3.2.0", + "source": { + "type": "git", + "url": "https://github.com/guzzle/psr7.git", + "reference": "cdc139396207a5eba10c242d241fb014cb098390" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/guzzle/psr7/zipball/cdc139396207a5eba10c242d241fb014cb098390", + "reference": "cdc139396207a5eba10c242d241fb014cb098390", + "shasum": "" + }, + "require": { + "php": "^7.4 || ^8.0", + "psr/http-factory": "^1.1", + "psr/http-message": "^2.0", + "symfony/polyfill-php80": "^1.25", + "symfony/polyfill-php82": "^1.27" + }, + "provide": { + "psr/http-factory-implementation": "1.1", + "psr/http-message-implementation": "2.0" + }, + "require-dev": { + "bamarni/composer-bin-plugin": "^1.8.2", + "http-interop/http-factory-tests": "1.1.0", + "jshttp/mime-db": "1.54.0.1", + "php-http/psr7-integration-tests": "^1.5.1", + "phpunit/phpunit": "^9.6.34" + }, + "suggest": { + "laminas/laminas-httphandlerrunner": "Emit PSR-7 responses" + }, + "type": "library", + "extra": { + "bamarni-bin": { + "bin-links": true, + "forward-command": false + } + }, + "autoload": { + "psr-4": { + "GuzzleHttp\\Psr7\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Graham Campbell", + "email": "hello@gjcampbell.co.uk", + "homepage": "https://github.com/GrahamCampbell" + }, + { + "name": "Michael Dowling", + "email": "mtdowling@gmail.com", + "homepage": "https://github.com/mtdowling" + }, + { + "name": "George Mponos", + "email": "gmponos@gmail.com", + "homepage": "https://github.com/gmponos" + }, + { + "name": "Tobias Nyholm", + "email": "tobias.nyholm@gmail.com", + "homepage": "https://github.com/Nyholm" + }, + { + "name": "Márk Sági-Kazár", + "email": "mark.sagikazar@gmail.com", + "homepage": "https://github.com/sagikazarmark" + }, + { + "name": "Tobias Schultze", + "email": "webmaster@tubo-world.de", + "homepage": "https://github.com/Tobion" + }, + { + "name": "Márk Sági-Kazár", + "email": "mark.sagikazar@gmail.com", + "homepage": "https://sagikazarmark.hu" + } + ], + "description": "PSR-7 message implementation that also provides common utility methods", + "keywords": [ + "http", + "message", + "psr-7", + "request", + "response", + "stream", + "uri", + "url" + ], + "support": { + "issues": "https://github.com/guzzle/psr7/issues", + "source": "https://github.com/guzzle/psr7/tree/3.2.0" + }, + "funding": [ + { + "url": "https://github.com/GrahamCampbell", + "type": "github" + }, + { + "url": "https://github.com/Nyholm", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/guzzlehttp/psr7", + "type": "tidelift" + } + ], + "time": "2026-10-17T23:57:19+00:00" + }, + { + "name": "psr/container", + "version": "2.0.2", + "source": { + "type": "git", + "url": "https://github.com/php-fig/container.git", + "reference": "c71ecc56dfe541dbd90c5360474fbc405f8d5963" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/php-fig/container/zipball/c71ecc56dfe541dbd90c5360474fbc405f8d5963", + "reference": "c71ecc56dfe541dbd90c5360474fbc405f8d5963", + "shasum": "" + }, + "require": { + "php": ">=7.4.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "2.0.x-dev" + } + }, + "autoload": { + "psr-4": { + "Psr\\Container\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "PHP-FIG", + "homepage": "https://www.php-fig.org/" + } + ], + "description": "Common Container Interface (PHP FIG PSR-11)", + "homepage": "https://github.com/php-fig/container", + "keywords": [ + "PSR-11", + "container", + "container-interface", + "container-interop", + "psr" + ], + "support": { + "issues": "https://github.com/php-fig/container/issues", + "source": "https://github.com/php-fig/container/tree/2.0.2" + }, + "time": "2021-11-05T16:47:00+00:00" + }, + { + "name": "psr/http-client", + "version": "1.0.3", + "source": { + "type": "git", + "url": "https://github.com/php-fig/http-client.git", + "reference": "bb5906edc1c324c9a05aa0873d40117941e5fa90" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/php-fig/http-client/zipball/bb5906edc1c324c9a05aa0873d40117941e5fa90", + "reference": "bb5906edc1c324c9a05aa0873d40117941e5fa90", + "shasum": "" + }, + "require": { + "php": "^7.0 || ^8.0", + "psr/http-message": "^1.0 || ^2.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "1.0.x-dev" + } + }, + "autoload": { + "psr-4": { + "Psr\\Http\\Client\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "PHP-FIG", + "homepage": "https://www.php-fig.org/" + } + ], + "description": "Common interface for HTTP clients", + "homepage": "https://github.com/php-fig/http-client", + "keywords": [ + "http", + "http-client", + "psr", + "psr-18" + ], + "support": { + "source": "https://github.com/php-fig/http-client" + }, + "time": "2023-09-23T14:17:50+00:00" + }, + { + "name": "psr/http-factory", + "version": "1.1.0", + "source": { + "type": "git", + "url": "https://github.com/php-fig/http-factory.git", + "reference": "2b4765fddfe3b508ac62f829e852b1501d3f6e8a" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/php-fig/http-factory/zipball/2b4765fddfe3b508ac62f829e852b1501d3f6e8a", + "reference": "2b4765fddfe3b508ac62f829e852b1501d3f6e8a", + "shasum": "" + }, + "require": { + "php": ">=7.1", + "psr/http-message": "^1.0 || ^2.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "1.0.x-dev" + } + }, + "autoload": { + "psr-4": { + "Psr\\Http\\Message\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "PHP-FIG", + "homepage": "https://www.php-fig.org/" + } + ], + "description": "PSR-17: Common interfaces for PSR-7 HTTP message factories", + "keywords": [ + "factory", + "http", + "message", + "psr", + "psr-17", + "psr-7", + "request", + "response" + ], + "support": { + "source": "https://github.com/php-fig/http-factory" + }, + "time": "2024-04-15T12:06:14+00:00" + }, + { + "name": "psr/http-message", + "version": "2.0", + "source": { + "type": "git", + "url": "https://github.com/php-fig/http-message.git", + "reference": "402d35bcb92c70c026d1a6a9883f06b2ead23d71" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/php-fig/http-message/zipball/402d35bcb92c70c026d1a6a9883f06b2ead23d71", + "reference": "402d35bcb92c70c026d1a6a9883f06b2ead23d71", + "shasum": "" + }, + "require": { + "php": "^7.2 || ^8.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "2.0.x-dev" + } + }, + "autoload": { + "psr-4": { + "Psr\\Http\\Message\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "PHP-FIG", + "homepage": "https://www.php-fig.org/" + } + ], + "description": "Common interface for HTTP messages", + "homepage": "https://github.com/php-fig/http-message", + "keywords": [ + "http", + "http-message", + "psr", + "psr-7", + "request", + "response" + ], + "support": { + "source": "https://github.com/php-fig/http-message/tree/2.0" + }, + "time": "2023-04-04T09:54:51+00:00" + }, + { + "name": "psr/log", + "version": "3.0.2", + "source": { + "type": "git", + "url": "https://github.com/php-fig/log.git", + "reference": "f16e1d5863e37f8d8c2a01719f5b34baa2b714d3" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/php-fig/log/zipball/f16e1d5863e37f8d8c2a01719f5b34baa2b714d3", + "reference": "f16e1d5863e37f8d8c2a01719f5b34baa2b714d3", + "shasum": "" + }, + "require": { + "php": ">=8.0.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "3.x-dev" + } + }, + "autoload": { + "psr-4": { + "Psr\\Log\\": "src" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "PHP-FIG", + "homepage": "https://www.php-fig.org/" + } + ], + "description": "Common interface for logging libraries", + "homepage": "https://github.com/php-fig/log", + "keywords": [ + "log", + "psr", + "psr-3" + ], + "support": { + "source": "https://github.com/php-fig/log/tree/3.0.2" + }, + "time": "2024-09-11T13:17:53+00:00" + }, + { + "name": "symfony/deprecation-contracts", + "version": "v3.7.1", + "source": { + "type": "git", + "url": "https://github.com/symfony/deprecation-contracts.git", + "reference": "f3202fa1b5097b0af062dc978b32ecf63404e31d" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/deprecation-contracts/zipball/f3202fa1b5097b0af062dc978b32ecf63404e31d", + "reference": "f3202fa1b5097b0af062dc978b32ecf63404e31d", + "shasum": "" + }, + "require": { + "php": ">=8.1" + }, + "type": "library", + "extra": { + "thanks": { + "url": "https://github.com/symfony/contracts", + "name": "symfony/contracts" + }, + "branch-alias": { + "dev-main": "3.7-dev" + } + }, + "autoload": { + "files": [ + "function.php" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Nicolas Grekas", + "email": "p@tchwork.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "A generic function and convention to trigger deprecation notices", + "homepage": "https://symfony.com", + "support": { + "source": "https://github.com/symfony/deprecation-contracts/tree/v3.7.1" + }, + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://github.com/nicolas-grekas", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], + "time": "2026-06-05T06:23:12+00:00" + }, + { + "name": "symfony/polyfill-php80", + "version": "v1.43.0", + "source": { + "type": "git", + "url": "https://github.com/symfony/polyfill-php80.git", + "reference": "9c6a5d6b01ca51d486c813e9a9c0ed55f208bb74" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/polyfill-php80/zipball/9c6a5d6b01ca51d486c813e9a9c0ed55f208bb74", + "reference": "9c6a5d6b01ca51d486c813e9a9c0ed55f208bb74", + "shasum": "" + }, + "require": { + "php": ">=7.2" + }, + "type": "library", + "extra": { + "thanks": { + "url": "https://github.com/symfony/polyfill", + "name": "symfony/polyfill" + } + }, + "autoload": { + "files": [ + "bootstrap.php" + ], + "psr-4": { + "Symfony\\Polyfill\\Php80\\": "" + }, + "classmap": [ + "Resources/stubs" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Ion Bazan", + "email": "ion.bazan@gmail.com" + }, + { + "name": "Nicolas Grekas", + "email": "p@tchwork.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "Symfony polyfill backporting some PHP 8.0+ features to lower PHP versions", + "homepage": "https://symfony.com", + "keywords": [ + "compatibility", + "polyfill", + "portable", + "shim" + ], + "support": { + "source": "https://github.com/symfony/polyfill-php80/tree/v1.43.0" + }, + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://github.com/nicolas-grekas", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], + "time": "2026-09-25T15:50:05+00:00" + }, + { + "name": "symfony/polyfill-php82", + "version": "v1.43.0", + "source": { + "type": "git", + "url": "https://github.com/symfony/polyfill-php82.git", + "reference": "843bbe8b2a7934307b749751eede0f111f5d1cfc" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/polyfill-php82/zipball/843bbe8b2a7934307b749751eede0f111f5d1cfc", + "reference": "843bbe8b2a7934307b749751eede0f111f5d1cfc", + "shasum": "" + }, + "require": { + "php": ">=7.2" + }, + "type": "library", + "extra": { + "thanks": { + "url": "https://github.com/symfony/polyfill", + "name": "symfony/polyfill" + } + }, + "autoload": { + "files": [ + "bootstrap.php" + ], + "psr-4": { + "Symfony\\Polyfill\\Php82\\": "" + }, + "classmap": [ + "Resources/stubs" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Nicolas Grekas", + "email": "p@tchwork.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "Symfony polyfill backporting some PHP 8.2+ features to lower PHP versions", + "homepage": "https://symfony.com", + "keywords": [ + "compatibility", + "polyfill", + "portable", + "shim" + ], + "support": { + "source": "https://github.com/symfony/polyfill-php82/tree/v1.43.0" + }, + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://github.com/nicolas-grekas", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], + "time": "2026-09-25T15:50:05+00:00" + } + ], + "packages-dev": [], + "aliases": [], + "minimum-stability": "stable", + "stability-flags": {}, + "prefer-stable": false, + "prefer-lowest": false, + "platform": {}, + "platform-dev": {}, + "plugin-api-version": "2.9.0" +} diff --git a/tests/Fixtures/PhpActivityLeaseConsumer/helpers.php b/tests/Fixtures/PhpActivityLeaseConsumer/helpers.php new file mode 100644 index 000000000..9133fc121 --- /dev/null +++ b/tests/Fixtures/PhpActivityLeaseConsumer/helpers.php @@ -0,0 +1,12 @@ + $helpers, 'count' => count($helpers)])."\n"; diff --git a/tests/Fixtures/PhpActivityLeaseConsumer/snapshot.php b/tests/Fixtures/PhpActivityLeaseConsumer/snapshot.php new file mode 100644 index 000000000..44dbdf8c9 --- /dev/null +++ b/tests/Fixtures/PhpActivityLeaseConsumer/snapshot.php @@ -0,0 +1,26 @@ + PDO::ERRMODE_EXCEPTION, + PDO::SQLITE_ATTR_OPEN_FLAGS => PDO::SQLITE_OPEN_READONLY, +]); +$db->exec('PRAGMA busy_timeout=1000'); +$runs = $db->query('SELECT id, workflow_instance_id, namespace, status FROM workflow_runs ORDER BY created_at') + ->fetchAll(PDO::FETCH_ASSOC); +$tasks = $db->query('SELECT id, workflow_run_id, task_type, status, attempt_count, repair_count, leased_at, lease_expires_at FROM workflow_tasks ORDER BY created_at') + ->fetchAll(PDO::FETCH_ASSOC); +$events = []; +foreach ($db->query('SELECT * FROM workflow_history_events ORDER BY workflow_run_id, sequence')->fetchAll(PDO::FETCH_ASSOC) as $event) { + $events[] = ['id' => $event['id'], 'run_id' => $event['workflow_run_id'], + 'sequence' => $event['sequence'], 'event_type' => $event['event_type'], + 'sha256' => hash('sha256', serialize($event))]; +} +$activities = $db->query('SELECT id, workflow_run_id, status, attempt_count, current_attempt_id, started_at, closed_at, last_heartbeat_at FROM activity_executions ORDER BY created_at') + ->fetchAll(PDO::FETCH_ASSOC); +$attempts = $db->query('SELECT id, workflow_run_id, activity_execution_id, attempt_number, status, lease_expires_at, last_heartbeat_at FROM activity_attempts ORDER BY created_at') + ->fetchAll(PDO::FETCH_ASSOC); +echo json_encode(['at' => gmdate('c'), 'runs' => $runs, 'tasks' => $tasks, + 'events' => $events, 'activities' => $activities, 'attempts' => $attempts, + 'failed_jobs' => (int) $db->query('SELECT COUNT(*) FROM failed_jobs')->fetchColumn()], JSON_THROW_ON_ERROR)."\n"; diff --git a/tests/Fixtures/PhpActivityLeaseConsumer/verify-long.php b/tests/Fixtures/PhpActivityLeaseConsumer/verify-long.php new file mode 100644 index 000000000..837a81a98 --- /dev/null +++ b/tests/Fixtures/PhpActivityLeaseConsumer/verify-long.php @@ -0,0 +1,47 @@ + json_decode($line, true, flags: JSON_THROW_ON_ERROR), + file('/evidence/state-long/invocations.jsonl', FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES)); +if ($result['result'] !== [9007199254740993, null, 'λ'] || $result['elapsed_seconds'] < 360 + || count($invocations) !== 1 || $invocations[0]['attempt'] !== 1) { + throw new RuntimeException('The real-duration activity/result contract failed.'); +} +if (count($final['runs']) !== 1 || $final['runs'][0]['status'] !== 'completed' + || $final['runs'][0]['namespace'] !== 'default' || count($final['tasks']) !== 3) { + throw new RuntimeException('The original workflow or task set did not complete.'); +} +foreach ($final['tasks'] as $task) { + if ($task['status'] !== 'completed' || $task['attempt_count'] !== 1 || $task['repair_count'] !== 0) { + throw new RuntimeException('An original task retried or required repair.'); + } +} +foreach (['activities', 'attempts'] as $key) { + if (count($final[$key]) !== 1 || $final[$key][0]['status'] !== 'completed' + || $final[$key][0]['last_heartbeat_at'] !== null) { + throw new RuntimeException('Activity cardinality, completion or heartbeat contract failed.'); + } +} +$counts = array_count_values(array_column($final['events'], 'event_type')); +foreach (['ActivityStarted', 'ActivityCompleted', 'WorkflowCompleted'] as $name) { + if (($counts[$name] ?? 0) !== 1) { + throw new RuntimeException('Unexpected event cardinality: '.$name); + } +} +if (($counts['ActivityHeartbeatRecorded'] ?? 0) !== 0 || $final['failed_jobs'] !== 0) { + throw new RuntimeException('Runtime maintenance produced progress events or failed jobs.'); +} +$finalById = array_column($final['events'], null, 'id'); +foreach ($prefix['events'] as $event) { + if (($finalById[$event['id']]['sha256'] ?? null) !== $event['sha256']) { + throw new RuntimeException('Previously recorded history bytes changed.'); + } +} +echo json_encode(['status' => 'PASS', 'elapsed_seconds' => $result['elapsed_seconds'], + 'tasks' => count($final['tasks']), 'events' => $counts, + 'preserved_history_rows' => count($prefix['events']), 'invocations' => count($invocations), + 'application_heartbeats' => 0, 'repairs' => 0], JSON_THROW_ON_ERROR)."\n"; diff --git a/tests/Fixtures/PhpActivityLeaseConsumer/verify-loss.php b/tests/Fixtures/PhpActivityLeaseConsumer/verify-loss.php new file mode 100644 index 000000000..a4ca3103e --- /dev/null +++ b/tests/Fixtures/PhpActivityLeaseConsumer/verify-loss.php @@ -0,0 +1,77 @@ + array_values(array_filter($rows, + static fn (array $row): bool => ($row['workflow_run_id'] ?? $row['run_id'] ?? null) === $runId)); +$run = array_values(array_filter($final['runs'], static fn (array $row): bool => $row['id'] === $runId)); +if ($result['result'] !== [9007199254740993, null, 'λ'] || $result['elapsed_seconds'] >= 30 + || count($run) !== 1 || $run[0]['status'] !== 'completed' || $run[0]['namespace'] !== 'lease-loss') { + throw new RuntimeException('The bounded worker-loss result contract failed.'); +} +$invocations = array_map(static fn (string $line): array => json_decode($line, true, flags: JSON_THROW_ON_ERROR), + file('/evidence/state-loss2/invocations.jsonl', FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES)); +if (array_column($invocations, 'attempt') !== [1, 2] + || readEvidence('loss2-helpers-before.json')['count'] !== 1 + || readEvidence('loss2-helpers-after.json')['count'] !== 0) { + throw new RuntimeException('Owner/helper lifecycle or invocation identity failed.'); +} +$taskId = $invocations[0]['task_id']; +$beforeTask = array_column($before['tasks'], null, 'id')[$taskId]; +$afterTask = array_column($after['tasks'], null, 'id')[$taskId]; +if ($beforeTask['lease_expires_at'] !== $afterTask['lease_expires_at']) { + throw new RuntimeException('The killed owner retained lease-renewal authority.'); +} +$attempts = $forRun($final['attempts']); +if (array_column($attempts, 'attempt_number') !== [1, 2] + || array_column($attempts, 'status') !== ['expired', 'completed'] + || count(array_unique(array_column($attempts, 'activity_execution_id'))) !== 1) { + throw new RuntimeException('Expiry/retry did not preserve the original activity.'); +} +foreach ($attempts as $attempt) { + if ($attempt['last_heartbeat_at'] !== null) { + throw new RuntimeException('The control recorded an application heartbeat.'); + } +} +$events = $forRun($final['events']); +$counts = array_count_values(array_column($events, 'event_type')); +if (($counts['ActivityStarted'] ?? 0) !== 2 || ($counts['ActivityCompleted'] ?? 0) !== 1 + || ($counts['ActivityRetryScheduled'] ?? 0) !== 1 || ($counts['WorkflowCompleted'] ?? 0) !== 1 + || ($counts['ActivityHeartbeatRecorded'] ?? 0) !== 0 || $final['failed_jobs'] !== 0) { + throw new RuntimeException('The durable retry outcomes or cardinality failed.'); +} +$retry = $forRun(readEvidence('retries.json')); +if (count($retry) !== 1 || $retry[0]['payload']['failure_reason'] !== 'lease_expired' + || $retry[0]['payload']['retry_of_task_id'] !== $invocations[0]['task_id'] + || $retry[0]['payload']['retry_task_id'] !== $invocations[1]['task_id'] + || $retry[0]['payload']['retry_policy']['max_attempts'] !== 2) { + throw new RuntimeException('The retry lost the original task/policy binding.'); +} +foreach ($forRun($final['tasks']) as $task) { + if ($task['repair_count'] !== 0) { + throw new RuntimeException('An unrelated task repair occurred.'); + } +} +$byId = array_column($events, null, 'id'); +foreach ($forRun($before['events']) as $event) { + if (($byId[$event['id']]['sha256'] ?? null) !== $event['sha256']) { + throw new RuntimeException('Previously recorded history changed.'); + } +} +$ready = readEvidence('loss-readiness.json'); +if ($ready['topology']['matching_role']['discovery_limits']['activity_task_lease_seconds'] !== 5) { + throw new RuntimeException('The runtime did not use the configured five-second lease.'); +} +echo json_encode(['status' => 'PASS', 'elapsed_seconds' => $result['elapsed_seconds'], + 'namespace' => $run[0]['namespace'], 'attempts' => [1, 2], 'expired_owner_helper_count' => 0, + 'failure_reason' => $retry[0]['payload']['failure_reason'], 'events' => $counts, + 'preserved_history_rows' => count($forRun($before['events'])), 'application_heartbeats' => 0], JSON_THROW_ON_ERROR)."\n"; diff --git a/tests/Fixtures/PhpActivityLeaseConsumer/worker.php b/tests/Fixtures/PhpActivityLeaseConsumer/worker.php new file mode 100644 index 000000000..77cc615fa --- /dev/null +++ b/tests/Fixtures/PhpActivityLeaseConsumer/worker.php @@ -0,0 +1,66 @@ +activity('qualification.blocking', [$seconds, $mode], [ + 'retry_policy' => ['max_attempts' => 2, 'backoff_seconds' => [0]], + ]); + } +} + +final class GeneralLeaseActivity +{ + #[Activity('qualification.blocking')] + public function run(ActivityContext $context, int $seconds, string $mode): array + { + file_put_contents('/state/invocations.jsonl', json_encode([ + 'mode' => $mode, + 'attempt' => $context->attemptNumber, + 'task_id' => $context->taskId, + 'activity_attempt_id' => $context->activityAttemptId, + 'pid' => getmypid(), + 'started_at' => gmdate('c'), + 'monotonic_ns' => hrtime(true), + ], JSON_THROW_ON_ERROR)."\n", FILE_APPEND | LOCK_EX); + if ($mode === 'long' || $context->attemptNumber === 1) { + sleep($seconds); + } + + return [9007199254740993, null, 'λ']; + } +} + +file_put_contents('/state/worker.pid', (string) getmypid()); +$client = new Client(getenv('SERVER_URL'), namespace: getenv('NAMESPACE') ?: 'default', + token: 'synthetic', workerProtocolVersion: '1.20'); +$worker = new Worker($client, 'lease-qualification', workerId: getenv('WORKER_ID') ?: 'lease-long', + diagnosticListener: static function (string $event, array $fields): void { + $record = ['event' => $event, 'at' => gmdate('c')]; + if (($fields['exception'] ?? null) instanceof Throwable) { + $record['exception'] = get_class($fields['exception']); + $record['message'] = substr($fields['exception']->getMessage(), 0, 512); + } + file_put_contents('/state/worker-events.jsonl', json_encode($record, JSON_THROW_ON_ERROR)."\n", + FILE_APPEND | LOCK_EX); + }); +$worker->register(GeneralLeaseWorkflow::class, GeneralLeaseActivity::class)->run(1); From d6ca57b179d503ee9de0bd95a664b0ffad5dbccf Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Wed, 21 Oct 2026 23:16:27 +0000 Subject: [PATCH 17/22] test: exercise application log streams in actual Server roles --- scripts/ci/verify-container-logging.mjs | 151 ++++++++++++++++++++++ tests/Fixtures/ContainerLogging/probe.php | 29 +++++ 2 files changed, 180 insertions(+) create mode 100644 scripts/ci/verify-container-logging.mjs create mode 100644 tests/Fixtures/ContainerLogging/probe.php diff --git a/scripts/ci/verify-container-logging.mjs b/scripts/ci/verify-container-logging.mjs new file mode 100644 index 000000000..87b0c5d2b --- /dev/null +++ b/scripts/ci/verify-container-logging.mjs @@ -0,0 +1,151 @@ +import assert from 'node:assert/strict'; +import {randomBytes} from 'node:crypto'; +import {execFileSync, spawnSync} from 'node:child_process'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import {fileURLToPath} from 'node:url'; + +const image = process.env.DW_SERVER_IMAGE; +assert.ok(image, 'DW_SERVER_IMAGE must identify the image under qualification'); +const scope = process.env.DW_LOGGING_SMOKE_SCOPE || `dw-log-${randomBytes(8).toString('hex')}`; +assert.match(scope, /^[A-Za-z0-9][A-Za-z0-9_.-]{0,63}$/); +const label = 'dev.durable-workflow.container-logging'; +const fixture = fileURLToPath(new URL('../../tests/Fixtures/ContainerLogging/probe.php', import.meta.url)); +const root = fs.mkdtempSync(path.join(process.env.RUNNER_TEMP || os.tmpdir(), `${scope}-`)); +const database = `${scope}-database`; +const ini = path.join(root, 'logging-probe.ini'); +fs.writeFileSync(ini, 'auto_prepend_file=/opt/container-logging-probe.php\n'); +const names = new Set(); +const failed = []; +const pause = milliseconds => new Promise(resolve => setTimeout(resolve, milliseconds)); + +function docker(args, timeout = 30000) { + return execFileSync('docker', args, {encoding: 'utf8', timeout, maxBuffer: 524288}).trim(); +} + +function owned(name) { + return docker(['inspect', '--format', `{{index .Config.Labels "${label}"}}`, name]) === scope; +} + +function remove(name) { + if (!names.has(name)) return; + assert.ok(owned(name), `refuse cleanup of foreign container ${name}`); + docker(['stop', '--time', '3', name]); + docker(['rm', name]); + names.delete(name); +} + +const common = [ + '--user', '1000:1000', '--cpus', '1', '--memory', '512m', '--memory-swap', '512m', + '--label', `${label}=${scope}`, + '--mount', `type=volume,source=${database},target=/app/database`, + '--env', 'DB_CONNECTION=sqlite', '--env', 'DB_DATABASE=/app/database/database.sqlite', + '--env', 'DB_SQLITE_JOURNAL_MODE=DELETE', '--env', 'CACHE_STORE=file', + '--env', 'QUEUE_CONNECTION=database', '--env', 'DW_AUTH_DRIVER=token', + '--env', 'DW_AUTH_TOKEN=container-logging-synthetic-token', +]; + +function copy(name, source, target) { + const result = spawnSync('docker', ['cp', `${name}:${source}`, target], { + encoding: 'utf8', timeout: 10000, maxBuffer: 524288, + }); + return result.status === 0 ? fs.readFileSync(target, 'utf8') : ''; +} + +async function check(role, mode) { + const name = `${scope}-${mode}-${role}`; + const nonce = randomBytes(16).toString('hex'); + const args = ['run', '--detach', '--name', name, ...common, + '--mount', `type=bind,source=${fixture},target=/opt/container-logging-probe.php,readonly`, + '--mount', `type=bind,source=${ini},target=/usr/local/etc/php/conf.d/zz-container-logging-probe.ini,readonly`, + '--env', `CONTAINER_LOGGING_PROBE_ROLE=${role}`, + '--env', `CONTAINER_LOGGING_PROBE_NONCE=${nonce}`]; + if (mode === 'json') args.push('--env', 'LOG_STDERR_FORMATTER=Monolog\\Formatter\\JsonFormatter'); + if (mode === 'file') args.push('--env', 'LOG_CHANNEL=single'); + if (mode === 'filtered') args.push('--env', 'LOG_LEVEL=critical'); + if (role === 'api') args.push('--publish', '127.0.0.1::8080'); + args.push(image); + if (role === 'worker') args.push('php', 'artisan', 'queue:work', '--once', '--stop-when-empty', '--max-time=2'); + if (role === 'scheduler') args.push('php', 'artisan', 'schedule:evaluate', '--limit=100', '--json'); + + assert.notEqual(spawnSync('docker', ['inspect', name], {stdio: 'ignore'}).status, 0, + `container name already exists: ${name}`); + docker(args); + names.add(name); + try { + if (role === 'api') { + const port = docker(['port', name, '8080/tcp']).split(':').at(-1); + let ready = false; + for (let attempt = 0; attempt < 100; attempt += 1) { + try { + const response = await fetch(`http://127.0.0.1:${port}/api/health`, {signal: AbortSignal.timeout(1000)}); + await response.text(); + if (response.status === 200) {ready = true; break;} + } catch { /* The real API may still be starting. */ } + await pause(100); + } + assert.ok(ready, 'API did not become live within its original probe budget'); + } else { + assert.equal(docker(['wait', name]), '0', `${role} command must exit successfully`); + } + + const proof = copy(name, '/tmp/dw-container-logging-probe', path.join(root, `${mode}-${role}.proof`)); + assert.equal(proof, nonce, `${role}: actual role must invoke the default Laravel logger`); + const logs = spawnSync('docker', ['logs', name], {encoding: 'utf8', timeout: 10000, maxBuffer: 524288}); + assert.equal(logs.status, 0); + const file = copy(name, '/app/storage/logs/laravel.log', path.join(root, `${mode}-${role}.log`)); + const stderr = logs.stderr.split('\n').filter(line => line.includes(nonce)); + assert.ok(!logs.stdout.includes(nonce), `${role}: application error must not go to stdout`); + if (mode === 'file') { + assert.ok(file.includes(nonce), 'operator-selected file channel must be honored'); + assert.equal(stderr.length, 0); + } else if (mode === 'filtered') { + assert.equal(stderr.length, 0, 'LOG_LEVEL must filter the emitted error'); + assert.ok(!file.includes(nonce)); + } else { + assert.ok(stderr.length > 0, `${role}/${mode}: application error missing from container stderr`); + assert.ok(!file.includes(nonce), `${role}: default stderr must not duplicate the error into a local file`); + if (mode === 'json') { + for (const line of stderr) { + const record = JSON.parse(line); + assert.equal(record.message, 'container-logging-probe'); + assert.equal(record.context.role, role); + assert.equal(record.context.nonce, nonce); + } + } + } + console.log(JSON.stringify({role, mode, outcome: 'PASS'})); + } finally { + remove(name); + } +} + +async function main() { +try { + docker(['volume', 'create', '--label', `${label}=${scope}`, database]); + const bootstrap = `${scope}-bootstrap`; + docker(['run', '--detach', '--name', bootstrap, ...common, image, 'server-bootstrap']); + names.add(bootstrap); + assert.equal(docker(['wait', bootstrap], 60000), '0', 'isolated database bootstrap must succeed'); + remove(bootstrap); + for (const mode of ['default', 'json']) { + for (const role of ['api', 'worker', 'scheduler']) { + try {await check(role, mode);} catch (error) {failed.push(`${role}/${mode}: ${error.message}`);} + } + } + for (const mode of ['file', 'filtered']) { + try {await check('worker', mode);} catch (error) {failed.push(`worker/${mode}: ${error.message}`);} + } + assert.deepEqual(failed, [], 'container logging acceptance failed'); + console.log('Container logging acceptance: 8 PASS'); +} finally { + for (const name of names) remove(name); + assert.equal(docker(['volume', 'inspect', '--format', `{{index .Labels "${label}"}}`, database]), scope, + 'refuse cleanup of a foreign volume'); + docker(['volume', 'rm', database]); + fs.rmSync(root, {recursive: true}); +} +} + +main().catch(error => {console.error(error.message.slice(0, 4000)); process.exitCode = 1;}); diff --git a/tests/Fixtures/ContainerLogging/probe.php b/tests/Fixtures/ContainerLogging/probe.php new file mode 100644 index 000000000..6607ef5b5 --- /dev/null +++ b/tests/Fixtures/ContainerLogging/probe.php @@ -0,0 +1,29 @@ + PHP_SAPI === 'apache2handler' && ($_SERVER['REQUEST_URI'] ?? null) === '/api/health', + 'worker' => $command === 'queue:work', + 'scheduler' => $command === 'schedule:evaluate', + default => false, +}; + +if ($selected) { + register_shutdown_function(static function () use ($role): void { + $nonce = getenv('CONTAINER_LOGGING_PROBE_NONCE'); + if (!is_string($nonce) || preg_match('/^[a-f0-9]{32}$/D', $nonce) !== 1) { + throw new RuntimeException('Invalid container logging probe identity.'); + } + + file_put_contents('/tmp/dw-container-logging-probe', $nonce); + \Illuminate\Support\Facades\Log::error('container-logging-probe', [ + 'role' => $role, + 'nonce' => $nonce, + ]); + }); +} From 52bc32091ae20e7e4ae85e0c3c4f53c06a724167 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Wed, 21 Oct 2026 23:36:13 +0000 Subject: [PATCH 18/22] Default container application logs to stderr and verify published roles --- .env.example | 2 +- .../workflows/published-release-recovery.yml | 17 ++ .github/workflows/release.yml | 2 + Dockerfile | 3 +- config/dw-contract.php | 2 +- k8s/README.md | 21 +++ k8s/helm/durable-workflow/README.md | 20 +++ k8s/helm/durable-workflow/values.yaml | 5 +- k8s/secret.yaml | 1 + scripts/ci/verify-container-logging.mjs | 162 ++++++++++-------- scripts/smoke-published-first-run.sh | 9 +- tests/Fixtures/ContainerLogging/probe.php | 2 +- ...eleaseImagePublishWorkflowContractTest.php | 15 ++ 13 files changed, 183 insertions(+), 78 deletions(-) diff --git a/.env.example b/.env.example index 2c2f7aeb7..4a807e1eb 100644 --- a/.env.example +++ b/.env.example @@ -4,7 +4,7 @@ APP_DEBUG=true APP_URL=http://localhost:8080 # APP_VERSION defaults to the checked-in source release identity. -LOG_CHANNEL=stack +LOG_CHANNEL=stderr LOG_LEVEL=debug DB_CONNECTION=mysql diff --git a/.github/workflows/published-release-recovery.yml b/.github/workflows/published-release-recovery.yml index 1985c7a87..469a4d3ce 100644 --- a/.github/workflows/published-release-recovery.yml +++ b/.github/workflows/published-release-recovery.yml @@ -66,6 +66,22 @@ jobs: TLS_TEST_ROOT: ${{ runner.temp }}/dw-server-tls-release-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }} run: bash scripts/ci/qualify-database-tls.sh + - name: Verify existing image first-run readiness and container logging + env: + DW_SERVER_IMAGE: ghcr.io/durable-workflow/server@${{ steps.release_verify.outputs.image_digest }} + DW_FIRST_RUN_SCOPE: recovery-first-run-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }} + run: scripts/smoke-published-first-run.sh + + - name: Clean up existing image first-run resources + if: always() + env: + DW_FIRST_RUN_SCOPE: recovery-first-run-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }} + run: | + docker ps -aq --filter "label=dev.durable-workflow.container-logging=${DW_FIRST_RUN_SCOPE}-logs" | xargs -r docker rm -f >/dev/null 2>&1 || true + docker volume ls -q --filter "label=dev.durable-workflow.container-logging=${DW_FIRST_RUN_SCOPE}-logs" | xargs -r docker volume rm >/dev/null 2>&1 || true + docker rm -f "${DW_FIRST_RUN_SCOPE}-server" "${DW_FIRST_RUN_SCOPE}-bootstrap" >/dev/null 2>&1 || true + docker volume rm "${DW_FIRST_RUN_SCOPE}-database" >/dev/null 2>&1 || true + - name: Clean up published image database TLS fixtures if: always() env: @@ -93,6 +109,7 @@ jobs: needs: verify if: >- always() && + needs.verify.result == 'success' && needs.verify.outputs.release_verification_outcome == 'success' && needs.verify.outputs.published_database_tls_outcome == 'success' && github.repository == 'durable-workflow/server' && diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index fd6d8ea96..bfb82da02 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -297,6 +297,8 @@ jobs: env: DW_FIRST_RUN_SCOPE: release-first-run-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }} run: | + docker ps -aq --filter "label=dev.durable-workflow.container-logging=${DW_FIRST_RUN_SCOPE}-logs" | xargs -r docker rm -f >/dev/null 2>&1 || true + docker volume ls -q --filter "label=dev.durable-workflow.container-logging=${DW_FIRST_RUN_SCOPE}-logs" | xargs -r docker volume rm >/dev/null 2>&1 || true docker rm -f \ "${DW_FIRST_RUN_SCOPE}-server" \ "${DW_FIRST_RUN_SCOPE}-bootstrap" >/dev/null 2>&1 || true diff --git a/Dockerfile b/Dockerfile index 190d11352..7f7a8b376 100644 --- a/Dockerfile +++ b/Dockerfile @@ -164,7 +164,8 @@ ENV DW_WORKER_LONG_POLL_MAX_CONCURRENT=2 \ DB_CONNECTION=sqlite \ DB_DATABASE=/app/database/database.sqlite \ QUEUE_CONNECTION=database \ - CACHE_STORE=file + CACHE_STORE=file \ + LOG_CHANNEL=stderr ENTRYPOINT ["server-entrypoint"] diff --git a/config/dw-contract.php b/config/dw-contract.php index c3353a9de..b97fee206 100644 --- a/config/dw-contract.php +++ b/config/dw-contract.php @@ -1101,7 +1101,7 @@ 'APP_NAME', 'APP_ENV', 'APP_KEY', 'APP_DEBUG', 'APP_URL', 'APP_VERSION', 'APP_LOCALE', 'APP_FALLBACK_LOCALE', 'APP_FAKER_LOCALE', 'APP_MAINTENANCE_DRIVER', 'APP_MAINTENANCE_STORE', 'APP_PREVIOUS_KEYS', 'APP_CIPHER', 'APP_TIMEZONE', - 'LOG_CHANNEL', 'LOG_LEVEL', 'LOG_DEPRECATIONS_CHANNEL', 'LOG_STACK', + 'LOG_CHANNEL', 'LOG_LEVEL', 'LOG_STDERR_FORMATTER', 'LOG_DEPRECATIONS_CHANNEL', 'LOG_STACK', 'LOG_DAILY_DAYS', 'LOG_SLACK_WEBHOOK_URL', 'LOG_PAPERTRAIL_URL', 'LOG_PAPERTRAIL_PORT', 'DB_CONNECTION', 'DB_HOST', 'DB_PORT', 'DB_DATABASE', 'DB_USERNAME', diff --git a/k8s/README.md b/k8s/README.md index 1f64202c7..162de8ba0 100644 --- a/k8s/README.md +++ b/k8s/README.md @@ -16,6 +16,27 @@ release and pin its Docker Hub tag: durableworkflow/server:2.10.1 ``` +## Application logs + +The raw ConfigMap sets `LOG_CHANNEL=stderr` for the API, worker and scheduler. +Read application logs with `kubectl logs` or your cluster's log collector. This +also works with existing published images when the setting is supplied explicitly. + +Set `LOG_LEVEL` to select the minimum severity. For newline-delimited JSON, add +this ConfigMap entry: + +```yaml +LOG_STDERR_FORMATTER: 'Monolog\Formatter\JsonFormatter' +``` + +An explicit `LOG_CHANNEL` override selects another Laravel channel. File channels +need their own persistent storage and collection; Kubernetes does not collect +`storage/logs/laravel.log` automatically. Restart persistent roles after changing +logging settings because configuration is cached at startup. `DW_LOG_FORMAT` is +not a supported setting. + +## Image selection + Before production use, patch every workload image to the exact published tag or digest you intend to run: diff --git a/k8s/helm/durable-workflow/README.md b/k8s/helm/durable-workflow/README.md index 4883a86fa..122a80e87 100644 --- a/k8s/helm/durable-workflow/README.md +++ b/k8s/helm/durable-workflow/README.md @@ -132,6 +132,26 @@ Driver, or any GitOps secret bridge that produces a Kubernetes `Secret`. | `ServiceAccount` (`*`) | Namespace-scoped SA for every workload. | `serviceAccount.create: false` | | `NetworkPolicy` (`*-server`) | Optional ingress allow-list to the API. | `networkPolicy.enabled: false` (default) | +## Application logs + +The chart defaults `config.extraEnv.LOG_CHANNEL` to `stderr` for the API, worker, +scheduler and bootstrap job. Use `kubectl logs` or the cluster's log collector. +To choose the minimum severity and emit newline-delimited JSON: + +```yaml +config: + extraEnv: + LOG_CHANNEL: "stderr" + LOG_LEVEL: "info" + LOG_STDERR_FORMATTER: 'Monolog\Formatter\JsonFormatter' +``` + +These explicit settings also work with existing published Server images. Override +`LOG_CHANNEL` to select a different Laravel channel. File channels require their +own storage and collection. Restart persistent roles after changing logging +settings because configuration is cached at startup. `DW_LOG_FORMAT` is not a +supported setting. + ## Database TLS For an authenticated encrypted database connection, pre-create a CA bundle diff --git a/k8s/helm/durable-workflow/values.yaml b/k8s/helm/durable-workflow/values.yaml index d521dd292..559c92309 100644 --- a/k8s/helm/durable-workflow/values.yaml +++ b/k8s/helm/durable-workflow/values.yaml @@ -77,10 +77,11 @@ config: # Free-form ConfigMap entries appended verbatim. Use this for environment # tuning that the chart does not expose as first-class settings (e.g. # DW_V2_MATCHING_ROLE_QUEUE_WAKE for dedicated matching rollouts). - extraEnv: {} + extraEnv: + LOG_CHANNEL: "stderr" # extraEnv: # DW_V2_MATCHING_ROLE_QUEUE_WAKE: "0" - # DW_LOG_FORMAT: "json" + # LOG_STDERR_FORMATTER: 'Monolog\Formatter\JsonFormatter' # --------------------------------------------------------------------------- # External persistence (REQUIRED — no in-cluster database is bundled) diff --git a/k8s/secret.yaml b/k8s/secret.yaml index 5236c860a..3d87fedae 100644 --- a/k8s/secret.yaml +++ b/k8s/secret.yaml @@ -15,6 +15,7 @@ data: APP_VERSION: "2.10.1" APP_ENV: production APP_DEBUG: "false" + LOG_CHANNEL: "stderr" DB_CONNECTION: mysql DB_HOST: mysql.durable-workflow.svc.cluster.local DB_PORT: "3306" diff --git a/scripts/ci/verify-container-logging.mjs b/scripts/ci/verify-container-logging.mjs index 87b0c5d2b..a8b044cc8 100644 --- a/scripts/ci/verify-container-logging.mjs +++ b/scripts/ci/verify-container-logging.mjs @@ -9,7 +9,7 @@ import {fileURLToPath} from 'node:url'; const image = process.env.DW_SERVER_IMAGE; assert.ok(image, 'DW_SERVER_IMAGE must identify the image under qualification'); const scope = process.env.DW_LOGGING_SMOKE_SCOPE || `dw-log-${randomBytes(8).toString('hex')}`; -assert.match(scope, /^[A-Za-z0-9][A-Za-z0-9_.-]{0,63}$/); +assert.match(scope, /^[A-Za-z0-9][A-Za-z0-9_.-]{0,95}$/); const label = 'dev.durable-workflow.container-logging'; const fixture = fileURLToPath(new URL('../../tests/Fixtures/ContainerLogging/probe.php', import.meta.url)); const root = fs.mkdtempSync(path.join(process.env.RUNNER_TEMP || os.tmpdir(), `${scope}-`)); @@ -18,10 +18,11 @@ const ini = path.join(root, 'logging-probe.ini'); fs.writeFileSync(ini, 'auto_prepend_file=/opt/container-logging-probe.php\n'); const names = new Set(); const failed = []; +let volumeCreated = false; const pause = milliseconds => new Promise(resolve => setTimeout(resolve, milliseconds)); function docker(args, timeout = 30000) { - return execFileSync('docker', args, {encoding: 'utf8', timeout, maxBuffer: 524288}).trim(); + return execFileSync('/usr/bin/docker', args, {encoding: 'utf8', timeout, maxBuffer: 524288}).trim(); } function owned(name) { @@ -47,21 +48,64 @@ const common = [ ]; function copy(name, source, target) { - const result = spawnSync('docker', ['cp', `${name}:${source}`, target], { + const result = spawnSync('/usr/bin/docker', ['cp', `${name}:${source}`, target], { encoding: 'utf8', timeout: 10000, maxBuffer: 524288, }); return result.status === 0 ? fs.readFileSync(target, 'utf8') : ''; } +async function waitForApi(name) { + const port = docker(['port', name, '8080/tcp']).split(':').at(-1); + // Poll serially so startup cannot accumulate concurrent health requests. + for (let attempt = 0; attempt < 100; attempt += 1) { + try { + const response = await fetch(`http://127.0.0.1:${port}/api/health`, {signal: AbortSignal.timeout(1000)}); + await response.text(); + if (response.status === 200) return; + } catch { /* The real API may still be starting. */ } + await pause(100); + } + assert.fail('API did not become live within its original probe budget'); +} + +function assertOutput(name, nonce, role, mode) { + const proof = copy(name, '/app/storage/framework/testing/dw-container-logging-probe', path.join(root, `${mode}-${role}.proof`)); + assert.equal(proof, nonce, `${role}: actual role must invoke the default Laravel logger`); + const logs = spawnSync('/usr/bin/docker', ['logs', name], {encoding: 'utf8', timeout: 10000, maxBuffer: 524288}); + assert.equal(logs.status, 0); + const file = copy(name, '/app/storage/logs/laravel.log', path.join(root, `${mode}-${role}.log`)); + const stderr = logs.stderr.split('\n').filter(line => line.includes(nonce)); + assert.ok(!logs.stdout.includes(nonce), `${role}: application error must not go to stdout`); + if (mode === 'file') { + assert.ok(file.includes(nonce), 'operator-selected file channel must be honored'); + assert.equal(stderr.length, 0); + } else if (mode === 'filtered') { + assert.equal(stderr.length, 0, 'LOG_LEVEL must filter the emitted error'); + assert.ok(!file.includes(nonce)); + } else { + assert.ok(stderr.length > 0, `${role}/${mode}: application error missing from container stderr`); + assert.ok(!file.includes(nonce), `${role}: default stderr must not duplicate the error into a local file`); + if (mode === 'json') { + for (const line of stderr) { + const record = JSON.parse(line); + assert.equal(record.message, 'container-logging-probe'); + assert.equal(record.context.role, role); + assert.equal(record.context.nonce, nonce); + } + } + } + console.log(JSON.stringify({role, mode, outcome: 'PASS'})); +} + async function check(role, mode) { const name = `${scope}-${mode}-${role}`; const nonce = randomBytes(16).toString('hex'); - const args = ['run', '--detach', '--name', name, ...common, + const args = ['create', '--name', name, ...common, '--mount', `type=bind,source=${fixture},target=/opt/container-logging-probe.php,readonly`, '--mount', `type=bind,source=${ini},target=/usr/local/etc/php/conf.d/zz-container-logging-probe.ini,readonly`, '--env', `CONTAINER_LOGGING_PROBE_ROLE=${role}`, '--env', `CONTAINER_LOGGING_PROBE_NONCE=${nonce}`]; - if (mode === 'json') args.push('--env', 'LOG_STDERR_FORMATTER=Monolog\\Formatter\\JsonFormatter'); + if (mode === 'json') args.push('--env', String.raw`LOG_STDERR_FORMATTER=Monolog\Formatter\JsonFormatter`); if (mode === 'file') args.push('--env', 'LOG_CHANNEL=single'); if (mode === 'filtered') args.push('--env', 'LOG_LEVEL=critical'); if (role === 'api') args.push('--publish', '127.0.0.1::8080'); @@ -69,83 +113,59 @@ async function check(role, mode) { if (role === 'worker') args.push('php', 'artisan', 'queue:work', '--once', '--stop-when-empty', '--max-time=2'); if (role === 'scheduler') args.push('php', 'artisan', 'schedule:evaluate', '--limit=100', '--json'); - assert.notEqual(spawnSync('docker', ['inspect', name], {stdio: 'ignore'}).status, 0, + assert.notEqual(spawnSync('/usr/bin/docker', ['inspect', name], {stdio: 'ignore'}).status, 0, `container name already exists: ${name}`); docker(args); names.add(name); try { - if (role === 'api') { - const port = docker(['port', name, '8080/tcp']).split(':').at(-1); - let ready = false; - for (let attempt = 0; attempt < 100; attempt += 1) { - try { - const response = await fetch(`http://127.0.0.1:${port}/api/health`, {signal: AbortSignal.timeout(1000)}); - await response.text(); - if (response.status === 200) {ready = true; break;} - } catch { /* The real API may still be starting. */ } - await pause(100); - } - assert.ok(ready, 'API did not become live within its original probe budget'); - } else { - assert.equal(docker(['wait', name]), '0', `${role} command must exit successfully`); - } - - const proof = copy(name, '/tmp/dw-container-logging-probe', path.join(root, `${mode}-${role}.proof`)); - assert.equal(proof, nonce, `${role}: actual role must invoke the default Laravel logger`); - const logs = spawnSync('docker', ['logs', name], {encoding: 'utf8', timeout: 10000, maxBuffer: 524288}); - assert.equal(logs.status, 0); - const file = copy(name, '/app/storage/logs/laravel.log', path.join(root, `${mode}-${role}.log`)); - const stderr = logs.stderr.split('\n').filter(line => line.includes(nonce)); - assert.ok(!logs.stdout.includes(nonce), `${role}: application error must not go to stdout`); - if (mode === 'file') { - assert.ok(file.includes(nonce), 'operator-selected file channel must be honored'); - assert.equal(stderr.length, 0); - } else if (mode === 'filtered') { - assert.equal(stderr.length, 0, 'LOG_LEVEL must filter the emitted error'); - assert.ok(!file.includes(nonce)); - } else { - assert.ok(stderr.length > 0, `${role}/${mode}: application error missing from container stderr`); - assert.ok(!file.includes(nonce), `${role}: default stderr must not duplicate the error into a local file`); - if (mode === 'json') { - for (const line of stderr) { - const record = JSON.parse(line); - assert.equal(record.message, 'container-logging-probe'); - assert.equal(record.context.role, role); - assert.equal(record.context.nonce, nonce); - } - } - } - console.log(JSON.stringify({role, mode, outcome: 'PASS'})); + docker(['start', name]); + if (role === 'api') await waitForApi(name); + else assert.equal(docker(['wait', name]), '0', `${role} command must exit successfully`); + assertOutput(name, nonce, role, mode); } finally { remove(name); } } async function main() { -try { - docker(['volume', 'create', '--label', `${label}=${scope}`, database]); - const bootstrap = `${scope}-bootstrap`; - docker(['run', '--detach', '--name', bootstrap, ...common, image, 'server-bootstrap']); - names.add(bootstrap); - assert.equal(docker(['wait', bootstrap], 60000), '0', 'isolated database bootstrap must succeed'); - remove(bootstrap); - for (const mode of ['default', 'json']) { - for (const role of ['api', 'worker', 'scheduler']) { - try {await check(role, mode);} catch (error) {failed.push(`${role}/${mode}: ${error.message}`);} + try { + assert.notEqual(spawnSync('/usr/bin/docker', ['volume', 'inspect', database], {stdio: 'ignore'}).status, 0, + `volume already exists: ${database}`); + docker(['volume', 'create', '--label', `${label}=${scope}`, database]); + assert.equal(docker(['volume', 'inspect', '--format', `{{index .Labels "${label}"}}`, database]), scope, + 'refuse use of a foreign volume'); + volumeCreated = true; + const bootstrap = `${scope}-bootstrap`; + docker(['create', '--name', bootstrap, ...common, image, 'server-bootstrap']); + names.add(bootstrap); + docker(['start', bootstrap]); + assert.equal(docker(['wait', bootstrap], 60000), '0', 'isolated database bootstrap must succeed'); + remove(bootstrap); + // Roles share one SQLite fixture and must not overlap or multiply resource use. + for (const mode of ['default', 'json']) { + for (const role of ['api', 'worker', 'scheduler']) { + try {await check(role, mode);} catch (error) {failed.push(`${role}/${mode}: ${error.message}`);} + } } + for (const mode of ['file', 'filtered']) { + try {await check('worker', mode);} catch (error) {failed.push(`worker/${mode}: ${error.message}`);} + } + assert.deepEqual(failed, [], 'container logging acceptance failed'); + console.log('Container logging acceptance: 8 PASS'); + } finally { + for (const name of names) remove(name); + if (volumeCreated) { + assert.equal(docker(['volume', 'inspect', '--format', `{{index .Labels "${label}"}}`, database]), scope, + 'refuse cleanup of a foreign volume'); + docker(['volume', 'rm', database]); + } + fs.rmSync(root, {recursive: true}); } - for (const mode of ['file', 'filtered']) { - try {await check('worker', mode);} catch (error) {failed.push(`worker/${mode}: ${error.message}`);} - } - assert.deepEqual(failed, [], 'container logging acceptance failed'); - console.log('Container logging acceptance: 8 PASS'); -} finally { - for (const name of names) remove(name); - assert.equal(docker(['volume', 'inspect', '--format', `{{index .Labels "${label}"}}`, database]), scope, - 'refuse cleanup of a foreign volume'); - docker(['volume', 'rm', database]); - fs.rmSync(root, {recursive: true}); -} } -main().catch(error => {console.error(error.message.slice(0, 4000)); process.exitCode = 1;}); +try { + await main(); +} catch (error) { + console.error(error.message.slice(0, 4000)); + process.exitCode = 1; +} diff --git a/scripts/smoke-published-first-run.sh b/scripts/smoke-published-first-run.sh index 958e1465e..47c3b1ecb 100755 --- a/scripts/smoke-published-first-run.sh +++ b/scripts/smoke-published-first-run.sh @@ -20,6 +20,10 @@ RECOVERED_SERVER_LOG="${ARTIFACT_DIR}/recovered-server.log" HEALTH_LOG="${ARTIFACT_DIR}/docker-health.log" cleanup() { + docker ps -aq --filter "label=dev.durable-workflow.container-logging=${SCOPE}-logs" \ + | xargs -r docker rm -f >/dev/null 2>&1 || true + docker volume ls -q --filter "label=dev.durable-workflow.container-logging=${SCOPE}-logs" \ + | xargs -r docker volume rm >/dev/null 2>&1 || true docker rm -f "$CONTAINER" "$BOOTSTRAP_CONTAINER" >/dev/null 2>&1 || true docker volume rm -f "$DATABASE_VOLUME" >/dev/null 2>&1 || true rm -rf "$ARTIFACT_DIR" @@ -190,4 +194,7 @@ assert ready.get("checks", {}).get("auth", {}).get("status") == "ok", ready assert os.environ["AUTH_TOKEN"] not in Path(os.environ["RECOVERED_SERVER_LOG"]).read_text() PY -echo "Published Server first-run readiness smoke passed" +DW_LOGGING_SMOKE_SCOPE="${SCOPE}-logs" \ + node "$(dirname "$0")/ci/verify-container-logging.mjs" + +echo "Published Server first-run readiness and container logging smoke passed" diff --git a/tests/Fixtures/ContainerLogging/probe.php b/tests/Fixtures/ContainerLogging/probe.php index 6607ef5b5..984275907 100644 --- a/tests/Fixtures/ContainerLogging/probe.php +++ b/tests/Fixtures/ContainerLogging/probe.php @@ -20,7 +20,7 @@ throw new RuntimeException('Invalid container logging probe identity.'); } - file_put_contents('/tmp/dw-container-logging-probe', $nonce); + file_put_contents('/app/storage/framework/testing/dw-container-logging-probe', $nonce); \Illuminate\Support\Facades\Log::error('container-logging-probe', [ 'role' => $role, 'nonce' => $nonce, diff --git a/tests/Unit/ReleaseImagePublishWorkflowContractTest.php b/tests/Unit/ReleaseImagePublishWorkflowContractTest.php index 22447bea2..f7d33ca31 100644 --- a/tests/Unit/ReleaseImagePublishWorkflowContractTest.php +++ b/tests/Unit/ReleaseImagePublishWorkflowContractTest.php @@ -19,6 +19,21 @@ protected function setUp(): void $this->repoRoot = dirname(__DIR__, 2); } + public function test_recovery_chart_publication_requires_all_image_checks_to_succeed(): void + { + $workflow = Yaml::parse($this->read('.github/workflows/published-release-recovery.yml')); + $condition = $workflow['jobs']['publish-chart']['if']; + $this->assertStringContainsString("needs.verify.result == 'success'", $condition); + + $checks = array_values(array_filter( + $workflow['jobs']['verify']['steps'], + static fn (array $step): bool => ($step['run'] ?? null) === 'scripts/smoke-published-first-run.sh', + )); + $this->assertCount(1, $checks); + $this->assertArrayNotHasKey('continue-on-error', $checks[0]); + $this->assertStringContainsString('verify-container-logging.mjs', $this->read('scripts/smoke-published-first-run.sh')); + } + public function test_release_publish_job_is_guarded_before_registry_login(): void { $workflow = $this->read('.github/workflows/release.yml'); From 7ac8d2be2a542f7c1eda0b077db26bee1578621e Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Wed, 21 Oct 2026 23:41:29 +0000 Subject: [PATCH 19/22] Keep historical immutable release recovery compatible with logging rollout --- .github/workflows/published-release-recovery.yml | 8 +++++++- tests/Unit/ReleaseImagePublishWorkflowContractTest.php | 3 ++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/published-release-recovery.yml b/.github/workflows/published-release-recovery.yml index 469a4d3ce..0ecbf1ab8 100644 --- a/.github/workflows/published-release-recovery.yml +++ b/.github/workflows/published-release-recovery.yml @@ -70,7 +70,13 @@ jobs: env: DW_SERVER_IMAGE: ghcr.io/durable-workflow/server@${{ steps.release_verify.outputs.image_digest }} DW_FIRST_RUN_SCOPE: recovery-first-run-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }} - run: scripts/smoke-published-first-run.sh + RELEASE_SOURCE_COMMIT: ${{ inputs.release_commit }} + run: | + if git cat-file -e "${RELEASE_SOURCE_COMMIT}:scripts/ci/verify-container-logging.mjs" 2>/dev/null; then + scripts/smoke-published-first-run.sh + else + echo "This immutable release predates container logging acceptance; historical verification remains unchanged." + fi - name: Clean up existing image first-run resources if: always() diff --git a/tests/Unit/ReleaseImagePublishWorkflowContractTest.php b/tests/Unit/ReleaseImagePublishWorkflowContractTest.php index f7d33ca31..2eaed0721 100644 --- a/tests/Unit/ReleaseImagePublishWorkflowContractTest.php +++ b/tests/Unit/ReleaseImagePublishWorkflowContractTest.php @@ -27,10 +27,11 @@ public function test_recovery_chart_publication_requires_all_image_checks_to_suc $checks = array_values(array_filter( $workflow['jobs']['verify']['steps'], - static fn (array $step): bool => ($step['run'] ?? null) === 'scripts/smoke-published-first-run.sh', + static fn (array $step): bool => ($step['name'] ?? null) === 'Verify existing image first-run readiness and container logging', )); $this->assertCount(1, $checks); $this->assertArrayNotHasKey('continue-on-error', $checks[0]); + $this->assertStringContainsString('scripts/smoke-published-first-run.sh', $checks[0]['run']); $this->assertStringContainsString('verify-container-logging.mjs', $this->read('scripts/smoke-published-first-run.sh')); } From ee079e78624f75a2b7b0d5b68593a8826c335522 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Thu, 22 Oct 2026 00:03:23 +0000 Subject: [PATCH 20/22] fix(release): version logging chart defaults as 0.2.4 --- k8s/helm/durable-workflow/Chart.yaml | 2 +- resources/release/source-release.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/k8s/helm/durable-workflow/Chart.yaml b/k8s/helm/durable-workflow/Chart.yaml index ee733b24b..3851cc6ae 100644 --- a/k8s/helm/durable-workflow/Chart.yaml +++ b/k8s/helm/durable-workflow/Chart.yaml @@ -5,7 +5,7 @@ type: application # The chart's own semver version. Bumped on every chart release; treated as # independent of the server image version (appVersion). Breaking-change rules # for this version live in docs/UPGRADING.md alongside the chart. -version: 0.2.3 +version: 0.2.4 # The immutable Durable Workflow Server identity this chart release packages. # The onboarding default in values.yaml and appVersion are generated from the # checked-in source release record. diff --git a/resources/release/source-release.json b/resources/release/source-release.json index 6d20c9916..66825c0ca 100644 --- a/resources/release/source-release.json +++ b/resources/release/source-release.json @@ -4,6 +4,6 @@ "version": "2.10.1" }, "helm_chart": { - "version": "0.2.3" + "version": "0.2.4" } } From 17f15b1143344d75644ec0720ff6e64c6f2885ac Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Thu, 22 Oct 2026 00:09:30 +0000 Subject: [PATCH 21/22] fix(ci): scope first-run cleanup to unconditional job steps --- .github/workflows/published-release-recovery.yml | 10 ++++++---- .github/workflows/release.yml | 7 +++++-- scripts/ci/verify-container-logging.mjs | 3 ++- scripts/smoke-published-first-run.sh | 11 ++++------- 4 files changed, 17 insertions(+), 14 deletions(-) diff --git a/.github/workflows/published-release-recovery.yml b/.github/workflows/published-release-recovery.yml index 0ecbf1ab8..9429a7544 100644 --- a/.github/workflows/published-release-recovery.yml +++ b/.github/workflows/published-release-recovery.yml @@ -72,7 +72,7 @@ jobs: DW_FIRST_RUN_SCOPE: recovery-first-run-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }} RELEASE_SOURCE_COMMIT: ${{ inputs.release_commit }} run: | - if git cat-file -e "${RELEASE_SOURCE_COMMIT}:scripts/ci/verify-container-logging.mjs" 2>/dev/null; then + if git rev-parse --verify "${RELEASE_SOURCE_COMMIT}:scripts/ci/verify-container-logging.mjs" >/dev/null 2>&1; then scripts/smoke-published-first-run.sh else echo "This immutable release predates container logging acceptance; historical verification remains unchanged." @@ -82,10 +82,12 @@ jobs: if: always() env: DW_FIRST_RUN_SCOPE: recovery-first-run-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }} + LOGGING_TEST_PROJECT: recovery-first-run-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }}-logs run: | - docker ps -aq --filter "label=dev.durable-workflow.container-logging=${DW_FIRST_RUN_SCOPE}-logs" | xargs -r docker rm -f >/dev/null 2>&1 || true - docker volume ls -q --filter "label=dev.durable-workflow.container-logging=${DW_FIRST_RUN_SCOPE}-logs" | xargs -r docker volume rm >/dev/null 2>&1 || true - docker rm -f "${DW_FIRST_RUN_SCOPE}-server" "${DW_FIRST_RUN_SCOPE}-bootstrap" >/dev/null 2>&1 || true + docker ps -aq --filter "label=com.docker.compose.project=$LOGGING_TEST_PROJECT" | xargs -r docker rm -f >/dev/null 2>&1 || true + docker network ls -q --filter "label=com.docker.compose.project=$LOGGING_TEST_PROJECT" | xargs -r docker network rm >/dev/null 2>&1 || true + docker volume ls -q --filter "label=com.docker.compose.project=$LOGGING_TEST_PROJECT" | xargs -r docker volume rm -f >/dev/null 2>&1 || true + docker rm -f "${DW_FIRST_RUN_SCOPE}-server" "${DW_FIRST_RUN_SCOPE}-bootstrap" "${DW_FIRST_RUN_SCOPE}-unbootstrapped" >/dev/null 2>&1 || true docker volume rm "${DW_FIRST_RUN_SCOPE}-database" >/dev/null 2>&1 || true - name: Clean up published image database TLS fixtures diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index bfb82da02..0ebd125ac 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -296,11 +296,14 @@ jobs: if: always() env: DW_FIRST_RUN_SCOPE: release-first-run-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }} + LOGGING_TEST_PROJECT: release-first-run-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }}-logs run: | - docker ps -aq --filter "label=dev.durable-workflow.container-logging=${DW_FIRST_RUN_SCOPE}-logs" | xargs -r docker rm -f >/dev/null 2>&1 || true - docker volume ls -q --filter "label=dev.durable-workflow.container-logging=${DW_FIRST_RUN_SCOPE}-logs" | xargs -r docker volume rm >/dev/null 2>&1 || true + docker ps -aq --filter "label=com.docker.compose.project=$LOGGING_TEST_PROJECT" | xargs -r docker rm -f >/dev/null 2>&1 || true + docker network ls -q --filter "label=com.docker.compose.project=$LOGGING_TEST_PROJECT" | xargs -r docker network rm >/dev/null 2>&1 || true + docker volume ls -q --filter "label=com.docker.compose.project=$LOGGING_TEST_PROJECT" | xargs -r docker volume rm -f >/dev/null 2>&1 || true docker rm -f \ "${DW_FIRST_RUN_SCOPE}-server" \ + "${DW_FIRST_RUN_SCOPE}-unbootstrapped" \ "${DW_FIRST_RUN_SCOPE}-bootstrap" >/dev/null 2>&1 || true docker volume rm -f \ "${DW_FIRST_RUN_SCOPE}-database" >/dev/null 2>&1 || true diff --git a/scripts/ci/verify-container-logging.mjs b/scripts/ci/verify-container-logging.mjs index a8b044cc8..4c65ba9e6 100644 --- a/scripts/ci/verify-container-logging.mjs +++ b/scripts/ci/verify-container-logging.mjs @@ -40,6 +40,7 @@ function remove(name) { const common = [ '--user', '1000:1000', '--cpus', '1', '--memory', '512m', '--memory-swap', '512m', '--label', `${label}=${scope}`, + '--label', `com.docker.compose.project=${scope}`, '--mount', `type=volume,source=${database},target=/app/database`, '--env', 'DB_CONNECTION=sqlite', '--env', 'DB_DATABASE=/app/database/database.sqlite', '--env', 'DB_SQLITE_JOURNAL_MODE=DELETE', '--env', 'CACHE_STORE=file', @@ -131,7 +132,7 @@ async function main() { try { assert.notEqual(spawnSync('/usr/bin/docker', ['volume', 'inspect', database], {stdio: 'ignore'}).status, 0, `volume already exists: ${database}`); - docker(['volume', 'create', '--label', `${label}=${scope}`, database]); + docker(['volume', 'create', '--label', `${label}=${scope}`, '--label', `com.docker.compose.project=${scope}`, database]); assert.equal(docker(['volume', 'inspect', '--format', `{{index .Labels "${label}"}}`, database]), scope, 'refuse use of a foreign volume'); volumeCreated = true; diff --git a/scripts/smoke-published-first-run.sh b/scripts/smoke-published-first-run.sh index 47c3b1ecb..aacaa8881 100755 --- a/scripts/smoke-published-first-run.sh +++ b/scripts/smoke-published-first-run.sh @@ -20,12 +20,8 @@ RECOVERED_SERVER_LOG="${ARTIFACT_DIR}/recovered-server.log" HEALTH_LOG="${ARTIFACT_DIR}/docker-health.log" cleanup() { - docker ps -aq --filter "label=dev.durable-workflow.container-logging=${SCOPE}-logs" \ - | xargs -r docker rm -f >/dev/null 2>&1 || true - docker volume ls -q --filter "label=dev.durable-workflow.container-logging=${SCOPE}-logs" \ - | xargs -r docker volume rm >/dev/null 2>&1 || true - docker rm -f "$CONTAINER" "$BOOTSTRAP_CONTAINER" >/dev/null 2>&1 || true - docker volume rm -f "$DATABASE_VOLUME" >/dev/null 2>&1 || true + # Workflow always-steps own deletion, including cancellation after partial setup. + docker stop "$CONTAINER" "$BOOTSTRAP_CONTAINER" >/dev/null 2>&1 || true rm -rf "$ARTIFACT_DIR" } @@ -150,7 +146,8 @@ for expected in ("server-bootstrap", "authentication", "/api/ready"): assert "server-bootstrap" in docker_health_log, docker_health_log PY -docker rm -f "$CONTAINER" >/dev/null +docker stop "$CONTAINER" >/dev/null +docker rename "$CONTAINER" "${SCOPE}-unbootstrapped" docker run --rm \ --name "$BOOTSTRAP_CONTAINER" \ From b0e425144fa47f39515ce466fbb42c5896e4f4c7 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Thu, 22 Oct 2026 00:27:17 +0000 Subject: [PATCH 22/22] test(release): require successful complete recovery verification --- scripts/ci/test_helm_chart_release.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/scripts/ci/test_helm_chart_release.py b/scripts/ci/test_helm_chart_release.py index 302cb4f65..8fc29830d 100644 --- a/scripts/ci/test_helm_chart_release.py +++ b/scripts/ci/test_helm_chart_release.py @@ -91,6 +91,7 @@ def evaluate_recovery_condition( ) -> bool: expression = job_condition(workflow_source(workflow), job) replacements = { + "needs.verify.result": context.get("verify_result", "success"), "needs.publish.outputs.exact_publish_outcome": context.get( "exact_publish_outcome", "" ), @@ -888,6 +889,22 @@ def test_chart_publication_requires_the_installed_database_tls_gate(self) -> Non ref="refs/heads/main", )) + def test_recovery_requires_the_complete_image_verification_job(self) -> None: + for result in ["success", "failure", "cancelled", "skipped", ""]: + with self.subTest(result=result): + self.assertEqual( + result == "success", + evaluate_recovery_condition( + "published-release-recovery.yml", + "publish-chart", + verify_result=result, + release_verification_outcome="success", + database_tls_outcome="success", + repository="durable-workflow/server", + ref="refs/heads/main", + ), + ) + def test_duplicate_publication_reuses_and_verifies_the_existing_package( self, ) -> None: