diff --git a/.github/workflows/phpunit-feature.yml b/.github/workflows/phpunit-feature.yml index 16b58a369..d403c73ed 100644 --- a/.github/workflows/phpunit-feature.yml +++ b/.github/workflows/phpunit-feature.yml @@ -220,6 +220,21 @@ jobs: done [ "$health" = healthy ] done + for mode in session transaction0 transaction200; do + pool_mode=transaction + prepared_statements=0 + [ "$mode" != session ] || pool_mode=session + [ "$mode" != transaction200 ] || prepared_statements=200 + docker run --detach \ + --name "dw-server-pooler-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }}-$mode" \ + --network "dw-server-feature-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }}" \ + --memory 128m --memory-swap 128m \ + -e DB_HOST="dw-server-backup-pgsql-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }}" \ + -e DB_NAME=postgres -e DB_USER=postgres -e DB_PASSWORD=pooler-disposable-password \ + -e AUTH_TYPE=trust -e "POOL_MODE=$pool_mode" -e "MAX_PREPARED_STATEMENTS=$prepared_statements" \ + -e DEFAULT_POOL_SIZE=2 -e MIN_POOL_SIZE=2 -e SERVER_ROUND_ROBIN=1 \ + edoburu/pgbouncer@sha256:b17551c776ef7e5769ef80b956d20f85e2fd25dd8912d31d58f782aad495b711 + done tar \ --exclude=vendor \ --exclude='*/vendor' \ @@ -236,6 +251,9 @@ jobs: -e DW_TEST_COMPLETION_MYSQL_HOST="dw-server-backup-mysql-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }}" \ -e DW_TEST_BACKUP_PGSQL_HOST="dw-server-backup-pgsql-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }}" \ -e DW_TEST_COMPLETION_PGSQL_HOST="dw-server-backup-pgsql-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }}" \ + -e DW_POOLER_TEST_HOST="dw-server-backup-pgsql-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }}" \ + -e DW_POOLER_TEST_PORT=5432 -e DW_POOLER_TEST_DATABASE=postgres -e DW_POOLER_TEST_USER=postgres \ + -e DW_POOLER_TEST_PREFIX="dw-server-pooler-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }}" \ -w /app \ --entrypoint sh \ durable-workflow-server-corpus-validator \ @@ -283,6 +301,9 @@ jobs: - name: Clean up test services if: always() run: | + for mode in session transaction0 transaction200; do + docker rm -f "dw-server-pooler-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }}-$mode" >/dev/null 2>&1 || true + done docker rm -f \ "dw-server-s3-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }}" \ "dw-server-backup-mysql-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }}" \ diff --git a/.github/workflows/server-parity.yml b/.github/workflows/server-parity.yml index 90a4dbe41..699b59641 100644 --- a/.github/workflows/server-parity.yml +++ b/.github/workflows/server-parity.yml @@ -377,6 +377,11 @@ jobs: - name: Bootstrap independent published PHP and native MySQL databases run: | set -euo pipefail + # Failed Rust tests are printed from their evidence files into this step's log. + rust_test_failures() { + bash scripts/conformance/server-parity/rust-test-failures.sh + } + trap rust_test_failures ERR mkdir -p cargo-home rust-target parity-evidence mysql-state chmod 0777 parity-evidence docker network create "$RESOURCE_SCOPE" @@ -663,14 +668,14 @@ jobs: cmp parity-evidence/catalog-before.json parity-evidence/catalog-after.json - name: Compare separate configured role-token cohort run: bash scripts/conformance/server-parity/role-profile.sh mysql - - name: Inspect separate published PHP upgrade source without takeover + - name: Back up, take over and resume a separate published PHP upgrade source timeout-minutes: 8 run: bash scripts/conformance/server-parity/upgrade-source-profile.sh mysql - name: Retire upgrade source cohort if: always() timeout-minutes: 3 run: bash scripts/conformance/server-parity/upgrade-source-cleanup.sh mysql - - name: Inspect published PHP 2.9.7 upgrade source without takeover + - name: Back up, take over and resume a published PHP 2.9.7 upgrade source timeout-minutes: 8 env: DW_PARITY_UPGRADE_LABEL: php297 @@ -871,6 +876,11 @@ jobs: POSTGRES_IMAGE: ${{ matrix.image }} run: | set -euo pipefail + # Failed Rust tests are printed from their evidence files into this step's log. + rust_test_failures() { + bash scripts/conformance/server-parity/rust-test-failures.sh + } + trap rust_test_failures ERR mkdir -p cargo-home rust-target parity-evidence pg-state chmod 0777 parity-evidence docker network create "$RESOURCE_SCOPE" @@ -1157,14 +1167,14 @@ jobs: env: POSTGRES_IMAGE: ${{ matrix.image }} run: bash scripts/conformance/server-parity/role-profile.sh pgsql - - name: Inspect separate published PHP upgrade source without takeover + - name: Back up, take over and resume a separate published PHP upgrade source timeout-minutes: 8 run: bash scripts/conformance/server-parity/upgrade-source-profile.sh pgsql - name: Retire upgrade source cohort if: always() timeout-minutes: 3 run: bash scripts/conformance/server-parity/upgrade-source-cleanup.sh pgsql - - name: Inspect published PHP 2.9.7 upgrade source without takeover + - name: Back up, take over and resume a published PHP 2.9.7 upgrade source timeout-minutes: 8 env: DW_PARITY_UPGRADE_LABEL: php297 @@ -1394,6 +1404,11 @@ jobs: - name: Execute official Rust codec and cross-decode with published PHP run: | set -euo pipefail + # Failed Rust tests are printed from their evidence files into this step's log. + rust_test_failures() { + bash scripts/conformance/server-parity/rust-test-failures.sh + } + trap rust_test_failures ERR mkdir -p cargo-home rust-target parity-evidence # Compiling the library tests peaks near 2.5 GB; a 2 GB limit OOM-killed rustc (exit 101). docker run --rm --user="$(id -u):$(id -g)" --cpus=2 --memory=4g --memory-swap=4g \ @@ -1642,7 +1657,7 @@ jobs: env: RESOURCE_SCOPE: dw-parity-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }} run: bash scripts/conformance/server-parity/role-profile.sh sqlite - - name: Inspect separate published PHP upgrade source without takeover + - name: Back up, take over and resume a separate published PHP upgrade source timeout-minutes: 8 env: RESOURCE_SCOPE: dw-parity-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }} @@ -1653,9 +1668,10 @@ jobs: env: RESOURCE_SCOPE: dw-parity-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }} run: bash scripts/conformance/server-parity/upgrade-source-cleanup.sh sqlite - - name: Inspect published PHP 2.9.7 upgrade source without takeover + - name: Back up, take over and resume a published PHP 2.9.7 upgrade source timeout-minutes: 8 env: + RESOURCE_SCOPE: dw-parity-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }} DW_PARITY_UPGRADE_LABEL: php297 DW_PARITY_UPGRADE_EXPECTED_LAYOUT: php-2.9.7 DW_PARITY_UPGRADE_PHP_IMAGE: ghcr.io/durable-workflow/server@sha256:30c34c5f3256a812fbccbe505a068309b58b0c14d01ffff0465d88604cb36d50 @@ -1664,6 +1680,7 @@ jobs: if: always() timeout-minutes: 3 env: + RESOURCE_SCOPE: dw-parity-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }} DW_PARITY_UPGRADE_LABEL: php297 DW_PARITY_UPGRADE_EXPECTED_LAYOUT: php-2.9.7 DW_PARITY_UPGRADE_PHP_IMAGE: ghcr.io/durable-workflow/server@sha256:30c34c5f3256a812fbccbe505a068309b58b0c14d01ffff0465d88604cb36d50 diff --git a/app/Http/Controllers/Api/TaskQueueController.php b/app/Http/Controllers/Api/TaskQueueController.php index f27274fda..907d18fe7 100644 --- a/app/Http/Controllers/Api/TaskQueueController.php +++ b/app/Http/Controllers/Api/TaskQueueController.php @@ -10,6 +10,7 @@ use App\Support\TaskQueueAdmission; use App\Support\TaskQueueBuildIdRolloutSnapshot; use App\Support\TaskQueuePriorityFairnessSurface; +use App\Support\WorkflowDefinitionRouting; use App\Support\WorkflowQueryTaskBroker; use Illuminate\Http\JsonResponse; use Illuminate\Http\Request; @@ -28,6 +29,7 @@ public function __construct( private readonly TaskQueueBuildIdRolloutSnapshot $buildIdRollouts, private readonly TaskQueuePriorityFairnessSurface $priorityFairness, private readonly DeploymentLifecycleService $deployments, + private readonly WorkflowDefinitionRouting $definitionRouting, ) {} public function index(Request $request): JsonResponse @@ -67,22 +69,26 @@ public function show(Request $request, string $taskQueue): JsonResponse $namespace = (string) $request->attributes->get('namespace'); - return ControlPlaneProtocol::json( - $this->withAdmission( + $payload = $this->withAdmission( + $namespace, + $this->withRecentTaskFlow( $namespace, - $this->withRecentTaskFlow( + $taskQueue, + StandaloneWorkerVisibility::queueDetail( $namespace, $taskQueue, - StandaloneWorkerVisibility::queueDetail( - $namespace, - $taskQueue, - WorkerRegistration::class, - now(), - $this->workerStaleAfterSeconds(), - )->toArray(), - ), + WorkerRegistration::class, + now(), + $this->workerStaleAfterSeconds(), + )->toArray(), ), ); + $payload['definition_routing'] = $this->definitionRouting->forQueue($namespace, $taskQueue); + if ($payload['definition_routing']['blocked_sample_count'] > 0) { + $payload['repair']['needs_attention'] = true; + } + + return ControlPlaneProtocol::json($payload); } /** @@ -263,7 +269,7 @@ private function setBuildIdDrainIntent(Request $request, string $taskQueue, stri } /** - * @param array $deployment + * @param array $deployment * @return array */ private function buildIdLifecyclePayload( @@ -351,7 +357,7 @@ private function normalizeDeploymentLifecycleTimestamps(array $deployment): arra } /** - * @param list> $blockages + * @param list> $blockages */ private function orderBlockages(array &$blockages): void { @@ -630,6 +636,7 @@ private function activityAttemptStateCounts(string $namespace, string $taskQueue if ($attemptStatus === ActivityAttemptStatus::Running->value) { $counts['running_count'] += $count; $counts['open_count'] += $count; + continue; } @@ -639,6 +646,7 @@ private function activityAttemptStateCounts(string $namespace, string $taskQueue ) { $counts['retrying_count'] += $count; $counts['open_count'] += $count; + continue; } @@ -646,6 +654,7 @@ private function activityAttemptStateCounts(string $namespace, string $taskQueue $counts['timed_out_count'] += $count; $counts['expired_count'] += $count; $counts['terminal_count'] += $count; + continue; } @@ -656,18 +665,21 @@ private function activityAttemptStateCounts(string $namespace, string $taskQueue ) { $counts['timed_out_count'] += $count; $counts['terminal_count'] += $count; + continue; } if ($attemptStatus === ActivityAttemptStatus::Failed->value) { $counts['failed_count'] += $count; $counts['terminal_count'] += $count; + continue; } if ($attemptStatus === ActivityAttemptStatus::Completed->value) { $counts['completed_count'] += $count; $counts['terminal_count'] += $count; + continue; } diff --git a/app/Http/Controllers/Api/WorkerController.php b/app/Http/Controllers/Api/WorkerController.php index 9d9e3c6ee..b192dc0f6 100644 --- a/app/Http/Controllers/Api/WorkerController.php +++ b/app/Http/Controllers/Api/WorkerController.php @@ -46,6 +46,7 @@ use App\Support\WorkflowHistoryPageToken; use App\Support\WorkflowMetadataCapabilityPolicy; use App\Support\WorkflowQueryTaskBroker; +use App\Support\WorkflowReplayEligibility; use App\Support\WorkflowStreamCommandProcessor; use App\Support\WorkflowStreamService; use App\Support\WorkflowTaskLeaseRecovery; @@ -160,6 +161,7 @@ class WorkerController 'version_marker_kind_mismatch', 'version_marker_sequence_invalid', 'version_range_invalid', + 'workflow_entry_arguments_mismatch', 'workflow_nondeterministic', ]; @@ -407,7 +409,10 @@ public function register(Request $request): JsonResponse $this->validateWorkerTaskCapacity($maxWorkflowTasks, $maxActivityTasks); $maxWorkerSessions = $validated['max_concurrent_worker_sessions'] ?? 10; $taskSlots = is_array($validated['task_slots'] ?? null) ? $validated['task_slots'] : []; - $processMetrics = $this->normalizeProcessMetrics($validated['process_metrics'] ?? null); + $processMetrics = [ + ...($this->normalizeProcessMetrics($validated['process_metrics'] ?? null) ?? []), + WorkflowReplayEligibility::PROTOCOL_METRIC => WorkerProtocol::requestVersion($request), + ]; $releaseLeasesForRegistration = $this->shouldReleaseLeasesForWorkerRegistration($existing, $processMetrics); try { @@ -935,6 +940,10 @@ public function heartbeat(Request $request): JsonResponse if (array_key_exists('process_metrics', $validated)) { $update['process_metrics'] = $this->normalizeProcessMetrics($validated['process_metrics']); } + $update['process_metrics'] = [ + ...(array_key_exists('process_metrics', $update) ? ($update['process_metrics'] ?? []) : ($worker->process_metrics ?? [])), + WorkflowReplayEligibility::PROTOCOL_METRIC => WorkerProtocol::requestVersion($request), + ]; if (array_key_exists('heartbeat_interval_seconds', $validated) && $validated['heartbeat_interval_seconds'] !== null) { diff --git a/app/Http/Controllers/Api/WorkflowController.php b/app/Http/Controllers/Api/WorkflowController.php index 7af65aa5b..d2c323820 100644 --- a/app/Http/Controllers/Api/WorkflowController.php +++ b/app/Http/Controllers/Api/WorkflowController.php @@ -1250,6 +1250,7 @@ private function formatRun(WorkflowRun $run, string $namespace, array $descripti 'compatibility_supported_in_fleet' => $this->compatibilitySupportedInFleet($namespace, $run), 'compatibility_fleet_reason' => $this->compatibilityFleetReason($namespace, $run), 'payload_codec' => $run->payload_codec, + 'definition_routing' => $description['definition_routing'] ?? null, 'execution_timeout_seconds' => $description['execution_timeout_seconds'] ?? null, 'run_timeout_seconds' => $runDescription['run_timeout_seconds'] ?? null, 'execution_deadline_at' => $runDescription['execution_deadline_at'] ?? null, diff --git a/app/Support/ServerReadiness.php b/app/Support/ServerReadiness.php index 43a4ce091..9dbedf537 100644 --- a/app/Support/ServerReadiness.php +++ b/app/Support/ServerReadiness.php @@ -40,7 +40,7 @@ public function snapshot(): array : $this->cacheCheck(); $checks = [ - 'database' => $this->databaseCheck(), + 'database' => $this->databaseCheck(probeQuery: true), 'migrations' => $this->migrationCheck(), 'queue' => $queue, 'default_namespace' => $this->defaultNamespaceCheck(), @@ -93,22 +93,38 @@ public function bootstrapStatus(): array ])); } - private function databaseCheck(): array + private function databaseCheck(bool $probeQuery = false): array { + $queryAttempted = false; try { - DB::connection()->getPdo(); + $connection = DB::connection(); + $connection->getPdo(); + if ($probeQuery) { + $queryAttempted = true; + $row = $connection->selectOne('SELECT ? AS readiness_probe', [1], useReadPdo: false); + if ((string) ($row->readiness_probe ?? '') !== '1') { + throw new \UnexpectedValueException('The database did not return the readiness probe value.'); + } + } return [ 'status' => 'ok', - 'check_scope' => 'connection_only', + 'check_scope' => $probeQuery ? 'bound_read_query' : 'connection_only', 'write_capacity_verified' => false, - 'message' => 'The database connection is available. Write capacity has not been verified.', + 'message' => $probeQuery + ? 'The primary database connection can execute a bound read query. Write capacity has not been verified.' + : 'The database connection is available. Write capacity has not been verified.', ]; } catch (\Throwable $exception) { + $reason = $queryAttempted ? 'database_query_failed' : null; + if (BackendUnavailable::is($exception)) { + $reason = 'backend_unavailable'; + } + return [ 'status' => 'unavailable', 'message' => $exception->getMessage(), - 'reason' => BackendUnavailable::is($exception) ? 'backend_unavailable' : null, + 'reason' => $reason, ]; } } diff --git a/app/Support/ServerWorkflowControlPlane.php b/app/Support/ServerWorkflowControlPlane.php index 162735706..caa2da4fe 100644 --- a/app/Support/ServerWorkflowControlPlane.php +++ b/app/Support/ServerWorkflowControlPlane.php @@ -26,6 +26,7 @@ public function __construct( private readonly ControlPlaneFailureDiagnostics $failureDiagnostics, private readonly NamespaceDurableStateQuota $durableStateQuota, private readonly WorkflowStartVersionPin $versionPin, + private readonly WorkflowDefinitionRouting $definitionRouting, ) {} public function start(string $workflowType, ?string $instanceId = null, array $options = []): array @@ -148,10 +149,12 @@ public function terminate(string $instanceId, array $options = []): array public function repair(string $instanceId, array $options = []): array { - return $this->mutateCommand( + $result = $this->mutateCommand( $instanceId, 'repair', HistoryEventType::RepairRequested, $options, fn (): array => $this->inner->repair($instanceId, $options), ); + + return $this->withDefinitionRouting($instanceId, $options, $result); } public function redrive(string $instanceId, string $failedRunId, array $options = []): array @@ -309,7 +312,27 @@ private function recoveredCommandResult(Throwable $exception, WorkflowCommand $c public function describe(string $instanceId, array $options = []): array { - return $this->inner->describe($instanceId, $options); + return $this->withDefinitionRouting($instanceId, $options, $this->inner->describe($instanceId, $options)); + } + + private function withDefinitionRouting(string $instanceId, array $options, array $result): array + { + $namespace = $this->namespace($options); + if ($namespace === null || ($result['found'] ?? true) === false + || in_array($result['reason'] ?? null, ['instance_not_found', 'run_not_found'], true)) { + return $result; + } + + $runId = data_get($result, 'run.run_id') ?? $result['resolved_run_id'] ?? $result['run_id'] + ?? $result['workflow_run_id'] ?? $options['run_id'] ?? null; + $run = is_string($runId) + ? NamespaceWorkflowScope::run($namespace, $instanceId, $runId) + : NamespaceWorkflowScope::currentRun($namespace, $instanceId); + if ($run instanceof WorkflowRun) { + $result['definition_routing'] = $this->definitionRouting->inspect($run); + } + + return $result; } /** diff --git a/app/Support/WorkflowDefinitionRouting.php b/app/Support/WorkflowDefinitionRouting.php new file mode 100644 index 000000000..0579f3ba0 --- /dev/null +++ b/app/Support/WorkflowDefinitionRouting.php @@ -0,0 +1,189 @@ + */ + public function forWorker( + WorkflowRun $run, + WorkerRegistration $worker, + ?string $taskQueue = null, + ?string $requiredCompatibility = null, + ): array { + $decision = $this->context($run, $taskQueue, $requiredCompatibility); + $recorded = $decision['recorded_fingerprint']; + $advertised = self::stringValue($worker->workflow_definition_fingerprints[$run->workflow_type] ?? null); + $decision['advertised_fingerprint'] = $advertised; + + $immediate = match (true) { + ! $decision['worker_pinned'] => ['allowed' => true, 'status' => 'not_worker_pinned'], + ! $decision['pinning_enabled'] => ['allowed' => true, 'status' => 'pinning_disabled'], + $recorded === null => ['allowed' => false, 'status' => 'recorded_definition_missing'], + $advertised === null => ['allowed' => false, 'status' => 'worker_definition_missing'], + hash_equals($recorded, $advertised) => ['allowed' => true, 'status' => 'matching_definition'], + default => null, + }; + if ($immediate !== null) { + return [...$decision, ...$immediate]; + } + + foreach ($this->eligibleWorkers($run, $decision['task_queue'], $decision['required_build_id']) as $peer) { + if (self::stringValue($peer->workflow_definition_fingerprints[$run->workflow_type] ?? null) === $recorded) { + return [...$decision, 'allowed' => false, 'status' => 'matching_definition_preferred', + 'preferred_worker_id' => $peer->worker_id]; + } + } + + // Match the embedded resolver: prefer the recorded source if it is + // available, otherwise let the current definition validate replay. + // This never changes the run's original fingerprint or build pin. + $entryValidation = in_array(self::ENTRY_VALIDATION_CAPABILITY, $worker->capabilities ?? [], true); + + return [...$decision, 'allowed' => $entryValidation, + 'status' => $entryValidation ? 'replay_required' : 'entry_validation_unavailable']; + } + + /** @return array */ + public function inspect(WorkflowRun $run, ?string $taskQueue = null, ?string $requiredCompatibility = null): array + { + $task = WorkflowTask::query()->where('workflow_run_id', $run->id) + ->where('task_type', TaskType::Workflow->value)->where('status', TaskStatus::Ready->value) + ->where(static fn ($query) => $query->whereNull('available_at')->orWhere('available_at', '<=', now()->format('Y-m-d H:i:s.u'))) + ->when($taskQueue !== null, static fn ($query) => $query->where('queue', $taskQueue)) + ->orderBy('id')->first(); + if ($task instanceof WorkflowTask) { + $taskQueue ??= $task->queue; + $requiredCompatibility ??= TaskCompatibility::resolve($task, $run); + } + $context = $this->context($run, $taskQueue, $requiredCompatibility); + $matching = $changed = $missing = $unsafeReplacement = 0; + $examples = []; + + foreach ($this->eligibleWorkers($run, (string) $context['task_queue'], $context['required_build_id']) as $worker) { + $fingerprint = self::stringValue($worker->workflow_definition_fingerprints[$run->workflow_type] ?? null); + if ($fingerprint === null) { + $missing++; + } elseif ($fingerprint === $context['recorded_fingerprint']) { + $matching++; + } else { + $changed++; + if (! in_array(self::ENTRY_VALIDATION_CAPABILITY, $worker->capabilities ?? [], true)) { + $unsafeReplacement++; + } + } + if (count($examples) < 10) { + $examples[] = ['worker_id' => $worker->worker_id, 'fingerprint' => $fingerprint]; + } + } + + $eligible = $matching + $changed + $missing; + $status = match (true) { + ! $context['worker_pinned'] => 'not_worker_pinned', + ! $context['pinning_enabled'] => 'pinning_disabled', + $context['recorded_fingerprint'] === null => 'recorded_definition_missing', + $matching > 0 => 'matching_definition_available', + $changed > $unsafeReplacement => 'replay_required', + $unsafeReplacement > 0 => 'entry_validation_unavailable', + $missing > 0 => 'worker_definition_missing', + default => 'no_eligible_worker', + }; + $runnable = ! $run->status->isTerminal() && $task instanceof WorkflowTask; + $blocked = $runnable && $context['worker_pinned'] && $context['pinning_enabled'] + && in_array($status, ['recorded_definition_missing', 'worker_definition_missing', 'no_eligible_worker', 'entry_validation_unavailable'], true); + $resolution = match (true) { + $blocked && $status === 'entry_validation_unavailable' => 'Upgrade the replacement worker SDK to an implementation advertising workflow_entry_arguments_validation, or restore the recorded definition. The current replacement cannot safely refuse incompatible entry input.', + $blocked => 'Register a fresh worker for the original namespace, task queue, workflow type, build and replay capabilities with its definition fingerprint. Do not rewrite the run history.', + $status === 'replay_required' => 'Current workers may replay the original history. Use versioned branches for command changes; incompatible replay still refuses.', + default => 'Recorded build, namespace, type and replay-capability guards remain in force.', + }; + + return [...$context, 'status' => $status, 'blocked' => $blocked, 'runnable_task' => $runnable, + 'eligible_worker_count' => $eligible, 'matching_worker_count' => $matching, + 'changed_worker_count' => $changed, 'missing_fingerprint_count' => $missing, + 'entry_validation_unavailable_count' => $unsafeReplacement, + 'workers' => $examples, 'workers_truncated' => $eligible > count($examples), + 'resolution' => $resolution, + ]; + } + + /** Bounded examples of due ready runs, never a repair or a payload read. */ + public function forQueue(string $namespace, string $taskQueue): array + { + $runs = WorkflowRun::query()->where('namespace', $namespace)->where('queue', $taskQueue) + ->whereIn('id', WorkflowTask::query()->select('workflow_run_id') + ->where('task_type', TaskType::Workflow->value)->where('status', TaskStatus::Ready->value) + ->where('queue', $taskQueue) + ->where(static fn ($query) => $query->whereNull('available_at')->orWhere('available_at', '<=', now()->format('Y-m-d H:i:s.u')))) + ->orderBy('id')->limit(11)->get(); + $examples = []; + foreach ($runs->take(10) as $run) { + if (! $run->status->isTerminal()) { + $examples[] = ['workflow_id' => $run->workflow_instance_id, 'run_id' => $run->id, + ...$this->inspect($run, $taskQueue)]; + } + } + + return ['sample_limit' => 10, 'has_more_ready_runs' => $runs->count() > 10, + 'blocked_sample_count' => count(array_filter($examples, static fn (array $row): bool => $row['blocked'])), + 'runs' => $examples]; + } + + /** @return array */ + private function context(WorkflowRun $run, ?string $taskQueue, ?string $requiredCompatibility): array + { + $started = WorkflowHistoryEvent::query()->where('workflow_run_id', $run->id) + ->where('event_type', 'WorkflowStarted')->orderBy('sequence')->first(); + $recorded = self::stringValue($started?->payload['workflow_definition_fingerprint'] ?? null); + + return [ + 'pinning_enabled' => (bool) config('workflows.v2.compatibility.pin_to_recorded_fingerprint', true), + 'worker_pinned' => ($started?->payload['workflow_definition_fingerprint_source'] ?? null) === 'worker', + 'recorded_fingerprint' => $recorded, + 'workflow_type' => $run->workflow_type, + 'task_queue' => $taskQueue ?? $run->queue, + 'required_build_id' => $requiredCompatibility ?? self::stringValue($run->compatibility), + ]; + } + + /** @return Generator */ + private function eligibleWorkers( + WorkflowRun $run, + string $taskQueue, + ?string $requiredCompatibility, + ): Generator { + $workers = WorkerRegistration::query()->where('namespace', $run->namespace) + ->where('task_queue', $taskQueue)->where('status', 'active'); + if ($requiredCompatibility !== null) { + $workers->where('build_id', $requiredCompatibility); + } + + foreach ($workers->cursor(['namespace', 'worker_id', 'task_queue', 'build_id', 'status', 'last_heartbeat_at', + 'workflow_definition_fingerprints', 'supported_workflow_types', 'max_concurrent_workflow_tasks', + 'capabilities', 'process_metrics']) as $worker) { + if (! in_array($run->workflow_type, $worker->supported_workflow_types ?? [], true) + || ($worker->max_concurrent_workflow_tasks !== null && (int) $worker->max_concurrent_workflow_tasks < 1)) { + continue; + } + if (WorkflowReplayEligibility::allows($worker, $run, WorkflowReplayEligibility::recordedProtocol($worker))) { + yield $worker; + } + } + } + + private static function stringValue(mixed $value): ?string + { + return is_string($value) && trim($value) !== '' ? trim($value) : null; + } +} diff --git a/app/Support/WorkflowReplayEligibility.php b/app/Support/WorkflowReplayEligibility.php new file mode 100644 index 000000000..da4744b2d --- /dev/null +++ b/app/Support/WorkflowReplayEligibility.php @@ -0,0 +1,37 @@ +namespace !== $run->namespace || ! WorkerPollFence::isFresh($worker)) { + return false; + } + + $capabilities = is_array($worker->capabilities) ? $worker->capabilities : []; + + if (is_string($run->cancellation_request_command_id) + && (! CooperativeCancellationPolicy::supports($capabilities, $protocolVersion) + || (NexusCooperativeCancellationPolicy::requiresCapabilityForReplay($run) + && ! NexusCooperativeCancellationPolicy::supports($capabilities, $protocolVersion)))) { + return false; + } + + return PreparedLocalActivityPolicy::canReplayRun($run->id, $capabilities, $protocolVersion) + && WorkflowMetadataCapabilityPolicy::canReplayRun($run->id, $capabilities, $protocolVersion); + } + + public static function recordedProtocol(WorkerRegistration $worker): ?string + { + $version = $worker->process_metrics[self::PROTOCOL_METRIC] ?? null; + + return is_string($version) && $version !== '' ? $version : null; + } +} diff --git a/app/Support/WorkflowRunDiagnostics.php b/app/Support/WorkflowRunDiagnostics.php index 6f6490561..810d3f1be 100644 --- a/app/Support/WorkflowRunDiagnostics.php +++ b/app/Support/WorkflowRunDiagnostics.php @@ -41,6 +41,8 @@ class WorkflowRunDiagnostics private const LAST_EVENT_PAYLOAD_KEY_LIMIT = 20; + public function __construct(private readonly WorkflowDefinitionRouting $definitionRouting) {} + /** * @return array */ @@ -84,6 +86,7 @@ public function forRun(string $namespace, WorkflowRun $run, bool $includeLastEve 'recent_failures_truncated' => $failureRows->count() > self::FAILURE_LIMIT, 'latest_workflow_task_failure' => $latestWorkflowTaskFailure, 'compatibility' => $this->compatibility($namespace, $run, $summary, $taskQueue), + 'definition_routing' => $this->definitionRouting->inspect($run), 'cancellation_cascade_supported' => class_exists(CancellationCascadeView::class), 'cancellation_cascade' => class_exists(CancellationCascadeView::class) && $run->namespace === $namespace ? CancellationCascadeView::forRun($run) @@ -95,6 +98,16 @@ public function forRun(string $namespace, WorkflowRun $run, bool $includeLastEve ...$this->pendingWorkflowRoutingFindings($namespace, $run, $taskRows->all(), $taskQueue), ]; + if ($payload['definition_routing']['blocked']) { + $payload['findings'][] = [ + 'severity' => 'warning', 'code' => 'workflow_definition_routing_blocked', + 'message' => 'A ready workflow task has no eligible definition route.', + 'routing_status' => $payload['definition_routing']['status'], + 'expected_resolution' => $payload['definition_routing']['resolution'], + ]; + $payload['diagnostic_status'] = 'needs_attention'; + } + return $payload; } diff --git a/app/Support/WorkflowTaskPoller.php b/app/Support/WorkflowTaskPoller.php index 6c21da0b8..f21fb6877 100644 --- a/app/Support/WorkflowTaskPoller.php +++ b/app/Support/WorkflowTaskPoller.php @@ -12,7 +12,6 @@ use Workflow\V2\Enums\TaskStatus; use Workflow\V2\Enums\TaskType; use Workflow\V2\Jobs\RunTimerTask; -use Workflow\V2\Models\WorkflowHistoryEvent; use Workflow\V2\Models\WorkflowRun; use Workflow\V2\Models\WorkflowSignal; use Workflow\V2\Models\WorkflowTask; @@ -42,6 +41,7 @@ public function __construct( private readonly PollRequestLeaseBinding $pollLeaseBindings, private readonly WorkflowUpdateValidationTaskBroker $updateValidationTasks, private readonly WorkflowTaskKindSelector $taskKindSelector, + private readonly WorkflowDefinitionRouting $definitionRouting, ) {} /** @@ -749,7 +749,6 @@ function () use ( historyPageSize: $historyPageSize, acceptHistoryEncoding: $acceptHistoryEncoding, supportedWorkflowTypes: $supportedWorkflowTypes, - workflowDefinitionFingerprints: $workflowDefinitionFingerprints, workerPollFence: $workerPollFence, ), ), @@ -798,7 +797,6 @@ function () use ( historyPageSize: $historyPageSize, acceptHistoryEncoding: $acceptHistoryEncoding, supportedWorkflowTypes: $supportedWorkflowTypes, - workflowDefinitionFingerprints: $workflowDefinitionFingerprints, workerPollFence: $workerPollFence, ), ), @@ -968,7 +966,6 @@ private function claimReadyTask( ?int $historyPageSize = null, ?string $acceptHistoryEncoding = null, array $supportedWorkflowTypes = [], - array $workflowDefinitionFingerprints = [], array $workerPollFence = [], ): ?array { $readyTasks = $this->pollReplayEligibleReadyTasks( @@ -1041,9 +1038,11 @@ private function claimReadyTask( } if (! $this->matchesRecordedWorkflowDefinition( + $namespace, + $leaseOwner, $runId, - $readyTask['workflow_type'] ?? null, - $workflowDefinitionFingerprints, + $taskQueue, + $effectiveCompatibility, )) { continue; } @@ -2325,63 +2324,25 @@ private function workerCanReplayRun( ->where('worker_id', $workerId) ->first(); - if (! $worker instanceof WorkerRegistration || ! WorkerPollFence::isFresh($worker)) { - return false; - } - - $capabilities = is_array($worker->capabilities) - ? array_values(array_filter( - $worker->capabilities, - static fn (mixed $capability): bool => is_string($capability) && trim($capability) !== '', - )) - : []; - $run = WorkflowRun::query()->where('namespace', $namespace)->find($runId); - if ($run instanceof WorkflowRun && is_string($run->cancellation_request_command_id) - && ! CooperativeCancellationPolicy::supports($capabilities, $protocolVersion)) { - return false; - } - if ($run instanceof WorkflowRun && is_string($run->cancellation_request_command_id) - && NexusCooperativeCancellationPolicy::requiresCapabilityForReplay($run) - && ! NexusCooperativeCancellationPolicy::supports($capabilities, $protocolVersion)) { - return false; - } - return PreparedLocalActivityPolicy::canReplayRun($runId, $capabilities, $protocolVersion) - && WorkflowMetadataCapabilityPolicy::canReplayRun( - $runId, - $capabilities, - $protocolVersion, - ); + return $worker instanceof WorkerRegistration && $run instanceof WorkflowRun + && WorkflowReplayEligibility::allows($worker, $run, $protocolVersion); } - /** @param array $workflowDefinitionFingerprints */ private function matchesRecordedWorkflowDefinition( + string $namespace, + string $workerId, string $runId, - mixed $workflowType, - array $workflowDefinitionFingerprints, + string $taskQueue, + ?string $requiredCompatibility, ): bool { - $started = WorkflowHistoryEvent::query() - ->where('workflow_run_id', $runId) - ->where('event_type', 'WorkflowStarted') - ->orderBy('sequence') - ->first(); - - if (($started?->payload['workflow_definition_fingerprint_source'] ?? null) !== 'worker') { - return true; - } - - $recorded = $this->nonEmptyString($started?->payload['workflow_definition_fingerprint'] ?? null); - - if ($recorded === null) { - return false; - } - - $advertised = is_string($workflowType) - ? $this->nonEmptyString($workflowDefinitionFingerprints[$workflowType] ?? null) - : null; + $run = WorkflowRun::query()->where('namespace', $namespace)->find($runId); + $worker = WorkerRegistration::query()->where('namespace', $namespace) + ->where('worker_id', $workerId)->first(); - return $advertised !== null && hash_equals($recorded, $advertised); + return $run instanceof WorkflowRun && $worker instanceof WorkerRegistration + && $this->definitionRouting->forWorker($run, $worker, $taskQueue, $requiredCompatibility)['allowed']; } /** diff --git a/benchmarks/server-port/v3/README.md b/benchmarks/server-port/v3/README.md index c99656586..f74ef55e1 100644 --- a/benchmarks/server-port/v3/README.md +++ b/benchmarks/server-port/v3/README.md @@ -5,7 +5,9 @@ run's native source, optimized binary and machine facts filled in. The workload, artifacts, limits and probe hashes are unchanged. The [October 20 observations](observations-2026-10-20.md) are one PHP/native pair on a disposable VM, run with `scripts/perf/run-server-port-matched.sh` -and summarized with `scripts/perf/summarize-server-port-matched.py`. +and summarized with `scripts/perf/summarize-server-port-matched.py`. The +[repeated observations](repeated-2026-10-20.md) are two alternating pairs +on a second host, after #516. To repeat it on a fresh dedicated host with Docker and Compose: diff --git a/benchmarks/server-port/v3/repeated-2026-10-20.md b/benchmarks/server-port/v3/repeated-2026-10-20.md new file mode 100644 index 000000000..f5d1eebed --- /dev/null +++ b/benchmarks/server-port/v3/repeated-2026-10-20.md @@ -0,0 +1,101 @@ +# Repeated matched observations, October 20 + +Two alternating PHP/native pairs on one dedicated experiment host, after #516 +landed on main. They repeat the [morning pair](observations-2026-10-20.md) +with the same workload, artifacts, limits and commands. These are development +observations on one host and one database family, not a release claim. + +## Conditions + +- **Source.** Server main `8957ba27a` (#516 merged), native tree `55aec4d8`. + The optimized binary was built with `cargo build --release --locked --bin + durable-workflow-server`, SHA-256 `edc283f1…6e2cc`. The PHP references, + PHP SDK, MySQL and Redis artifacts are the profile's, unchanged. +- **Host.** A fresh Vultr `vhp-4c-8gb-amd` VM: 4 vCPU AMD EPYC-Rome, 7.7 GiB, + Ubuntu 24.04.5 with kernel 6.8, Docker 29.1.3 and Compose 2.40.3, no swap. + No other workload ran. It ran from 13:15 to 15:48 UTC, was deleted at the + end and its deletion was verified, at about $0.20. +- **Procedure.** A native smoke check, then PHP, native, PHP and native full + runs in that order. Each run uses its own Compose project and fresh + database. Each run is the morning procedure: warmup, three 120 s windows at + 0.25, 0.5 and 1 workflow/s, the 0.25/s windows repeated after as a drift + check, and mixed idle polls. The four runs' summaries are in + `summary-repeated-{php,native}-{1,2}-2026-10-20.json`. + +All four runs finished with status 0. Every window's workflows verified, with +nothing pending. No container restarted or was OOM-killed. + +## Offered work and latency + +Each cell gives the range of the three per-window values. + +| Run / rate | In-window completions | p50 s | p95 s | p99 s | +| --- | --- | --- | --- | --- | +| PHP 1 / 0.25 before | 30/30/30 | 1.129–1.188 | 1.459–1.620 | 1.515–1.652 | +| PHP 1 / 0.5 | 60/60/60 | 1.129–1.215 | 1.517–1.545 | 1.633–1.675 | +| PHP 1 / 1 | 69/68/66 of 120 | 44.782–48.343 | 64.771–67.389 | 66.026–69.027 | +| PHP 1 / 0.25 after | 30/30/30 | 1.195–1.226 | 1.568–1.732 | 1.686–1.818 | +| Native 1 / 0.25 before | 30/30/30 | 0.815–1.171 | 1.485–1.785 | 1.867–2.202 | +| Native 1 / 0.5 | 60/60/60 | 0.806–0.862 | 1.361–1.434 | 1.490–1.730 | +| Native 1 / 1 | 120/120/120 | 0.642–0.688 | 1.139–2.111 | 1.604–2.545 | +| Native 1 / 0.25 after | 30/30/30 | 1.061–1.139 | 1.691–1.887 | 1.916–2.042 | +| PHP 2 / 0.25 before | 30/30/30 | 1.236–1.306 | 1.649–1.680 | 1.697–1.818 | +| PHP 2 / 0.5 | 60/60/60 | 1.179–1.268 | 1.557–1.635 | 1.694–1.867 | +| PHP 2 / 1 | 62/67/65 of 120 | 48.434–55.744 | 65.491–72.870 | 66.325–74.189 | +| PHP 2 / 0.25 after | 30/30/30 | 1.164–1.258 | 1.441–1.643 | 1.531–1.731 | +| Native 2 / 0.25 before | 30/30/30 | 0.942–0.989 | 1.655–1.876 | 1.860–1.944 | +| Native 2 / 0.5 | 60/60/60 | 0.874–0.985 | 1.380–1.555 | 1.618–1.918 | +| Native 2 / 1 | 120/119/120 | 0.675–0.715 | 1.121–1.567 | 1.759–2.945 | +| Native 2 / 0.25 after | 30/30/30 | 0.951–1.049 | 1.503–1.822 | 1.716–1.958 | + +PHP still does not sustain 1/s. Its in-window completions at 1/s fell from +93–96 this morning to 62–69 here, and its 1/s median latency grew from 15–18 s +to 45–56 s. Its CPU per workflow also rose about a quarter on this host, as the +next table shows. Native completed all 120 in both pairs here and this +morning. The two PHP runs agree with each other, and so do the two native runs. +So the change is between hosts, not drift within this one. + +## CPU and memory + +Core-seconds per verified workflow, and the highest container memory sampled. + +| Run / rate | Runtime core-s/wf | MySQL core-s/wf | Runtime max MiB | +| --- | --- | --- | --- | +| PHP 1 / 0.25 before | 2.574–2.755 | 0.662–0.713 | 192.0 | +| PHP 1 / 1 | 1.599–1.638 | 0.554–0.566 | 201.2 | +| PHP 2 / 0.25 before | 2.737–2.829 | 0.710–0.748 | 203.0 | +| PHP 2 / 1 | 1.603–1.676 | 0.549–0.573 | 213.1 | +| Native 1 / 0.25 before | 0.239–0.247 | 0.489–0.506 | 6.3 | +| Native 1 / 1 | 0.102–0.104 | 0.241–0.245 | 7.0 | +| Native 2 / 0.25 before | 0.245–0.256 | 0.503–0.518 | 6.9 | +| Native 2 / 1 | 0.110–0.116 | 0.264–0.275 | 6.8 | + +At 1/s, native uses about a fifteenth of PHP's runtime CPU per workflow and +about half of its database CPU. Its peak memory is about 7 MiB against PHP's +190–215 MiB. + +## Idle mixed long polls + +Runtime core-seconds per idle window, three windows each. + +| Run | 6 polls | 12 polls | 24 polls | Health failures | +| --- | --- | --- | --- | --- | +| PHP 1 | 6.117–6.877 | 7.279–8.642 | 9.160–9.564 | 0, 1, 2 per window | +| Native 1 | 0.645–0.683 | 0.790–0.917 | 1.206–1.256 | none | +| PHP 2 | 5.699–6.651 | 7.803–8.198 | 9.408–9.695 | 0, 1, 2–3 per window | +| Native 2 | 0.684–0.755 | 0.822–0.922 | 1.189–1.241 | none | + +PHP answers part of its 12- and 24-poll windows with backpressure, and its +health checks fail in them. Native answers every poll empty, as expected. + +## What this shows and what it does not + +On this profile, native meets or beats PHP on throughput, CPU, memory and idle +long-poll cost in every window of two repeated pairs. The before and after +windows show no drift within any run. + +It does not show the result on PostgreSQL or SQLite, on other machine types, +across nodes, or under the CPU-heavy deadline profile. The host-to-host change +in PHP's capacity also says one host is not enough for a release claim. The +driver records host load and swap but not CPU steal. Sampling steal in +`host-state.tsv` is the next change to the driver. diff --git a/benchmarks/server-port/v3/summary-repeated-native-1-2026-10-20.json b/benchmarks/server-port/v3/summary-repeated-native-1-2026-10-20.json new file mode 100644 index 000000000..89b505fd0 --- /dev/null +++ b/benchmarks/server-port/v3/summary-repeated-native-1-2026-10-20.json @@ -0,0 +1,911 @@ +{ + "identity": { + "implementation": "native", + "compose_project": "dw-port-native-20261020t141652", + "profile": "/home/linuxuser/profile.json", + "profile_sha256": "5e7946a6a506e391510e9222021e60c80d184742b030ee6e2978ab0c0b1e8ac5", + "source_revision": "8957ba27aa24b79ec2ea59e0c37b5ce662d434ab", + "native_binary_sha256": "edc283f130ff5c802fe5a80d56730cc9fe377c01c9c90340e08a68eb0866e2cc", + "smoke": false, + "idle_only": false, + "machine": { + "kernel": "6.8.0-146-generic", + "logical_cpus": 4, + "physical_memory_bytes": 8320786432, + "cpu_model": "AMD EPYC-Rome Processor", + "docker": "29.1.3", + "cgroup_driver": "systemd", + "docker_storage_driver": "overlayfs", + "docker_root_filesystem": "ext4" + } + }, + "windows": { + "setup": { + "seconds": 5.4, + "health": { + "replies": 8, + "failures": 0, + "slowest_ok_seconds": 0.302 + }, + "cpu_core_seconds": { + "mysql": 0.876176, + "native-server": 0.408746, + "probe": 0.0, + "sdk-worker": 0.039165 + }, + "max_memory_bytes": { + "mysql": 527196160, + "native-server": 4235264, + "probe": 16654336, + "sdk-worker": 19292160 + }, + "max_runtime_memory_bytes": 4235264, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 2, + "accepted_starts": 2, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 2, + "pending_at_deadline": 0, + "closed_during_offer_window": 2, + "latency_p50_seconds": 0.2922959327697754, + "latency_p95_seconds": 0.5931649208068848, + "latency_p99_seconds": 0.5931649208068848, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.2044, + "mysql_core_seconds_per_workflow": 0.4381 + }, + "warmup": { + "seconds": 59.3, + "health": { + "replies": 88, + "failures": 0, + "slowest_ok_seconds": 0.476 + }, + "cpu_core_seconds": { + "mysql": 7.408028, + "native-server": 3.565426, + "probe": 0.383084, + "sdk-worker": 0.367419 + }, + "max_memory_bytes": { + "mysql": 540975104, + "native-server": 6533120, + "probe": 16683008, + "sdk-worker": 19447808 + }, + "max_runtime_memory_bytes": 6533120, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 15, + "accepted_starts": 15, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 15, + "pending_at_deadline": 0, + "closed_during_offer_window": 15, + "latency_p50_seconds": 1.2313079833984375, + "latency_p95_seconds": 2.112252950668335, + "latency_p99_seconds": 2.112252950668335, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.2377, + "mysql_core_seconds_per_workflow": 0.4939 + }, + "low-before-1": { + "seconds": 120.5, + "health": { + "replies": 181, + "failures": 0, + "slowest_ok_seconds": 0.431 + }, + "cpu_core_seconds": { + "mysql": 14.753484, + "native-server": 7.199888, + "probe": 0.365237, + "sdk-worker": 0.774945 + }, + "max_memory_bytes": { + "mysql": 556244992, + "native-server": 6656000, + "probe": 16691200, + "sdk-worker": 21241856 + }, + "max_runtime_memory_bytes": 6656000, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 30, + "accepted_starts": 30, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 30, + "pending_at_deadline": 0, + "closed_during_offer_window": 30, + "latency_p50_seconds": 0.8146710395812988, + "latency_p95_seconds": 1.784688949584961, + "latency_p99_seconds": 1.8666729927062988, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.24, + "mysql_core_seconds_per_workflow": 0.4918 + }, + "low-before-2": { + "seconds": 119.5, + "health": { + "replies": 178, + "failures": 0, + "slowest_ok_seconds": 0.435 + }, + "cpu_core_seconds": { + "mysql": 15.181942, + "native-server": 7.39961, + "probe": 0.023146, + "sdk-worker": 0.804998 + }, + "max_memory_bytes": { + "mysql": 569720832, + "native-server": 5312512, + "probe": 16683008, + "sdk-worker": 23449600 + }, + "max_runtime_memory_bytes": 5312512, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 30, + "accepted_starts": 30, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 30, + "pending_at_deadline": 0, + "closed_during_offer_window": 30, + "latency_p50_seconds": 1.170957088470459, + "latency_p95_seconds": 1.6039619445800781, + "latency_p99_seconds": 2.2019379138946533, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.2467, + "mysql_core_seconds_per_workflow": 0.5061 + }, + "low-before-3": { + "seconds": 120.1, + "health": { + "replies": 183, + "failures": 0, + "slowest_ok_seconds": 0.342 + }, + "cpu_core_seconds": { + "mysql": 14.658095, + "native-server": 7.169432, + "probe": -0.017477, + "sdk-worker": 0.760215 + }, + "max_memory_bytes": { + "mysql": 580112384, + "native-server": 5115904, + "probe": 16936960, + "sdk-worker": 24776704 + }, + "max_runtime_memory_bytes": 5115904, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 30, + "accepted_starts": 30, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 30, + "pending_at_deadline": 0, + "closed_during_offer_window": 30, + "latency_p50_seconds": 0.8877580165863037, + "latency_p95_seconds": 1.4847869873046875, + "latency_p99_seconds": 2.122774839401245, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.239, + "mysql_core_seconds_per_workflow": 0.4886 + }, + "rate-0.5-1": { + "seconds": 120.2, + "health": { + "replies": 174, + "failures": 0, + "slowest_ok_seconds": 0.527 + }, + "cpu_core_seconds": { + "mysql": 19.666724, + "native-server": 8.952809, + "probe": 0.261888, + "sdk-worker": 1.170249 + }, + "max_memory_bytes": { + "mysql": 604008448, + "native-server": 7004160, + "probe": 16703488, + "sdk-worker": 28123136 + }, + "max_runtime_memory_bytes": 7004160, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.5, + "planned_starts": 60, + "accepted_starts": 60, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 60, + "pending_at_deadline": 0, + "closed_during_offer_window": 60, + "latency_p50_seconds": 0.8057119846343994, + "latency_p95_seconds": 1.4087231159210205, + "latency_p99_seconds": 1.4897749423980713, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.1492, + "mysql_core_seconds_per_workflow": 0.3278 + }, + "rate-0.5-2": { + "seconds": 120.6, + "health": { + "replies": 169, + "failures": 0, + "slowest_ok_seconds": 0.606 + }, + "cpu_core_seconds": { + "mysql": 20.18183, + "native-server": 9.157594, + "probe": -0.046921, + "sdk-worker": 1.216679 + }, + "max_memory_bytes": { + "mysql": 626311168, + "native-server": 6995968, + "probe": 16932864, + "sdk-worker": 30769152 + }, + "max_runtime_memory_bytes": 6995968, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.5, + "planned_starts": 60, + "accepted_starts": 60, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 60, + "pending_at_deadline": 0, + "closed_during_offer_window": 60, + "latency_p50_seconds": 0.8616900444030762, + "latency_p95_seconds": 1.3606820106506348, + "latency_p99_seconds": 1.7296950817108154, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.1526, + "mysql_core_seconds_per_workflow": 0.3364 + }, + "rate-0.5-3": { + "seconds": 119.8, + "health": { + "replies": 168, + "failures": 0, + "slowest_ok_seconds": 0.595 + }, + "cpu_core_seconds": { + "mysql": 19.68832, + "native-server": 8.950379, + "probe": -0.012808, + "sdk-worker": 1.201273 + }, + "max_memory_bytes": { + "mysql": 647774208, + "native-server": 7008256, + "probe": 16949248, + "sdk-worker": 33390592 + }, + "max_runtime_memory_bytes": 7008256, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.5, + "planned_starts": 60, + "accepted_starts": 60, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 60, + "pending_at_deadline": 0, + "closed_during_offer_window": 60, + "latency_p50_seconds": 0.8583979606628418, + "latency_p95_seconds": 1.4343540668487549, + "latency_p99_seconds": 1.5413579940795898, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.1492, + "mysql_core_seconds_per_workflow": 0.3281 + }, + "rate-1-1": { + "seconds": 121.3, + "health": { + "replies": 158, + "failures": 0, + "slowest_ok_seconds": 0.763 + }, + "cpu_core_seconds": { + "mysql": 28.940648, + "native-server": 12.366946, + "probe": 0.301992, + "sdk-worker": 2.071431 + }, + "max_memory_bytes": { + "mysql": 691490816, + "native-server": 7041024, + "probe": 16936960, + "sdk-worker": 36601856 + }, + "max_runtime_memory_bytes": 7041024, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 1, + "planned_starts": 120, + "accepted_starts": 120, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 120, + "pending_at_deadline": 0, + "closed_during_offer_window": 120, + "latency_p50_seconds": 0.642268180847168, + "latency_p95_seconds": 1.1391160488128662, + "latency_p99_seconds": 1.6041810512542725, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.1031, + "mysql_core_seconds_per_workflow": 0.2412 + }, + "rate-1-2": { + "seconds": 121.3, + "health": { + "replies": 156, + "failures": 0, + "slowest_ok_seconds": 0.764 + }, + "cpu_core_seconds": { + "mysql": 29.034662, + "native-server": 12.278764, + "probe": -0.090032, + "sdk-worker": 2.080174 + }, + "max_memory_bytes": { + "mysql": 735203328, + "native-server": 7110656, + "probe": 18374656, + "sdk-worker": 36524032 + }, + "max_runtime_memory_bytes": 7110656, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 1, + "planned_starts": 120, + "accepted_starts": 120, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 120, + "pending_at_deadline": 0, + "closed_during_offer_window": 120, + "latency_p50_seconds": 0.6831719875335693, + "latency_p95_seconds": 2.1111860275268555, + "latency_p99_seconds": 2.5451819896698, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.1023, + "mysql_core_seconds_per_workflow": 0.242 + }, + "rate-1-3": { + "seconds": 121.6, + "health": { + "replies": 156, + "failures": 0, + "slowest_ok_seconds": 0.717 + }, + "cpu_core_seconds": { + "mysql": 29.387882, + "native-server": 12.440495, + "probe": 0.039508, + "sdk-worker": 2.065757 + }, + "max_memory_bytes": { + "mysql": 777027584, + "native-server": 7311360, + "probe": 16703488, + "sdk-worker": 37605376 + }, + "max_runtime_memory_bytes": 7311360, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 1, + "planned_starts": 120, + "accepted_starts": 120, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 120, + "pending_at_deadline": 0, + "closed_during_offer_window": 120, + "latency_p50_seconds": 0.6876001358032227, + "latency_p95_seconds": 1.6906061172485352, + "latency_p99_seconds": 2.5086309909820557, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.1037, + "mysql_core_seconds_per_workflow": 0.2449 + }, + "low-after-1": { + "seconds": 120.1, + "health": { + "replies": 180, + "failures": 0, + "slowest_ok_seconds": 0.387 + }, + "cpu_core_seconds": { + "mysql": 15.418894, + "native-server": 7.249945, + "probe": -0.469773, + "sdk-worker": 0.772241 + }, + "max_memory_bytes": { + "mysql": 788262912, + "native-server": 6823936, + "probe": 16687104, + "sdk-worker": 37871616 + }, + "max_runtime_memory_bytes": 6823936, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 30, + "accepted_starts": 30, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 30, + "pending_at_deadline": 0, + "closed_during_offer_window": 30, + "latency_p50_seconds": 1.120251178741455, + "latency_p95_seconds": 1.8871901035308838, + "latency_p99_seconds": 1.9591500759124756, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.2417, + "mysql_core_seconds_per_workflow": 0.514 + }, + "low-after-2": { + "seconds": 120.2, + "health": { + "replies": 180, + "failures": 0, + "slowest_ok_seconds": 0.429 + }, + "cpu_core_seconds": { + "mysql": 15.669081, + "native-server": 7.285336, + "probe": 0.020311, + "sdk-worker": 0.811745 + }, + "max_memory_bytes": { + "mysql": 794497024, + "native-server": 7094272, + "probe": 16699392, + "sdk-worker": 38146048 + }, + "max_runtime_memory_bytes": 7094272, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 30, + "accepted_starts": 30, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 30, + "pending_at_deadline": 0, + "closed_during_offer_window": 30, + "latency_p50_seconds": 1.0607280731201172, + "latency_p95_seconds": 1.8657190799713135, + "latency_p99_seconds": 1.9156861305236816, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.2428, + "mysql_core_seconds_per_workflow": 0.5223 + }, + "low-after-3": { + "seconds": 120.2, + "health": { + "replies": 180, + "failures": 0, + "slowest_ok_seconds": 0.388 + }, + "cpu_core_seconds": { + "mysql": 15.794167, + "native-server": 7.405617, + "probe": 0.017688, + "sdk-worker": 0.800382 + }, + "max_memory_bytes": { + "mysql": 800604160, + "native-server": 6619136, + "probe": 16941056, + "sdk-worker": 38514688 + }, + "max_runtime_memory_bytes": 6619136, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 30, + "accepted_starts": 30, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 30, + "pending_at_deadline": 0, + "closed_during_offer_window": 30, + "latency_p50_seconds": 1.1393530368804932, + "latency_p95_seconds": 1.6913490295410156, + "latency_p99_seconds": 2.042344093322754, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.2469, + "mysql_core_seconds_per_workflow": 0.5265 + } + }, + "idle": { + "idle-6-1": { + "seconds": 12.3, + "health": { + "replies": 20, + "failures": 0, + "slowest_ok_seconds": 0.104 + }, + "cpu_core_seconds": { + "mysql": 1.271815, + "native-server": 0.678925, + "probe": -0.55062, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 801001472, + "native-server": 5025792, + "probe": 31444992, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 5025792, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "requested_polls": 6, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "empty": 6 + } + }, + "idle-6-2": { + "seconds": 11.2, + "health": { + "replies": 18, + "failures": 0, + "slowest_ok_seconds": 0.104 + }, + "cpu_core_seconds": { + "mysql": 1.277346, + "native-server": 0.682507, + "probe": -0.004253, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 804032512, + "native-server": 6746112, + "probe": 31248384, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 6746112, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "requested_polls": 6, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "empty": 6 + } + }, + "idle-6-3": { + "seconds": 11.3, + "health": { + "replies": 17, + "failures": 0, + "slowest_ok_seconds": 0.105 + }, + "cpu_core_seconds": { + "mysql": 1.187882, + "native-server": 0.644606, + "probe": 0.024741, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 802537472, + "native-server": 6971392, + "probe": 31236096, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 6971392, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "requested_polls": 6, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "empty": 6 + } + }, + "idle-12-1": { + "seconds": 11.5, + "health": { + "replies": 17, + "failures": 0, + "slowest_ok_seconds": 0.105 + }, + "cpu_core_seconds": { + "mysql": 1.518828, + "native-server": 0.80815, + "probe": 0.14792, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 801198080, + "native-server": 5578752, + "probe": 45658112, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 5578752, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "requested_polls": 12, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "empty": 12 + } + }, + "idle-12-2": { + "seconds": 11.3, + "health": { + "replies": 18, + "failures": 0, + "slowest_ok_seconds": 0.016 + }, + "cpu_core_seconds": { + "mysql": 1.467164, + "native-server": 0.789659, + "probe": -0.105134, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 801349632, + "native-server": 5386240, + "probe": 44597248, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 5386240, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "requested_polls": 12, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "empty": 12 + } + }, + "idle-12-3": { + "seconds": 11.5, + "health": { + "replies": 16, + "failures": 0, + "slowest_ok_seconds": 0.104 + }, + "cpu_core_seconds": { + "mysql": 1.661435, + "native-server": 0.916948, + "probe": 0.13414, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 801509376, + "native-server": 5398528, + "probe": 43933696, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 5398528, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "requested_polls": 12, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "empty": 12 + } + }, + "idle-24-1": { + "seconds": 11.9, + "health": { + "replies": 18, + "failures": 0, + "slowest_ok_seconds": 0.019 + }, + "cpu_core_seconds": { + "mysql": 2.294075, + "native-server": 1.256186, + "probe": 0.249966, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 801611776, + "native-server": 5935104, + "probe": 68399104, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 5935104, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "requested_polls": 24, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "empty": 24 + } + }, + "idle-24-2": { + "seconds": 12.2, + "health": { + "replies": 18, + "failures": 0, + "slowest_ok_seconds": 0.017 + }, + "cpu_core_seconds": { + "mysql": 2.315425, + "native-server": 1.232444, + "probe": -0.043559, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 801931264, + "native-server": 5935104, + "probe": 67612672, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 5935104, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "requested_polls": 24, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "empty": 24 + } + }, + "idle-24-3": { + "seconds": 12.2, + "health": { + "replies": 19, + "failures": 0, + "slowest_ok_seconds": 0.019 + }, + "cpu_core_seconds": { + "mysql": 2.217726, + "native-server": 1.206354, + "probe": 0.083076, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 801779712, + "native-server": 6131712, + "probe": 73183232, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 6131712, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "requested_polls": 24, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "empty": 24 + } + } + } +} diff --git a/benchmarks/server-port/v3/summary-repeated-native-2-2026-10-20.json b/benchmarks/server-port/v3/summary-repeated-native-2-2026-10-20.json new file mode 100644 index 000000000..36aa1e61c --- /dev/null +++ b/benchmarks/server-port/v3/summary-repeated-native-2-2026-10-20.json @@ -0,0 +1,911 @@ +{ + "identity": { + "implementation": "native", + "compose_project": "dw-port-native-20261020t151845", + "profile": "/home/linuxuser/profile.json", + "profile_sha256": "5e7946a6a506e391510e9222021e60c80d184742b030ee6e2978ab0c0b1e8ac5", + "source_revision": "8957ba27aa24b79ec2ea59e0c37b5ce662d434ab", + "native_binary_sha256": "edc283f130ff5c802fe5a80d56730cc9fe377c01c9c90340e08a68eb0866e2cc", + "smoke": false, + "idle_only": false, + "machine": { + "kernel": "6.8.0-146-generic", + "logical_cpus": 4, + "physical_memory_bytes": 8320786432, + "cpu_model": "AMD EPYC-Rome Processor", + "docker": "29.1.3", + "cgroup_driver": "systemd", + "docker_storage_driver": "overlayfs", + "docker_root_filesystem": "ext4" + } + }, + "windows": { + "setup": { + "seconds": 5.5, + "health": { + "replies": 8, + "failures": 0, + "slowest_ok_seconds": 0.185 + }, + "cpu_core_seconds": { + "mysql": 0.78864, + "native-server": 0.406774, + "probe": 0.006179, + "sdk-worker": 0.046886 + }, + "max_memory_bytes": { + "mysql": 529104896, + "native-server": 4411392, + "probe": 16662528, + "sdk-worker": 18796544 + }, + "max_runtime_memory_bytes": 4411392, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 2, + "accepted_starts": 2, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 2, + "pending_at_deadline": 0, + "closed_during_offer_window": 2, + "latency_p50_seconds": 0.264415979385376, + "latency_p95_seconds": 0.4201939105987549, + "latency_p99_seconds": 0.4201939105987549, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.2034, + "mysql_core_seconds_per_workflow": 0.3943 + }, + "warmup": { + "seconds": 60.4, + "health": { + "replies": 90, + "failures": 0, + "slowest_ok_seconds": 0.434 + }, + "cpu_core_seconds": { + "mysql": 7.903876, + "native-server": 3.848973, + "probe": 0.431635, + "sdk-worker": 0.430859 + }, + "max_memory_bytes": { + "mysql": 544555008, + "native-server": 6356992, + "probe": 16945152, + "sdk-worker": 18952192 + }, + "max_runtime_memory_bytes": 6356992, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 15, + "accepted_starts": 15, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 15, + "pending_at_deadline": 0, + "closed_during_offer_window": 15, + "latency_p50_seconds": 0.7263832092285156, + "latency_p95_seconds": 1.335334062576294, + "latency_p99_seconds": 1.335334062576294, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.2566, + "mysql_core_seconds_per_workflow": 0.5269 + }, + "low-before-1": { + "seconds": 120.3, + "health": { + "replies": 180, + "failures": 0, + "slowest_ok_seconds": 0.344 + }, + "cpu_core_seconds": { + "mysql": 15.542938, + "native-server": 7.685158, + "probe": 0.347865, + "sdk-worker": 0.825953 + }, + "max_memory_bytes": { + "mysql": 557219840, + "native-server": 7204864, + "probe": 18534400, + "sdk-worker": 20742144 + }, + "max_runtime_memory_bytes": 7204864, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 30, + "accepted_starts": 30, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 30, + "pending_at_deadline": 0, + "closed_during_offer_window": 30, + "latency_p50_seconds": 0.9674599170684814, + "latency_p95_seconds": 1.6804499626159668, + "latency_p99_seconds": 1.8725829124450684, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.2562, + "mysql_core_seconds_per_workflow": 0.5181 + }, + "low-before-2": { + "seconds": 119.7, + "health": { + "replies": 180, + "failures": 0, + "slowest_ok_seconds": 0.428 + }, + "cpu_core_seconds": { + "mysql": 15.094367, + "native-server": 7.400587, + "probe": 0.013393, + "sdk-worker": 0.78088 + }, + "max_memory_bytes": { + "mysql": 569982976, + "native-server": 5160960, + "probe": 16703488, + "sdk-worker": 24485888 + }, + "max_runtime_memory_bytes": 5160960, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 30, + "accepted_starts": 30, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 30, + "pending_at_deadline": 0, + "closed_during_offer_window": 30, + "latency_p50_seconds": 0.989130973815918, + "latency_p95_seconds": 1.6551799774169922, + "latency_p99_seconds": 1.8601720333099365, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.2467, + "mysql_core_seconds_per_workflow": 0.5031 + }, + "low-before-3": { + "seconds": 120.0, + "health": { + "replies": 178, + "failures": 0, + "slowest_ok_seconds": 0.519 + }, + "cpu_core_seconds": { + "mysql": 15.215566, + "native-server": 7.359013, + "probe": -0.015717, + "sdk-worker": 0.806679 + }, + "max_memory_bytes": { + "mysql": 581537792, + "native-server": 5169152, + "probe": 16691200, + "sdk-worker": 24260608 + }, + "max_runtime_memory_bytes": 5169152, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 30, + "accepted_starts": 30, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 30, + "pending_at_deadline": 0, + "closed_during_offer_window": 30, + "latency_p50_seconds": 0.9421291351318359, + "latency_p95_seconds": 1.8761250972747803, + "latency_p99_seconds": 1.94413423538208, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.2453, + "mysql_core_seconds_per_workflow": 0.5072 + }, + "rate-0.5-1": { + "seconds": 119.9, + "health": { + "replies": 170, + "failures": 0, + "slowest_ok_seconds": 0.513 + }, + "cpu_core_seconds": { + "mysql": 21.17952, + "native-server": 9.658587, + "probe": 0.251825, + "sdk-worker": 1.308499 + }, + "max_memory_bytes": { + "mysql": 604405760, + "native-server": 6815744, + "probe": 16932864, + "sdk-worker": 27607040 + }, + "max_runtime_memory_bytes": 6815744, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.5, + "planned_starts": 60, + "accepted_starts": 60, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 60, + "pending_at_deadline": 0, + "closed_during_offer_window": 60, + "latency_p50_seconds": 0.874359130859375, + "latency_p95_seconds": 1.5553700923919678, + "latency_p99_seconds": 1.6183700561523438, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.161, + "mysql_core_seconds_per_workflow": 0.353 + }, + "rate-0.5-2": { + "seconds": 120.4, + "health": { + "replies": 166, + "failures": 0, + "slowest_ok_seconds": 0.548 + }, + "cpu_core_seconds": { + "mysql": 21.947495, + "native-server": 9.851621, + "probe": 0.012017, + "sdk-worker": 1.336684 + }, + "max_memory_bytes": { + "mysql": 628731904, + "native-server": 6828032, + "probe": 18669568, + "sdk-worker": 30281728 + }, + "max_runtime_memory_bytes": 6828032, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.5, + "planned_starts": 60, + "accepted_starts": 60, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 60, + "pending_at_deadline": 0, + "closed_during_offer_window": 60, + "latency_p50_seconds": 0.8867721557617188, + "latency_p95_seconds": 1.3797781467437744, + "latency_p99_seconds": 1.711742877960205, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.1642, + "mysql_core_seconds_per_workflow": 0.3658 + }, + "rate-0.5-3": { + "seconds": 120.6, + "health": { + "replies": 165, + "failures": 0, + "slowest_ok_seconds": 0.518 + }, + "cpu_core_seconds": { + "mysql": 24.106008, + "native-server": 10.781074, + "probe": 0.075858, + "sdk-worker": 1.460856 + }, + "max_memory_bytes": { + "mysql": 649342976, + "native-server": 7081984, + "probe": 16957440, + "sdk-worker": 32976896 + }, + "max_runtime_memory_bytes": 7081984, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.5, + "planned_starts": 60, + "accepted_starts": 60, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 60, + "pending_at_deadline": 0, + "closed_during_offer_window": 60, + "latency_p50_seconds": 0.9854328632354736, + "latency_p95_seconds": 1.5133790969848633, + "latency_p99_seconds": 1.9184279441833496, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.1797, + "mysql_core_seconds_per_workflow": 0.4018 + }, + "rate-1-1": { + "seconds": 121.0, + "health": { + "replies": 150, + "failures": 0, + "slowest_ok_seconds": 0.719 + }, + "cpu_core_seconds": { + "mysql": 33.022403, + "native-server": 13.945428, + "probe": 0.224814, + "sdk-worker": 2.466644 + }, + "max_memory_bytes": { + "mysql": 692928512, + "native-server": 7155712, + "probe": 16941056, + "sdk-worker": 36147200 + }, + "max_runtime_memory_bytes": 7155712, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 1, + "planned_starts": 120, + "accepted_starts": 120, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 120, + "pending_at_deadline": 0, + "closed_during_offer_window": 120, + "latency_p50_seconds": 0.7154889106750488, + "latency_p95_seconds": 1.5674889087677002, + "latency_p99_seconds": 2.94549298286438, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.1162, + "mysql_core_seconds_per_workflow": 0.2752 + }, + "rate-1-2": { + "seconds": 122.0, + "health": { + "replies": 153, + "failures": 0, + "slowest_ok_seconds": 0.86 + }, + "cpu_core_seconds": { + "mysql": 32.588333, + "native-server": 13.690904, + "probe": -0.024711, + "sdk-worker": 2.364287 + }, + "max_memory_bytes": { + "mysql": 736538624, + "native-server": 7127040, + "probe": 16945152, + "sdk-worker": 36696064 + }, + "max_runtime_memory_bytes": 7127040, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 1, + "planned_starts": 120, + "accepted_starts": 120, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 120, + "pending_at_deadline": 0, + "closed_during_offer_window": 119, + "latency_p50_seconds": 0.6948199272155762, + "latency_p95_seconds": 1.394827127456665, + "latency_p99_seconds": 1.7588160037994385, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.1141, + "mysql_core_seconds_per_workflow": 0.2716 + }, + "rate-1-3": { + "seconds": 121.1, + "health": { + "replies": 150, + "failures": 0, + "slowest_ok_seconds": 0.728 + }, + "cpu_core_seconds": { + "mysql": 31.711915, + "native-server": 13.246497, + "probe": -0.056814, + "sdk-worker": 2.334332 + }, + "max_memory_bytes": { + "mysql": 778280960, + "native-server": 7151616, + "probe": 16691200, + "sdk-worker": 37113856 + }, + "max_runtime_memory_bytes": 7151616, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 1, + "planned_starts": 120, + "accepted_starts": 120, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 120, + "pending_at_deadline": 0, + "closed_during_offer_window": 120, + "latency_p50_seconds": 0.6748061180114746, + "latency_p95_seconds": 1.1207969188690186, + "latency_p99_seconds": 2.0817408561706543, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.1104, + "mysql_core_seconds_per_workflow": 0.2643 + }, + "low-after-1": { + "seconds": 119.9, + "health": { + "replies": 174, + "failures": 0, + "slowest_ok_seconds": 0.48 + }, + "cpu_core_seconds": { + "mysql": 18.335757, + "native-server": 8.539634, + "probe": -0.419468, + "sdk-worker": 0.943311 + }, + "max_memory_bytes": { + "mysql": 789663744, + "native-server": 7131136, + "probe": 16699392, + "sdk-worker": 37392384 + }, + "max_runtime_memory_bytes": 7131136, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 30, + "accepted_starts": 30, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 30, + "pending_at_deadline": 0, + "closed_during_offer_window": 30, + "latency_p50_seconds": 1.0487730503082275, + "latency_p95_seconds": 1.6618030071258545, + "latency_p99_seconds": 1.9577748775482178, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.2847, + "mysql_core_seconds_per_workflow": 0.6112 + }, + "low-after-2": { + "seconds": 120.2, + "health": { + "replies": 176, + "failures": 0, + "slowest_ok_seconds": 0.483 + }, + "cpu_core_seconds": { + "mysql": 17.32578, + "native-server": 8.043092, + "probe": -0.002517, + "sdk-worker": 0.906545 + }, + "max_memory_bytes": { + "mysql": 795537408, + "native-server": 7127040, + "probe": 16687104, + "sdk-worker": 37650432 + }, + "max_runtime_memory_bytes": 7127040, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 30, + "accepted_starts": 30, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 30, + "pending_at_deadline": 0, + "closed_during_offer_window": 30, + "latency_p50_seconds": 0.989509105682373, + "latency_p95_seconds": 1.8215291500091553, + "latency_p99_seconds": 1.8415191173553467, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.2681, + "mysql_core_seconds_per_workflow": 0.5775 + }, + "low-after-3": { + "seconds": 119.1, + "health": { + "replies": 174, + "failures": 0, + "slowest_ok_seconds": 0.356 + }, + "cpu_core_seconds": { + "mysql": 17.427466, + "native-server": 8.173384, + "probe": -0.029393, + "sdk-worker": 0.915135 + }, + "max_memory_bytes": { + "mysql": 801325056, + "native-server": 7168000, + "probe": 18608128, + "sdk-worker": 39112704 + }, + "max_runtime_memory_bytes": 7168000, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 30, + "accepted_starts": 30, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 30, + "pending_at_deadline": 0, + "closed_during_offer_window": 30, + "latency_p50_seconds": 0.9510529041290283, + "latency_p95_seconds": 1.502997875213623, + "latency_p99_seconds": 1.7160539627075195, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 0.2724, + "mysql_core_seconds_per_workflow": 0.5809 + } + }, + "idle": { + "idle-6-1": { + "seconds": 12.4, + "health": { + "replies": 18, + "failures": 0, + "slowest_ok_seconds": 0.601 + }, + "cpu_core_seconds": { + "mysql": 1.388882, + "native-server": 0.694483, + "probe": -0.556331, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 801333248, + "native-server": 6881280, + "probe": 31330304, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 6881280, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "requested_polls": 6, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "empty": 6 + } + }, + "idle-6-2": { + "seconds": 11.2, + "health": { + "replies": 17, + "failures": 0, + "slowest_ok_seconds": 0.107 + }, + "cpu_core_seconds": { + "mysql": 1.305827, + "native-server": 0.755374, + "probe": -0.025734, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 801746944, + "native-server": 5115904, + "probe": 30846976, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 5115904, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "requested_polls": 6, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "empty": 6 + } + }, + "idle-6-3": { + "seconds": 11.4, + "health": { + "replies": 18, + "failures": 0, + "slowest_ok_seconds": 0.09 + }, + "cpu_core_seconds": { + "mysql": 1.233457, + "native-server": 0.683825, + "probe": 0.043972, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 801890304, + "native-server": 5021696, + "probe": 31555584, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 5021696, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "requested_polls": 6, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "empty": 6 + } + }, + "idle-12-1": { + "seconds": 11.8, + "health": { + "replies": 19, + "failures": 0, + "slowest_ok_seconds": 0.101 + }, + "cpu_core_seconds": { + "mysql": 1.61893, + "native-server": 0.822211, + "probe": 0.205678, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 802017280, + "native-server": 5345280, + "probe": 45867008, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 5345280, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "requested_polls": 12, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "empty": 12 + } + }, + "idle-12-2": { + "seconds": 11.4, + "health": { + "replies": 18, + "failures": 0, + "slowest_ok_seconds": 0.105 + }, + "cpu_core_seconds": { + "mysql": 1.645461, + "native-server": 0.921932, + "probe": -0.173109, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 802181120, + "native-server": 5292032, + "probe": 42070016, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 5292032, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "requested_polls": 12, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "empty": 12 + } + }, + "idle-12-3": { + "seconds": 11.5, + "health": { + "replies": 18, + "failures": 0, + "slowest_ok_seconds": 0.098 + }, + "cpu_core_seconds": { + "mysql": 1.590614, + "native-server": 0.873969, + "probe": 0.067627, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 802652160, + "native-server": 5328896, + "probe": 42647552, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 5328896, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "requested_polls": 12, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "empty": 12 + } + }, + "idle-24-1": { + "seconds": 12.2, + "health": { + "replies": 20, + "failures": 0, + "slowest_ok_seconds": 0.102 + }, + "cpu_core_seconds": { + "mysql": 2.273613, + "native-server": 1.241196, + "probe": 0.300321, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 802402304, + "native-server": 7852032, + "probe": 70094848, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 7852032, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "requested_polls": 24, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "empty": 24 + } + }, + "idle-24-2": { + "seconds": 11.7, + "health": { + "replies": 17, + "failures": 0, + "slowest_ok_seconds": 0.107 + }, + "cpu_core_seconds": { + "mysql": 2.244408, + "native-server": 1.18888, + "probe": -0.185103, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 802770944, + "native-server": 8011776, + "probe": 58925056, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 8011776, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "requested_polls": 24, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "empty": 24 + } + }, + "idle-24-3": { + "seconds": 12.2, + "health": { + "replies": 18, + "failures": 0, + "slowest_ok_seconds": 0.101 + }, + "cpu_core_seconds": { + "mysql": 2.191166, + "native-server": 1.235365, + "probe": 0.126906, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 802361344, + "native-server": 5984256, + "probe": 69505024, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 5984256, + "oom_kills": { + "mysql": 0, + "native-server": 0, + "probe": 0, + "sdk-worker": 0 + }, + "requested_polls": 24, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "empty": 24 + } + } + } +} diff --git a/benchmarks/server-port/v3/summary-repeated-php-1-2026-10-20.json b/benchmarks/server-port/v3/summary-repeated-php-1-2026-10-20.json new file mode 100644 index 000000000..62de913d8 --- /dev/null +++ b/benchmarks/server-port/v3/summary-repeated-php-1-2026-10-20.json @@ -0,0 +1,1124 @@ +{ + "identity": { + "implementation": "php", + "compose_project": "dw-port-php-20261020t134400", + "profile": "/home/linuxuser/profile.json", + "profile_sha256": "5e7946a6a506e391510e9222021e60c80d184742b030ee6e2978ab0c0b1e8ac5", + "source_revision": "8957ba27aa24b79ec2ea59e0c37b5ce662d434ab", + "native_binary_sha256": "edc283f130ff5c802fe5a80d56730cc9fe377c01c9c90340e08a68eb0866e2cc", + "smoke": false, + "idle_only": false, + "machine": { + "kernel": "6.8.0-146-generic", + "logical_cpus": 4, + "physical_memory_bytes": 8320786432, + "cpu_model": "AMD EPYC-Rome Processor", + "docker": "29.1.3", + "cgroup_driver": "systemd", + "docker_storage_driver": "overlayfs", + "docker_root_filesystem": "ext4" + } + }, + "windows": { + "setup": { + "seconds": 6.6, + "health": { + "replies": 8, + "failures": 0, + "slowest_ok_seconds": 0.874 + }, + "cpu_core_seconds": { + "mysql": 1.504809, + "php-scheduler": 1.649437, + "php-server": 5.056903, + "php-worker": 0.023939, + "probe": 0.007121, + "redis": 0.182108, + "sdk-worker": 0.052484 + }, + "max_memory_bytes": { + "mysql": 569229312, + "php-scheduler": 43175936, + "php-server": 91193344, + "php-worker": 49610752, + "probe": 19148800, + "redis": 3719168, + "sdk-worker": 20959232 + }, + "max_runtime_memory_bytes": 187621376, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 2, + "accepted_starts": 2, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 2, + "pending_at_deadline": 0, + "closed_during_offer_window": 1, + "latency_p50_seconds": 1.3568880558013916, + "latency_p95_seconds": 2.585994005203247, + "latency_p99_seconds": 2.585994005203247, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 3.4562, + "mysql_core_seconds_per_workflow": 0.7524 + }, + "warmup": { + "seconds": 59.2, + "health": { + "replies": 78, + "failures": 0, + "slowest_ok_seconds": 0.863 + }, + "cpu_core_seconds": { + "mysql": 9.797015, + "php-scheduler": 6.664555, + "php-server": 30.112366, + "php-worker": 0.394338, + "probe": 0.365943, + "redis": 1.656685, + "sdk-worker": 0.485625 + }, + "max_memory_bytes": { + "mysql": 585056256, + "php-scheduler": 48877568, + "php-server": 97693696, + "php-worker": 49614848, + "probe": 19161088, + "redis": 4448256, + "sdk-worker": 22761472 + }, + "max_runtime_memory_bytes": 195813376, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 15, + "accepted_starts": 15, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 15, + "pending_at_deadline": 0, + "closed_during_offer_window": 15, + "latency_p50_seconds": 1.0513579845428467, + "latency_p95_seconds": 1.5864241123199463, + "latency_p99_seconds": 1.5864241123199463, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 2.5885, + "mysql_core_seconds_per_workflow": 0.6531 + }, + "low-before-1": { + "seconds": 120.2, + "health": { + "replies": 158, + "failures": 0, + "slowest_ok_seconds": 0.584 + }, + "cpu_core_seconds": { + "mysql": 20.040341, + "php-scheduler": 13.497456, + "php-server": 60.623072, + "php-worker": 0.798438, + "probe": 0.37686, + "redis": 3.363789, + "sdk-worker": 0.944666 + }, + "max_memory_bytes": { + "mysql": 605970432, + "php-scheduler": 52334592, + "php-server": 109006848, + "php-worker": 50028544, + "probe": 20021248, + "redis": 6221824, + "sdk-worker": 25751552 + }, + "max_runtime_memory_bytes": 198287360, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 30, + "accepted_starts": 30, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 30, + "pending_at_deadline": 0, + "closed_during_offer_window": 30, + "latency_p50_seconds": 1.128861904144287, + "latency_p95_seconds": 1.4585249423980713, + "latency_p99_seconds": 1.537226915359497, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 2.6094, + "mysql_core_seconds_per_workflow": 0.668 + }, + "low-before-2": { + "seconds": 120.2, + "health": { + "replies": 158, + "failures": 0, + "slowest_ok_seconds": 0.702 + }, + "cpu_core_seconds": { + "mysql": 19.857788, + "php-scheduler": 12.698215, + "php-server": 60.391955, + "php-worker": 0.785859, + "probe": 0.01672, + "redis": 3.357225, + "sdk-worker": 0.950209 + }, + "max_memory_bytes": { + "mysql": 621260800, + "php-scheduler": 51154944, + "php-server": 104452096, + "php-worker": 50839552, + "probe": 19791872, + "redis": 6172672, + "sdk-worker": 28938240 + }, + "max_runtime_memory_bytes": 195870720, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 30, + "accepted_starts": 30, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 30, + "pending_at_deadline": 0, + "closed_during_offer_window": 30, + "latency_p50_seconds": 1.1518518924713135, + "latency_p95_seconds": 1.4961800575256348, + "latency_p99_seconds": 1.5145728588104248, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 2.5744, + "mysql_core_seconds_per_workflow": 0.6619 + }, + "low-before-3": { + "seconds": 120.2, + "health": { + "replies": 156, + "failures": 0, + "slowest_ok_seconds": 0.858 + }, + "cpu_core_seconds": { + "mysql": 21.376865, + "php-scheduler": 14.672424, + "php-server": 63.623704, + "php-worker": 0.823863, + "probe": 0.05507, + "redis": 3.531226, + "sdk-worker": 1.027329 + }, + "max_memory_bytes": { + "mysql": 638160896, + "php-scheduler": 51429376, + "php-server": 112340992, + "php-worker": 51687424, + "probe": 21409792, + "redis": 6946816, + "sdk-worker": 31318016 + }, + "max_runtime_memory_bytes": 201371648, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 30, + "accepted_starts": 30, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 30, + "pending_at_deadline": 0, + "closed_during_offer_window": 30, + "latency_p50_seconds": 1.187798023223877, + "latency_p95_seconds": 1.6202080249786377, + "latency_p99_seconds": 1.651698112487793, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 2.755, + "mysql_core_seconds_per_workflow": 0.7126 + }, + "rate-0.5-1": { + "seconds": 122.4, + "health": { + "replies": 154, + "failures": 0, + "slowest_ok_seconds": 0.787 + }, + "cpu_core_seconds": { + "mysql": 34.159995, + "php-scheduler": 14.081556, + "php-server": 90.804063, + "php-worker": 0.827854, + "probe": 0.356474, + "redis": 4.042836, + "sdk-worker": 1.722026 + }, + "max_memory_bytes": { + "mysql": 672313344, + "php-scheduler": 50196480, + "php-server": 106352640, + "php-worker": 53936128, + "probe": 20967424, + "redis": 7548928, + "sdk-worker": 35209216 + }, + "max_runtime_memory_bytes": 204369920, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.5, + "planned_starts": 60, + "accepted_starts": 60, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 60, + "pending_at_deadline": 0, + "closed_during_offer_window": 60, + "latency_p50_seconds": 1.1291561126708984, + "latency_p95_seconds": 1.526949167251587, + "latency_p99_seconds": 1.633439064025879, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 1.8293, + "mysql_core_seconds_per_workflow": 0.5693 + }, + "rate-0.5-2": { + "seconds": 122.9, + "health": { + "replies": 151, + "failures": 0, + "slowest_ok_seconds": 0.809 + }, + "cpu_core_seconds": { + "mysql": 34.1152, + "php-scheduler": 13.440402, + "php-server": 90.610046, + "php-worker": 0.799173, + "probe": 0.01692, + "redis": 4.091798, + "sdk-worker": 1.702909 + }, + "max_memory_bytes": { + "mysql": 702459904, + "php-scheduler": 49590272, + "php-server": 110915584, + "php-worker": 52420608, + "probe": 20811776, + "redis": 6348800, + "sdk-worker": 38711296 + }, + "max_runtime_memory_bytes": 202567680, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.5, + "planned_starts": 60, + "accepted_starts": 60, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 60, + "pending_at_deadline": 0, + "closed_during_offer_window": 60, + "latency_p50_seconds": 1.1356868743896484, + "latency_p95_seconds": 1.5167322158813477, + "latency_p99_seconds": 1.6396760940551758, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 1.8157, + "mysql_core_seconds_per_workflow": 0.5686 + }, + "rate-0.5-3": { + "seconds": 122.1, + "health": { + "replies": 147, + "failures": 0, + "slowest_ok_seconds": 0.868 + }, + "cpu_core_seconds": { + "mysql": 35.826468, + "php-scheduler": 14.224871, + "php-server": 93.479474, + "php-worker": 0.827446, + "probe": -0.005189, + "redis": 4.158363, + "sdk-worker": 1.809416 + }, + "max_memory_bytes": { + "mysql": 733597696, + "php-scheduler": 51208192, + "php-server": 120877056, + "php-worker": 52404224, + "probe": 19980288, + "redis": 7733248, + "sdk-worker": 38957056 + }, + "max_runtime_memory_bytes": 226398208, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.5, + "planned_starts": 60, + "accepted_starts": 60, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 60, + "pending_at_deadline": 0, + "closed_during_offer_window": 60, + "latency_p50_seconds": 1.2154319286346436, + "latency_p95_seconds": 1.5453448295593262, + "latency_p99_seconds": 1.6753089427947998, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 1.8782, + "mysql_core_seconds_per_workflow": 0.5971 + }, + "rate-1-1": { + "seconds": 176.8, + "health": { + "replies": 204, + "failures": 0, + "slowest_ok_seconds": 0.884 + }, + "cpu_core_seconds": { + "mysql": 66.727563, + "php-scheduler": 21.013864, + "php-server": 164.107583, + "php-worker": 1.188324, + "probe": 0.830886, + "redis": 6.423366, + "sdk-worker": 3.396652 + }, + "max_memory_bytes": { + "mysql": 781090816, + "php-scheduler": 51212288, + "php-server": 118976512, + "php-worker": 52404224, + "probe": 19783680, + "redis": 8462336, + "sdk-worker": 39272448 + }, + "max_runtime_memory_bytes": 207253504, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 1, + "planned_starts": 120, + "accepted_starts": 120, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 120, + "pending_at_deadline": 0, + "closed_during_offer_window": 69, + "latency_p50_seconds": 44.781501054763794, + "latency_p95_seconds": 64.77137804031372, + "latency_p99_seconds": 66.02623295783997, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 1.6061, + "mysql_core_seconds_per_workflow": 0.5561 + }, + "rate-1-2": { + "seconds": 178.3, + "health": { + "replies": 210, + "failures": 0, + "slowest_ok_seconds": 0.787 + }, + "cpu_core_seconds": { + "mysql": 66.481232, + "php-scheduler": 20.010903, + "php-server": 164.224564, + "php-worker": 1.228025, + "probe": 0.038332, + "redis": 6.447414, + "sdk-worker": 3.377742 + }, + "max_memory_bytes": { + "mysql": 818790400, + "php-scheduler": 48791552, + "php-server": 114503680, + "php-worker": 52760576, + "probe": 20025344, + "redis": 8364032, + "sdk-worker": 40349696 + }, + "max_runtime_memory_bytes": 210096128, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 1, + "planned_starts": 120, + "accepted_starts": 120, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 120, + "pending_at_deadline": 0, + "closed_during_offer_window": 68, + "latency_p50_seconds": 46.07006311416626, + "latency_p95_seconds": 65.88363909721375, + "latency_p99_seconds": 67.44143986701965, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 1.5993, + "mysql_core_seconds_per_workflow": 0.554 + }, + "rate-1-3": { + "seconds": 181.3, + "health": { + "replies": 208, + "failures": 0, + "slowest_ok_seconds": 0.875 + }, + "cpu_core_seconds": { + "mysql": 67.915662, + "php-scheduler": 20.903777, + "php-server": 167.68432, + "php-worker": 1.2719, + "probe": 0.061674, + "redis": 6.673205, + "sdk-worker": 3.479508 + }, + "max_memory_bytes": { + "mysql": 856571904, + "php-scheduler": 52236288, + "php-server": 118317056, + "php-worker": 54628352, + "probe": 19775488, + "redis": 8306688, + "sdk-worker": 41897984 + }, + "max_runtime_memory_bytes": 210964480, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 1, + "planned_starts": 120, + "accepted_starts": 120, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 120, + "pending_at_deadline": 0, + "closed_during_offer_window": 66, + "latency_p50_seconds": 48.342694997787476, + "latency_p95_seconds": 67.38858103752136, + "latency_p99_seconds": 69.02710199356079, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 1.6378, + "mysql_core_seconds_per_workflow": 0.566 + }, + "low-after-1": { + "seconds": 119.4, + "health": { + "replies": 152, + "failures": 0, + "slowest_ok_seconds": 0.782 + }, + "cpu_core_seconds": { + "mysql": 22.179925, + "php-scheduler": 14.484918, + "php-server": 64.695478, + "php-worker": 0.829711, + "probe": -1.265806, + "redis": 3.54183, + "sdk-worker": 1.127742 + }, + "max_memory_bytes": { + "mysql": 866603008, + "php-scheduler": 47493120, + "php-server": 114991104, + "php-worker": 54050816, + "probe": 19767296, + "redis": 7155712, + "sdk-worker": 41742336 + }, + "max_runtime_memory_bytes": 201912320, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 30, + "accepted_starts": 30, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 30, + "pending_at_deadline": 0, + "closed_during_offer_window": 30, + "latency_p50_seconds": 1.2256431579589844, + "latency_p95_seconds": 1.61228609085083, + "latency_p99_seconds": 1.7355849742889404, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 2.7851, + "mysql_core_seconds_per_workflow": 0.7393 + }, + "low-after-2": { + "seconds": 120.2, + "health": { + "replies": 154, + "failures": 0, + "slowest_ok_seconds": 0.683 + }, + "cpu_core_seconds": { + "mysql": 21.854782, + "php-scheduler": 13.381766, + "php-server": 63.860201, + "php-worker": 0.806617, + "probe": -0.048811, + "redis": 3.524795, + "sdk-worker": 1.107556 + }, + "max_memory_bytes": { + "mysql": 875786240, + "php-scheduler": 52068352, + "php-server": 111415296, + "php-worker": 52908032, + "probe": 19775488, + "redis": 6717440, + "sdk-worker": 41689088 + }, + "max_runtime_memory_bytes": 197271552, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 30, + "accepted_starts": 30, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 30, + "pending_at_deadline": 0, + "closed_during_offer_window": 30, + "latency_p50_seconds": 1.197011947631836, + "latency_p95_seconds": 1.7318499088287354, + "latency_p99_seconds": 1.8175239562988281, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 2.7191, + "mysql_core_seconds_per_workflow": 0.7285 + }, + "low-after-3": { + "seconds": 120.0, + "health": { + "replies": 156, + "failures": 0, + "slowest_ok_seconds": 0.686 + }, + "cpu_core_seconds": { + "mysql": 21.31017, + "php-scheduler": 13.8905, + "php-server": 62.803105, + "php-worker": 0.825999, + "probe": 0.016519, + "redis": 3.496282, + "sdk-worker": 1.113688 + }, + "max_memory_bytes": { + "mysql": 885342208, + "php-scheduler": 47980544, + "php-server": 122150912, + "php-worker": 52764672, + "probe": 20742144, + "redis": 7487488, + "sdk-worker": 43237376 + }, + "max_runtime_memory_bytes": 195055616, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 30, + "accepted_starts": 30, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 30, + "pending_at_deadline": 0, + "closed_during_offer_window": 30, + "latency_p50_seconds": 1.1954889297485352, + "latency_p95_seconds": 1.56825590133667, + "latency_p99_seconds": 1.68644118309021, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 2.7005, + "mysql_core_seconds_per_workflow": 0.7103 + } + }, + "idle": { + "idle-6-1": { + "seconds": 12.9, + "health": { + "replies": 16, + "failures": 0, + "slowest_ok_seconds": 1.431 + }, + "cpu_core_seconds": { + "mysql": 1.119762, + "php-scheduler": 1.189401, + "php-server": 4.417996, + "php-worker": 0.084415, + "probe": -0.557917, + "redis": 0.425557, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 886943744, + "php-scheduler": 43503616, + "php-server": 117972992, + "php-worker": 54337536, + "probe": 31399936, + "redis": 4829184, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 205033472, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "requested_polls": 6, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "empty": 6 + } + }, + "idle-6-2": { + "seconds": 13.0, + "health": { + "replies": 16, + "failures": 0, + "slowest_ok_seconds": 1.217 + }, + "cpu_core_seconds": { + "mysql": 1.085944, + "php-scheduler": 2.023353, + "php-server": 4.351547, + "php-worker": 0.100568, + "probe": -0.046043, + "redis": 0.401379, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 887758848, + "php-scheduler": 43229184, + "php-server": 104980480, + "php-worker": 52760576, + "probe": 30400512, + "redis": 4653056, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 190066688, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "requested_polls": 6, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "empty": 6 + } + }, + "idle-6-3": { + "seconds": 13.1, + "health": { + "replies": 16, + "failures": 0, + "slowest_ok_seconds": 0.549 + }, + "cpu_core_seconds": { + "mysql": 1.005154, + "php-scheduler": 1.77791, + "php-server": 4.344363, + "php-worker": 0.079783, + "probe": -0.002653, + "redis": 0.39114, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 887844864, + "php-scheduler": 42561536, + "php-server": 101949440, + "php-worker": 52764672, + "probe": 31424512, + "redis": 4689920, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 183525376, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "requested_polls": 6, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "empty": 6 + } + }, + "idle-12-1": { + "seconds": 15.3, + "health": { + "replies": 16, + "failures": 1, + "slowest_ok_seconds": 1.301 + }, + "cpu_core_seconds": { + "mysql": 1.450981, + "php-scheduler": 1.344831, + "php-server": 5.344161, + "php-worker": 0.089381, + "probe": 0.11171, + "redis": 0.501114, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 925487104, + "php-scheduler": 46772224, + "php-server": 139833344, + "php-worker": 52760576, + "probe": 46059520, + "redis": 4550656, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 198746112, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "requested_polls": 12, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "backpressure": 4, + "empty": 8 + } + }, + "idle-12-2": { + "seconds": 16.1, + "health": { + "replies": 16, + "failures": 1, + "slowest_ok_seconds": 1.708 + }, + "cpu_core_seconds": { + "mysql": 1.617205, + "php-scheduler": 2.118904, + "php-server": 5.849974, + "php-worker": 0.090799, + "probe": 0.101958, + "redis": 0.582651, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 925876224, + "php-scheduler": 36306944, + "php-server": 131977216, + "php-worker": 52764672, + "probe": 46710784, + "redis": 4497408, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 218853376, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "requested_polls": 12, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "empty": 8, + "backpressure": 4 + } + }, + "idle-12-3": { + "seconds": 16.1, + "health": { + "replies": 17, + "failures": 1, + "slowest_ok_seconds": 1.476 + }, + "cpu_core_seconds": { + "mysql": 1.445036, + "php-scheduler": 1.548843, + "php-server": 5.645729, + "php-worker": 0.130054, + "probe": 0.086832, + "redis": 0.520426, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 924483584, + "php-scheduler": 44150784, + "php-server": 134873088, + "php-worker": 52760576, + "probe": 46292992, + "redis": 4390912, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 232247296, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "requested_polls": 12, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "empty": 8, + "backpressure": 4 + } + }, + "idle-24-1": { + "seconds": 17.2, + "health": { + "replies": 18, + "failures": 2, + "slowest_ok_seconds": 0.896 + }, + "cpu_core_seconds": { + "mysql": 1.803645, + "php-scheduler": 2.060201, + "php-server": 6.451095, + "php-worker": 0.092034, + "probe": 0.219455, + "redis": 0.55685, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 931160064, + "php-scheduler": 51216384, + "php-server": 132136960, + "php-worker": 52756480, + "probe": 67342336, + "redis": 4497408, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 224530432, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "requested_polls": 24, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "backpressure": 16, + "empty": 8 + } + }, + "idle-24-2": { + "seconds": 17.1, + "health": { + "replies": 17, + "failures": 2, + "slowest_ok_seconds": 1.676 + }, + "cpu_core_seconds": { + "mysql": 1.906241, + "php-scheduler": 1.901284, + "php-server": 6.692275, + "php-worker": 0.122455, + "probe": -0.035132, + "redis": 0.579389, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 936239104, + "php-scheduler": 35033088, + "php-server": 152723456, + "php-worker": 52756480, + "probe": 69390336, + "redis": 5758976, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 217575424, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "requested_polls": 24, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "backpressure": 16, + "empty": 8 + } + }, + "idle-24-3": { + "seconds": 17.2, + "health": { + "replies": 16, + "failures": 2, + "slowest_ok_seconds": 0.916 + }, + "cpu_core_seconds": { + "mysql": 1.92142, + "php-scheduler": 2.029289, + "php-server": 6.846726, + "php-worker": 0.124736, + "probe": 0.083039, + "redis": 0.563508, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 937267200, + "php-scheduler": 43589632, + "php-server": 168722432, + "php-worker": 52764672, + "probe": 72994816, + "redis": 4243456, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 246112256, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "requested_polls": 24, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "backpressure": 16, + "empty": 8 + } + } + } +} diff --git a/benchmarks/server-port/v3/summary-repeated-php-2-2026-10-20.json b/benchmarks/server-port/v3/summary-repeated-php-2-2026-10-20.json new file mode 100644 index 000000000..5d054322e --- /dev/null +++ b/benchmarks/server-port/v3/summary-repeated-php-2-2026-10-20.json @@ -0,0 +1,1124 @@ +{ + "identity": { + "implementation": "php", + "compose_project": "dw-port-php-20261020t144545", + "profile": "/home/linuxuser/profile.json", + "profile_sha256": "5e7946a6a506e391510e9222021e60c80d184742b030ee6e2978ab0c0b1e8ac5", + "source_revision": "8957ba27aa24b79ec2ea59e0c37b5ce662d434ab", + "native_binary_sha256": "edc283f130ff5c802fe5a80d56730cc9fe377c01c9c90340e08a68eb0866e2cc", + "smoke": false, + "idle_only": false, + "machine": { + "kernel": "6.8.0-146-generic", + "logical_cpus": 4, + "physical_memory_bytes": 8320786432, + "cpu_model": "AMD EPYC-Rome Processor", + "docker": "29.1.3", + "cgroup_driver": "systemd", + "docker_storage_driver": "overlayfs", + "docker_root_filesystem": "ext4" + } + }, + "windows": { + "setup": { + "seconds": 6.1, + "health": { + "replies": 8, + "failures": 0, + "slowest_ok_seconds": 0.403 + }, + "cpu_core_seconds": { + "mysql": 1.170159, + "php-scheduler": 1.375, + "php-server": 3.530644, + "php-worker": 0.017897, + "probe": 0.007011, + "redis": 0.170855, + "sdk-worker": 0.042902 + }, + "max_memory_bytes": { + "mysql": 572542976, + "php-scheduler": 43524096, + "php-server": 88588288, + "php-worker": 49451008, + "probe": 19144704, + "redis": 3624960, + "sdk-worker": 20951040 + }, + "max_runtime_memory_bytes": 182153216, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 2, + "accepted_starts": 2, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 2, + "pending_at_deadline": 0, + "closed_during_offer_window": 2, + "latency_p50_seconds": 0.8652470111846924, + "latency_p95_seconds": 1.8545799255371094, + "latency_p99_seconds": 1.8545799255371094, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 2.5472, + "mysql_core_seconds_per_workflow": 0.5851 + }, + "warmup": { + "seconds": 60.8, + "health": { + "replies": 78, + "failures": 0, + "slowest_ok_seconds": 0.723 + }, + "cpu_core_seconds": { + "mysql": 10.668871, + "php-scheduler": 6.668423, + "php-server": 32.386096, + "php-worker": 0.41917, + "probe": 0.372215, + "redis": 1.763566, + "sdk-worker": 0.500587 + }, + "max_memory_bytes": { + "mysql": 593006592, + "php-scheduler": 44797952, + "php-server": 96911360, + "php-worker": 50581504, + "probe": 21094400, + "redis": 5603328, + "sdk-worker": 22888448 + }, + "max_runtime_memory_bytes": 189968384, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 15, + "accepted_starts": 15, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 15, + "pending_at_deadline": 0, + "closed_during_offer_window": 15, + "latency_p50_seconds": 1.2237191200256348, + "latency_p95_seconds": 1.5795209407806396, + "latency_p99_seconds": 1.5795209407806396, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 2.7492, + "mysql_core_seconds_per_workflow": 0.7113 + }, + "low-before-1": { + "seconds": 120.3, + "health": { + "replies": 154, + "failures": 0, + "slowest_ok_seconds": 0.713 + }, + "cpu_core_seconds": { + "mysql": 21.296271, + "php-scheduler": 13.985756, + "php-server": 63.837221, + "php-worker": 0.833778, + "probe": 0.438814, + "redis": 3.467972, + "sdk-worker": 1.000538 + }, + "max_memory_bytes": { + "mysql": 614154240, + "php-scheduler": 47267840, + "php-server": 114008064, + "php-worker": 50053120, + "probe": 20639744, + "redis": 6180864, + "sdk-worker": 25747456 + }, + "max_runtime_memory_bytes": 199602176, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 30, + "accepted_starts": 30, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 30, + "pending_at_deadline": 0, + "closed_during_offer_window": 30, + "latency_p50_seconds": 1.2357759475708008, + "latency_p95_seconds": 1.679650068283081, + "latency_p99_seconds": 1.8180029392242432, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 2.7375, + "mysql_core_seconds_per_workflow": 0.7099 + }, + "low-before-2": { + "seconds": 120.4, + "health": { + "replies": 152, + "failures": 0, + "slowest_ok_seconds": 0.695 + }, + "cpu_core_seconds": { + "mysql": 21.78718, + "php-scheduler": 13.668353, + "php-server": 64.700526, + "php-worker": 0.852338, + "probe": 0.01564, + "redis": 3.596517, + "sdk-worker": 1.001945 + }, + "max_memory_bytes": { + "mysql": 631521280, + "php-scheduler": 52035584, + "php-server": 120164352, + "php-worker": 50896896, + "probe": 19759104, + "redis": 6782976, + "sdk-worker": 28925952 + }, + "max_runtime_memory_bytes": 212869120, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 30, + "accepted_starts": 30, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 30, + "pending_at_deadline": 0, + "closed_during_offer_window": 30, + "latency_p50_seconds": 1.2468769550323486, + "latency_p95_seconds": 1.6789839267730713, + "latency_p99_seconds": 1.6973340511322021, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 2.7606, + "mysql_core_seconds_per_workflow": 0.7262 + }, + "low-before-3": { + "seconds": 120.7, + "health": { + "replies": 152, + "failures": 0, + "slowest_ok_seconds": 0.806 + }, + "cpu_core_seconds": { + "mysql": 22.43567, + "php-scheduler": 14.76506, + "php-server": 65.670336, + "php-worker": 0.866131, + "probe": -0.000866, + "redis": 3.581031, + "sdk-worker": 1.043077 + }, + "max_memory_bytes": { + "mysql": 647872512, + "php-scheduler": 49594368, + "php-server": 113401856, + "php-worker": 52457472, + "probe": 19767296, + "redis": 6266880, + "sdk-worker": 30670848 + }, + "max_runtime_memory_bytes": 201875456, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 30, + "accepted_starts": 30, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 30, + "pending_at_deadline": 0, + "closed_during_offer_window": 30, + "latency_p50_seconds": 1.3055198192596436, + "latency_p95_seconds": 1.649399995803833, + "latency_p99_seconds": 1.7223989963531494, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 2.8294, + "mysql_core_seconds_per_workflow": 0.7479 + }, + "rate-0.5-1": { + "seconds": 122.9, + "health": { + "replies": 150, + "failures": 0, + "slowest_ok_seconds": 0.825 + }, + "cpu_core_seconds": { + "mysql": 35.34101, + "php-scheduler": 13.665336, + "php-server": 93.721084, + "php-worker": 0.846103, + "probe": 0.383153, + "redis": 4.218765, + "sdk-worker": 1.686126 + }, + "max_memory_bytes": { + "mysql": 681295872, + "php-scheduler": 49901568, + "php-server": 117063680, + "php-worker": 54022144, + "probe": 20889600, + "redis": 6488064, + "sdk-worker": 35143680 + }, + "max_runtime_memory_bytes": 207396864, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.5, + "planned_starts": 60, + "accepted_starts": 60, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 60, + "pending_at_deadline": 0, + "closed_during_offer_window": 60, + "latency_p50_seconds": 1.1792030334472656, + "latency_p95_seconds": 1.609131097793579, + "latency_p99_seconds": 1.6938180923461914, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 1.8742, + "mysql_core_seconds_per_workflow": 0.589 + }, + "rate-0.5-2": { + "seconds": 122.1, + "health": { + "replies": 146, + "failures": 0, + "slowest_ok_seconds": 0.793 + }, + "cpu_core_seconds": { + "mysql": 36.534184, + "php-scheduler": 14.546593, + "php-server": 96.016586, + "php-worker": 0.927426, + "probe": 0.072128, + "redis": 4.226208, + "sdk-worker": 1.752287 + }, + "max_memory_bytes": { + "mysql": 713097216, + "php-scheduler": 48873472, + "php-server": 99962880, + "php-worker": 52498432, + "probe": 20951040, + "redis": 8265728, + "sdk-worker": 39034880 + }, + "max_runtime_memory_bytes": 201519104, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.5, + "planned_starts": 60, + "accepted_starts": 60, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 60, + "pending_at_deadline": 0, + "closed_during_offer_window": 60, + "latency_p50_seconds": 1.2057840824127197, + "latency_p95_seconds": 1.6348719596862793, + "latency_p99_seconds": 1.7296020984649658, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 1.9286, + "mysql_core_seconds_per_workflow": 0.6089 + }, + "rate-0.5-3": { + "seconds": 121.9, + "health": { + "replies": 144, + "failures": 0, + "slowest_ok_seconds": 0.812 + }, + "cpu_core_seconds": { + "mysql": 36.688681, + "php-scheduler": 14.632643, + "php-server": 95.642259, + "php-worker": 0.866217, + "probe": -0.083048, + "redis": 4.215405, + "sdk-worker": 1.7989 + }, + "max_memory_bytes": { + "mysql": 742236160, + "php-scheduler": 50565120, + "php-server": 118951936, + "php-worker": 54308864, + "probe": 21495808, + "redis": 7311360, + "sdk-worker": 39960576 + }, + "max_runtime_memory_bytes": 204169216, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.5, + "planned_starts": 60, + "accepted_starts": 60, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 60, + "pending_at_deadline": 0, + "closed_during_offer_window": 60, + "latency_p50_seconds": 1.2680859565734863, + "latency_p95_seconds": 1.557013988494873, + "latency_p99_seconds": 1.8666460514068604, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 1.9226, + "mysql_core_seconds_per_workflow": 0.6115 + }, + "rate-1-1": { + "seconds": 185.9, + "health": { + "replies": 212, + "failures": 0, + "slowest_ok_seconds": 0.911 + }, + "cpu_core_seconds": { + "mysql": 68.790216, + "php-scheduler": 22.00776, + "php-server": 171.008503, + "php-worker": 1.330057, + "probe": 0.87524, + "redis": 6.809588, + "sdk-worker": 3.413805 + }, + "max_memory_bytes": { + "mysql": 788774912, + "php-scheduler": 52244480, + "php-server": 118599680, + "php-worker": 52498432, + "probe": 19771392, + "redis": 7471104, + "sdk-worker": 40652800 + }, + "max_runtime_memory_bytes": 214515712, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 1, + "planned_starts": 120, + "accepted_starts": 120, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 120, + "pending_at_deadline": 0, + "closed_during_offer_window": 62, + "latency_p50_seconds": 55.74392104148865, + "latency_p95_seconds": 72.86986804008484, + "latency_p99_seconds": 74.18884301185608, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 1.6763, + "mysql_core_seconds_per_workflow": 0.5733 + }, + "rate-1-2": { + "seconds": 178.4, + "health": { + "replies": 204, + "failures": 0, + "slowest_ok_seconds": 0.834 + }, + "cpu_core_seconds": { + "mysql": 65.858819, + "php-scheduler": 20.794849, + "php-server": 163.897122, + "php-worker": 1.237972, + "probe": -0.027616, + "redis": 6.473824, + "sdk-worker": 3.309925 + }, + "max_memory_bytes": { + "mysql": 827719680, + "php-scheduler": 51093504, + "php-server": 119918592, + "php-worker": 52858880, + "probe": 21254144, + "redis": 9441280, + "sdk-worker": 41283584 + }, + "max_runtime_memory_bytes": 209801216, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 1, + "planned_starts": 120, + "accepted_starts": 120, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 120, + "pending_at_deadline": 0, + "closed_during_offer_window": 67, + "latency_p50_seconds": 48.43381905555725, + "latency_p95_seconds": 65.49115705490112, + "latency_p99_seconds": 66.32482099533081, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 1.6034, + "mysql_core_seconds_per_workflow": 0.5488 + }, + "rate-1-3": { + "seconds": 181.6, + "health": { + "replies": 208, + "failures": 0, + "slowest_ok_seconds": 0.805 + }, + "cpu_core_seconds": { + "mysql": 67.678031, + "php-scheduler": 21.939475, + "php-server": 167.652277, + "php-worker": 1.27479, + "probe": 0.059808, + "redis": 6.565551, + "sdk-worker": 3.383119 + }, + "max_memory_bytes": { + "mysql": 864178176, + "php-scheduler": 50556928, + "php-server": 119033856, + "php-worker": 52858880, + "probe": 20824064, + "redis": 8126464, + "sdk-worker": 41496576 + }, + "max_runtime_memory_bytes": 223461376, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 1, + "planned_starts": 120, + "accepted_starts": 120, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 120, + "pending_at_deadline": 0, + "closed_during_offer_window": 65, + "latency_p50_seconds": 49.91317009925842, + "latency_p95_seconds": 69.5694899559021, + "latency_p99_seconds": 71.26597094535828, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 1.6453, + "mysql_core_seconds_per_workflow": 0.564 + }, + "low-after-1": { + "seconds": 119.3, + "health": { + "replies": 152, + "failures": 0, + "slowest_ok_seconds": 0.826 + }, + "cpu_core_seconds": { + "mysql": 21.861028, + "php-scheduler": 13.144639, + "php-server": 64.432165, + "php-worker": 0.842851, + "probe": -1.237331, + "redis": 3.57914, + "sdk-worker": 1.100568 + }, + "max_memory_bytes": { + "mysql": 873193472, + "php-scheduler": 51814400, + "php-server": 104218624, + "php-worker": 52879360, + "probe": 19767296, + "redis": 8040448, + "sdk-worker": 40128512 + }, + "max_runtime_memory_bytes": 200564736, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 30, + "accepted_starts": 30, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 30, + "pending_at_deadline": 0, + "closed_during_offer_window": 30, + "latency_p50_seconds": 1.2581219673156738, + "latency_p95_seconds": 1.5643978118896484, + "latency_p99_seconds": 1.730724811553955, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 2.7333, + "mysql_core_seconds_per_workflow": 0.7287 + }, + "low-after-2": { + "seconds": 120.4, + "health": { + "replies": 156, + "failures": 0, + "slowest_ok_seconds": 0.816 + }, + "cpu_core_seconds": { + "mysql": 21.271663, + "php-scheduler": 14.092062, + "php-server": 63.232411, + "php-worker": 0.818295, + "probe": -0.057113, + "redis": 3.492146, + "sdk-worker": 1.08394 + }, + "max_memory_bytes": { + "mysql": 881983488, + "php-scheduler": 50245632, + "php-server": 112705536, + "php-worker": 52858880, + "probe": 19750912, + "redis": 7168000, + "sdk-worker": 40996864 + }, + "max_runtime_memory_bytes": 209444864, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 30, + "accepted_starts": 30, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 30, + "pending_at_deadline": 0, + "closed_during_offer_window": 30, + "latency_p50_seconds": 1.1640570163726807, + "latency_p95_seconds": 1.6432580947875977, + "latency_p99_seconds": 1.6643199920654297, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 2.7212, + "mysql_core_seconds_per_workflow": 0.7091 + }, + "low-after-3": { + "seconds": 120.0, + "health": { + "replies": 152, + "failures": 0, + "slowest_ok_seconds": 0.699 + }, + "cpu_core_seconds": { + "mysql": 21.127632, + "php-scheduler": 14.040819, + "php-server": 62.560674, + "php-worker": 0.81184, + "probe": -0.023795, + "redis": 3.422797, + "sdk-worker": 1.044698 + }, + "max_memory_bytes": { + "mysql": 891281408, + "php-scheduler": 49897472, + "php-server": 114806784, + "php-worker": 54472704, + "probe": 20803584, + "redis": 6688768, + "sdk-worker": 41263104 + }, + "max_runtime_memory_bytes": 207220736, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "offered_rate_per_second": 0.25, + "planned_starts": 30, + "accepted_starts": 30, + "skipped_offer_slots": 0, + "late_starts_over_one_slot": 0, + "completed_by_observation_deadline": 30, + "pending_at_deadline": 0, + "closed_during_offer_window": 30, + "latency_p50_seconds": 1.232442855834961, + "latency_p95_seconds": 1.440964937210083, + "latency_p99_seconds": 1.5305070877075195, + "poll_errors": 0, + "errors": 0, + "runtime_core_seconds_per_workflow": 2.6945, + "mysql_core_seconds_per_workflow": 0.7043 + } + }, + "idle": { + "idle-6-1": { + "seconds": 13.7, + "health": { + "replies": 16, + "failures": 0, + "slowest_ok_seconds": 1.375 + }, + "cpu_core_seconds": { + "mysql": 1.104864, + "php-scheduler": 1.495123, + "php-server": 4.535996, + "php-worker": 0.126425, + "probe": -0.518748, + "redis": 0.493259, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 893804544, + "php-scheduler": 48877568, + "php-server": 104476672, + "php-worker": 52858880, + "probe": 31391744, + "redis": 6451200, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 208138240, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "requested_polls": 6, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "empty": 6 + } + }, + "idle-6-2": { + "seconds": 13.0, + "health": { + "replies": 17, + "failures": 0, + "slowest_ok_seconds": 0.516 + }, + "cpu_core_seconds": { + "mysql": 1.089478, + "php-scheduler": 0.78164, + "php-server": 4.397601, + "php-worker": 0.078596, + "probe": -0.063075, + "redis": 0.441157, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 894009344, + "php-scheduler": 34197504, + "php-server": 124526592, + "php-worker": 52858880, + "probe": 31281152, + "redis": 6389760, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 183857152, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "requested_polls": 6, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "empty": 6 + } + }, + "idle-6-3": { + "seconds": 13.0, + "health": { + "replies": 16, + "failures": 0, + "slowest_ok_seconds": 1.093 + }, + "cpu_core_seconds": { + "mysql": 1.008821, + "php-scheduler": 1.749659, + "php-server": 3.779992, + "php-worker": 0.081498, + "probe": 0.004168, + "redis": 0.373502, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 894189568, + "php-scheduler": 43732992, + "php-server": 121151488, + "php-worker": 52854784, + "probe": 30355456, + "redis": 4386816, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 201003008, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "requested_polls": 6, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "empty": 6 + } + }, + "idle-12-1": { + "seconds": 15.6, + "health": { + "replies": 16, + "failures": 1, + "slowest_ok_seconds": 1.104 + }, + "cpu_core_seconds": { + "mysql": 1.584993, + "php-scheduler": 2.108578, + "php-server": 5.409933, + "php-worker": 0.128848, + "probe": 0.099836, + "redis": 0.543311, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 932577280, + "php-scheduler": 50565120, + "php-server": 124182528, + "php-worker": 52858880, + "probe": 45785088, + "redis": 4153344, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 213549056, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "requested_polls": 12, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "backpressure": 4, + "empty": 8 + } + }, + "idle-12-2": { + "seconds": 16.1, + "health": { + "replies": 16, + "failures": 1, + "slowest_ok_seconds": 1.15 + }, + "cpu_core_seconds": { + "mysql": 1.471128, + "php-scheduler": 1.61049, + "php-server": 5.549401, + "php-worker": 0.099235, + "probe": 0.076546, + "redis": 0.543377, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 934100992, + "php-scheduler": 36921344, + "php-server": 132620288, + "php-worker": 52854784, + "probe": 45920256, + "redis": 5201920, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 196657152, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "requested_polls": 12, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "backpressure": 4, + "empty": 8 + } + }, + "idle-12-3": { + "seconds": 15.9, + "health": { + "replies": 16, + "failures": 1, + "slowest_ok_seconds": 1.554 + }, + "cpu_core_seconds": { + "mysql": 1.400935, + "php-scheduler": 2.024423, + "php-server": 5.508323, + "php-worker": 0.121851, + "probe": 0.015692, + "redis": 0.543894, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 935186432, + "php-scheduler": 44457984, + "php-server": 126164992, + "php-worker": 52858880, + "probe": 46223360, + "redis": 5644288, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 204046336, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "requested_polls": 12, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "backpressure": 4, + "empty": 8 + } + }, + "idle-24-1": { + "seconds": 17.3, + "health": { + "replies": 18, + "failures": 2, + "slowest_ok_seconds": 0.844 + }, + "cpu_core_seconds": { + "mysql": 1.972297, + "php-scheduler": 1.896292, + "php-server": 6.795804, + "php-worker": 0.128665, + "probe": 0.437206, + "redis": 0.587129, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 941764608, + "php-scheduler": 52084736, + "php-server": 137228288, + "php-worker": 52858880, + "probe": 77619200, + "redis": 4280320, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 198823936, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "requested_polls": 24, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "backpressure": 16, + "empty": 8 + } + }, + "idle-24-2": { + "seconds": 17.2, + "health": { + "replies": 16, + "failures": 2, + "slowest_ok_seconds": 0.937 + }, + "cpu_core_seconds": { + "mysql": 2.037846, + "php-scheduler": 2.212841, + "php-server": 6.790381, + "php-worker": 0.097124, + "probe": -0.146052, + "redis": 0.594822, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 947671040, + "php-scheduler": 39317504, + "php-server": 157556736, + "php-worker": 52858880, + "probe": 66371584, + "redis": 4366336, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 235614208, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "requested_polls": 24, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "empty": 8, + "backpressure": 16 + } + }, + "idle-24-3": { + "seconds": 17.3, + "health": { + "replies": 15, + "failures": 3, + "slowest_ok_seconds": 0.441 + }, + "cpu_core_seconds": { + "mysql": 2.059356, + "php-scheduler": 1.905943, + "php-server": 6.98372, + "php-worker": 0.123942, + "probe": -0.076262, + "redis": 0.59964, + "sdk-worker": null + }, + "max_memory_bytes": { + "mysql": 947970048, + "php-scheduler": 50569216, + "php-server": 147697664, + "php-worker": 52854784, + "probe": 61833216, + "redis": 4112384, + "sdk-worker": 0 + }, + "max_runtime_memory_bytes": 206655488, + "oom_kills": { + "mysql": 0, + "php-scheduler": 0, + "php-server": 0, + "php-worker": 0, + "probe": 0, + "redis": 0, + "sdk-worker": 0 + }, + "requested_polls": 24, + "all_registered": true, + "cleanup_errors": 0, + "outcomes": { + "backpressure": 16, + "empty": 8 + } + } + } +} diff --git a/composer.json b/composer.json index 00e3ccfba..2de1c9710 100644 --- a/composer.json +++ b/composer.json @@ -48,7 +48,7 @@ }, "extra": { "durable-workflow": { - "product-train": "2.9.7" + "product-train": "2.9.9" }, "laravel": { "dont-discover": [] diff --git a/composer.lock b/composer.lock index 4f4553fcb..6ffbeab65 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "aa78b1f467750bfc7db005c71eb290e7", + "content-hash": "1230e17c71101ee7340e52d0c1fe2e35", "packages": [ { "name": "apache/avro", diff --git a/config/database.php b/config/database.php index 6fafd2094..fd1f198af 100644 --- a/config/database.php +++ b/config/database.php @@ -1,6 +1,20 @@ = 80500 ? Mysql::ATTR_SSL_CA : PDO::MYSQL_ATTR_SSL_CA; + $mysqlOptions = array_filter([$sslCaAttribute => env('MYSQL_ATTR_SSL_CA')]); +} + +$pgsqlOptions = []; +if (extension_loaded('pdo_pgsql')) { + $disablePreparesAttribute = PHP_VERSION_ID >= 80500 ? Pgsql::ATTR_DISABLE_PREPARES : PDO::PGSQL_ATTR_DISABLE_PREPARES; + $pgsqlOptions = [$disablePreparesAttribute => true]; +} return [ @@ -60,9 +74,7 @@ 'prefix_indexes' => true, 'strict' => true, 'engine' => null, - 'options' => extension_loaded('pdo_mysql') ? array_filter([ - (PHP_VERSION_ID >= 80500 ? Pdo\Mysql::ATTR_SSL_CA : PDO::MYSQL_ATTR_SSL_CA) => env('MYSQL_ATTR_SSL_CA'), - ]) : [], + 'options' => $mysqlOptions, ], 'mariadb' => [ @@ -80,9 +92,7 @@ 'prefix_indexes' => true, 'strict' => true, 'engine' => null, - 'options' => extension_loaded('pdo_mysql') ? array_filter([ - (PHP_VERSION_ID >= 80500 ? Pdo\Mysql::ATTR_SSL_CA : PDO::MYSQL_ATTR_SSL_CA) => env('MYSQL_ATTR_SSL_CA'), - ]) : [], + 'options' => $mysqlOptions, ], 'pgsql' => [ @@ -98,6 +108,9 @@ 'prefix_indexes' => true, 'search_path' => 'public', 'sslmode' => 'prefer', + // Keep native parameter binding without session-scoped statement + // names, so transaction poolers can safely change backend sessions. + 'options' => $pgsqlOptions, ], 'sqlsrv' => [ diff --git a/config/dw-contract.php b/config/dw-contract.php index f9f23786c..2e8abf2f3 100644 --- a/config/dw-contract.php +++ b/config/dw-contract.php @@ -889,7 +889,7 @@ 'legacy' => 'WORKFLOW_V2_COMPATIBILITY_HEARTBEAT_TTL', ], 'DW_V2_PIN_TO_RECORDED_FINGERPRINT' => [ - 'description' => 'When true (default), in-flight runs resolve their workflow class from the fingerprint recorded at WorkflowStarted.', + 'description' => 'Prefer an eligible worker matching the recorded definition. If none remains, current code must validate strict replay. False removes only this source preference, not build pins or replay checks.', 'default' => 'true', 'since' => '2.0.0', 'legacy' => 'WORKFLOW_V2_PIN_TO_RECORDED_FINGERPRINT', diff --git a/docker-compose.dedicated-matching.yml b/docker-compose.dedicated-matching.yml index 071822046..48c69e0ab 100644 --- a/docker-compose.dedicated-matching.yml +++ b/docker-compose.dedicated-matching.yml @@ -32,13 +32,13 @@ name: durable-workflow-server # daemon reports `shape: dedicated`. # Generated by scripts/ci/sync-source-release.mjs. Do not edit the fallback. -x-server-image: &server-image ${DW_SERVER_IMAGE:-durableworkflow/server:${DW_SERVER_TAG:-2.9.7}} +x-server-image: &server-image ${DW_SERVER_IMAGE:-durableworkflow/server:${DW_SERVER_TAG:-2.9.9}} x-server-environment: &server-environment APP_NAME: "Durable Workflow Server" APP_ENV: ${APP_ENV:-local} DW_SERVER_KEY: ${DW_SERVER_KEY:-} - APP_VERSION: ${APP_VERSION:-${DW_SERVER_TAG:-2.9.7}} + APP_VERSION: ${APP_VERSION:-${DW_SERVER_TAG:-2.9.9}} APP_DEBUG: ${APP_DEBUG:-false} DB_CONNECTION: mysql DB_HOST: mysql diff --git a/docker-compose.memo-rolling.yml b/docker-compose.memo-rolling.yml index 0fc31e22c..4c77c8946 100644 --- a/docker-compose.memo-rolling.yml +++ b/docker-compose.memo-rolling.yml @@ -49,14 +49,14 @@ services: command: ["server-bootstrap"] environment: <<: *runtime-environment - APP_VERSION: ${APP_VERSION:-2.9.7} + APP_VERSION: ${APP_VERSION:-2.9.9} successor: image: ${DW_MEMO_SUCCESSOR_IMAGE:-durable-workflow/server-memo-rolling:local} ports: !override [] environment: <<: *runtime-environment - APP_VERSION: ${APP_VERSION:-2.9.7} + APP_VERSION: ${APP_VERSION:-2.9.9} DW_SERVER_ID: memo-successor DW_SERVER_TOPOLOGY_SHAPE: standalone_server DW_SERVER_PROCESS_CLASS: server_http_node diff --git a/docker-compose.published.yml b/docker-compose.published.yml index e6fed642f..e5cc0b811 100644 --- a/docker-compose.published.yml +++ b/docker-compose.published.yml @@ -1,13 +1,13 @@ name: durable-workflow-server # Generated by scripts/ci/sync-source-release.mjs. Do not edit the fallback. -x-server-image: &server-image ${DW_SERVER_IMAGE:-durableworkflow/server:${DW_SERVER_TAG:-2.9.7}} +x-server-image: &server-image ${DW_SERVER_IMAGE:-durableworkflow/server:${DW_SERVER_TAG:-2.9.9}} x-server-environment: &server-environment APP_NAME: "Durable Workflow Server" APP_ENV: ${APP_ENV:-local} DW_SERVER_KEY: ${DW_SERVER_KEY:-} - APP_VERSION: ${APP_VERSION:-${DW_SERVER_TAG:-2.9.7}} + APP_VERSION: ${APP_VERSION:-${DW_SERVER_TAG:-2.9.9}} APP_DEBUG: ${APP_DEBUG:-false} LOG_CHANNEL: ${LOG_CHANNEL:-stderr} LOG_LEVEL: ${LOG_LEVEL:-info} diff --git a/docker-compose.small-cluster.yml b/docker-compose.small-cluster.yml index b5a994c31..a6a242017 100644 --- a/docker-compose.small-cluster.yml +++ b/docker-compose.small-cluster.yml @@ -12,7 +12,7 @@ x-server-build: &server-build x-server-environment: &server-environment APP_NAME: "Durable Workflow Server" APP_ENV: testing - APP_VERSION: ${APP_VERSION:-2.9.7} + APP_VERSION: ${APP_VERSION:-2.9.9} APP_DEBUG: "false" DW_SERVER_KEY: ${DW_SERVER_KEY:-base64:5Zt4nUhlCm3DD0nLXZJQdHiwPfb56yGo9gNV/g3jYbY=} DB_CONNECTION: ${DW_SMALL_CLUSTER_DB:-mysql} diff --git a/docker-compose.yml b/docker-compose.yml index 5d587e3bb..95d538a38 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,7 +15,7 @@ services: DW_SERVER_KEY: "${DW_SERVER_KEY:-}" DW_SERVER_TOPOLOGY_SHAPE: standalone_server DW_SERVER_PROCESS_CLASS: server_http_node - APP_VERSION: "${APP_VERSION:-2.9.7}" + APP_VERSION: "${APP_VERSION:-2.9.9}" APP_DEBUG: "false" DB_CONNECTION: mysql DB_HOST: mysql @@ -62,7 +62,7 @@ services: APP_NAME: "Durable Workflow Server" APP_ENV: local DW_SERVER_KEY: "${DW_SERVER_KEY:-}" - APP_VERSION: "${APP_VERSION:-2.9.7}" + APP_VERSION: "${APP_VERSION:-2.9.9}" APP_DEBUG: "false" DB_CONNECTION: mysql DB_HOST: mysql @@ -124,7 +124,7 @@ services: DW_SERVER_KEY: "${DW_SERVER_KEY:-}" DW_SERVER_TOPOLOGY_SHAPE: standalone_server DW_SERVER_PROCESS_CLASS: worker_node - APP_VERSION: "${APP_VERSION:-2.9.7}" + APP_VERSION: "${APP_VERSION:-2.9.9}" DB_CONNECTION: mysql DB_HOST: mysql DB_PORT: 3306 @@ -178,7 +178,7 @@ services: DW_SERVER_KEY: "${DW_SERVER_KEY:-}" DW_SERVER_TOPOLOGY_SHAPE: standalone_server DW_SERVER_PROCESS_CLASS: scheduler_node - APP_VERSION: "${APP_VERSION:-2.9.7}" + APP_VERSION: "${APP_VERSION:-2.9.9}" DB_CONNECTION: mysql DB_HOST: mysql DB_PORT: 3306 diff --git a/docs/rust-server-storage-audit.md b/docs/rust-server-storage-audit.md index b1cfd921e..cd97df381 100644 --- a/docs/rust-server-storage-audit.md +++ b/docs/rust-server-storage-audit.md @@ -96,5 +96,7 @@ and takeover flag stays false. There is no mixed PHP/Rust fleet or PHP rollback after upgrade. Restoring a pre-upgrade backup is a separate recovery action with its own data boundary. -All gates above remain open. HTTP export/import remains a supported deployment +All gates above remain open. The [takeover design](rust-server-upgrade-takeover.md) +proposes how the takeover command proves the backup, excludes other writers and +resumes after interruption. HTTP export/import remains a supported deployment transfer surface, but it does not substitute for in-place upgrade qualification. diff --git a/docs/rust-server-transition-locking.md b/docs/rust-server-transition-locking.md index e593eab1a..f46ded9b4 100644 --- a/docs/rust-server-transition-locking.md +++ b/docs/rust-server-transition-locking.md @@ -26,6 +26,16 @@ read-only snapshot (`Store::begin_read()`, `READ_SNAPSHOT_BEGIN` in `backend.rs`). Control-plane reads such as describe and run history query the pool without a transaction. +A held long poll takes the lock for its first probe, and afterwards only when +a lock-free read finds something to claim or refuse: an eligible task (or +query), a receipt for the poll's own request ID, or a removed, moved or +replaced registration (`Store::task_poll_ready` and `Store::query_poll_ready`). The read runs at once when a +commit on the same node wakes the poll. Otherwise it runs 50 ms after a probe +and then at doubling intervals up to 250 ms, so a poll observes work committed +by another server process within 250 ms. An idle poll therefore neither waits +for the lock nor delays transitions, and an unrelated commit does not send +every waiting poll to the lock. + ## Measured cost Under the CPU deadline-health profile on a local MySQL 8.0 cell, per-step @@ -36,6 +46,21 @@ scheduler from sleeping after each pass to a fixed tick made no measurable difference. Deadline health and throughput under load are bounded by how long other transitions hold the lock. +A scratch build that logged lock wait and hold per call site then ran the same +cell on a host shared with other work, so absolute times are approximate: + +- A cancellation request held the lock 0.65 to 0.86 s, almost all of it + waiting for the single codec permit behind describe decodes of the dense + input (0.58 to 0.80 s). With a permit of its own the hold fell to 0.05 to + 0.09 s. Codec work under the lock now has its own permit. +- Activity status checks spent about 1.6 s per run waiting for the lock. They + no longer take it. +- Cancellation delivery and expiry, workflow completions, heartbeats and + activity expiry held the lock 0.2 to 0.9 s during the read wave without codec + work, several times longer than the same transitions before it. Their summary + projections took under 30 ms each. Lowering the decode thread's CPU priority + changed nothing measurable. This remaining cost is not yet attributed. + ## What lengthens lock hold time - **Summary projection at commit.** `Transaction::commit` reorders open waits @@ -43,10 +68,12 @@ other transitions hold the lock. `summary_history` pages through the run's entire history to do so (`terminal_summary.rs`). Every heartbeat and claim therefore costs time proportional to history length while holding the lock. -- **Codec work inside transactions.** Several paths take the single codec - permit and run Avro work on the blocking pool while the transaction is open, - for example the rejected-signal path, continue-as-new, cooperative requests - and repair. +- **Codec work inside transactions.** Five paths run Avro work on the blocking + pool while the transaction is open: the rejected-signal command, signal + arguments applied by a resumed workflow task, cooperative cancellation + requests, task repair and fenced worker deregistration. They take + `transition_codec`, never the read permit `signal_codec`, so they do not + queue behind reads. The CPU work itself still runs under the lock. - **Claim reads.** Workflow claims load the first history page (the requested page size, 100 by default) and compute the history budget under the lock. Query claims parse cached query JSON values. diff --git a/docs/rust-server-upgrade-takeover.md b/docs/rust-server-upgrade-takeover.md new file mode 100644 index 000000000..dfa5d7003 --- /dev/null +++ b/docs/rust-server-upgrade-takeover.md @@ -0,0 +1,163 @@ +# Native upgrade takeover + +This note designs the one-way, backup-first takeover of an existing PHP Server +database by the native runtime. It is a design input for +[Server #325](https://github.com/durable-workflow/server/issues/325), not a +qualified procedure. The gates it must satisfy are in the +[storage and upgrade audit](rust-server-storage-audit.md). Today +`upgrade-inspect`, `upgrade-queue-inspect` and `upgrade-payload-inspect` only +read, and their `backup_verified`, `php_quiescence_verified`, +`stored_representation_compatibility_verified` and `takeover_allowed` flags are +always false. This note says what makes each one true. + +## Shape + +Takeover is an explicit command, never a side effect of starting the server. +Native startup keeps refusing a PHP or unknown database. + +1. The installation runs Server 2.5.14 or later, the first release with the + ownership fence, so any PHP role that restarts later refuses the upgraded + database. The operator then stops every PHP write-capable role: HTTP, + bootstrap and migration, queue workers and the scheduler. +2. `upgrade-fingerprint` prints a digest of the database's complete logical + contents (below). +3. The operator takes a backup with the + [existing procedure](self-hosted-backup-and-restore.md), restores it into a + separate database, and runs `upgrade-fingerprint` there. Equal digests show + that the backup restores to exactly the stopped database. +4. `upgrade-takeover` runs with `DW_UPGRADE_BACKUP_FINGERPRINT` set to that + digest. It refuses unless the live database still has that digest inside + the same exclusive transaction that writes the ownership marker. A write + after the backup, by PHP or anything else, changes the digest and stops the + takeover before any mutation. +5. Native starts normally on the upgraded database. PHP refuses it from the + marker onward ([PHP ownership checks](php-database-ownership.md)). Earlier + PHP releases lack that check, which is why step 1 requires 2.5.14. + +There is no PHP rollback after step 4. Recovery restores the verified backup +into a new database for a PHP installation, with the data boundary of that +backup. + +## Fingerprint + +The digest covers every relation in the database, not only the ones PHP +knows: the sorted relation names, each relation's column names and declared +types, and every row in primary-key order (all columns when there is no +primary key). Values are encoded canonically, so a dump and restore reproduces +the same bytes: integers in decimal, text and JSON as stored text, timestamps +at their stored precision, binary as hex, NULL distinct from empty. It is read +in one read-only snapshot (`READ_SNAPSHOT_BEGIN`). The command reports the +digest, the per-relation row counts and the elapsed time, never row contents. + +Takeover reads every row once more, inside its exclusive transaction. That +cost is accepted for a one-time operation. Its duration is reported so an +operator can plan the stop window from the fingerprint run. + +External payload objects are not in the database. Until a qualified object +check exists, takeover refuses when `upgrade-payload-inspect` reports any +external reference, and the operator's object manifest from the backup +procedure remains the record for those bytes. + +## Exclusive transaction per backend + +The marker, the fingerprint check and the schema changes must not interleave +with any other writer. + +| Backend | Exclusion | Atomicity | +| --- | --- | --- | +| SQLite | `BEGIN EXCLUSIVE` on the database file | Fingerprint, marker, native relations and conversions commit in one transaction. | +| PostgreSQL | Native's bootstrap advisory key, then `LOCK TABLE … IN ACCESS EXCLUSIVE MODE` on every table, with a short lock wait. The later DDL needs that mode anyway, so no lock is upgraded while another session waits. | DDL is transactional, so everything commits in one transaction. | +| MySQL and MariaDB | Native's bootstrap lock and a check that no other session uses the database (which needs the `PROCESS` privilege), repeated after the marker is written. `LOCK TABLES` cannot be used: under it MySQL refuses to create or drop any table it does not hold. | DDL commits implicitly, so the marker records a step journal (below). | + +On SQLite and PostgreSQL, a PHP connection that slipped past the stop blocks on +these locks, and a write it committed earlier changes the fingerprint. Neither +leaves a mixed state. + +## Marker and interruption + +On SQLite and PostgreSQL the marker commits with everything else, so an +interruption leaves the database unchanged PHP and nothing needs a journal. +The operator reruns `upgrade-takeover` with the same fingerprint. A killed +SQLite takeover leaves a rollback journal; read-only inspection refuses it, +and the rerun lets SQLite roll it back before the fingerprint check. + +MySQL and MariaDB commit DDL implicitly, so there native's marker table is +written first, with one row `('rust-takeover:', , +)`. Native startup refuses it with +`native_takeover_incomplete`. The sessions and the fingerprint are checked +again with the marker set aside, then each step runs and records its +completion in that row; row changes commit with their step, and DDL steps +check what is left to do. Rerunning `upgrade-takeover` with the same +fingerprint resumes after the last recorded step; a different fingerprint is +refused. The ledger and native's final marker `('rust-development', 4, +)` commit together. The PHP fence refuses the database from the +first step, because the marker table exists. + +## SQLite sources in WAL mode + +PHP Server keeps SQLite in WAL mode, and a read-only SQLite open can still +create WAL and shared-memory sidecars, so inspection, fingerprint and takeover +require the rollback journal. With PHP stopped, `upgrade-prepare-sqlite` opens +the file as its only connection and switches it to the rollback journal. +SQLite first checkpoints every committed WAL frame into the file, so the stored +rows are unchanged. The command refuses while another process holds the file, +and runs before the fingerprint and the backup. + +## Steps + +1. Refuse unless every inspection command reports `qualified` for this + database and the source catalog matches a qualified published PHP release. +2. Check the fingerprint (above) and write the marker. +3. Create the native relations and indexes (`upgrade.rs`, `native_object`). +4. Convert or adopt stored representations the inspections qualified: + poll-request bindings in task payloads, worker registrations (an + incarnation per active registration), cached query state, and queued + timer wakeups mapped to their durable task rows. Anything not qualified + was already refused by step 1. +5. Record the native migration ledger and set the marker to `ready`. + +Native's frozen physical layout predates the current PHP release, so step 3 +alone does not yield a database native accepts. PHP 2.8.0 adds its own +`workflow_worker_registration_incarnations` table, a `registration_token` +column on `workflow_worker_registrations` and a `call_deadline_at` column on +`workflow_service_calls` (`rust/upgrade/php-2.8.0/`). Server 2.9.7, with Core +2.7.2, also adds `workflow_tasks.queue_delivery_reference`. Native keeps its +own layout, and step 4 converts each qualified PHP release's layout to it +with a recorded recipe: + +- PHP's registration incarnations and tokens are copied into native's + incarnation table, so a running SDK worker's fenced deregistration still + matches. The PHP table and column are then removed. +- `queue_delivery_reference` is optional lookup metadata for PHP's database + queue delivery, and PHP itself grants it no task authority. Native does not + use PHP's queue, and the queue inspection has already bound every queued + job to its durable task row, so the column is removed and the takeover + report counts the references it held. +- Service calls are not part of native yet, so a source with service calls + is refused before any change, and `call_deadline_at` is removed only from + an empty table. + +A release whose layout has no recipe is refused by step 1. + +## Tests + +Each backend gets an upgrade fixture written by the last published PHP release +with running and waiting workflows, a pending timer, a leased activity, an +acknowledged signal and an expired lease. The tests check: + +- refusal without a fingerprint, with a stale one, and after one extra row; +- interruption after each step, then resumption to the same final state; +- native completes the original work with its original histories, deadlines + and identities, and refuses the stale attempt; +- PHP refuses the upgraded database at startup, bootstrap, migration, worker + and scheduler entry points; +- a backup restored into a new database fingerprints equal to the stopped + source. + +## Open questions + +- Whether the declared column types print identically after a restore into + another minor server version. Until that is shown, the restore check uses + the same server version as the source. +- How long the fingerprint takes on a large installation, measured on a + disposable host before the stop window is documented. diff --git a/docs/server-reference.md b/docs/server-reference.md index 8d836aa79..2c3fe5e44 100644 --- a/docs/server-reference.md +++ b/docs/server-reference.md @@ -418,6 +418,29 @@ Workers that omit fingerprints during re-registration cannot clear previously stored fingerprints for workflow types they still advertise; the server keeps the stored value until a new worker id is used. +Existing runs retain their original definition fingerprint, build pin, history +and deadlines. By default, the Server prefers a fresh eligible worker with the +recorded definition. When that definition is no longer available in the run's +namespace, queue, workflow type and build, a replacement definition can claim +the original task only when it advertises `workflow_entry_arguments_validation`, +and must pass the SDK's strict replay checks. This worker capability declares +pre-entry refusal without failing the business run; older worker implementations +do not gain it from their runtime or protocol version. `entry_validation_unavailable` +in `definition_routing` identifies a replacement that needs an SDK upgrade. +Use versioned +branches such as `get_version` for changes to durable commands. A different +source fingerprint alone is not proof of incompatible replay. + +`DW_V2_PIN_TO_RECORDED_FINGERPRINT=false` removes the matching-source preference. +It does not remove build pins, namespace or queue isolation, workflow-type and +capability checks, immutable registration identity, lease fences or strict replay. +Run detail, run debug, repair responses and queue detail expose +`definition_routing`. `replay_required` is an allowed fallback, while +`blocked=true` means a due ready workflow task has no eligible definition route. +Repair can fix transport state; it cannot manufacture a compatible worker or +rewrite the original definition. Queue observations contain at most ten run +examples and explicitly report whether more ready runs exist. + ### 5. Poll for Workflow Tasks The server holds the connection open (long-poll) until a task is ready or @@ -1599,6 +1622,37 @@ overlay or with `kubectl scale`. ### Configuration +#### PostgreSQL connection pooling + +The PostgreSQL connection uses native bound parameters without session-scoped +prepared-statement names. This lets PgBouncer transaction pooling change the +physical database session between transactions. Direct and session-pooled +connections use the same binding path. Statement pooling cannot support the +Server's multi-statement transactions. + +The PHP feature job runs the same alternating-client and rollback controls +against direct PostgreSQL, session pooling, and transaction pooling with +`max_prepared_statements=0` and `200`. The transaction controls require actual +backend PID changes. To run against an isolated existing endpoint: + +```bash +DW_POOLER_TEST_HOST=pooler DW_POOLER_TEST_PORT=6432 \ +DW_POOLER_TEST_DATABASE=isolated_test DW_POOLER_TEST_USER=test_user \ +vendor/bin/phpunit tests/Feature/PostgresPoolerTest.php --fail-on-skipped +``` + +These controls do not validate custom session settings, temporary tables, +advisory-lock policies, or arbitrary proxy configurations. Qualify the exact +published image, pooler version, authentication, schema and TLS settings used +by your deployment before routing production traffic. + +`/api/ready` executes a bound read on the primary database connection and +reports `checks.database.check_scope=bound_read_query`. A failed query reports +`database_query_failed`, or `backend_unavailable` when the backend is known to +be unavailable. This remains a read probe: `write_capacity_verified=false`. +Use an actual workflow canary to verify writes and the complete worker path. +The runtime's lightweight bootstrap guard remains connection-only. + All operator-facing configuration is via `DW_*` environment variables. `config/dw-contract.php` is the authoritative machine-checkable contract; CI (`tests/Unit/EnvContractTest.php`) diffs it against `.env.example`, @@ -1715,7 +1769,7 @@ inside the package's `config/workflows.php` via | `DW_V2_SUPPORTED_COMPATIBILITIES` | (unset) | Comma-separated marker list the worker accepts, or `*` for any. | | `DW_V2_COMPATIBILITY_NAMESPACE` | (unset) | Compatibility namespace for independent fleets sharing one database. | | `DW_V2_COMPATIBILITY_HEARTBEAT_TTL` | `30` | Seconds a worker-compatibility heartbeat remains valid. | -| `DW_V2_PIN_TO_RECORDED_FINGERPRINT` | `true` | Resolve in-flight runs from the fingerprint recorded at WorkflowStarted. | +| `DW_V2_PIN_TO_RECORDED_FINGERPRINT` | `true` | Prefer an eligible recorded-definition worker; otherwise require strict replay by current code. False removes this preference, not build pins or replay checks. | | `DW_V2_CONTINUE_AS_NEW_EVENT_THRESHOLD` | `10000` | History event count at which the package signals continue-as-new. | | `DW_V2_CONTINUE_AS_NEW_SIZE_BYTES_THRESHOLD` | `5242880` | Serialized-history byte count at which the package signals continue-as-new. | | `DW_V2_HISTORY_EVENT_WARNING_THRESHOLD` | `8000` | Event count at which history pressure becomes `approaching`. | diff --git a/k8s/README.md b/k8s/README.md index cf7c9a540..e1a7dd001 100644 --- a/k8s/README.md +++ b/k8s/README.md @@ -13,7 +13,7 @@ The checked-in manifests are synchronized with the repository's stable source release and pin its Docker Hub tag: ```text -durableworkflow/server:2.9.7 +durableworkflow/server:2.9.9 ``` Before production use, patch every workload image to the exact published tag or @@ -21,15 +21,15 @@ digest you intend to run: ```bash kubectl set image -n durable-workflow deploy/durable-workflow-server \ - server=durableworkflow/server:2.9.7 + server=durableworkflow/server:2.9.9 kubectl set image -n durable-workflow deploy/durable-workflow-worker \ - worker=durableworkflow/server:2.9.7 + worker=durableworkflow/server:2.9.9 kubectl set image -n durable-workflow cronjob/durable-workflow-scheduler \ - scheduler=durableworkflow/server:2.9.7 + scheduler=durableworkflow/server:2.9.9 ``` GitHub Container Registry publishes the same release line at -`ghcr.io/durable-workflow/server:2.9.7`. Digest pinning is preferred for strict +`ghcr.io/durable-workflow/server:2.9.9`. Digest pinning is preferred for strict change control. The manifests expect you to provide: diff --git a/k8s/helm/durable-workflow/Chart.yaml b/k8s/helm/durable-workflow/Chart.yaml index 3c1884699..321c99ce0 100644 --- a/k8s/helm/durable-workflow/Chart.yaml +++ b/k8s/helm/durable-workflow/Chart.yaml @@ -5,11 +5,11 @@ type: application # The chart's own semver version. Bumped on every chart release; treated as # independent of the server image version (appVersion). Breaking-change rules # for this version live in docs/helm-upgrading.md alongside the chart. -version: 0.1.183 +version: 0.1.185 # The immutable Durable Workflow Server identity this chart release packages. # The onboarding default in values.yaml and appVersion are generated from the # checked-in source release record. -appVersion: "2.9.7" +appVersion: "2.9.9" kubeVersion: ">=1.27.0-0" home: https://durable-workflow.github.io/docs/2.0/deployment sources: @@ -30,10 +30,10 @@ annotations: # exact commit that most recently changed the packaged chart. org.opencontainers.image.source: https://github.com/durable-workflow/server dev.durable-workflow.source-revision: "unreleased" - dev.durable-workflow.image-reference: "docker.io/durableworkflow/server:2.9.7" + dev.durable-workflow.image-reference: "docker.io/durableworkflow/server:2.9.9" artifacthub.io/license: MIT artifacthub.io/category: integration-delivery # Free-form changelog for the current chart release shown by Artifact Hub. artifacthub.io/changes: | - kind: fixed - description: Preserve accepted signals at terminal workflow boundaries and exclude intentionally pruned run summaries from projection drift. + description: Allow compatible workflow-code replacement with validated workers and preserve incompatible entry failures for supported same-run repair. diff --git a/k8s/helm/durable-workflow/README.md b/k8s/helm/durable-workflow/README.md index e286e2c20..42c39a15c 100644 --- a/k8s/helm/durable-workflow/README.md +++ b/k8s/helm/durable-workflow/README.md @@ -63,7 +63,7 @@ helm install durable-workflow ./k8s/helm/durable-workflow \ ```yaml image: - tag: "2.9.7" + tag: "2.9.9" # Pin a digest in production: # digest: "sha256:abc123..." # memoPayloadStorage: "raw-json-v1" # Required for a digest or custom image. diff --git a/k8s/helm/durable-workflow/ci/existing-secrets-values.yaml b/k8s/helm/durable-workflow/ci/existing-secrets-values.yaml index 2aa2fb07c..11f807c49 100644 --- a/k8s/helm/durable-workflow/ci/existing-secrets-values.yaml +++ b/k8s/helm/durable-workflow/ci/existing-secrets-values.yaml @@ -1,7 +1,7 @@ # CI fixture: GitOps / externally-managed-secret path. The chart consumes # existing Secrets and renders no Secret resources of its own. image: - tag: "2.9.7" + tag: "2.9.9" externalDatabase: connection: pgsql diff --git a/k8s/helm/durable-workflow/ci/ingress-and-hpa-values.yaml b/k8s/helm/durable-workflow/ci/ingress-and-hpa-values.yaml index bfc1fa4a5..70d9fa26c 100644 --- a/k8s/helm/durable-workflow/ci/ingress-and-hpa-values.yaml +++ b/k8s/helm/durable-workflow/ci/ingress-and-hpa-values.yaml @@ -1,6 +1,6 @@ # CI fixture: ingress + autoscaling enabled. Exercises optional templates. image: - tag: "2.9.7" + tag: "2.9.9" externalDatabase: connection: mysql diff --git a/k8s/helm/durable-workflow/ci/inline-secrets-values.yaml b/k8s/helm/durable-workflow/ci/inline-secrets-values.yaml index ab0467cf2..8160b1c33 100644 --- a/k8s/helm/durable-workflow/ci/inline-secrets-values.yaml +++ b/k8s/helm/durable-workflow/ci/inline-secrets-values.yaml @@ -2,7 +2,7 @@ # chart's render path is exercised end-to-end. Real deployments should use # existingSecret instead. image: - tag: "2.9.7" + tag: "2.9.9" externalDatabase: connection: mysql diff --git a/k8s/helm/durable-workflow/templates/_helpers.tpl b/k8s/helm/durable-workflow/templates/_helpers.tpl index d09b528d2..fb7c05772 100644 --- a/k8s/helm/durable-workflow/templates/_helpers.tpl +++ b/k8s/helm/durable-workflow/templates/_helpers.tpl @@ -88,7 +88,7 @@ resolved by an explicit capability declaration or an existing workload marker. {{- define "durable-workflow.memoPayloadStorageForImage" -}} {{- $image := toString . -}} {{- $normalized := regexReplaceAll "^index\\.docker\\.io/" $image "docker.io/" -}} -{{- if eq $normalized "docker.io/durableworkflow/server:2.9.7" -}} +{{- if eq $normalized "docker.io/durableworkflow/server:2.9.9" -}} dual-v1 {{- else if regexMatch "^docker\\.io/durableworkflow/server:2\\.0\\.0-rc\\.[0-9]+$" $normalized -}} {{- $releaseCandidate := atoi (regexFind "[0-9]+$" $normalized) -}} diff --git a/k8s/helm/durable-workflow/values.yaml b/k8s/helm/durable-workflow/values.yaml index 33e0ae404..3aac505f6 100644 --- a/k8s/helm/durable-workflow/values.yaml +++ b/k8s/helm/durable-workflow/values.yaml @@ -21,7 +21,7 @@ image: registry: docker.io repository: durableworkflow/server # Generated by scripts/ci/sync-source-release.mjs. Do not edit this default. - tag: "2.9.7" + tag: "2.9.9" # Optional digest pin. When set, takes precedence over tag for change control. # Example: "sha256:abc123..." digest: "" diff --git a/k8s/helm/examples/values-dev.yaml b/k8s/helm/examples/values-dev.yaml index 3a413fe4a..871391cbe 100644 --- a/k8s/helm/examples/values-dev.yaml +++ b/k8s/helm/examples/values-dev.yaml @@ -3,7 +3,7 @@ # shape in production. image: - tag: "2.9.7" + tag: "2.9.9" externalDatabase: connection: mysql diff --git a/k8s/helm/examples/values-external-secrets-operator.yaml b/k8s/helm/examples/values-external-secrets-operator.yaml index 881442606..2bbfd25c3 100644 --- a/k8s/helm/examples/values-external-secrets-operator.yaml +++ b/k8s/helm/examples/values-external-secrets-operator.yaml @@ -5,7 +5,7 @@ # concern. image: - tag: "2.9.7" + tag: "2.9.9" externalDatabase: connection: pgsql diff --git a/k8s/helm/examples/values-production-existing-secrets.yaml b/k8s/helm/examples/values-production-existing-secrets.yaml index 248b16bb3..e1faeeff0 100644 --- a/k8s/helm/examples/values-production-existing-secrets.yaml +++ b/k8s/helm/examples/values-production-existing-secrets.yaml @@ -10,7 +10,7 @@ image: repository: durable-workflow/server # Pin a digest in production for change-control auditability. digest: "" # e.g. "sha256:abc123..." - tag: "2.9.7" + tag: "2.9.9" externalDatabase: connection: pgsql diff --git a/k8s/migration-job.yaml b/k8s/migration-job.yaml index 6d6e2ed71..a514b2e47 100644 --- a/k8s/migration-job.yaml +++ b/k8s/migration-job.yaml @@ -13,7 +13,7 @@ spec: restartPolicy: OnFailure containers: - name: migrate - image: durableworkflow/server:2.9.7 + image: durableworkflow/server:2.9.9 command: ["server-entrypoint"] args: ["server-bootstrap"] envFrom: diff --git a/k8s/scheduler-cronjob.yaml b/k8s/scheduler-cronjob.yaml index fd8f44b8c..6017ed7c3 100644 --- a/k8s/scheduler-cronjob.yaml +++ b/k8s/scheduler-cronjob.yaml @@ -24,7 +24,7 @@ spec: restartPolicy: Never containers: - name: scheduler - image: durableworkflow/server:2.9.7 + image: durableworkflow/server:2.9.9 command: ["server-entrypoint"] args: - sh diff --git a/k8s/secret.yaml b/k8s/secret.yaml index 303e2bf60..753b17c9e 100644 --- a/k8s/secret.yaml +++ b/k8s/secret.yaml @@ -12,7 +12,7 @@ metadata: app.kubernetes.io/name: durable-workflow data: APP_NAME: "Durable Workflow Server" - APP_VERSION: "2.9.7" + APP_VERSION: "2.9.9" APP_ENV: production APP_DEBUG: "false" DB_CONNECTION: mysql diff --git a/k8s/server-deployment.yaml b/k8s/server-deployment.yaml index 5b7a8b68f..15c71b386 100644 --- a/k8s/server-deployment.yaml +++ b/k8s/server-deployment.yaml @@ -23,7 +23,7 @@ spec: spec: containers: - name: server - image: durableworkflow/server:2.9.7 + image: durableworkflow/server:2.9.9 ports: - containerPort: 8080 name: http diff --git a/k8s/worker-deployment.yaml b/k8s/worker-deployment.yaml index 0e933e8b5..910009def 100644 --- a/k8s/worker-deployment.yaml +++ b/k8s/worker-deployment.yaml @@ -19,7 +19,7 @@ spec: spec: containers: - name: worker - image: durableworkflow/server:2.9.7 + image: durableworkflow/server:2.9.9 command: ["server-entrypoint"] args: ["php", "artisan", "queue:work", "--sleep=1", "--tries=3", "--max-time=3600"] envFrom: diff --git a/resources/release/source-release.json b/resources/release/source-release.json index 6397bb46f..234add484 100644 --- a/resources/release/source-release.json +++ b/resources/release/source-release.json @@ -1,9 +1,9 @@ { "schema": "durable-workflow.server.source-release/v1", "server": { - "version": "2.9.7" + "version": "2.9.9" }, "helm_chart": { - "version": "0.1.183" + "version": "0.1.185" } } diff --git a/rust/README.md b/rust/README.md index 1e4aa7dbb..679d895d9 100644 --- a/rust/README.md +++ b/rust/README.md @@ -412,31 +412,58 @@ version-1/version-2/version-3 development schemas are refused without conversion needed old development data separately; this unpublished slice supplies no older-development converter. Use a new isolated file for version 4. -This refusal is not the upgrade mechanism. A stopped PHP SQLite database can -be taken over with the development `upgrade-takeover` command, following the +This refusal is not the upgrade mechanism. A stopped PHP database can be taken +over with the development `upgrade-takeover` command, following the [takeover design](../docs/rust-server-upgrade-takeover.md): 1. Run PHP Server 2.5.14 or later, which refuses a Rust-owned database, and stop every PHP role. -2. Run `upgrade-fingerprint` on the source, mounted read-only. Back the file - up, restore the backup to a separate path, and fingerprint that too. The - digests must be equal. + PHP Server keeps SQLite in WAL mode, which the upgrade commands refuse. With + PHP stopped, run `upgrade-prepare-sqlite` on the source file (writable). + It has SQLite checkpoint every committed WAL frame into the file and switch + it to the rollback journal, refusing with `upgrade_source_in_use` while any + other process has the file open. The rows are unchanged; the backup below + is taken afterwards. A journal left by a crashed writer is rolled back the + same way. +2. Run `upgrade-fingerprint` on the source: a SQLite file mounted read-only, + or the database through a read-only account. Back the database up (a file + copy, `pg_dump` or `mysqldump --single-transaction`), restore the backup + separately, and fingerprint that too. The digests must be equal. 3. Run `upgrade-takeover` with `DW_UPGRADE_BACKUP_FINGERPRINT` set to that digest and the original queue settings (`DW_RUST_UPGRADE_QUEUE_*`, as for - `upgrade-queue-inspect`). - -Inspection, the fingerprint check, the conversion of the recorded PHP release -layout, the native relations, the migration ledger and native's own startup -verification all happen in one exclusive transaction. A refusal or an -interruption leaves the PHP file unchanged. The command refuses: + `upgrade-queue-inspect`). PostgreSQL uses the owner's `DB_*` settings. + MySQL and MariaDB use an account that owns the database and has the + `PROCESS` privilege, so the takeover can see every other session. + +On SQLite and PostgreSQL, inspection, the fingerprint check, the conversion +of the recorded PHP release layout, the native relations, the migration ledger +and native's own startup verification all happen in one transaction. SQLite +holds the file +exclusively; PostgreSQL takes native's bootstrap lock and locks every table +before reading, and refuses with `upgrade_source_in_use` if a session still +holds one after five seconds. A refusal leaves the PHP database unchanged. An +interrupted takeover commits nothing: rerun it with the same digest. On +SQLite, the rerun lets SQLite roll back the journal the interrupted run left, +and the fingerprint check then proves the file is the backed-up one. + +MySQL and MariaDB commit DDL implicitly, so their takeover runs in steps under +native's bootstrap lock. It refuses with `upgrade_source_in_use` while any +other session uses the database. Once the source is verified, it first writes +native's marker with a `rust-takeover:` row: PHP refuses the database +from then on, and native refuses it with `native_takeover_incomplete` until +the takeover finishes. It then checks the sessions and the fingerprint again, +and runs each conversion step, recording each in that row. An interrupted +takeover is rerun with the same digest and continues after the last recorded +step; a different digest is refused. + +The command refuses: - a digest that differs, so any write after the backup stops it; - unrecognized queued jobs, external payloads and service calls; -- a layout other than the frozen one, PHP 2.8.0 or PHP 2.9.7, or a database - native already owns. +- a layout other than the frozen one, PHP 2.8.0 or PHP 2.9.7 (with either + Laravel MariaDB driver), or a database native already owns. -PostgreSQL and MySQL/MariaDB takeover, nested payload columns and multi-node -qualification remain open. **Never connect PHP to a Rust development database.** +Nested payload columns and multi-node qualification remain open. **Never connect PHP to a Rust development database.** Named namespace isolation and configured worker/operator/admin roles have representative shared qualification. Principal/custom-provider credentials and the full authorization contract remain open. Broader timer/signal semantics, failure/retry policy, cancellation and the rest of the API also diff --git a/rust/src/main.rs b/rust/src/main.rs index 79365e07a..ae8efabcb 100644 --- a/rust/src/main.rs +++ b/rust/src/main.rs @@ -133,21 +133,42 @@ async fn main() -> Result<(), Box> { | "upgrade-queue-inspect" | "upgrade-payload-inspect" | "upgrade-fingerprint" + | "upgrade-prepare-sqlite" | "upgrade-takeover" ) { return Err("unknown development command".into()); } let backend = env::var("DB_CONNECTION").unwrap_or_default(); + if command == "upgrade-prepare-sqlite" { + if backend != "sqlite" { + return Err("upgrade-prepare-sqlite requires DB_CONNECTION=sqlite".into()); + } + let path = env::var("DB_DATABASE").map_err(|_| "DB_DATABASE is required")?; + return match durable_workflow_server::runtime::upgrade::prepare_sqlite_source(&path).await + { + Ok(report) => { + println!("{report}"); + Ok(()) + } + Err(error) => { + let reason = match error { + RuntimeError::Refused { reason, .. } => reason, + _ => "upgrade_prepare_unavailable", + }; + eprintln!("{reason}"); + Err("SQLite source preparation refused".into()) + } + }; + } if command == "upgrade-takeover" { use durable_workflow_server::runtime::upgrade::{ queue_binding::QueueSourceConfig, takeover, }; - if backend != "sqlite" { + if !matches!(backend.as_str(), "sqlite" | "pgsql" | "mysql" | "mariadb") { eprintln!("upgrade_takeover_backend_unqualified"); return Err("upgrade takeover refused".into()); } - let path = env::var("DB_DATABASE").map_err(|_| "DB_DATABASE is required")?; let fingerprint = env::var("DW_UPGRADE_BACKUP_FINGERPRINT") .map_err(|_| "DW_UPGRADE_BACKUP_FINGERPRINT is required")?; // The original Laravel queue transport, as for upgrade-queue-inspect. @@ -164,7 +185,15 @@ async fn main() -> Result<(), Box> { default_connection: &default_connection, default_queue: &default_queue, }; - return match takeover::take_over_sqlite(&path, &fingerprint, &queues).await { + let result = if backend == "pgsql" { + takeover::take_over_postgres(postgres_options()?, &fingerprint, &queues).await + } else if backend != "sqlite" { + takeover::take_over_mysql(mysql_options()?, &fingerprint, &queues).await + } else { + let path = env::var("DB_DATABASE").map_err(|_| "DB_DATABASE is required")?; + takeover::take_over_sqlite(&path, &fingerprint, &queues).await + }; + return match result { Ok(report) => { println!("{report}"); Ok(()) diff --git a/rust/src/runtime/backend.rs b/rust/src/runtime/backend.rs index 7aaa69405..b31a8349d 100644 --- a/rust/src/runtime/backend.rs +++ b/rust/src/runtime/backend.rs @@ -337,8 +337,8 @@ impl Backend for Postgres { async fn begin(pool: &Pool) -> Result> { let mut tx = pool.begin().await?; // The first single-namespace slice serializes its durable transitions - // across independent pools/processes. Empty long polls release this - // transaction before waiting. Throughput/idle cost remain unqualified. + // across independent pools/processes. Held long polls wait without it + // (Store::task_poll_ready). Throughput/idle cost remain unqualified. sqlx::query("SELECT pg_advisory_xact_lock(4924501473315676163)") .execute(&mut *tx) .await?; diff --git a/rust/src/runtime/children.rs b/rust/src/runtime/children.rs index 73fc654b7..c6d194fe2 100644 --- a/rust/src/runtime/children.rs +++ b/rust/src/runtime/children.rs @@ -155,6 +155,11 @@ where { let contracts = DB::document_row(&worker, "workflow_command_contracts")?; if let Some(contract) = contracts.get(workflow_type) { + // Older development registrations may predate validation. + // Never turn a malformed entry into authoritative empties. + let Ok(contract) = super::worker_contracts::declaration(contract) else { + continue; + }; for (source, target) in [ ("signals", "declared_signals"), ("signal_contracts", "declared_signal_contracts"), @@ -164,7 +169,9 @@ where ("update_contracts", "declared_update_contracts"), ("update_validators", "declared_update_validators"), ] { - started[target] = contract[source].clone(); + if let Some(value) = contract.get(source) { + started[target] = value.clone(); + } } break; } diff --git a/rust/src/runtime/execution.rs b/rust/src/runtime/execution.rs index edb69ac8e..5715137b8 100644 --- a/rust/src/runtime/execution.rs +++ b/rust/src/runtime/execution.rs @@ -25,6 +25,10 @@ enum Storage { #[path = "query_codec_tests.rs"] mod query_codec_tests; +#[cfg(test)] +#[path = "transition_codec_tests.rs"] +mod transition_codec_tests; + #[derive(Clone)] pub struct Runtime { storage: Arc, @@ -32,6 +36,10 @@ pub struct Runtime { pub(crate) role_tokens: [Option>; 3], scheduler: Arc, signal_codec: Arc, + // Codec work done while holding the transition lock takes this permit, + // never `signal_codec`, so a transition never queues behind reads for + // codec capacity. + transition_codec: Arc, pub(super) external: Arc, queries: Arc, } @@ -194,6 +202,7 @@ impl Runtime { role_tokens: [None, None, None], scheduler: Arc::new(TimerScheduler::new()), signal_codec: Arc::new(Semaphore::new(1)), + transition_codec: Arc::new(Semaphore::new(1)), external: Arc::new(external), queries: Arc::new(queries), }; @@ -262,6 +271,7 @@ impl Runtime { role_tokens: self.role_tokens.clone(), scheduler: self.scheduler.clone(), signal_codec: self.signal_codec.clone(), + transition_codec: self.transition_codec.clone(), external: self.external.clone(), queries: self.queries.clone(), } @@ -376,7 +386,7 @@ impl Runtime { name, body, prepared, - self.signal_codec.clone(), + self.transition_codec.clone(), ) .await } @@ -388,7 +398,7 @@ impl Runtime { name, body, prepared, - self.signal_codec.clone(), + self.transition_codec.clone(), ) .await } @@ -400,7 +410,7 @@ impl Runtime { name, body, prepared, - self.signal_codec.clone(), + self.transition_codec.clone(), ) .await } @@ -421,7 +431,7 @@ impl Runtime { run_id, body, context, - self.signal_codec.clone(), + self.transition_codec.clone(), ) .await } @@ -432,7 +442,7 @@ impl Runtime { run_id, body, context, - self.signal_codec.clone(), + self.transition_codec.clone(), ) .await } @@ -443,7 +453,7 @@ impl Runtime { run_id, body, context, - self.signal_codec.clone(), + self.transition_codec.clone(), ) .await } @@ -658,9 +668,8 @@ impl Runtime { ) -> Result<(Vec, Vec)> ); pub(crate) async fn activities(&self, workflow_id: &str, run_id: &str) -> Result { - // Writers take this permit while holding the writer transaction. Finish - // every read before taking it so a listing can never wait on a writer - // that is itself waiting on the permit. + // Finish every read before taking the codec permit, so a listing never + // holds codec capacity while it waits on the database. let (events, executions) = self.activity_metadata_input(workflow_id, run_id).await?; let permit = self .signal_codec @@ -700,7 +709,7 @@ impl Runtime { delegate!(worker_heartbeat(body: Value) -> Result); delegate!(deregister(worker_id: &str) -> Result); pub(crate) async fn deregister_fenced(&self, worker_id: &str, body: Value) -> Result { - let codec = self.signal_codec.clone(); + let codec = self.transition_codec.clone(); match &*self.storage { Storage::Sqlite(store) => store.deregister_fenced(worker_id, body, codec).await, Storage::Postgres(store) => store.deregister_fenced(worker_id, body, codec).await, @@ -761,17 +770,17 @@ impl Runtime { match &*self.storage { Storage::Sqlite(store) => { store - .complete_workflow(task_id, body, self.signal_codec.clone(), external) + .complete_workflow(task_id, body, self.transition_codec.clone(), external) .await } Storage::Postgres(store) => { store - .complete_workflow(task_id, body, self.signal_codec.clone(), external) + .complete_workflow(task_id, body, self.transition_codec.clone(), external) .await } Storage::MySql(store) => { store - .complete_workflow(task_id, body, self.signal_codec.clone(), external) + .complete_workflow(task_id, body, self.transition_codec.clone(), external) .await } } @@ -817,7 +826,7 @@ impl Runtime { run_id, reason, budget, - self.signal_codec.clone(), + self.transition_codec.clone(), ) .await } @@ -828,7 +837,7 @@ impl Runtime { run_id, reason, budget, - self.signal_codec.clone(), + self.transition_codec.clone(), ) .await } @@ -839,7 +848,7 @@ impl Runtime { run_id, reason, budget, - self.signal_codec.clone(), + self.transition_codec.clone(), ) .await } diff --git a/rust/src/runtime/mod.rs b/rust/src/runtime/mod.rs index 6eb8fb731..72c5cb512 100644 --- a/rust/src/runtime/mod.rs +++ b/rust/src/runtime/mod.rs @@ -36,6 +36,7 @@ mod memos; pub mod mysql; mod namespaces; pub mod postgres; +mod poll_readiness; mod previews; mod queries; mod query_authority; @@ -64,6 +65,7 @@ pub mod upgrade; mod visibility; mod wait_projection; mod wait_timeouts; +mod worker_contracts; mod worker_incarnations; mod workflow_failures; diff --git a/rust/src/runtime/mysql.rs b/rust/src/runtime/mysql.rs index 35e68ac0d..a4a76dbf5 100644 --- a/rust/src/runtime/mysql.rs +++ b/rust/src/runtime/mysql.rs @@ -13,15 +13,19 @@ pub const VERSION: i64 = 4; type Catalog = Vec<(String, String)>; pub const MARKER_SQL: &str = "CREATE TABLE dw_server_schema (engine VARCHAR(64) NOT NULL PRIMARY KEY,version BIGINT NOT NULL,bootstrap_checksum VARBINARY(48) NOT NULL) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin"; -pub const LEDGER_SQL: &str = "CREATE TABLE IF NOT EXISTS _sqlx_migrations (version BIGINT PRIMARY KEY,description TEXT NOT NULL,installed_on TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,success BOOLEAN NOT NULL,checksum BLOB NOT NULL,execution_time BIGINT NOT NULL) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci"; +/// The marker row of a takeover in progress: `rust-takeover:`, with +/// the completed step count as its version and the backup fingerprint's +/// digest as its checksum. Native startup refuses it and PHP sees the marker. +pub(super) const TAKEOVER_ENGINE: &str = "rust-takeover:"; +pub const LEDGER_SQL: &str ="CREATE TABLE IF NOT EXISTS _sqlx_migrations (version BIGINT PRIMARY KEY,description TEXT NOT NULL,installed_on TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,success BOOLEAN NOT NULL,checksum BLOB NOT NULL,execution_time BIGINT NOT NULL) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci"; #[derive(Clone, Copy)] -enum Flavor { +pub(super) enum Flavor { MySql, MariaDb, } impl Flavor { - async fn detect(connection: &mut MySqlConnection) -> Result { + pub(super) async fn detect(connection: &mut MySqlConnection) -> Result { let version: String = sqlx::query_scalar("SELECT VERSION()") .fetch_one(connection) .await?; @@ -72,7 +76,35 @@ impl Flavor { Self::MariaDb => include_str!("../../migrations/mariadb/0004_catalog.json"), }))?) } - fn checksum(self) -> &'static [u8] { + /// The single native migration, which a takeover records as applied. + pub(super) fn migration(self) -> &'static Migration { + self.migrator().iter().next().expect("one native MySQL migration") + } + /// The native relations a fresh bootstrap adds after the frozen PHP + /// layout, each created only if missing so a resumed takeover can repeat + /// them. The poll index on `workflow_tasks` is added separately. + pub(super) fn native_relations_sql(self) -> Result { + let (frozen, incarnations) = match self { + Self::MySql => ( + include_str!("../../migrations/mysql/0003_full_schema.sql"), + include_str!("../../migrations/mysql/0004_worker_incarnations.sql"), + ), + Self::MariaDb => ( + include_str!("../../migrations/mariadb/0003_full_schema.sql"), + include_str!("../../migrations/mariadb/0004_worker_incarnations.sql"), + ), + }; + Ok(super::upgrade::native_relations( + frozen, + "-- Native receipts are separate", + incarnations, + )? + .replace( + "CREATE TABLE dw_worker_registration_incarnations", + "CREATE TABLE IF NOT EXISTS dw_worker_registration_incarnations", + )) + } + pub(super) fn checksum(self) -> &'static [u8] { self.migrator().iter().next().unwrap().checksum.as_ref() } } @@ -83,7 +115,7 @@ fn options(options: MySqlConnectOptions) -> MySqlConnectOptions { .collation("utf8mb4_unicode_ci") .timezone(Some("+00:00".into())) } -async fn configure(connection: &mut MySqlConnection) -> Result<()> { +pub(super) async fn configure(connection: &mut MySqlConnection) -> Result<()> { sqlx::query("SET sql_quote_show_create=1,lock_wait_timeout=5,innodb_lock_wait_timeout=5") .execute(connection) .await?; @@ -205,13 +237,44 @@ pub(super) async fn php_source_catalog(connection: &mut MySqlConnection) -> Resu )); } } + let actual = catalog(connection).await?; + source_layout(connection, actual).await +} + +/// The PHP source catalog of a database whose takeover has written its +/// marker and nothing else yet: native's exact marker table, set aside. +pub(super) async fn fenced_source_catalog(connection: &mut MySqlConnection) -> Result<(Catalog, &'static str)> { + let flavor = Flavor::detect(connection).await?; + let marker = flavor + .expected()? + .into_iter() + .find(|(table, _)| table == "dw_server_schema"); + let mut actual = catalog(connection).await?; + let position = actual.iter().position(|(table, _)| table == "dw_server_schema"); + match position { + Some(position) if Some(&actual[position]) == marker.as_ref() => { + actual.remove(position); + } + _ => { + return Err(refuse( + StatusCode::CONFLICT, + "upgrade_takeover_marker_unqualified", + )); + } + } + source_layout(connection, actual).await +} + +async fn source_layout( + connection: &mut MySqlConnection, + mut actual: Catalog, +) -> Result<(Catalog, &'static str)> { let flavor = Flavor::detect(connection).await?; let expected: Catalog = flavor.expected()?.into_iter() .filter(|(table, _)| !super::upgrade::native_relation(table)) .map(|(table, ddl)| (table, ddl.replace( ",\n KEY `dw_workflow_tasks_poll` (`namespace`(64),`queue`(64),`task_type`(16),`status`(16),`available_at`)", ""))) .collect(); - let mut actual = catalog(connection).await?; // MariaDB's information_schema collation ignores some underscores. Only // canonicalize catalog presentation; preserve every full definition. actual.sort_by(|left, right| left.0.cmp(&right.0)); @@ -322,7 +385,7 @@ pub(super) async fn verify_history(connection: &mut MySqlConnection) -> Result<( Ok(()) } -async fn verify(connection: &mut MySqlConnection, flavor: Flavor) -> Result<()> { +pub(super) async fn verify(connection: &mut MySqlConnection, flavor: Flavor) -> Result<()> { history(connection, flavor, false).await?; if catalog(connection).await? != flavor.expected()? { return Err(refuse( @@ -388,6 +451,9 @@ async fn preflight(connection: &mut MySqlConnection, flavor: Flavor) -> Result<( if engines == ["rust-initializing"] { verify_initializing(connection, flavor).await?; Ok(()) + } else if engines.iter().any(|engine| engine.starts_with(TAKEOVER_ENGINE)) { + // An interrupted takeover resumes only through upgrade-takeover. + Err(refuse(StatusCode::CONFLICT, "native_takeover_incomplete")) } else { verify(connection, flavor).await } @@ -1038,4 +1104,356 @@ mod tests { db.remove().await; } } + + use crate::runtime::upgrade::{fingerprint, queue_binding::QueueSourceConfig, takeover}; + + const TAKEOVER_QUEUES: QueueSourceConfig<'static> = QueueSourceConfig { + connection: "database", + default_connection: "database", + default_queue: "default", + }; + + /// The frozen PHP layout, or a release layout rebuilt from the recorded + /// deltas (each changed table exactly as that release's PHP created it). + async fn php_layout(db: &Database, release: Option<&str>) -> Flavor { + let mut connection = db.connection().await; + let flavor = Flavor::detect(&mut connection).await.unwrap(); + let (frozen, deltas) = match flavor { + Flavor::MySql => ( + include_str!("../../migrations/mysql/0003_full_schema.sql"), + [ + include_str!("../../upgrade/php-2.8.0/mysql.json"), + include_str!("../../upgrade/php-2.9.7/mysql.json"), + ], + ), + Flavor::MariaDb => ( + include_str!("../../migrations/mariadb/0003_full_schema.sql"), + [ + include_str!("../../upgrade/php-2.8.0/mariadb.json"), + include_str!("../../upgrade/php-2.9.7/mariadb.json"), + ], + ), + }; + let php = &frozen[..frozen.find("-- Native receipts are separate").unwrap()]; + connection.execute(php).await.unwrap(); + connection + .execute("SET FOREIGN_KEY_CHECKS=1; ALTER TABLE workflow_tasks DROP KEY dw_workflow_tasks_poll") + .await + .unwrap(); + let releases = match release { + None => 0, + Some("php-2.8.0") => 1, + Some(_) => 2, + }; + for delta in &deltas[..releases] { + let delta: Value = serde_json::from_str(delta).unwrap(); + for added in delta["add"].as_array().unwrap() { + let (table, ddl) = (added[0].as_str().unwrap(), added[1].as_str().unwrap()); + connection + .execute(table_sql("DROP TABLE IF EXISTS ", table, "").unwrap()) + .await + .unwrap(); + connection.execute(AssertSqlSafe(ddl.to_owned())).await.unwrap(); + } + } + connection.close().await.unwrap(); + flavor + } + + async fn backup_fingerprint(db: &Database) -> String { + fingerprint::fingerprint_mysql(db.options.clone()) + .await + .unwrap()["fingerprint"] + .as_str() + .unwrap() + .to_owned() + } + + /// The physical schema and the fingerprint of every PHP row. + async fn source_state(db: &Database) -> (Catalog, String) { + let mut connection = db.connection().await; + let schema = catalog(&mut connection).await.unwrap(); + connection.close().await.unwrap(); + (schema, backup_fingerprint(db).await) + } + + async fn sessions_ended(db: &Database) { + let deadline = tokio::time::Instant::now() + Duration::from_secs(15); + loop { + let mut connection = db.connection().await; + let others: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM information_schema.PROCESSLIST WHERE DB=DATABASE() AND ID<>CONNECTION_ID()") + .fetch_one(&mut connection) + .await + .unwrap(); + connection.close().await.unwrap(); + if others == 0 { + return; + } + assert!(tokio::time::Instant::now() < deadline, "a takeover session survived"); + tokio::time::sleep(Duration::from_millis(50)).await; + } + } + + fn takeover_refused(result: Result, expected: &str) { + match result { + Err(super::super::RuntimeError::Refused { reason, .. }) => assert_eq!(reason, expected), + Err(error) => panic!("unexpected takeover error: {error}"), + Ok(report) => panic!("takeover was accepted: {report}"), + } + } + + async fn registrations(db: &Database) { + let mut connection = db.connection().await; + connection.execute( + "INSERT INTO workflow_worker_registrations(worker_id,namespace,task_queue,runtime,registration_token,created_at,updated_at) VALUES ('current','default','q','php','0123456789abcdef0123456789abcdef','2026-10-20 09:00:00','2026-10-20 09:00:00'); + INSERT INTO workflow_worker_registrations(worker_id,namespace,task_queue,runtime,created_at,updated_at) VALUES ('older','default','q','php','2026-10-20 09:00:00','2026-10-20 09:00:00'); + INSERT INTO workflow_worker_registration_incarnations VALUES ('0123456789abcdef0123456789abcdef','default','current','active',0,NULL,'2026-10-20 09:00:00','2026-10-20 09:00:00'); + INSERT INTO workflow_worker_registration_incarnations VALUES ('11111111111111111111111111111111','default','current','superseded',0,'2026-10-20 08:00:00','2026-10-20 07:00:00','2026-10-20 08:00:00'); + INSERT INTO workflow_worker_registration_incarnations VALUES ('22222222222222222222222222222222','default','gone','deregistered',3,'2026-10-20 08:30:00','2026-10-20 07:00:00','2026-10-20 08:30:00');", + ) + .await + .unwrap(); + connection.close().await.unwrap(); + } + + /// Native opens the database and holds the converted incarnations. + async fn converted(db: &Database) { + let storage = MySqlStorage::open(db.options.clone()).await.unwrap(); + storage.close().await; + MySqlStorage::check(db.options.clone()).await.unwrap(); + let mut connection = db.connection().await; + let incarnations: Vec<(String, String, String, Option, bool)> = sqlx::query_as( + "SELECT token,worker_id,status,recovered_workflow_task_count,finished_at IS NOT NULL FROM dw_worker_registration_incarnations ORDER BY worker_id,status", + ) + .fetch_all(&mut connection) + .await + .unwrap(); + connection.close().await.unwrap(); + assert_eq!(incarnations.len(), 4, "{incarnations:?}"); + assert!(incarnations.contains(&( + "0123456789abcdef0123456789abcdef".into(), + "current".into(), + "active".into(), + None, + false + ))); + assert!(incarnations.contains(&( + "22222222222222222222222222222222".into(), + "gone".into(), + "deregistered".into(), + Some(3), + true + ))); + assert!(incarnations.iter().any(|row| row.1 == "older" && row.2 == "active" && row.0.len() == 32)); + } + + #[tokio::test] + #[ignore = "requires an isolated MySQL/MariaDB create-database role"] + async fn qualification_takeover_converts_php_297_and_native_opens_it() { + let db = Database::new().await; + php_layout(&db, Some("php-2.9.7")).await; + registrations(&db).await; + let fingerprint = backup_fingerprint(&db).await; + let report = takeover::take_over_mysql(db.options.clone(), &fingerprint, &TAKEOVER_QUEUES) + .await + .unwrap(); + assert_eq!(report["status"], "taken_over", "{report}"); + assert_eq!(report["source_layout"], "php-2.9.7"); + assert_eq!(report["converted"]["incarnations_copied"], 3, "{report}"); + assert_eq!(report["converted"]["incarnations_created"], 1, "{report}"); + converted(&db).await; + takeover_refused( + takeover::take_over_mysql(db.options.clone(), &fingerprint, &TAKEOVER_QUEUES).await, + "upgrade_source_already_owned", + ); + db.remove().await; + } + + #[tokio::test] + #[ignore = "requires an isolated MySQL/MariaDB create-database role"] + async fn qualification_takeover_converts_the_mariadb_driver_layout() { + let db = Database::new().await; + if let Flavor::MySql = php_layout(&db, Some("php-2.9.7")).await { + db.remove().await; + return; + } + // Laravel's mariadb connection stores the backup hold owner as uuid. + let mut connection = db.connection().await; + connection + .execute("ALTER TABLE runtime_external_payload_backup_hold MODIFY owner uuid DEFAULT NULL") + .await + .unwrap(); + connection.close().await.unwrap(); + let fingerprint = backup_fingerprint(&db).await; + let report = takeover::take_over_mysql(db.options.clone(), &fingerprint, &TAKEOVER_QUEUES) + .await + .unwrap(); + assert_eq!(report["source_layout"], "php-2.9.7+mariadb-driver", "{report}"); + MySqlStorage::check(db.options.clone()).await.unwrap(); + db.remove().await; + } + + #[tokio::test] + #[ignore = "requires an isolated MySQL/MariaDB create-database role"] + async fn qualification_takeover_of_the_frozen_layout_adds_only_native_relations() { + let db = Database::new().await; + php_layout(&db, None).await; + let fingerprint = backup_fingerprint(&db).await; + let report = takeover::take_over_mysql(db.options.clone(), &fingerprint, &TAKEOVER_QUEUES) + .await + .unwrap(); + assert_eq!(report["source_layout"], "frozen", "{report}"); + MySqlStorage::check(db.options.clone()).await.unwrap(); + db.remove().await; + } + + #[tokio::test] + #[ignore = "requires an isolated MySQL/MariaDB create-database role"] + async fn qualification_takeover_refusals_before_the_marker_leave_php_unchanged() { + let db = Database::new().await; + php_layout(&db, Some("php-2.9.7")).await; + let fingerprint = backup_fingerprint(&db).await; + let original = source_state(&db).await; + takeover_refused( + takeover::take_over_mysql(db.options.clone(), &format!("sha256:{}", "0".repeat(64)), &TAKEOVER_QUEUES).await, + "upgrade_backup_fingerprint_mismatch", + ); + takeover_refused( + takeover::take_over_mysql(db.options.clone(), &format!("sha256:{}", "z".repeat(64)), &TAKEOVER_QUEUES).await, + "upgrade_backup_fingerprint_required", + ); + // Any other session on the database refuses the takeover. + let php = db.connection().await; + takeover_refused( + takeover::take_over_mysql(db.options.clone(), &fingerprint, &TAKEOVER_QUEUES).await, + "upgrade_source_in_use", + ); + php.close().await.unwrap(); + assert_eq!(source_state(&db).await, original); + // A write after the backup changes the digest, so the old one is refused. + let mut php = db.connection().await; + php.execute("INSERT INTO migrations(migration,batch) VALUES ('after-backup',9)") + .await + .unwrap(); + php.close().await.unwrap(); + let written = source_state(&db).await; + takeover_refused( + takeover::take_over_mysql(db.options.clone(), &fingerprint, &TAKEOVER_QUEUES).await, + "upgrade_backup_fingerprint_mismatch", + ); + assert_eq!(source_state(&db).await, written); + db.remove().await; + for (statement, expected) in [ + ( + "INSERT INTO workflow_service_calls(id,endpoint_name,service_name,operation_name,status,operation_mode) VALUES ('call','e','s','o','accepted','sync')", + "upgrade_source_service_calls_unsupported", + ), + ( + "INSERT INTO runtime_external_payloads(id,namespace,storage_uri,storage_uri_sha256,codec,sha256,size_bytes) VALUES ('p','default','s3://b/k','x','avro','y',1)", + "upgrade_source_external_payloads_unsupported", + ), + ] { + let db = Database::new().await; + php_layout(&db, Some("php-2.9.7")).await; + let mut connection = db.connection().await; + connection.execute(statement).await.unwrap(); + connection.close().await.unwrap(); + let fingerprint = backup_fingerprint(&db).await; + let before = source_state(&db).await; + takeover_refused( + takeover::take_over_mysql(db.options.clone(), &fingerprint, &TAKEOVER_QUEUES).await, + expected, + ); + assert_eq!(source_state(&db).await, before, "{expected}"); + db.remove().await; + } + } + + // Named under bootstrap_child so the qualification run's existing skip of + // that subprocess helper also skips this one. + #[tokio::test] + #[ignore = "invoked only as a subprocess by the takeover interruption qualification"] + async fn bootstrap_child_takeover() { + let url = std::env::var("DW_TAKEOVER_TEST_DATABASE").unwrap(); + let fingerprint = std::env::var("DW_TAKEOVER_TEST_FINGERPRINT").unwrap(); + // Parks at DW_TAKEOVER_TEST_BOUNDARY, where the parent kills it. + let report = takeover::take_over_mysql( + MySqlConnectOptions::from_str(&url).unwrap(), + &fingerprint, + &TAKEOVER_QUEUES, + ) + .await; + panic!("takeover returned instead of parking: {report:?}"); + } + + #[cfg(unix)] + #[tokio::test] + #[ignore = "requires an isolated MySQL/MariaDB create-database role"] + async fn qualification_takeover_killed_at_each_step_resumes_with_the_same_fingerprint() { + use std::os::unix::process::ExitStatusExt; + for boundary in [ + "before-marker-row", + "after-step-1", + "after-step-3", + "after-step-8", + ] { + let db = Database::new().await; + php_layout(&db, Some("php-2.9.7")).await; + registrations(&db).await; + let fingerprint = backup_fingerprint(&db).await; + let directory = tempfile::tempdir().unwrap(); + let receipt = directory.path().join("boundary"); + let mut child = Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "runtime::mysql::tests::bootstrap_child_takeover", + "--ignored", + "--nocapture", + ]) + .env("DW_TAKEOVER_TEST_DATABASE", &db.url) + .env("DW_TAKEOVER_TEST_FINGERPRINT", &fingerprint) + .env("DW_TAKEOVER_TEST_BOUNDARY", boundary) + .env("DW_TAKEOVER_TEST_RECEIPT", &receipt) + .spawn() + .unwrap(); + let deadline = tokio::time::Instant::now() + Duration::from_secs(90); + while !receipt.exists() && tokio::time::Instant::now() < deadline { + if let Some(status) = child.try_wait().unwrap() { + panic!("child exited before {boundary}: {status}"); + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + if !receipt.exists() { + child.kill().unwrap(); + child.wait().unwrap(); + panic!("child did not reach {boundary}"); + } + child.kill().unwrap(); + assert_eq!(child.wait().unwrap().signal(), Some(9)); + sessions_ended(&db).await; + // The marker already fences PHP, and native will not open the + // database until the takeover finishes. + let mut connection = db.connection().await; + let marked: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM information_schema.TABLES WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME='dw_server_schema'") + .fetch_one(&mut connection) + .await + .unwrap(); + connection.close().await.unwrap(); + assert_eq!(marked, 1, "{boundary}"); + assert!(MySqlStorage::open(db.options.clone()).await.is_err(), "{boundary}"); + if boundary == "after-step-3" { + takeover_refused( + takeover::take_over_mysql(db.options.clone(), &format!("sha256:{}", "0".repeat(64)), &TAKEOVER_QUEUES).await, + "upgrade_backup_fingerprint_mismatch", + ); + } + let report = takeover::take_over_mysql(db.options.clone(), &fingerprint, &TAKEOVER_QUEUES) + .await + .unwrap(); + assert_eq!(report["status"], "taken_over", "{boundary}: {report}"); + assert_eq!(report["source_layout"], "php-2.9.7", "{boundary}"); + converted(&db).await; + db.remove().await; + } + } } diff --git a/rust/src/runtime/poll_readiness.rs b/rust/src/runtime/poll_readiness.rs new file mode 100644 index 000000000..6c0265829 --- /dev/null +++ b/rust/src/runtime/poll_readiness.rs @@ -0,0 +1,37 @@ +//! Lock-free waiting for held long polls. +//! +//! A held poll takes the transition lock only to claim or refuse. After its +//! first claim it reads, without the lock, whether anything now exists that it +//! could claim or must refuse: an eligible task, a receipt for its own request, +//! or a changed registration (`Store::task_poll_ready`, `Store::query_poll_ready`). +//! A same-node commit wakes it to read at once; timed reads observe other +//! server processes. A false positive costs one locked claim that finds +//! nothing, and a commit racing a read is seen on the next one. +use std::time::Duration; +use tokio::{sync::futures::Notified, time::Instant}; + +/// Paces a held poll's timed reads, which are how it observes another server +/// process. The first read comes 50 ms after a probe, and each later one waits +/// twice as long, up to 250 ms. A same-node wake reads at once and starts again +/// from 50 ms. No wait runs past the poll's deadline. +pub(super) struct ReadPace { + next: Duration, +} + +impl ReadPace { + const FIRST: Duration = Duration::from_millis(50); + const LONGEST: Duration = Duration::from_millis(250); + + pub(super) fn new() -> Self { + Self { next: Self::FIRST } + } + + /// Returns at a same-node wake or when the next timed read is due. + pub(super) async fn wait(&mut self, wake: Notified<'_>, deadline: Instant) { + let pause = self.next.min(deadline.saturating_duration_since(Instant::now())); + tokio::select! { + _ = wake => self.next = Self::FIRST, + _ = tokio::time::sleep(pause) => self.next = (self.next * 2).min(Self::LONGEST), + } + } +} diff --git a/rust/src/runtime/postgres.rs b/rust/src/runtime/postgres.rs index fa763e307..06af1a9ce 100644 --- a/rust/src/runtime/postgres.rs +++ b/rust/src/runtime/postgres.rs @@ -39,6 +39,23 @@ fn migrator() -> &'static Migrator { }) } +/// The single native migration, which a takeover records as applied once +/// conversion has produced its schema. +pub(super) fn migration() -> &'static Migration { + migrator().iter().next().expect("one native PostgreSQL migration") +} + +/// The native relations, indexes and ownership marker that a fresh +/// bootstrap adds after the frozen PHP layout. Takeover applies them to a +/// converted PHP database instead of running the whole migration. +pub(super) fn native_relations_sql() -> Result { + super::upgrade::native_relations( + include_str!("../../migrations/postgres/0003_full_schema.sql"), + "CREATE TABLE public.dw_server_schema ", + include_str!("../../migrations/postgres/0004_worker_incarnations.sql"), + ) +} + fn options(options: PgConnectOptions) -> PgConnectOptions { options .application_name("durable-workflow-rust-development") @@ -156,7 +173,7 @@ pub(super) async fn verify_history(connection: &mut PgConnection) -> Result<()> Ok(()) } -async fn verify(connection: &mut PgConnection) -> Result<()> { +pub(super) async fn verify(connection: &mut PgConnection) -> Result<()> { verify_scope(connection).await?; verify_history(connection).await?; // A canonical catalog is captured with PostgreSQL itself, not inferred @@ -712,4 +729,289 @@ mod tests { db.remove().await; } } + + use crate::runtime::upgrade::{fingerprint, queue_binding::QueueSourceConfig, takeover}; + + const TAKEOVER_QUEUES: QueueSourceConfig<'static> = QueueSourceConfig { + connection: "database", + default_connection: "database", + default_queue: "default", + }; + + /// The frozen PHP layout moved to PHP 2.9.7's by the changes PHP's own + /// migrations make (rust/upgrade/php-2.8.0 and php-2.9.7). Inspection + /// must call it php-2.9.7. + async fn php_297(db: &Database) -> PgConnection { + let frozen = include_str!("../../migrations/postgres/0003_full_schema.sql"); + let php = &frozen[..frozen.find("-- Native bookkeeping").unwrap()]; + let mut connection = db.connection().await; + sqlx::raw_sql(php).execute(&mut connection).await.unwrap(); + sqlx::raw_sql( + "ALTER TABLE workflow_worker_registrations ADD COLUMN registration_token varchar(32); + CREATE TABLE workflow_worker_registration_incarnations (token varchar(32) NOT NULL, namespace varchar(255) NOT NULL, worker_id varchar(255) NOT NULL, status varchar(32) NOT NULL, recovered_workflow_task_count integer NOT NULL DEFAULT 0, finished_at timestamp(6) without time zone, created_at timestamp(6) without time zone, updated_at timestamp(6) without time zone, PRIMARY KEY (token)); + CREATE INDEX workflow_worker_registration_incarnations_finished_at_index ON workflow_worker_registration_incarnations (finished_at); + CREATE INDEX dw_worker_incarnation_scope_idx ON workflow_worker_registration_incarnations (namespace, worker_id); + ALTER TABLE workflow_service_calls ADD COLUMN call_deadline_at timestamp(6) without time zone; + CREATE INDEX wf_service_calls_deadline_idx ON workflow_service_calls (status, call_deadline_at); + ALTER TABLE workflow_tasks ADD COLUMN queue_delivery_reference json;", + ) + .execute(&mut connection) + .await + .unwrap(); + connection + } + + async fn backup_fingerprint(db: &Database) -> String { + fingerprint::fingerprint_postgres(db.options.clone()) + .await + .unwrap()["fingerprint"] + .as_str() + .unwrap() + .to_owned() + } + + /// Every row and the physical schema, for byte-for-byte comparisons. + async fn source_state(db: &Database) -> (Catalog, String) { + let mut connection = db.connection().await; + let state = ( + catalog(&mut connection).await.unwrap(), + backup_fingerprint(db).await, + ); + connection.close().await.unwrap(); + state + } + + fn takeover_refused(result: Result, expected: &str) { + match result { + Err(super::super::RuntimeError::Refused { reason, .. }) => assert_eq!(reason, expected), + Err(error) => panic!("unexpected takeover error: {error}"), + Ok(report) => panic!("takeover was accepted: {report}"), + } + } + + #[tokio::test] + #[ignore = "requires a disposable PostgreSQL; run the qualification_ tests explicitly"] + async fn qualification_takeover_converts_php_297_and_native_opens_it() { + let db = Database::new().await; + let mut connection = php_297(&db).await; + sqlx::raw_sql( + "INSERT INTO workflow_worker_registrations(worker_id,namespace,task_queue,runtime,registration_token,created_at,updated_at) VALUES ('current','default','q','php','0123456789abcdef0123456789abcdef','2026-10-20 09:00:00','2026-10-20 09:00:00'); + INSERT INTO workflow_worker_registrations(worker_id,namespace,task_queue,runtime,created_at,updated_at) VALUES ('older','default','q','php','2026-10-20 09:00:00','2026-10-20 09:00:00'); + INSERT INTO workflow_worker_registration_incarnations VALUES ('0123456789abcdef0123456789abcdef','default','current','active',0,NULL,'2026-10-20 09:00:00','2026-10-20 09:00:00'); + INSERT INTO workflow_worker_registration_incarnations VALUES ('11111111111111111111111111111111','default','current','superseded',0,'2026-10-20 08:00:00','2026-10-20 07:00:00','2026-10-20 08:00:00'); + INSERT INTO workflow_worker_registration_incarnations VALUES ('22222222222222222222222222222222','default','gone','deregistered',3,'2026-10-20 08:30:00','2026-10-20 07:00:00','2026-10-20 08:30:00');", + ) + .execute(&mut connection) + .await + .unwrap(); + connection.close().await.unwrap(); + let fingerprint = backup_fingerprint(&db).await; + let report = takeover::take_over_postgres(db.options.clone(), &fingerprint, &TAKEOVER_QUEUES) + .await + .unwrap(); + assert_eq!(report["status"], "taken_over", "{report}"); + assert_eq!(report["backend"], "pgsql"); + assert_eq!(report["source_layout"], "php-2.9.7"); + assert_eq!(report["converted"]["incarnations_copied"], 3, "{report}"); + assert_eq!(report["converted"]["incarnations_created"], 1, "{report}"); + let mut connection = db.connection().await; + let incarnations: Vec<(String, String, String, Option, bool)> = sqlx::query_as( + "SELECT token,worker_id,status,recovered_workflow_task_count,finished_at IS NOT NULL FROM dw_worker_registration_incarnations ORDER BY worker_id,status", + ) + .fetch_all(&mut connection) + .await + .unwrap(); + assert_eq!(incarnations.len(), 4, "{incarnations:?}"); + // The current worker keeps its PHP token; the retired ones keep their state. + assert!(incarnations.contains(&( + "0123456789abcdef0123456789abcdef".into(), + "current".into(), + "active".into(), + None, + false + ))); + assert!(incarnations.contains(&( + "22222222222222222222222222222222".into(), + "gone".into(), + "deregistered".into(), + Some(3), + true + ))); + assert!(incarnations.iter().any(|row| row.1 == "current" && row.2 == "superseded" && row.3.is_none() && row.4)); + assert!(incarnations.iter().any(|row| row.1 == "older" && row.2 == "active" && row.0.len() == 32)); + connection.close().await.unwrap(); + let storage = PostgresStorage::open(db.options.clone()).await.unwrap(); + storage.close().await; + PostgresStorage::check(db.options.clone()).await.unwrap(); + // A second takeover finds a native database. + takeover_refused( + takeover::take_over_postgres(db.options.clone(), &fingerprint, &TAKEOVER_QUEUES).await, + "upgrade_source_already_owned", + ); + db.remove().await; + } + + #[tokio::test] + #[ignore = "requires a disposable PostgreSQL; run the qualification_ tests explicitly"] + async fn qualification_takeover_of_the_frozen_layout_adds_only_native_relations() { + let db = Database::new().await; + let frozen = include_str!("../../migrations/postgres/0003_full_schema.sql"); + let mut connection = db.connection().await; + sqlx::raw_sql(&frozen[..frozen.find("-- Native bookkeeping").unwrap()]) + .execute(&mut connection) + .await + .unwrap(); + connection.close().await.unwrap(); + let fingerprint = backup_fingerprint(&db).await; + let report = takeover::take_over_postgres(db.options.clone(), &fingerprint, &TAKEOVER_QUEUES) + .await + .unwrap(); + assert_eq!(report["source_layout"], "frozen", "{report}"); + PostgresStorage::check(db.options.clone()).await.unwrap(); + db.remove().await; + } + + #[tokio::test] + #[ignore = "requires a disposable PostgreSQL; run the qualification_ tests explicitly"] + async fn qualification_takeover_refusals_leave_the_php_database_unchanged() { + let db = Database::new().await; + php_297(&db).await.close().await.unwrap(); + let fingerprint = backup_fingerprint(&db).await; + let original = source_state(&db).await; + takeover_refused( + takeover::take_over_postgres(db.options.clone(), &format!("sha256:{}", "0".repeat(64)), &TAKEOVER_QUEUES).await, + "upgrade_backup_fingerprint_mismatch", + ); + takeover_refused( + takeover::take_over_postgres(db.options.clone(), "not-a-fingerprint", &TAKEOVER_QUEUES).await, + "upgrade_backup_fingerprint_required", + ); + assert_eq!(source_state(&db).await, original); + // A PHP connection still holding a table refuses the takeover within + // the lock wait instead of stalling it. + let mut php = db.connection().await; + sqlx::raw_sql("BEGIN; LOCK TABLE jobs IN ROW EXCLUSIVE MODE") + .execute(&mut php) + .await + .unwrap(); + takeover_refused( + takeover::take_over_postgres(db.options.clone(), &fingerprint, &TAKEOVER_QUEUES).await, + "upgrade_source_in_use", + ); + sqlx::raw_sql("ROLLBACK").execute(&mut php).await.unwrap(); + // A write after the backup changes the digest, so the old one is refused. + sqlx::raw_sql("INSERT INTO migrations(migration,batch) VALUES ('after-backup',9)") + .execute(&mut php) + .await + .unwrap(); + let written = source_state(&db).await; + takeover_refused( + takeover::take_over_postgres(db.options.clone(), &fingerprint, &TAKEOVER_QUEUES).await, + "upgrade_backup_fingerprint_mismatch", + ); + assert_eq!(source_state(&db).await, written); + php.close().await.unwrap(); + db.remove().await; + for (statement, expected) in [ + ( + "INSERT INTO workflow_service_calls(id,endpoint_name,service_name,operation_name,status,operation_mode) VALUES ('call','e','s','o','accepted','sync')", + "upgrade_source_service_calls_unsupported", + ), + ( + "INSERT INTO runtime_external_payloads(id,namespace,storage_uri,storage_uri_sha256,codec,sha256,size_bytes) VALUES ('p','default','s3://b/k','x','avro','y',1)", + "upgrade_source_external_payloads_unsupported", + ), + ] { + let db = Database::new().await; + let mut connection = php_297(&db).await; + sqlx::raw_sql(statement).execute(&mut connection).await.unwrap(); + connection.close().await.unwrap(); + let fingerprint = backup_fingerprint(&db).await; + let before = source_state(&db).await; + takeover_refused( + takeover::take_over_postgres(db.options.clone(), &fingerprint, &TAKEOVER_QUEUES).await, + expected, + ); + assert_eq!(source_state(&db).await, before, "{expected}"); + db.remove().await; + } + } + + #[tokio::test] + #[ignore = "only invoked as a real subprocess by the takeover interruption qualification"] + async fn takeover_child() { + let url = std::env::var("DW_TAKEOVER_TEST_DATABASE").unwrap(); + let fingerprint = std::env::var("DW_TAKEOVER_TEST_FINGERPRINT").unwrap(); + // Parks just before the commit, where the parent kills this process. + let report = takeover::take_over_postgres( + PgConnectOptions::from_str(&url).unwrap(), + &fingerprint, + &TAKEOVER_QUEUES, + ) + .await; + panic!("takeover returned instead of parking: {report:?}"); + } + + #[cfg(unix)] + #[tokio::test] + #[ignore = "requires a disposable PostgreSQL; run the qualification_ tests explicitly"] + async fn qualification_takeover_killed_before_commit_leaves_php_and_retries() { + use std::os::unix::process::ExitStatusExt; + let db = Database::new().await; + php_297(&db).await.close().await.unwrap(); + let fingerprint = backup_fingerprint(&db).await; + let original = source_state(&db).await; + let directory = tempfile::tempdir().unwrap(); + let receipt = directory.path().join("boundary"); + let mut child = Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "runtime::postgres::tests::takeover_child", + "--ignored", + "--nocapture", + ]) + .env("DW_TAKEOVER_TEST_DATABASE", &db.url) + .env("DW_TAKEOVER_TEST_FINGERPRINT", &fingerprint) + .env("DW_TAKEOVER_TEST_RECEIPT", &receipt) + .spawn() + .unwrap(); + let deadline = tokio::time::Instant::now() + Duration::from_secs(60); + while !receipt.exists() && tokio::time::Instant::now() < deadline { + if let Some(status) = child.try_wait().unwrap() { + panic!("child exited before the commit point: {status}"); + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + if !receipt.exists() { + child.kill().unwrap(); + child.wait().unwrap(); + panic!("child did not reach the commit point"); + } + assert_eq!(std::fs::read_to_string(&receipt).unwrap(), "before-commit"); + child.kill().unwrap(); + assert_eq!(child.wait().unwrap().signal(), Some(9)); + // PostgreSQL ends the killed session's transaction and its locks. + let released = tokio::time::Instant::now() + Duration::from_secs(15); + loop { + let mut connection = db.connection().await; + let sessions: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM pg_stat_activity WHERE datname=current_database() AND pid<>pg_backend_pid()") + .fetch_one(&mut connection) + .await + .unwrap(); + connection.close().await.unwrap(); + if sessions == 0 { + break; + } + assert!(tokio::time::Instant::now() < released, "killed takeover session survived"); + tokio::time::sleep(Duration::from_millis(50)).await; + } + assert_eq!(source_state(&db).await, original, "partial takeover survived"); + // The same verified backup fingerprint still takes the database over. + let report = takeover::take_over_postgres(db.options.clone(), &fingerprint, &TAKEOVER_QUEUES) + .await + .unwrap(); + assert_eq!(report["status"], "taken_over", "{report}"); + PostgresStorage::check(db.options.clone()).await.unwrap(); + db.remove().await; + } } diff --git a/rust/src/runtime/queries.rs b/rust/src/runtime/queries.rs index f91c4e04e..9a6b17ec6 100644 --- a/rust/src/runtime/queries.rs +++ b/rust/src/runtime/queries.rs @@ -354,17 +354,23 @@ where } else { None }; + // Only the first probe claims under the lock unconditionally; later + // probes claim after a lock-free read finds something to claim or refuse. + let mut first = true; + let mut pace = super::poll_readiness::ReadPace::new(); loop { let wake = self.wake.notified(); - match self.claim_query(&body, config, &proof).await? { - QueryClaim::Task(task) => return Ok(poll_result(Some(task), "task")), - QueryClaim::StaleWorker => { - return Ok(poll_result(None, "stale_worker_registration")); + if std::mem::take(&mut first) || self.query_poll_ready(&body, config, &proof).await? { + match self.claim_query(&body, config, &proof).await? { + QueryClaim::Task(task) => return Ok(poll_result(Some(task), "leased")), + QueryClaim::StaleWorker => { + return Ok(poll_result(None, "stale_worker_registration")); + } + QueryClaim::Empty => {} } - QueryClaim::Empty => {} } if tokio::time::Instant::now() >= deadline { - return Ok(poll_result(None, "timeout")); + return Ok(poll_result(None, "empty")); } if self .poll_queue_changed(&body, "query", &mut cross_kind) @@ -375,7 +381,7 @@ where json!({"task":null,"poll_status":"task_queue_changed","server_capabilities":super::store::capabilities()}), ); } - tokio::select! { _ = wake => {}, _ = tokio::time::sleep(Duration::from_millis(50)) => {} } + pace.wait(wake, deadline).await; } } @@ -431,6 +437,36 @@ where delivered } + /// Whether a held query poll should claim again. `proof` is the + /// registration this poll belongs to. + async fn query_poll_ready( + &self, + body: &Value, + config: &QueryTaskConfig, + proof: &str, + ) -> Result { + let owner = text(body, "worker_id")?; + // The claim refuses a replaced, removed or stale registration. + if Self::query_worker_proof(&self.pool, self.namespace.as_ref(), owner, config) + .await? + .as_deref() + != Some(proof) + { + return Ok(true); + } + let at = super::store::now(); + let query_now = authority::wire(at); + Ok(Self::query(DB::POLL_WAKE_QUERIES_SQL) + .bind(at.timestamp()) + .bind(self.namespace.as_ref()) + .bind(text(body, "task_queue")?) + .bind(query_now.clone()) + .bind(query_now) + .fetch_optional(&self.pool) + .await? + .is_some()) + } + async fn claim_query( &self, body: &Value, diff --git a/rust/src/runtime/schema.rs b/rust/src/runtime/schema.rs index 6b6ebaf0e..9b602f319 100644 --- a/rust/src/runtime/schema.rs +++ b/rust/src/runtime/schema.rs @@ -35,36 +35,17 @@ fn migrator() -> &'static Migrator { /// bootstrap adds after the frozen PHP layout. Takeover applies them to a /// converted PHP database instead of running the whole migration. pub(super) fn native_relations_sql() -> Result { - let frozen = include_str!("../../migrations/sqlite/0003_full_schema.sql"); - let start = frozen - .find("CREATE TABLE dw_server_schema ") - .ok_or_else(|| refuse(StatusCode::CONFLICT, "native_schema_catalog_mismatch"))?; - Ok(format!( - "{}\n{}", - &frozen[start..], - include_str!("../../migrations/sqlite/0004_worker_incarnations.sql") - )) + super::upgrade::native_relations( + include_str!("../../migrations/sqlite/0003_full_schema.sql"), + "CREATE TABLE dw_server_schema ", + include_str!("../../migrations/sqlite/0004_worker_incarnations.sql"), + ) } -/// Records the native migration as applied in the caller's transaction, for a -/// takeover whose schema came from conversion. `verify` then checks the -/// marker, this ledger and the complete catalog as for any native database. -pub(super) async fn record_migration(connection: &mut SqliteConnection) -> Result<()> { - use sqlx::migrate::Migrate; - connection - .ensure_migrations_table("_sqlx_migrations") - .await?; - let migration = migrator() - .iter() - .next() - .ok_or_else(|| refuse(StatusCode::CONFLICT, "native_migration_history_mismatch"))?; - sqlx::query("INSERT INTO _sqlx_migrations(version,description,success,checksum,execution_time) VALUES ($1,$2,TRUE,$3,0)") - .bind(migration.version) - .bind(migration.description.as_ref()) - .bind(migration.checksum.as_ref()) - .execute(&mut *connection) - .await?; - Ok(()) +/// The single native migration, which a takeover records as applied once +/// conversion has produced its schema. +pub(super) fn migration() -> &'static Migration { + migrator().iter().next().expect("one native SQLite migration") } type Catalog = Vec<(String, String, String, String)>; diff --git a/rust/src/runtime/store.rs b/rust/src/runtime/store.rs index 0ce48aabc..e3cdd8659 100644 --- a/rust/src/runtime/store.rs +++ b/rust/src/runtime/store.rs @@ -403,7 +403,7 @@ where name: &str, body: Value, mut prepared: PreparedControlArguments, - signal_codec: Arc, + transition_codec: Arc, ) -> Result { reject_fields(&body, &["input", "request_id"])?; if name.trim().is_empty() || name.len() > 255 { @@ -506,7 +506,7 @@ where "message":format!("Workflow signal [{name}] is unknown for workflow type [{workflow_type}]: the durable command contract does not declare that handler. Command contract source [durable_history], declared signals [{labels}].")}); } prepared.command = prepare_rejected_signal_command( - signal_codec, + transition_codec, name.to_owned(), prepared.values.clone(), validation_errors.clone(), @@ -697,7 +697,7 @@ where run_id: &str, task_id: &str, task: &DB::Row, - signal_codec: Arc, + transition_codec: Arc, ) -> Result<()> { let payload = DB::document_row(task, "payload")?; if payload["resume_source_kind"] != "workflow_signal" { @@ -724,7 +724,7 @@ where return Err(refuse(StatusCode::CONFLICT, "signal_wait_mismatch")); } let prepared = prepare_control_arguments( - signal_codec, + transition_codec, name.clone(), DB::string(&signal, "arguments")?, "invalid_signal_arguments", @@ -814,27 +814,29 @@ where } } let runtime = text(&body, "runtime")?; - let contracts = body + let supplied_contracts = body .get("workflow_command_contracts") - .cloned() - .unwrap_or_else(|| json!({})); - // PHP encodes an empty associative map as []; activity-only published - // workers send this shape. Preserve a map internally for lookups. - let contracts = if contracts.as_array().is_some_and(|values| values.is_empty()) { - json!({}) - } else { - contracts - }; - if !contracts.is_object() { - return Err(refuse( - StatusCode::UNPROCESSABLE_ENTITY, - "invalid_worker_definition", - )); - } + .map(|contracts| { + super::worker_contracts::normalize(contracts, &body["supported_workflow_types"]) + }) + .transpose()?; let token = super::worker_incarnations::token().await?; let mut tx = self.begin().await?; - let previous = Self::query("SELECT worker_id FROM workflow_worker_registrations WHERE namespace=$1 AND worker_id=$2") + let previous = Self::query("SELECT worker_id,workflow_command_contracts FROM workflow_worker_registrations WHERE namespace=$1 AND worker_id=$2") .bind(self.namespace.as_ref()).bind(worker_id).fetch_optional(&mut *tx).await?; + let contracts = match supplied_contracts { + Some(contracts) => contracts, + None => { + let existing = match &previous { + Some(worker) => { + DB::optional_document_row(worker, "workflow_command_contracts")? + .unwrap_or(Value::Null) + } + None => Value::Null, + }; + super::worker_contracts::normalize(&existing, &body["supported_workflow_types"])? + } + }; if previous.is_some() { // Match PHP's registration boundary: return original workflow leases // to ready without a repair command/history event. Token rotation @@ -934,18 +936,25 @@ where } else { None }; + // Only the first probe claims under the lock unconditionally. Later + // probes, on a same-node wake or a timed read that also observes + // other server processes, claim only after a lock-free read finds + // something to claim or refuse. + let mut first = true; + let mut pace = super::poll_readiness::ReadPace::new(); loop { - // Register the notification before probing; periodic probing also - // observes work committed by an independent process. + // Register the notification before probing. let wake = self.wake.notified(); - if let Some(task) = self.claim(&body, kind, protocol_version).await? { + if (std::mem::take(&mut first) || self.task_poll_ready(&body, kind).await?) + && let Some(task) = self.claim(&body, kind, protocol_version).await? + { return Ok( - json!({"task": task, "poll_status": "task", "server_capabilities": capabilities()}), + json!({"task": task, "poll_status": "leased", "server_capabilities": capabilities()}), ); } if tokio::time::Instant::now() >= deadline { return Ok( - json!({"task": null, "poll_status": "timeout", "server_capabilities": capabilities()}), + json!({"task": null, "poll_status": "empty", "server_capabilities": capabilities()}), ); } if self @@ -957,8 +966,45 @@ where json!({"task":null,"poll_status":"task_queue_changed","server_capabilities":capabilities()}), ); } - tokio::select! { _ = wake => {}, _ = tokio::time::sleep(Duration::from_millis(50)) => {} } + pace.wait(wake, deadline).await; + } + } + + /// Whether a held workflow or activity poll should claim again. + async fn task_poll_ready(&self, body: &Value, kind: &str) -> Result { + let worker_id = text(body, "worker_id")?; + let queue = text(body, "task_queue")?; + // The claim refuses a removed or moved registration. + let registration = Self::query( + "SELECT task_queue FROM workflow_worker_registrations WHERE worker_id=$1 AND namespace=$2", + ) + .bind(worker_id) + .bind(self.namespace.as_ref()) + .fetch_optional(&self.pool) + .await?; + match registration { + Some(row) if DB::string(&row, "task_queue")? == queue => {} + _ => return Ok(true), + } + // A concurrent poll with the same request ID may have claimed already; + // the claim replays its receipt. + if let Some(request_id) = body.get("poll_request_id").and_then(Value::as_str) + && Self::query("SELECT task_id FROM dw_poll_receipts WHERE worker_id=$1 AND kind=$2 AND request_id=$3 AND namespace=$4") + .bind(worker_id).bind(kind).bind(request_id).bind(self.namespace.as_ref()) + .fetch_optional(&self.pool).await?.is_some() + { + return Ok(true); } + let at = now(); + Ok(Self::query(DB::TASK_CANDIDATES_SQL) + .bind(self.namespace.as_ref()) + .bind(queue) + .bind(kind) + .bind(DB::bind_time(at)) + .bind(DB::bind_time(at)) + .fetch_optional(&self.pool) + .await? + .is_some()) } async fn claim( @@ -1500,7 +1546,7 @@ where &self, task_id: &str, body: Value, - signal_codec: Arc, + transition_codec: Arc, external: super::external_resolution::PreparedCommands, ) -> Result { let commands = body["commands"] @@ -1700,7 +1746,7 @@ where return Err(refuse(StatusCode::CONFLICT, "update_task_mismatch")); } let mut sequence = Self::authored_sequence(&mut tx, &run_id).await?; - Self::apply_task_signal(&mut tx, &run_id, task_id, &task, signal_codec).await?; + Self::apply_task_signal(&mut tx, &run_id, task_id, &task, transition_codec).await?; for (index, command) in commands.iter().enumerate() { if command["type"] == "fail_workflow" { // Published parseCommands applies every nonterminal command diff --git a/rust/src/runtime/transition_codec_tests.rs b/rust/src/runtime/transition_codec_tests.rs new file mode 100644 index 000000000..9f02385d6 --- /dev/null +++ b/rust/src/runtime/transition_codec_tests.rs @@ -0,0 +1,204 @@ +//! Transitions that encode or decode while holding the transition lock use +//! their own codec permit. Each test holds the read codec permit, as a long +//! describe decode would, and requires the transition to finish anyway. +use super::{Result, Runtime}; +use crate::codec::{Value as Payload, ValueCodec}; +use axum::http::StatusCode; +use serde_json::{Value, json}; +use std::time::Duration; + +fn envelope(value: Payload) -> Value { + json!({"codec":"avro","blob":ValueCodec::new().unwrap().encode(&value).unwrap()}) +} + +const DECISIVE: Duration = Duration::from_secs(1); + +async fn runtime(directory: &tempfile::TempDir) -> Runtime { + let runtime = Runtime::open( + directory.path().join("runtime.sqlite").to_str().unwrap(), + "test-token".into(), + ) + .await + .unwrap(); + runtime + .register(json!({"worker_id":"codec-worker","task_queue":"test","runtime":"php", + "supported_workflow_types":["echo"],"supported_activity_types":[], + "workflow_command_contracts":{"echo":{"signals":["payload"],"signal_contracts":[{"name":"payload", + "parameters":[{"name":"value","position":0,"type":"array","required":true,"variadic":false, + "default_available":false,"allows_null":false}]}]}}})) + .await + .unwrap(); + runtime +} + +async fn start(runtime: &Runtime, workflow_id: &str) { + runtime + .start(json!({"workflow_id":workflow_id,"workflow_type":"echo","task_queue":"test", + "input":envelope(Payload::Array(vec![]))})) + .await + .unwrap(); +} + +async fn poll(runtime: &Runtime) -> Value { + let task = runtime + .poll( + json!({"worker_id":"codec-worker","task_queue":"test","timeout_seconds":0}), + "workflow", + "1.20", + ) + .await + .unwrap()["task"] + .clone(); + assert!(!task.is_null(), "workflow task is ready"); + task +} + +async fn complete(runtime: &Runtime, task: &Value, commands: Value) -> Result { + runtime + .complete_workflow( + task["task_id"].as_str().unwrap(), + json!({"lease_owner":task["lease_owner"],"workflow_task_attempt":task["workflow_task_attempt"], + "commands":commands}), + ) + .await +} + +#[tokio::test] +async fn cancellation_request_does_not_wait_for_read_codec_capacity() { + let directory = tempfile::tempdir().unwrap(); + let runtime = runtime(&directory).await; + start(&runtime, "codec-cancel").await; + let _held = runtime.signal_codec.clone().acquire_owned().await.unwrap(); + let (status, accepted) = tokio::time::timeout( + DECISIVE, + runtime.request_cancellation( + "codec-cancel", + None, + json!({"reason":"held read permit","cleanup_timeout_seconds":30}), + ), + ) + .await + .expect("the cancellation encode must not wait for read codec capacity") + .unwrap(); + assert_eq!(status, StatusCode::ACCEPTED, "{accepted}"); + runtime.close().await; +} + +#[tokio::test] +async fn signal_resumed_completion_does_not_wait_for_read_codec_capacity() { + let directory = tempfile::tempdir().unwrap(); + let runtime = runtime(&directory).await; + start(&runtime, "codec-signal").await; + let waiting = poll(&runtime).await; + complete( + &runtime, + &waiting, + json!([{"type":"open_signal_wait","signal_name":"payload"}]), + ) + .await + .unwrap(); + runtime + .signal( + "codec-signal", + None, + "payload", + json!({"input":envelope(Payload::Array(vec![Payload::Map( + std::collections::BTreeMap::from([("count".into(), Payload::Long(7))]), + )]))}), + ) + .await + .unwrap(); + let resumed = poll(&runtime).await; + let _held = runtime.signal_codec.clone().acquire_owned().await.unwrap(); + let completed = tokio::time::timeout( + DECISIVE, + complete( + &runtime, + &resumed, + json!([{"type":"complete_workflow","result":envelope(Payload::Long(7))}]), + ), + ) + .await + .expect("applying the signal must not wait for read codec capacity") + .unwrap(); + assert_eq!(completed["recorded"], true, "{completed}"); + runtime.close().await; +} + +#[tokio::test] +async fn fenced_deregistration_repair_does_not_wait_for_read_codec_capacity() { + let directory = tempfile::tempdir().unwrap(); + let runtime = Runtime::open( + directory.path().join("runtime.sqlite").to_str().unwrap(), + "test-token".into(), + ) + .await + .unwrap(); + let registration = runtime + .register(json!({"worker_id":"codec-worker","task_queue":"test","runtime":"php", + "supported_workflow_types":["echo"],"supported_activity_types":[]})) + .await + .unwrap(); + start(&runtime, "codec-repair").await; + poll(&runtime).await; + let _held = runtime.signal_codec.clone().acquire_owned().await.unwrap(); + let deregistered = tokio::time::timeout( + DECISIVE, + runtime.deregister_fenced( + "codec-worker", + json!({"registration_token":registration["registration_token"]}), + ), + ) + .await + .expect("repairing the leased task must not wait for read codec capacity") + .unwrap(); + assert_eq!(deregistered["recovered_workflow_task_count"], 1, "{deregistered}"); + runtime.close().await; +} + +#[tokio::test] +async fn range_repair_does_not_wait_for_read_codec_capacity() { + use axum::{ + body::{Body, to_bytes}, + http::Request, + }; + use tower::ServiceExt; + let directory = tempfile::tempdir().unwrap(); + let runtime = runtime(&directory).await; + start(&runtime, "codec-range").await; + let task = poll(&runtime).await; + let reported = runtime + .fail_workflow_task( + task["task_id"].as_str().unwrap(), + json!({"lease_owner":task["lease_owner"],"workflow_task_attempt":task["workflow_task_attempt"], + "failure":{"message":"Recorded version 1 is outside supported range 2..2.", + "type":"DurableWorkflow\\Exception\\NonDeterministicWorkflow", + "reason":"version_marker_incompatible_range","sequence":1}}), + ) + .await + .unwrap(); + assert_eq!(reported["recorded"], true, "{reported}"); + // The repair context comes from the authenticated request, so use the route. + let request = Request::builder() + .method("POST") + .uri("/api/workflows/codec-range/repair") + .header("authorization", "Bearer test-token") + .header("content-type", "application/json") + .header("x-namespace", "default") + .header("x-durable-workflow-control-plane-version", "2") + .body(Body::empty()) + .unwrap(); + let _held = runtime.signal_codec.clone().acquire_owned().await.unwrap(); + let response = tokio::time::timeout( + DECISIVE, + super::super::router(runtime.clone()).oneshot(request), + ) + .await + .expect("the repair payload encode must not wait for read codec capacity") + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let repaired: Value = + serde_json::from_slice(&to_bytes(response.into_body(), 1 << 20).await.unwrap()).unwrap(); + assert_eq!(repaired["outcome"], "repair_dispatched", "{repaired}"); + runtime.close().await; +} diff --git a/rust/src/runtime/upgrade.rs b/rust/src/runtime/upgrade.rs index 068fa2014..e3f3f7b44 100644 --- a/rust/src/runtime/upgrade.rs +++ b/rust/src/runtime/upgrade.rs @@ -407,6 +407,130 @@ pub(super) async fn open_sqlite_source(path: &str, deadline: Instant) -> Result< open_sqlite_file(path, deadline, false).await } +/// The file format version bytes of a SQLite header: rollback journal or WAL. +const ROLLBACK_JOURNAL: [u8; 2] = [1, 1]; +const WRITE_AHEAD_LOG: [u8; 2] = [2, 2]; + +/// The journal format bytes of a regular SQLite file, read without SQLite. +async fn journal_format(path: &str) -> Result<[u8; 2]> { + let regular_file_required = || { + refuse( + StatusCode::BAD_REQUEST, + "upgrade_source_regular_file_required", + ) + }; + if !tokio::fs::metadata(path).await?.is_file() { + return Err(regular_file_required()); + } + let mut options = tokio::fs::OpenOptions::new(); + options.read(true); + // Tokio filesystem opens use blocking workers. A replaced FIFO must + // not strand one beyond the observation budget or prevent CLI exit. + #[cfg(unix)] + options.custom_flags(libc::O_NONBLOCK); + let mut file = options.open(path).await?; + if !file.metadata().await?.is_file() { + return Err(regular_file_required()); + } + let mut header = [0_u8; 20]; + file.read_exact(&mut header).await?; + if &header[..16] != b"SQLite format 3\0" { + return Err(refuse( + StatusCode::CONFLICT, + "upgrade_source_sqlite_journal_unqualified", + )); + } + Ok([header[18], header[19]]) +} + +/// Whether a SQLite file has a WAL, shared-memory or rollback journal beside it. +async fn sidecars(path: &str) -> Result { + for suffix in ["-wal", "-shm", "-journal"] { + if tokio::fs::try_exists(format!("{path}{suffix}")).await? { + return Ok(true); + } + } + Ok(false) +} + +/// Prepares a stopped PHP SQLite source for the upgrade commands, which need +/// the rollback journal. PHP Server's default is WAL: switching it makes +/// SQLite checkpoint every committed WAL frame into the file first, so the +/// stored rows are unchanged. A journal left by a crashed writer is rolled +/// back the same way. Refuses while another process has the file open. +pub async fn prepare_sqlite_source(path: &str) -> Result { + if path.is_empty() || path.contains(['?', ':']) { + return Err(refuse( + StatusCode::BAD_REQUEST, + "upgrade_source_file_required", + )); + } + let deadline = Instant::now() + BUDGET; + let previous = match timeout_at(deadline, journal_format(path)) + .await + .map_err(|_| expired())?? + { + WRITE_AHEAD_LOG => "wal", + ROLLBACK_JOURNAL => "delete", + _ => { + return Err(refuse( + StatusCode::CONFLICT, + "upgrade_source_sqlite_journal_unqualified", + )); + } + }; + let switched = previous == "wal" || sidecars(path).await?; + if switched { + let in_use = || refuse(StatusCode::CONFLICT, "upgrade_source_in_use"); + let opened = timeout_at( + deadline, + SqliteConnection::connect_with( + &SqliteConnectOptions::new() + .filename(path) + .create_if_missing(false) + .foreign_keys(false) + .busy_timeout(Duration::from_secs(2)) + .journal_mode(sqlx::sqlite::SqliteJournalMode::Delete), + ), + ) + .await + .map_err(|_| expired())?; + // Leaving WAL needs the only connection. SQLite answers busy when another + // still has the file after a short wait, which also covers a writer + // that is just closing. + let mut connection = match opened { + Ok(connection) => connection, + Err(sqlx::Error::Database(error)) + if error + .code() + .and_then(|code| code.parse::().ok()) + .is_some_and(|code| code & 0xff == 5) => + { + return Err(in_use()); + } + Err(error) => return Err(error.into()), + }; + let mode: String = sqlx::query_scalar("PRAGMA journal_mode") + .fetch_one(&mut connection) + .await?; + close(connection.close()).await?; + if mode != "delete" { + return Err(in_use()); + } + } + if journal_format(path).await? != ROLLBACK_JOURNAL || sidecars(path).await? { + return Err(refuse( + StatusCode::CONFLICT, + "upgrade_source_sqlite_journal_unqualified", + )); + } + Ok(json!({ + "schema":"durable-workflow.rust-upgrade-sqlite-prepare.v1", + "status":"rollback_journal", "previous_journal_mode":previous, + "source_mutated":switched, "takeover_allowed":false, "backup_verified":false + })) +} + /// Opens a qualified rollback-journal source file, writable only for takeover. /// It keeps the rollback journal: native enables WAL itself after takeover. pub(super) async fn open_sqlite_file( @@ -424,35 +548,24 @@ pub(super) async fn open_sqlite_file( // not open an unqualified WAL source or attempt journal recovery. Unlike // immutable=1 this preserves normal SQLite locking/change detection for // the supported rollback-journal source, rather than ignoring live WAL. + // A WAL source is first switched with upgrade-prepare-sqlite. timeout_at(deadline, async { - let regular_file_required = || { - refuse( - StatusCode::BAD_REQUEST, - "upgrade_source_regular_file_required", - ) - }; - if !tokio::fs::metadata(path).await?.is_file() { - return Err(regular_file_required()); - } - let mut options = tokio::fs::OpenOptions::new(); - options.read(true); - // Tokio filesystem opens use blocking workers. A replaced FIFO must - // not strand one beyond the observation budget or prevent CLI exit. - #[cfg(unix)] - options.custom_flags(libc::O_NONBLOCK); - let mut file = options.open(path).await?; - if !file.metadata().await?.is_file() { - return Err(regular_file_required()); - } - let mut header = [0_u8; 20]; - file.read_exact(&mut header).await?; - if &header[..16] != b"SQLite format 3\0" || header[18..20] != [1, 1] { + if journal_format(path).await? != ROLLBACK_JOURNAL { return Err(refuse( StatusCode::CONFLICT, "upgrade_source_sqlite_journal_unqualified", )); } - for suffix in ["-wal", "-shm", "-journal"] { + // A killed takeover leaves its rollback journal. Takeover opens the + // file writable, so SQLite rolls that journal back before anything is + // read, and the fingerprint then proves the result is the backed-up + // PHP database. Read-only opens never attempt that recovery. + let sidecars: &[&str] = if writable { + &["-wal", "-shm"] + } else { + &["-wal", "-shm", "-journal"] + }; + for suffix in sidecars { if tokio::fs::try_exists(format!("{path}{suffix}")).await? { return Err(refuse( StatusCode::CONFLICT, @@ -502,46 +615,68 @@ async fn inspect_mysql_inner( ) -> Result { queue_config(queues)?; let deadline = Instant::now() + BUDGET; - let (mut connection, backend) = mysql_source(options, deadline, false, 30).await?; - let result = timeout_at(deadline, async { + let (connection, backend) = mysql_source(options, deadline, false, 30).await?; + bounded(connection, deadline, async |connection| { let mut tx = connection.begin_with("START TRANSACTION READ ONLY").await?; - let (catalog, layout) = mysql::php_source_catalog(&mut tx).await?; - let tables: Vec = catalog.iter().map(|(name, _)| name.clone()).collect(); - let counts = count_tables!(tx, tables, '`'); - let queue_inspection = match queues { - Some(config) => { - Some(queue_scan::inspect::(&mut *tx, config, counts["jobs"]).await?) - } - None => None, - }; - let payload_inspection = if payloads { - Some(payload_scan::inspect::(&mut *tx, &counts).await?) - } else { - None - }; - // MySQL's metadata is not an MVCC snapshot. Refuse a changed visible - // catalog rather than joining counts to a different schema. - if mysql::php_source_catalog(&mut tx).await?.0 != catalog { - return Err(refuse( - StatusCode::CONFLICT, - "upgrade_source_catalog_changed", - )); - } + let report = mysql_inventory(&mut tx, backend, queues, payloads, false).await; tx.rollback().await?; - report( - backend, - &catalog, - layout, - counts, - queue_inspection, - payload_inspection, - ) + report }) .await - .map_err(|_| expired()) - .and_then(|result| result); - close(connection.close()).await?; - result +} + +/// The PHP source catalog, or with `fenced` the catalog of a source whose +/// takeover marker is all that has been written, with that marker set aside. +pub(super) async fn mysql_source_catalog( + connection: &mut MySqlConnection, + fenced: bool, +) -> Result<(Vec<(String, String)>, &'static str)> { + if fenced { + mysql::fenced_source_catalog(connection).await + } else { + mysql::php_source_catalog(connection).await + } +} + +/// The MySQL or MariaDB inspection report, read inside the caller's +/// transaction. +pub(super) async fn mysql_inventory( + tx: &mut MySqlConnection, + backend: &str, + queues: Option<&QueueSourceConfig<'_>>, + payloads: bool, + fenced: bool, +) -> Result { + let (catalog, layout) = mysql_source_catalog(&mut *tx, fenced).await?; + let tables: Vec = catalog.iter().map(|(name, _)| name.clone()).collect(); + let counts = count_tables!(tx, tables, '`'); + let queue_inspection = match queues { + Some(config) => { + Some(queue_scan::inspect::(&mut *tx, config, counts["jobs"]).await?) + } + None => None, + }; + let payload_inspection = if payloads { + Some(payload_scan::inspect::(&mut *tx, &counts).await?) + } else { + None + }; + // MySQL's metadata is not an MVCC snapshot. Refuse a changed visible + // catalog rather than joining counts to a different schema. + if mysql_source_catalog(&mut *tx, fenced).await?.0 != catalog { + return Err(refuse( + StatusCode::CONFLICT, + "upgrade_source_catalog_changed", + )); + } + report( + backend, + &catalog, + layout, + counts, + queue_inspection, + payload_inspection, + ) } pub async fn inspect_postgres(options: PgConnectOptions) -> Result { @@ -566,47 +701,77 @@ async fn inspect_postgres_inner( ) -> Result { queue_config(queues)?; let deadline = Instant::now() + BUDGET; - let mut connection = timeout_at( + let connection = timeout_at( deadline, PgConnection::connect_with(&postgres::source_options(options)), ) .await .map_err(|_| expired())??; - let result = timeout_at(deadline, async { + bounded(connection, deadline, async |connection| { let mut tx = connection .begin_with("BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY") .await?; - let (catalog, layout) = postgres::php_source_catalog(&mut tx).await?; - let tables: Vec = catalog - .iter() - .filter(|(kind, _, data)| kind == "relation" && data.0["kind"] == "r") - .map(|(_, name, _)| name.clone()) - .collect(); - let counts = count_tables!(tx, tables, '"'); - let queue_inspection = match queues { - Some(config) => { - Some(queue_scan::inspect::(&mut *tx, config, counts["jobs"]).await?) - } - None => None, - }; - let payload_inspection = if payloads { - Some(payload_scan::inspect::(&mut *tx, &counts).await?) - } else { - None - }; + let report = postgres_inventory(&mut tx, queues, payloads).await; tx.rollback().await?; - report( - "pgsql", - &catalog, - layout, - counts, - queue_inspection, - payload_inspection, - ) + report }) .await - .map_err(|_| expired()) - .and_then(|result| result); +} + +/// The PostgreSQL inspection report, read inside the caller's transaction. +pub(super) async fn postgres_inventory( + tx: &mut PgConnection, + queues: Option<&QueueSourceConfig<'_>>, + payloads: bool, +) -> Result { + let (catalog, layout) = postgres::php_source_catalog(&mut *tx).await?; + let tables: Vec = catalog + .iter() + .filter(|(kind, _, data)| kind == "relation" && data.0["kind"] == "r") + .map(|(_, name, _)| name.clone()) + .collect(); + let counts = count_tables!(tx, tables, '"'); + let queue_inspection = match queues { + Some(config) => { + Some(queue_scan::inspect::(&mut *tx, config, counts["jobs"]).await?) + } + None => None, + }; + let payload_inspection = if payloads { + Some(payload_scan::inspect::(&mut *tx, &counts).await?) + } else { + None + }; + report( + "pgsql", + &catalog, + layout, + counts, + queue_inspection, + payload_inspection, + ) +} + +/// The frozen migration from its ownership marker on, then the incarnation +/// migration: what a fresh native bootstrap adds after the frozen PHP layout. +pub(super) fn native_relations(frozen: &str, marker: &str, incarnations: &str) -> Result { + let start = frozen + .find(marker) + .ok_or_else(|| refuse(StatusCode::CONFLICT, "native_schema_catalog_mismatch"))?; + Ok(format!("{}\n{incarnations}", &frozen[start..])) +} + +/// Runs work on a dedicated source connection within the deadline, then +/// closes the connection whatever the work returned. +pub(super) async fn bounded( + mut connection: C, + deadline: Instant, + work: impl AsyncFnOnce(&mut C) -> Result, +) -> Result { + let result = timeout_at(deadline, work(&mut connection)) + .await + .map_err(|_| expired()) + .and_then(|result| result); close(connection.close()).await?; result } diff --git a/rust/src/runtime/upgrade/fingerprint.rs b/rust/src/runtime/upgrade/fingerprint.rs index a13ed89ed..df82bb3b8 100644 --- a/rust/src/runtime/upgrade/fingerprint.rs +++ b/rust/src/runtime/upgrade/fingerprint.rs @@ -8,8 +8,8 @@ //! per-table row counts leave the database. The fingerprint grants nothing: //! every compatibility, backup, quiescence and takeover flag stays false. use super::{ - Result, close, expired, mysql, mysql_source, open_sqlite_source, postgres, readonly_source, - refuse, schema, + Result, bounded, expired, mysql_source, mysql_source_catalog, open_sqlite_source, postgres, + readonly_source, refuse, schema, }; use axum::http::StatusCode; use futures_util::TryStreamExt; @@ -17,8 +17,8 @@ use serde::Serialize; use serde_json::{Value, json}; use sha2::{Digest as _, Sha256}; use sqlx::{ - AssertSqlSafe, Connection, PgConnection, Row, SqliteConnection, mysql::MySqlConnectOptions, - postgres::PgConnectOptions, + AssertSqlSafe, Connection, MySqlConnection, PgConnection, Row, SqliteConnection, + mysql::MySqlConnectOptions, postgres::PgConnectOptions, }; use std::{collections::BTreeMap, time::Duration}; use tokio::time::{Instant, timeout, timeout_at}; @@ -80,7 +80,7 @@ impl Digest { /// Names come from a catalog equal to a qualified PHP layout. Anything else is /// refused rather than quoted. -fn identifier(name: &str) -> Result<&str> { +pub(super) fn identifier(name: &str) -> Result<&str> { if name.is_empty() || !name.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'_') { return Err(refuse( StatusCode::CONFLICT, @@ -123,21 +123,6 @@ fn row_select( )) } -/// Runs a read on a dedicated source connection within the deadline, then -/// closes the connection whatever the read returned. -async fn bounded_read( - mut connection: C, - deadline: Instant, - read: impl AsyncFnOnce(&mut C) -> Result, -) -> Result { - let result = timeout_at(deadline, read(&mut connection)) - .await - .map_err(|_| expired()) - .and_then(|result| result); - close(connection.close()).await?; - result -} - // Every rendered value is text, framed in column order. macro_rules! hash_rows { ($tx:ident, $digest:ident, $table:expr, $sql:expr, $width:expr) => {{ @@ -169,7 +154,7 @@ pub async fn fingerprint_sqlite_cli(path: &str) -> Result { pub async fn fingerprint_sqlite(path: &str) -> Result { let deadline = Instant::now() + BUDGET; let connection = open_sqlite_source(path, deadline).await?; - bounded_read(connection, deadline, async |connection| { + bounded(connection, deadline, async |connection| { // One deferred read transaction keeps one snapshot of the file. let mut tx = connection.begin().await?; let report = sqlite_digest(&mut tx).await; @@ -214,80 +199,96 @@ pub async fn fingerprint_postgres(options: PgConnectOptions) -> Result { ) .await .map_err(|_| expired())??; - bounded_read(connection, deadline, async |connection| { + bounded(connection, deadline, async |connection| { let mut tx = connection .begin_with("BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY") .await?; - // Fixed text forms for timestamps, intervals, floats and bytes. - sqlx::raw_sql("SET LOCAL TimeZone='UTC'; SET LOCAL DateStyle='ISO, YMD'; SET LOCAL IntervalStyle='postgres'; SET LOCAL extra_float_digits=1; SET LOCAL bytea_output='hex'") - .execute(&mut *tx) - .await?; - let (catalog, layout) = postgres::php_source_catalog(&mut tx).await?; - let mut digest = Digest::new("pgsql", &catalog)?; - let tables: Vec = catalog - .iter() - .filter(|(kind, _, data)| kind == "relation" && data.0["kind"] == "r") - .map(|(_, name, _)| name.clone()) - .collect(); - for table in &tables { - let table = identifier(table)?; - let columns: Vec = sqlx::query_scalar("SELECT column_name::text FROM information_schema.columns WHERE table_schema='public' AND table_name=$1 ORDER BY ordinal_position") - .bind(table).fetch_all(&mut *tx).await?; - let key: Vec = sqlx::query_scalar("SELECT a.attname::text FROM pg_index i CROSS JOIN LATERAL unnest(i.indkey::int2[]) WITH ORDINALITY AS k(attnum, position) JOIN pg_attribute a ON a.attrelid=i.indrelid AND a.attnum=k.attnum WHERE i.indrelid=('public.'||quote_ident($1))::regclass AND i.indisprimary ORDER BY k.position") - .bind(table).fetch_all(&mut *tx).await?; - digest.table(table, &columns); - let sql = row_select(&format!("public.\"{table}\""), &columns, &key, '"', |c, _| { - format!("quote_nullable(\"{c}\")") - })?; - hash_rows!(tx, digest, table, sql, columns.len()); - } + let report = postgres_digest(&mut tx).await; tx.rollback().await?; - Ok(digest.report("pgsql", layout)) + report }) .await } +/// The PostgreSQL fingerprint report, read inside the caller's transaction. +/// Its settings last until that transaction ends. +pub(super) async fn postgres_digest(tx: &mut PgConnection) -> Result { + // Fixed text forms for timestamps, intervals, floats and bytes. Long table + // reads are expected, so only the client deadline bounds them. + sqlx::raw_sql("SET LOCAL TimeZone='UTC'; SET LOCAL DateStyle='ISO, YMD'; SET LOCAL IntervalStyle='postgres'; SET LOCAL extra_float_digits=1; SET LOCAL bytea_output='hex'; SET LOCAL statement_timeout=0") + .execute(&mut *tx) + .await?; + let (catalog, layout) = postgres::php_source_catalog(&mut *tx).await?; + let mut digest = Digest::new("pgsql", &catalog)?; + let tables: Vec = catalog + .iter() + .filter(|(kind, _, data)| kind == "relation" && data.0["kind"] == "r") + .map(|(_, name, _)| name.clone()) + .collect(); + for table in &tables { + let table = identifier(table)?; + let columns: Vec = sqlx::query_scalar("SELECT column_name::text FROM information_schema.columns WHERE table_schema='public' AND table_name=$1 ORDER BY ordinal_position") + .bind(table).fetch_all(&mut *tx).await?; + let key: Vec = sqlx::query_scalar("SELECT a.attname::text FROM pg_index i CROSS JOIN LATERAL unnest(i.indkey::int2[]) WITH ORDINALITY AS k(attnum, position) JOIN pg_attribute a ON a.attrelid=i.indrelid AND a.attnum=k.attnum WHERE i.indrelid=('public.'||quote_ident($1))::regclass AND i.indisprimary ORDER BY k.position") + .bind(table).fetch_all(&mut *tx).await?; + digest.table(table, &columns); + let sql = row_select(&format!("public.\"{table}\""), &columns, &key, '"', |c, _| { + format!("quote_nullable(\"{c}\")") + })?; + hash_rows!(tx, digest, table, sql, columns.len()); + } + Ok(digest.report("pgsql", layout)) +} + pub async fn fingerprint_mysql(options: MySqlConnectOptions) -> Result { let deadline = Instant::now() + BUDGET; // TIMESTAMP values render in UTC. Long table reads are expected, but lock // waits stay short; the client deadline bounds the whole read. let (connection, backend) = mysql_source(options, deadline, true, 0).await?; - bounded_read(connection, deadline, async |connection| { + bounded(connection, deadline, async |connection| { let mut tx = connection .begin_with("START TRANSACTION WITH CONSISTENT SNAPSHOT, READ ONLY") .await?; - let (catalog, layout) = mysql::php_source_catalog(&mut tx).await?; - let mut digest = Digest::new(backend, &catalog)?; - for (table, _) in &catalog { - let table = identifier(table)?; - let columns: Vec<(String, String)> = sqlx::query_as("SELECT CAST(COLUMN_NAME AS CHAR),CAST(DATA_TYPE AS CHAR) FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? ORDER BY ORDINAL_POSITION") - .bind(table).fetch_all(&mut *tx).await?; - let key: Vec = sqlx::query_scalar("SELECT CAST(COLUMN_NAME AS CHAR) FROM information_schema.KEY_COLUMN_USAGE WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? AND CONSTRAINT_NAME='PRIMARY' ORDER BY ORDINAL_POSITION") - .bind(table).fetch_all(&mut *tx).await?; - let (names, kinds): (Vec, Vec) = columns.into_iter().unzip(); - digest.table(table, &names); - // Binary values render as hex literals; everything else as its - // quoted string form. - let sql = row_select(&format!("`{table}`"), &names, &key, '`', |c, index| { - match kinds[index].to_ascii_lowercase().as_str() { - "binary" | "varbinary" | "tinyblob" | "blob" | "mediumblob" | "longblob" | "bit" => format!( - "CAST(IF(`{c}` IS NULL,'NULL',CONCAT('X''',HEX(`{c}`),'''')) AS CHAR CHARACTER SET utf8mb4)" - ), - _ => format!("CAST(QUOTE(`{c}`) AS CHAR CHARACTER SET utf8mb4)"), - } - })?; - hash_rows!(tx, digest, table, sql, names.len()); - } - // MySQL's metadata is not an MVCC snapshot. Refuse a catalog that - // changed while the rows were read. - if mysql::php_source_catalog(&mut tx).await?.0 != catalog { - return Err(refuse( - StatusCode::CONFLICT, - "upgrade_source_catalog_changed", - )); - } + let report = mysql_digest(&mut tx, backend, false).await; tx.rollback().await?; - Ok(digest.report(backend, layout)) + report }) .await } + +/// The MySQL or MariaDB fingerprint report, read inside the caller's +/// consistent snapshot. With `fenced`, a takeover's own marker is set aside, +/// so the digest is the one the backup had. +pub(super) async fn mysql_digest(tx: &mut MySqlConnection, backend: &str, fenced: bool) -> Result { + let (catalog, layout) = mysql_source_catalog(&mut *tx, fenced).await?; + let mut digest = Digest::new(backend, &catalog)?; + for (table, _) in &catalog { + let table = identifier(table)?; + let columns: Vec<(String, String)> = sqlx::query_as("SELECT CAST(COLUMN_NAME AS CHAR),CAST(DATA_TYPE AS CHAR) FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? ORDER BY ORDINAL_POSITION") + .bind(table).fetch_all(&mut *tx).await?; + let key: Vec = sqlx::query_scalar("SELECT CAST(COLUMN_NAME AS CHAR) FROM information_schema.KEY_COLUMN_USAGE WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? AND CONSTRAINT_NAME='PRIMARY' ORDER BY ORDINAL_POSITION") + .bind(table).fetch_all(&mut *tx).await?; + let (names, kinds): (Vec, Vec) = columns.into_iter().unzip(); + digest.table(table, &names); + // Binary values render as hex literals; everything else as its + // quoted string form. + let sql = row_select(&format!("`{table}`"), &names, &key, '`', |c, index| { + match kinds[index].to_ascii_lowercase().as_str() { + "binary" | "varbinary" | "tinyblob" | "blob" | "mediumblob" | "longblob" | "bit" => format!( + "CAST(IF(`{c}` IS NULL,'NULL',CONCAT('X''',HEX(`{c}`),'''')) AS CHAR CHARACTER SET utf8mb4)" + ), + _ => format!("CAST(QUOTE(`{c}`) AS CHAR CHARACTER SET utf8mb4)"), + } + })?; + hash_rows!(tx, digest, table, sql, names.len()); + } + // MySQL's metadata is not an MVCC snapshot. Refuse a catalog that + // changed while the rows were read. + if mysql_source_catalog(&mut *tx, fenced).await?.0 != catalog { + return Err(refuse( + StatusCode::CONFLICT, + "upgrade_source_catalog_changed", + )); + } + Ok(digest.report(backend, layout)) +} diff --git a/rust/src/runtime/upgrade/payload_scan.rs b/rust/src/runtime/upgrade/payload_scan.rs index 555eea3b2..412c2b91e 100644 --- a/rust/src/runtime/upgrade/payload_scan.rs +++ b/rust/src/runtime/upgrade/payload_scan.rs @@ -72,11 +72,15 @@ macro_rules! column { }; } -const COLUMNS: [Column; 4] = [ +const COLUMNS: [Column; 8] = [ column!("workflow_runs", "arguments", "payload_codec"), column!("workflow_runs", "output", "output_payload_codec"), column!("activity_executions", "arguments", "payload_codec"), column!("activity_executions", "result", "payload_codec"), + column!("workflow_commands", "payload", "payload_codec"), + column!("workflow_signal_records", "arguments", "payload_codec"), + column!("workflow_updates", "arguments", "payload_codec"), + column!("workflow_updates", "result", "payload_codec"), ]; #[derive(Default, Serialize)] @@ -218,9 +222,10 @@ where "references_metadata_supported":references_supported, "diagnostics":diagnostics, "diagnostics_omitted":omitted, "inline_values_verified":inline_supported, "nested_payloads_verified":false, - // Published migrations also give these tables codec-tagged payloads. + // These tables also hold codec-tagged payloads; takeover refuses a + // source with any of their rows, since native lacks the features. "other_root_payload_columns_verified":false, - "unscanned_payload_codec_tables":["workflow_commands","workflow_signal_records","workflow_updates"], + "unscanned_payload_codec_tables":["workflow_durable_stream_items","workflow_inbound_stream_items","workflow_update_validation_tasks"], "inline_validation_limits":{"encoded_frame_bytes":65536,"traversal_depth":64, "traversal_visits":16384,"scalar_allocation_bytes":apache_avro::util::max_allocation_bytes(apache_avro::util::DEFAULT_MAX_ALLOCATION_BYTES)}, "provider_bytes_verified":false, "registry_binding_verified":false, diff --git a/rust/src/runtime/upgrade/takeover.rs b/rust/src/runtime/upgrade/takeover.rs index a0e71b9b9..0d9188b83 100644 --- a/rust/src/runtime/upgrade/takeover.rs +++ b/rust/src/runtime/upgrade/takeover.rs @@ -1,30 +1,81 @@ -//! Backup-first, one-way takeover of a stopped PHP SQLite database, following +//! Backup-first, one-way takeover of a stopped PHP database, following //! docs/rust-server-upgrade-takeover.md. //! -//! Everything happens in one `BEGIN EXCLUSIVE` transaction: the inspections, -//! the fingerprint check against the operator's verified backup, the layout -//! conversion, the native relations, the migration ledger and native's own -//! startup verification. An interrupted or refused takeover therefore leaves -//! the PHP database exactly as it was, and a committed one is a native -//! database that PHP 2.5.14 and later refuse to open. +//! On SQLite and PostgreSQL everything happens in one transaction that no +//! other writer can interleave with: the inspections, the fingerprint check +//! against the operator's verified backup, the layout conversion, the native +//! relations, the migration ledger and native's own startup verification. An +//! interrupted or refused takeover therefore leaves the PHP database exactly +//! as it was, and a committed one is a native database that PHP 2.5.14 and +//! later refuse to open. use super::{ - Result, close, expired, fingerprint, open_sqlite_file, queue_binding::QueueSourceConfig, - queue_config, refuse, schema, sqlite_inventory, + Result, bounded, fingerprint, open_sqlite_file, postgres, postgres_inventory, + queue_binding::QueueSourceConfig, queue_config, refuse, schema, sqlite_inventory, }; +use crate::runtime::backend::Backend; use axum::http::StatusCode; use serde_json::{Value, json}; -use sqlx::{AssertSqlSafe, Connection, SqliteConnection}; +use sqlx::{ + AssertSqlSafe, Connection, Encode, Executor, IntoArguments, PgConnection, Postgres, Sqlite, + SqliteConnection, Transaction, Type, + migrate::{Migrate, Migration}, + postgres::PgConnectOptions, +}; use std::time::Duration; use tokio::time::{Instant, timeout_at}; +mod mysql_journal; +pub use mysql_journal::take_over_mysql; + /// Takeover reads every row twice (inspection and fingerprint). const BUDGET: Duration = Duration::from_secs(3600); +/// Native's PostgreSQL bootstrap key, so a native node cannot open the +/// database while it is taken over. +const POSTGRES_BOOTSTRAP_KEY: i64 = 4924501473315676162; pub async fn take_over_sqlite( path: &str, backup_fingerprint: &str, queues: &QueueSourceConfig<'_>, ) -> Result { + preconditions(backup_fingerprint, queues)?; + let deadline = Instant::now() + BUDGET; + let connection = open_sqlite_file(path, deadline, true).await?; + bounded(connection, deadline, async |connection| { + let mut tx = connection.begin_with("BEGIN EXCLUSIVE").await?; + let outcome = take_over_sqlite_in(&mut tx, backup_fingerprint, queues).await; + settle(tx, outcome).await + }) + .await +} + +/// Takes over with the owner's credentials. The connection keeps native's +/// short lock wait, so a PHP process still holding a table refuses the +/// takeover instead of stalling it. +pub async fn take_over_postgres( + options: PgConnectOptions, + backup_fingerprint: &str, + queues: &QueueSourceConfig<'_>, +) -> Result { + preconditions(backup_fingerprint, queues)?; + let deadline = Instant::now() + BUDGET; + let connection = timeout_at( + deadline, + PgConnection::connect_with(&postgres::source_options(options)), + ) + .await + .map_err(|_| super::expired())??; + bounded(connection, deadline, async |connection| { + // Read committed: once every table is locked, each statement reads + // the latest committed state and nothing else can change it. + let mut tx = connection.begin().await?; + let outcome = take_over_postgres_in(&mut tx, backup_fingerprint, queues).await; + settle(tx, outcome).await + }) + .await +} + +fn preconditions(backup_fingerprint: &str, queues: &QueueSourceConfig<'_>) -> Result<()> { queue_config(Some(queues))?; if !backup_fingerprint.starts_with("sha256:") || backup_fingerprint.len() != 71 { return Err(refuse( @@ -32,68 +83,181 @@ pub async fn take_over_sqlite( "upgrade_backup_fingerprint_required", )); } - let deadline = Instant::now() + BUDGET; - let mut connection = open_sqlite_file(path, deadline, true).await?; - let result = timeout_at(deadline, async { - let mut tx = connection.begin_with("BEGIN EXCLUSIVE").await?; - match take_over(&mut tx, backup_fingerprint, queues).await { - Ok(report) => { - tx.commit().await?; - Ok(report) - } - Err(error) => { - tx.rollback().await?; - Err(error) - } + Ok(()) +} + +/// Commits a completed takeover, and rolls back anything else. +async fn settle(tx: Transaction<'_, DB>, outcome: Result) -> Result { + match outcome { + Ok(report) => { + tx.commit().await?; + Ok(report) } - }) - .await - .map_err(|_| expired()) - .and_then(|result| result); - close(connection.close()).await?; - result + Err(error) => { + tx.rollback().await?; + Err(error) + } + } } -async fn take_over( +async fn take_over_sqlite_in( tx: &mut SqliteConnection, backup_fingerprint: &str, queues: &QueueSourceConfig<'_>, ) -> Result { let inventory = sqlite_inventory(&mut *tx, Some(queues), true).await?; - qualified(&inventory)?; + let layout = qualified(&inventory)?; // The same digest the operator compared with the restored backup. Any // write after that backup changes it, so nothing below can run. let digest = fingerprint::sqlite_digest(&mut *tx).await?; - if digest["fingerprint"] != backup_fingerprint { + matches_backup(&digest, backup_fingerprint)?; + sqlx::raw_sql(AssertSqlSafe(schema::native_relations_sql()?)) + .execute(&mut *tx) + .await?; + let converted = convert::(&mut *tx, layout).await?; + record_migration::(&mut *tx, schema::migration()).await?; + // Exactly the check native startup makes: marker, ledger and catalog. + schema::verify(&mut *tx).await?; + #[cfg(test)] + interruption_point("before-commit"); + Ok(report("sqlite", layout, &digest, &inventory, converted)) +} + +async fn take_over_postgres_in( + tx: &mut PgConnection, + backup_fingerprint: &str, + queues: &QueueSourceConfig<'_>, +) -> Result { + // Whole-table reads and index builds are expected; the client deadline + // bounds them, while lock waits stay short. + sqlx::raw_sql("SET LOCAL statement_timeout=0") + .execute(&mut *tx) + .await?; + sqlx::query("SELECT pg_advisory_xact_lock($1)") + .bind(POSTGRES_BOOTSTRAP_KEY) + .execute(&mut *tx) + .await?; + // The later DDL needs ACCESS EXCLUSIVE anyway; taking it first means no + // lock is ever upgraded while another session waits. + let tables: Vec = sqlx::query_scalar("SELECT c.relname::text FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname='public' AND c.relkind IN ('r','p') ORDER BY c.relname") + .fetch_all(&mut *tx) + .await?; + if tables.is_empty() { return Err(refuse( StatusCode::CONFLICT, - "upgrade_backup_fingerprint_mismatch", + "upgrade_source_catalog_unqualified", )); } - let layout = inventory["source_layout"] - .as_str() - .ok_or_else(|| refuse(StatusCode::CONFLICT, "upgrade_source_catalog_unqualified"))?; - sqlx::raw_sql(AssertSqlSafe(schema::native_relations_sql()?)) + let tables = tables + .iter() + .map(|table| fingerprint::identifier(table).map(|table| format!("public.\"{table}\""))) + .collect::>>()? + .join(","); + sqlx::raw_sql(AssertSqlSafe(format!( + "LOCK TABLE {tables} IN ACCESS EXCLUSIVE MODE" + ))) + .execute(&mut *tx) + .await + .map_err(|error| match &error { + sqlx::Error::Database(database) if database.code().as_deref() == Some("55P03") => { + refuse(StatusCode::CONFLICT, "upgrade_source_in_use") + } + _ => error.into(), + })?; + let inventory = postgres_inventory(&mut *tx, Some(queues), true).await?; + let layout = qualified(&inventory)?; + let digest = fingerprint::postgres_digest(&mut *tx).await?; + matches_backup(&digest, backup_fingerprint)?; + sqlx::raw_sql(AssertSqlSafe(postgres::native_relations_sql()?)) .execute(&mut *tx) .await?; - let converted = convert(&mut *tx, layout).await?; - schema::record_migration(&mut *tx).await?; - // Exactly the check native startup makes: marker, ledger and catalog. - schema::verify(&mut *tx).await?; - Ok(json!({ + let converted = convert::(&mut *tx, layout).await?; + record_migration::(&mut *tx, postgres::migration()).await?; + postgres::verify(&mut *tx).await?; + #[cfg(test)] + interruption_point("before-commit"); + Ok(report("pgsql", layout, &digest, &inventory, converted)) +} + +/// Lets the interruption qualification kill a real takeover process at a +/// named boundary: `before-commit` unless `DW_TAKEOVER_TEST_BOUNDARY` names +/// another. +#[cfg(test)] +fn interruption_point(boundary: &str) { + let wanted = std::env::var("DW_TAKEOVER_TEST_BOUNDARY"); + if let Ok(receipt) = std::env::var("DW_TAKEOVER_TEST_RECEIPT") + && wanted.as_deref().unwrap_or("before-commit") == boundary + { + std::fs::write(format!("{receipt}.partial"), boundary).unwrap(); + std::fs::rename(format!("{receipt}.partial"), &receipt).unwrap(); + loop { + std::thread::park(); + } + } +} + +fn matches_backup(digest: &Value, backup_fingerprint: &str) -> Result<()> { + if digest["fingerprint"] != backup_fingerprint { + return Err(refuse( + StatusCode::CONFLICT, + "upgrade_backup_fingerprint_mismatch", + )); + } + Ok(()) +} + +fn report(backend: &str, layout: &str, digest: &Value, inventory: &Value, converted: Value) -> Value { + json!({ "schema":"durable-workflow.rust-upgrade-takeover.v1", - "status":"taken_over", "backend":"sqlite", "source_layout":layout, + "status":"taken_over", "backend":backend, "source_layout":layout, "fingerprint":digest["fingerprint"], "table_rows":inventory["table_rows"], "converted":converted, "backup_fingerprint_matched":true, // What the inspections cannot yet establish stays explicit. "nested_payloads_verified":false, "other_root_payload_columns_verified":false, "stored_representation_compatibility_verified":false - })) + }) } +/// Published PHP features native does not implement yet. A source holding any +/// of their rows is refused, rather than taken over without them. Derived +/// projections and PHP's own queue and poll bookkeeping are left in place. +const UNSUPPORTED_FEATURES: [(&str, &[&str]); 7] = [ + ("upgrade_source_runtime_credentials_unsupported", &["runtime_credentials"]), + ( + "upgrade_source_message_streams_unsupported", + &[ + "workflow_messages", + "workflow_durable_streams", + "workflow_durable_stream_items", + "workflow_inbound_streams", + "workflow_inbound_stream_items", + ], + ), + ( + "upgrade_source_services_unsupported", + &["workflow_services", "workflow_service_endpoints", "workflow_service_operations"], + ), + ("upgrade_source_update_validation_unsupported", &["workflow_update_validation_tasks"]), + ("upgrade_source_worker_sessions_unsupported", &["workflow_worker_sessions"]), + ("upgrade_source_build_id_rollouts_unsupported", &["workflow_worker_build_id_rollouts"]), + ( + "upgrade_source_search_attribute_definitions_unsupported", + &["search_attribute_definitions"], + ), +]; + /// Takeover needs every inspection result it relies on, and refuses what -/// native cannot represent yet: external payloads and service calls. -fn qualified(inventory: &Value) -> Result<()> { +/// native cannot represent yet: external payloads, service calls and the +/// other unsupported features above. Returns the qualified source layout. +fn qualified(inventory: &Value) -> Result<&str> { + for (reason, tables) in UNSUPPORTED_FEATURES { + if tables + .iter() + .any(|table| inventory["table_rows"][*table].as_i64().unwrap_or(0) != 0) + { + return Err(refuse(StatusCode::CONFLICT, reason)); + } + } if inventory["queue_inspection"]["status"] != "known_subset_bound" { return Err(refuse( StatusCode::CONFLICT, @@ -126,76 +290,81 @@ fn qualified(inventory: &Value) -> Result<()> { "upgrade_source_external_payloads_unsupported", )); } - Ok(()) + inventory["source_layout"] + .as_str() + .ok_or_else(|| refuse(StatusCode::CONFLICT, "upgrade_source_catalog_unqualified")) } -/// Converts a recorded PHP release layout to native's (the design's recipes). -async fn convert(tx: &mut SqliteConnection, layout: &str) -> Result { - let (incarnation_table, token_column) = match layout { - "frozen" => (false, false), - "php-2.8.0" | "php-2.9.7" => (true, true), - _ => { - return Err(refuse( - StatusCode::CONFLICT, - "upgrade_source_layout_unconverted", - )); - } - }; - let calls: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM workflow_service_calls") - .fetch_one(&mut *tx) +fn query(sql: &'static str) -> sqlx::query::Query<'static, DB, DB::Arguments> +where + DB::Arguments: IntoArguments, +{ + sqlx::query(AssertSqlSafe(DB::statement(sql))) +} + +/// Records the native migration as applied, for a schema that came from +/// conversion. Native verification then checks this ledger like any other. +async fn record_migration( + tx: &mut DB::Connection, + migration: &'static Migration, +) -> Result<()> +where + DB::Connection: Migrate, + for<'c> &'c mut DB::Connection: Executor<'c, Database = DB>, + DB::Arguments: IntoArguments, + for<'q> i64: Encode<'q, DB> + Type, + for<'q> &'q str: Encode<'q, DB> + Type, + for<'q> &'q [u8]: Encode<'q, DB> + Type, +{ + tx.ensure_migrations_table("_sqlx_migrations").await?; + query::("INSERT INTO _sqlx_migrations(version,description,success,checksum,execution_time) VALUES ($1,$2,TRUE,$3,0)") + .bind(migration.version) + .bind(migration.description.as_ref()) + .bind(migration.checksum.as_ref()) + .execute(&mut *tx) .await?; - if calls != 0 { + Ok(()) +} + +/// Whether a recorded PHP release layout has PHP's own registration +/// incarnations. Layouts without a recipe are refused. +fn php_incarnations(layout: &str) -> Result { + match layout.trim_end_matches("+mariadb-driver") { + "frozen" => Ok(false), + "php-2.8.0" | "php-2.9.7" => Ok(true), + _ => Err(refuse( + StatusCode::CONFLICT, + "upgrade_source_layout_unconverted", + )), + } +} + +/// Service calls are not part of native yet. +async fn service_calls_absent(tx: &mut DB::Connection) -> Result<()> +where + for<'c> &'c mut DB::Connection: Executor<'c, Database = DB>, + DB::Arguments: IntoArguments, +{ + if count::(tx, "SELECT COUNT(*) AS n FROM workflow_service_calls").await? != 0 { return Err(refuse( StatusCode::CONFLICT, "upgrade_source_service_calls_unsupported", )); } - let now = super::super::store::now() - .format("%Y-%m-%d %H:%M:%S%.6f") - .to_string(); - let mut copied = 0; - if incarnation_table { - // Same statuses as native. Native keeps the recovered count only on - // deregistration and a finish time on every retired incarnation. - copied = sqlx::query("INSERT INTO dw_worker_registration_incarnations(token,namespace,worker_id,status,recovered_workflow_task_count,finished_at,created_at,updated_at) SELECT token,namespace,worker_id,status,CASE WHEN status='deregistered' THEN recovered_workflow_task_count END,CASE WHEN status='active' THEN NULL ELSE COALESCE(finished_at,updated_at,created_at,$1) END,COALESCE(created_at,$1),COALESCE(updated_at,created_at,$1) FROM workflow_worker_registration_incarnations WHERE status IN ('active','superseded','deregistered')") - .bind(&now) - .execute(&mut *tx) - .await? - .rows_affected(); - let total: i64 = - sqlx::query_scalar("SELECT COUNT(*) FROM workflow_worker_registration_incarnations") - .fetch_one(&mut *tx) - .await?; - if copied != total as u64 { - return Err(refuse( - StatusCode::CONFLICT, - "upgrade_source_incarnation_unqualified", - )); - } - } - // Every active registration gets the authority native expects. A PHP - // 2.8.0+ worker keeps the token it was given; an older one gets a new token. - let missing: Vec<(String, String, Option)> = sqlx::query_as(AssertSqlSafe(format!( - "SELECT r.namespace,r.worker_id,{} FROM workflow_worker_registrations r WHERE r.status='active' AND NOT EXISTS (SELECT 1 FROM dw_worker_registration_incarnations i WHERE i.namespace=r.namespace AND i.worker_id=r.worker_id AND i.status='active') ORDER BY r.namespace,r.worker_id", - if token_column { "r.registration_token" } else { "NULL" } - ))) - .fetch_all(&mut *tx) - .await?; - for (namespace, worker_id, token) in &missing { - let token = match token { - Some(token) => token.clone(), - None => super::super::worker_incarnations::token().await?, - }; - sqlx::query("INSERT INTO dw_worker_registration_incarnations(token,namespace,worker_id,status,created_at,updated_at) VALUES ($1,$2,$3,'active',$4,$5)") - .bind(token) - .bind(namespace) - .bind(worker_id) - .bind(&now) - .bind(&now) - .execute(&mut *tx) - .await?; - } - if incarnation_table { + Ok(()) +} + +/// Converts a recorded PHP release layout to native's (the design's recipes). +async fn convert(tx: &mut DB::Connection, layout: &str) -> Result +where + for<'c> &'c mut DB::Connection: Executor<'c, Database = DB>, + DB::Arguments: IntoArguments, + for<'q> String: Encode<'q, DB> + Type, +{ + let php_incarnations = php_incarnations(layout)?; + service_calls_absent::(tx).await?; + let (copied, created) = adopt_incarnations::(tx, php_incarnations).await?; + if php_incarnations { sqlx::raw_sql( "DROP TABLE workflow_worker_registration_incarnations; \ ALTER TABLE workflow_worker_registrations DROP COLUMN registration_token; \ @@ -206,12 +375,9 @@ async fn convert(tx: &mut SqliteConnection, layout: &str) -> Result { .await?; } let delivery_references = if layout == "php-2.9.7" { - // PHP's own queue delivery bookkeeping; native does not use the jobs. - let references: i64 = sqlx::query_scalar( - "SELECT COUNT(*) FROM workflow_tasks WHERE queue_delivery_reference IS NOT NULL", - ) - .fetch_one(&mut *tx) - .await?; + // PHP's own queue delivery lookup metadata; native does not use PHP's + // queue, and the queue inspection bound every job to its task row. + let references = count::(tx, DELIVERY_REFERENCES).await?; sqlx::raw_sql("ALTER TABLE workflow_tasks DROP COLUMN queue_delivery_reference") .execute(&mut *tx) .await?; @@ -220,7 +386,82 @@ async fn convert(tx: &mut SqliteConnection, layout: &str) -> Result { 0 }; Ok(json!({ - "incarnations_copied":copied, "incarnations_created":missing.len(), + "incarnations_copied":copied, "incarnations_created":created, "queue_delivery_references_removed":delivery_references })) } + +const DELIVERY_REFERENCES: &str = + "SELECT COUNT(*) AS n FROM workflow_tasks WHERE queue_delivery_reference IS NOT NULL"; + +/// Gives every registration the incarnation native expects. PHP's own +/// incarnations and tokens are copied, so a running SDK worker's fenced +/// deregistration still matches; an active registration without one gets a +/// new incarnation. Returns how many were copied and created. +async fn adopt_incarnations( + tx: &mut DB::Connection, + php_incarnations: bool, +) -> Result<(u64, usize)> +where + for<'c> &'c mut DB::Connection: Executor<'c, Database = DB>, + DB::Arguments: IntoArguments, + for<'q> String: Encode<'q, DB> + Type, +{ + let now = super::super::store::now(); + let mut copied = 0; + if php_incarnations { + // Same statuses as native. Native keeps the recovered count only on + // deregistration and a finish time on every retired incarnation. + copied = DB::affected( + query::("INSERT INTO dw_worker_registration_incarnations(token,namespace,worker_id,status,recovered_workflow_task_count,finished_at,created_at,updated_at) SELECT token,namespace,worker_id,status,CASE WHEN status='deregistered' THEN recovered_workflow_task_count END,CASE WHEN status='active' THEN NULL ELSE COALESCE(finished_at,updated_at,created_at,$1) END,COALESCE(created_at,$2),COALESCE(updated_at,created_at,$3) FROM workflow_worker_registration_incarnations WHERE status IN ('active','superseded','deregistered')") + .bind(DB::bind_time(now)) + .bind(DB::bind_time(now)) + .bind(DB::bind_time(now)) + .execute(&mut *tx) + .await?, + ); + let total = count::( + tx, + "SELECT COUNT(*) AS n FROM workflow_worker_registration_incarnations", + ) + .await?; + if copied != total as u64 { + return Err(refuse( + StatusCode::CONFLICT, + "upgrade_source_incarnation_unqualified", + )); + } + } + // A PHP 2.8.0+ worker keeps the token it was given; an older one gets a + // new token. + let missing = query::(if php_incarnations { + "SELECT r.namespace,r.worker_id,r.registration_token AS token FROM workflow_worker_registrations r WHERE r.status='active' AND NOT EXISTS (SELECT 1 FROM dw_worker_registration_incarnations i WHERE i.namespace=r.namespace AND i.worker_id=r.worker_id AND i.status='active') ORDER BY r.namespace,r.worker_id" + } else { + "SELECT r.namespace,r.worker_id,NULL AS token FROM workflow_worker_registrations r WHERE r.status='active' AND NOT EXISTS (SELECT 1 FROM dw_worker_registration_incarnations i WHERE i.namespace=r.namespace AND i.worker_id=r.worker_id AND i.status='active') ORDER BY r.namespace,r.worker_id" + }) + .fetch_all(&mut *tx) + .await?; + for row in &missing { + let token = match DB::optional_string(row, "token")? { + Some(token) => token, + None => super::super::worker_incarnations::token().await?, + }; + query::("INSERT INTO dw_worker_registration_incarnations(token,namespace,worker_id,status,created_at,updated_at) VALUES ($1,$2,$3,'active',$4,$5)") + .bind(token) + .bind(DB::string(row, "namespace")?) + .bind(DB::string(row, "worker_id")?) + .bind(DB::bind_time(now)) + .bind(DB::bind_time(now)) + .execute(&mut *tx) + .await?; + } + Ok((copied, missing.len())) +} + +async fn count(tx: &mut DB::Connection, sql: &'static str) -> Result +where + for<'c> &'c mut DB::Connection: Executor<'c, Database = DB>, + DB::Arguments: IntoArguments, +{ + DB::number(&query::(sql).fetch_one(&mut *tx).await?, "n") +} diff --git a/rust/src/runtime/upgrade/takeover/mysql_journal.rs b/rust/src/runtime/upgrade/takeover/mysql_journal.rs new file mode 100644 index 000000000..15b3d8ce5 --- /dev/null +++ b/rust/src/runtime/upgrade/takeover/mysql_journal.rs @@ -0,0 +1,397 @@ +//! MySQL and MariaDB takeover. Their DDL commits implicitly, and `LOCK +//! TABLES` refuses to create or drop the tables it does not hold, so the +//! takeover cannot be one transaction. It writes native's marker first, with +//! a `rust-takeover:` row whose version counts the completed steps +//! and whose checksum holds the backup fingerprint's digest. From then on PHP +//! refuses the database and native refuses it until the last step. Every step +//! is idempotent, so a rerun with the same fingerprint resumes after the last +//! recorded one. +use super::{ + BUDGET, DELIVERY_REFERENCES, Result, adopt_incarnations, count, matches_backup, + php_incarnations, preconditions, qualified, report, service_calls_absent, +}; +use crate::runtime::{ + mysql::{self, Flavor, LEDGER_SQL, MARKER_SQL, TAKEOVER_ENGINE}, + upgrade::{ + bounded, fingerprint, mysql_inventory, mysql_source, queue_binding::QueueSourceConfig, + refuse, + }, +}; +use axum::http::StatusCode; +use serde_json::{Value, json}; +use sqlx::{Connection, Executor, MySql, MySqlConnection, mysql::MySqlConnectOptions}; +use tokio::time::Instant; + +/// Completed-step counts recorded in the marker row. +const MARKED: i64 = 0; +/// Sessions rechecked and the source fingerprinted again after the marker: +/// nothing PHP wrote can be missing from the backup. +const FENCED: i64 = 1; +const LAST: i64 = 8; + +/// One takeover run: what its steps need and what its report says. +struct Run<'a> { + backend: &'static str, + flavor: Flavor, + digest: Vec, + layout: String, + fingerprint: &'a str, + queues: &'a QueueSourceConfig<'a>, + inventory: Value, + converted: serde_json::Map, +} + +enum Journal { + /// No marker: a PHP database. + Absent, + /// The marker table without its row: a takeover killed between the two. + Empty, + /// A takeover in progress, with its layout, step count and digest. + Takeover(String, i64, Vec), + /// Native's own database, or one native is still initializing. + Owned, +} + +pub async fn take_over_mysql( + options: MySqlConnectOptions, + backup_fingerprint: &str, + queues: &QueueSourceConfig<'_>, +) -> Result { + preconditions(backup_fingerprint, queues)?; + let digest = fingerprint_digest(backup_fingerprint)?; + let deadline = Instant::now() + BUDGET; + // TIMESTAMP values render in UTC for the fingerprint, and lock waits stay + // short; the client deadline bounds whole-table work. + let (connection, backend) = mysql_source(options, deadline, true, 0).await?; + bounded(connection, deadline, async |connection| { + // The same session lock as native bootstrap, so neither a native node + // nor a second takeover can run on this database meanwhile. + let locked: Option = sqlx::query_scalar( + "SELECT GET_LOCK(CONCAT('dw-native-bootstrap:',SHA1(DATABASE())),5)", + ) + .fetch_one(&mut *connection) + .await?; + if locked != Some(1) { + return Err(refuse( + StatusCode::SERVICE_UNAVAILABLE, + "native_initialization_busy", + )); + } + let mut run = Run { + backend, + flavor: Flavor::detect(connection).await?, + digest, + layout: String::new(), + fingerprint: backup_fingerprint, + queues, + inventory: Value::Null, + converted: serde_json::Map::new(), + }; + let completed = match journal(connection).await? { + Journal::Absent => { + mark(connection, &mut run).await?; + MARKED + } + Journal::Empty => { + // Only a takeover adds the marker to a PHP database. Its layout + // is read again with the marker set aside. + let mut tx = connection + .begin_with("START TRANSACTION WITH CONSISTENT SNAPSHOT, READ ONLY") + .await?; + let layout = mysql_inventory(&mut tx, backend, Some(queues), true, true).await; + tx.rollback().await?; + run.layout = qualified(&layout?)?.to_owned(); + record(connection, &run).await?; + MARKED + } + Journal::Takeover(layout, completed, digest) => { + if digest != run.digest { + return Err(refuse( + StatusCode::CONFLICT, + "upgrade_backup_fingerprint_mismatch", + )); + } + php_incarnations(&layout)?; + run.layout = layout; + run.converted + .insert("resumed_after_step".into(), json!(completed)); + completed + } + Journal::Owned => { + return Err(refuse(StatusCode::CONFLICT, "upgrade_source_already_owned")); + } + }; + for step in completed + 1..=LAST { + perform(connection, &mut run, step).await?; + #[cfg(test)] + super::interruption_point(&format!("after-step-{step}")); + } + finish(connection, &run).await?; + mysql::verify(connection, run.flavor).await?; + let digest = json!({"fingerprint": run.fingerprint}); + Ok(report( + run.backend, + &run.layout, + &digest, + &run.inventory, + Value::Object(run.converted), + )) + }) + .await +} + +/// `sha256:` as the 32 bytes the marker row keeps. +fn fingerprint_digest(fingerprint: &str) -> Result> { + fingerprint["sha256:".len()..] + .as_bytes() + .chunks(2) + .map(|pair| { + std::str::from_utf8(pair) + .ok() + .and_then(|pair| u8::from_str_radix(pair, 16).ok()) + }) + .collect::>>() + .ok_or_else(|| { + refuse( + StatusCode::BAD_REQUEST, + "upgrade_backup_fingerprint_required", + ) + }) +} + +async fn journal(connection: &mut MySqlConnection) -> Result { + let marked: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM information_schema.TABLES WHERE TABLE_SCHEMA=DATABASE() AND LOWER(TABLE_NAME)='dw_server_schema'") + .fetch_one(&mut *connection) + .await?; + if marked == 0 { + return Ok(Journal::Absent); + } + let mut rows: Vec<(String, i64, Vec)> = + sqlx::query_as("SELECT engine,version,bootstrap_checksum FROM dw_server_schema") + .fetch_all(&mut *connection) + .await?; + Ok(match rows.pop() { + None => Journal::Empty, + Some((engine, completed, digest)) if rows.is_empty() => { + match engine.strip_prefix(TAKEOVER_ENGINE) { + Some(layout) => Journal::Takeover(layout.to_owned(), completed, digest), + None => Journal::Owned, + } + } + Some(_) => Journal::Owned, + }) +} + +/// No other session may use the database. That needs the PROCESS privilege, +/// since without it MySQL shows only the account's own sessions. +async fn sessions_clear(connection: &mut MySqlConnection) -> Result<()> { + let visible: bool = sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM information_schema.USER_PRIVILEGES WHERE PRIVILEGE_TYPE='PROCESS')", + ) + .fetch_one(&mut *connection) + .await?; + if !visible { + return Err(refuse( + StatusCode::CONFLICT, + "upgrade_source_sessions_unverifiable", + )); + } + let others: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM information_schema.PROCESSLIST WHERE DB=DATABASE() AND ID<>CONNECTION_ID()") + .fetch_one(&mut *connection) + .await?; + if others != 0 { + return Err(refuse(StatusCode::CONFLICT, "upgrade_source_in_use")); + } + Ok(()) +} + +/// The inventory and fingerprint from one consistent snapshot, refused +/// unless the source is qualified and matches the backup. +async fn verified_snapshot( + connection: &mut MySqlConnection, + run: &Run<'_>, + fenced: bool, +) -> Result<(Value, String)> { + sessions_clear(connection).await?; + let mut tx = connection + .begin_with("START TRANSACTION WITH CONSISTENT SNAPSHOT, READ ONLY") + .await?; + let read = async { + let inventory = + mysql_inventory(&mut tx, run.backend, Some(run.queues), true, fenced).await?; + let digest = fingerprint::mysql_digest(&mut tx, run.backend, fenced).await?; + Ok::<_, crate::runtime::RuntimeError>((inventory, digest)) + } + .await; + tx.rollback().await?; + let (inventory, digest) = read?; + let layout = qualified(&inventory)?.to_owned(); + matches_backup(&digest, run.fingerprint)?; + php_incarnations(&layout)?; + service_calls_absent::(connection).await?; + Ok((inventory, layout)) +} + +/// The first write: native's marker with the takeover row, once the source +/// is verified. PHP refuses the database from here on. +async fn mark(connection: &mut MySqlConnection, run: &mut Run<'_>) -> Result<()> { + let (inventory, layout) = verified_snapshot(connection, run, false).await?; + run.inventory = inventory; + run.layout = layout; + connection.execute(MARKER_SQL).await?; + #[cfg(test)] + super::interruption_point("before-marker-row"); + record(connection, run).await +} + +async fn record(connection: &mut MySqlConnection, run: &Run<'_>) -> Result<()> { + sqlx::query("INSERT INTO dw_server_schema(engine,version,bootstrap_checksum) VALUES (?,?,?)") + .bind(format!("{TAKEOVER_ENGINE}{}", run.layout)) + .bind(MARKED) + .bind(&run.digest[..]) + .execute(&mut *connection) + .await?; + Ok(()) +} + +/// Records one more completed step, in the caller's transaction or alone. +async fn advance<'c, E>(executor: E, run: &Run<'_>, step: i64) -> Result<()> +where + E: Executor<'c, Database = MySql>, +{ + let advanced = sqlx::query("UPDATE dw_server_schema SET version=? WHERE engine=? AND version=?") + .bind(step) + .bind(format!("{TAKEOVER_ENGINE}{}", run.layout)) + .bind(step - 1) + .execute(executor) + .await? + .rows_affected(); + if advanced != 1 { + return Err(refuse( + StatusCode::CONFLICT, + "upgrade_takeover_journal_changed", + )); + } + Ok(()) +} + +async fn column_exists(connection: &mut MySqlConnection, table: &str, column: &str) -> Result { + let found: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? AND COLUMN_NAME=?") + .bind(table) + .bind(column) + .fetch_one(&mut *connection) + .await?; + Ok(found != 0) +} + +/// Each step checks what it has to do, so repeating an interrupted one is +/// harmless, and then records itself. +async fn perform(connection: &mut MySqlConnection, run: &mut Run<'_>, step: i64) -> Result<()> { + let php_incarnations = php_incarnations(&run.layout)?; + match step { + FENCED => { + // A session that was still open when the marker appeared could + // have written; none is left, and the source is still the backup. + let (inventory, layout) = verified_snapshot(connection, run, true).await?; + if layout != run.layout { + return Err(refuse( + StatusCode::CONFLICT, + "upgrade_source_catalog_unqualified", + )); + } + run.inventory = inventory; + } + 2 => { + sqlx::raw_sql(sqlx::AssertSqlSafe(run.flavor.native_relations_sql()?)) + .execute(&mut *connection) + .await?; + let indexed: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME='workflow_tasks' AND INDEX_NAME='dw_workflow_tasks_poll'") + .fetch_one(&mut *connection) + .await?; + if indexed == 0 { + connection.execute("ALTER TABLE workflow_tasks ADD KEY dw_workflow_tasks_poll (namespace(64),queue(64),task_type(16),status(16),available_at)").await?; + } + } + 3 => { + // Rows, unlike DDL, commit with the step that records them. + let mut tx = connection.begin().await?; + let (copied, created) = adopt_incarnations::(&mut tx, php_incarnations).await?; + advance(&mut *tx, run, step).await?; + tx.commit().await?; + run.converted + .insert("incarnations_copied".into(), json!(copied)); + run.converted + .insert("incarnations_created".into(), json!(created)); + return Ok(()); + } + 4 => { + if php_incarnations { + connection + .execute("DROP TABLE IF EXISTS workflow_worker_registration_incarnations") + .await?; + } + } + 5 => { + if column_exists(connection, "workflow_worker_registrations", "registration_token").await? { + connection + .execute("ALTER TABLE workflow_worker_registrations DROP COLUMN registration_token") + .await?; + } + } + 6 => { + if column_exists(connection, "workflow_service_calls", "call_deadline_at").await? { + // Dropping only the column would leave its index on status alone. + connection.execute("ALTER TABLE workflow_service_calls DROP INDEX wf_service_calls_deadline_idx, DROP COLUMN call_deadline_at").await?; + } + } + 7 => { + if column_exists(connection, "workflow_tasks", "queue_delivery_reference").await? { + let references = count::(connection, DELIVERY_REFERENCES).await?; + run.converted + .insert("queue_delivery_references_removed".into(), json!(references)); + connection + .execute("ALTER TABLE workflow_tasks DROP COLUMN queue_delivery_reference") + .await?; + } + } + 8 => { + if run.layout.ends_with("+mariadb-driver") { + // Laravel's mariadb connection made the backup hold owner a uuid. + connection + .execute("ALTER TABLE runtime_external_payload_backup_hold MODIFY owner char(36) DEFAULT NULL") + .await?; + } + } + _ => {} + } + advance(&mut *connection, run, step).await +} + +/// The ledger row and native's final marker commit together. +async fn finish(connection: &mut MySqlConnection, run: &Run<'_>) -> Result<()> { + connection.execute(LEDGER_SQL).await?; + let migration = run.flavor.migration(); + let mut tx = connection.begin().await?; + sqlx::query("INSERT INTO _sqlx_migrations(version,description,success,checksum,execution_time) VALUES (?,?,TRUE,?,0)") + .bind(migration.version) + .bind(migration.description.as_ref()) + .bind(migration.checksum.as_ref()) + .execute(&mut *tx) + .await?; + let marked = sqlx::query("UPDATE dw_server_schema SET engine='rust-development',version=?,bootstrap_checksum=? WHERE engine=? AND version=?") + .bind(mysql::VERSION) + .bind(run.flavor.checksum()) + .bind(format!("{TAKEOVER_ENGINE}{}", run.layout)) + .bind(LAST) + .execute(&mut *tx) + .await? + .rows_affected(); + if marked != 1 { + return Err(refuse( + StatusCode::CONFLICT, + "upgrade_takeover_journal_changed", + )); + } + tx.commit().await?; + Ok(()) +} diff --git a/rust/src/runtime/upgrade_tests.rs b/rust/src/runtime/upgrade_tests.rs index d45f60094..35e563a5a 100644 --- a/rust/src/runtime/upgrade_tests.rs +++ b/rust/src/runtime/upgrade_tests.rs @@ -406,7 +406,11 @@ async fn payload_scan_validates_inline_pages_and_reports_late_bad_values_without assert_eq!(report["payload_inspection"]["other_root_payload_columns_verified"], false); assert_eq!( report["payload_inspection"]["unscanned_payload_codec_tables"], - json!(["workflow_commands", "workflow_signal_records", "workflow_updates"]) + json!([ + "workflow_durable_stream_items", + "workflow_inbound_stream_items", + "workflow_update_validation_tasks" + ]) ); assert_eq!(report["payload_inspection"]["diagnostics"], json!([])); for (table, column, count) in [ @@ -935,6 +939,71 @@ async fn takeover_converts_a_php_297_database_that_native_then_opens() { connection.close().await.unwrap(); } +#[tokio::test] +#[ignore = "only invoked as a real subprocess by the takeover interruption test"] +async fn sqlite_takeover_child() { + let path = std::env::var("DW_TAKEOVER_TEST_DATABASE").unwrap(); + let fingerprint = std::env::var("DW_TAKEOVER_TEST_FINGERPRINT").unwrap(); + // Parks just before the commit, where the parent kills this process. + let report = takeover::take_over_sqlite(&path, &fingerprint, &queue_config()).await; + panic!("takeover returned instead of parking: {report:?}"); +} + +#[cfg(unix)] +#[tokio::test] +async fn takeover_killed_before_commit_leaves_php_data_and_can_be_rerun() { + use std::os::unix::process::ExitStatusExt; + let (directory, path) = php297_layout().await; + let fingerprint = backup_fingerprint(&path).await; + let original = std::fs::read(&path).unwrap(); + let receipt = directory.path().join("boundary"); + let mut child = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "runtime::upgrade::tests::sqlite_takeover_child", + "--ignored", + "--nocapture", + ]) + .env("DW_TAKEOVER_TEST_DATABASE", &path) + .env("DW_TAKEOVER_TEST_FINGERPRINT", &fingerprint) + .env("DW_TAKEOVER_TEST_RECEIPT", &receipt) + .spawn() + .unwrap(); + let deadline = tokio::time::Instant::now() + Duration::from_secs(60); + while !receipt.exists() && tokio::time::Instant::now() < deadline { + if let Some(status) = child.try_wait().unwrap() { + panic!("child exited before the commit point: {status}"); + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + if !receipt.exists() { + child.kill().unwrap(); + child.wait().unwrap(); + panic!("child did not reach the commit point"); + } + child.kill().unwrap(); + assert_eq!(child.wait().unwrap().signal(), Some(9)); + // Either SQLite had not written the file yet, or its rollback journal + // still holds the original pages. Read-only inspection never recovers it. + if Path::new(&format!("{path}-journal")).exists() { + reason( + inspect_sqlite(&path).await, + "upgrade_source_sqlite_journal_unqualified", + ); + } else { + assert_eq!(std::fs::read(&path).unwrap(), original); + } + // Rerunning with the same verified fingerprint rolls the journal back, + // finds the backed-up PHP database and takes it over. + let report = takeover::take_over_sqlite(&path, &fingerprint, &queue_config()) + .await + .unwrap(); + assert_eq!(report["status"], "taken_over", "{report}"); + assert_eq!(report["fingerprint"], fingerprint.as_str()); + let runtime = crate::runtime::Runtime::open(&path, "test-token".into()).await.unwrap(); + runtime.close().await; +} + #[tokio::test] async fn takeover_of_the_frozen_layout_adds_only_native_relations() { let (_directory, path) = php_layout().await; @@ -977,6 +1046,15 @@ async fn takeover_refusals_leave_the_php_database_unchanged() { "INSERT INTO runtime_external_payloads(id,namespace,storage_uri,storage_uri_sha256,codec,sha256,size_bytes) VALUES ('p','default','s3://b/k','x','avro','y',1)", "upgrade_source_external_payloads_unsupported", ), + // Features native lacks: their rows would be left behind unread. + ( + "INSERT INTO runtime_credentials(id,subject,roles,tenant,token_prefix,token_hash) VALUES ('c','worker','[]','default','dw_','h')", + "upgrade_source_runtime_credentials_unsupported", + ), + ( + "INSERT INTO workflow_messages(workflow_instance_id,direction,channel,stream_key,sequence) VALUES ('w','inbound','c','k',1)", + "upgrade_source_message_streams_unsupported", + ), ] { let (_directory, path) = php297_layout().await; mutate(&path, statement).await; @@ -989,3 +1067,92 @@ async fn takeover_refusals_leave_the_php_database_unchanged() { assert_eq!(std::fs::read(&path).unwrap(), before, "{expected}"); } } + +/// The frozen layout in PHP Server's default WAL mode, copied while its +/// writer was still open: one committed row is only in the WAL, as a crashed +/// PHP would leave it. +async fn wal_source() -> (tempfile::TempDir, String) { + let (directory, path) = php_layout().await; + let mut writer = SqliteConnection::connect_with( + &SqliteConnectOptions::new() + .filename(&path) + .journal_mode(SqliteJournalMode::Wal), + ) + .await + .unwrap(); + writer.execute("PRAGMA wal_autocheckpoint=0").await.unwrap(); + writer + .execute("INSERT INTO migrations(migration,batch) VALUES ('only-in-wal',7)") + .await + .unwrap(); + let copy = directory.path().join("wal.sqlite").to_str().unwrap().to_owned(); + for suffix in ["", "-wal", "-shm"] { + std::fs::copy(format!("{path}{suffix}"), format!("{copy}{suffix}")).unwrap(); + } + writer.close().await.unwrap(); + (directory, copy) +} + +#[tokio::test] +async fn prepare_switches_a_wal_source_to_the_rollback_journal_and_keeps_its_rows() { + let (_directory, path) = wal_source().await; + assert!(Path::new(&format!("{path}-wal")).exists()); + reason( + inspect_sqlite(&path).await, + "upgrade_source_sqlite_journal_unqualified", + ); + let report = prepare_sqlite_source(&path).await.unwrap(); + assert_eq!(report["status"], "rollback_journal", "{report}"); + assert_eq!(report["previous_journal_mode"], "wal"); + assert_eq!(report["source_mutated"], true); + for suffix in ["-wal", "-shm", "-journal"] { + assert!(!Path::new(&format!("{path}{suffix}")).exists(), "{suffix}"); + } + assert_eq!(journal_format(&path).await.unwrap(), ROLLBACK_JOURNAL); + let inventory = inspect_sqlite(&path).await.unwrap(); + assert_eq!(inventory["table_rows"]["migrations"], 1, "{inventory}"); + let mut connection = SqliteConnection::connect(&format!("sqlite://{path}?mode=ro")) + .await + .unwrap(); + let batch: i64 = sqlx::query_scalar("SELECT batch FROM migrations WHERE migration='only-in-wal'") + .fetch_one(&mut connection) + .await + .unwrap(); + assert_eq!(batch, 7); + connection.close().await.unwrap(); + // Preparing again changes nothing. + let original = std::fs::read(&path).unwrap(); + assert_eq!(prepare_sqlite_source(&path).await.unwrap()["source_mutated"], false); + assert_eq!(std::fs::read(&path).unwrap(), original); +} + +#[tokio::test] +async fn prepare_refuses_while_another_connection_holds_the_source() { + let (_directory, path) = php_layout().await; + let mut writer = SqliteConnection::connect_with( + &SqliteConnectOptions::new() + .filename(&path) + .journal_mode(SqliteJournalMode::Wal), + ) + .await + .unwrap(); + writer + .execute("INSERT INTO migrations(migration,batch) VALUES ('open',8)") + .await + .unwrap(); + reason(prepare_sqlite_source(&path).await, "upgrade_source_in_use"); + assert_eq!(journal_format(&path).await.unwrap(), WRITE_AHEAD_LOG); + writer.close().await.unwrap(); + let report = prepare_sqlite_source(&path).await.unwrap(); + assert_eq!(report["previous_journal_mode"], "wal", "{report}"); +} + +#[tokio::test] +async fn prepare_leaves_a_rollback_journal_source_byte_identical() { + let (_directory, path) = php_layout().await; + let original = std::fs::read(&path).unwrap(); + let report = prepare_sqlite_source(&path).await.unwrap(); + assert_eq!(report["previous_journal_mode"], "delete", "{report}"); + assert_eq!(report["source_mutated"], false); + assert_eq!(std::fs::read(&path).unwrap(), original); +} diff --git a/rust/src/runtime/worker_contracts.rs b/rust/src/runtime/worker_contracts.rs new file mode 100644 index 000000000..871249dc3 --- /dev/null +++ b/rust/src/runtime/worker_contracts.rs @@ -0,0 +1,93 @@ +//! Registration declarations become immutable admission snapshots. A missing +//! workflow entry stays unknown; a valid empty entry declares empty lists. +use super::{Result, refuse}; +use axum::http::StatusCode; +use serde_json::{Value, json}; +use std::collections::BTreeSet; + +fn invalid() -> super::RuntimeError { + refuse( + StatusCode::UNPROCESSABLE_ENTITY, + "invalid_worker_definition", + ) +} + +fn name(value: &Value) -> Result<&str> { + let text = value.as_str().ok_or_else(invalid)?; + let text = text.trim_matches([' ', '\t', '\n', '\r', '\0', '\u{b}']); + if text.is_empty() || text.chars().count() > 255 { + return Err(invalid()); + } + Ok(text) +} + +fn list<'a>(entry: &'a Value, field: &str) -> Result<&'a [Value]> { + match entry.get(field) { + None => Ok(&[]), + Some(Value::Array(values)) => Ok(values), + Some(_) => Err(invalid()), + } +} + +pub(super) fn declaration(entry: &Value) -> Result { + if !entry.is_object() && !entry.as_array().is_some_and(Vec::is_empty) { + return Err(invalid()); + } + let mut result = json!({}); + for (names, handlers) in [ + ("queries", "query_contracts"), + ("signals", "signal_contracts"), + ("updates", "update_contracts"), + ] { + let mut declared = BTreeSet::new(); + for value in list(entry, names)? { + declared.insert(name(value)?.to_owned()); + } + let mut seen = BTreeSet::new(); + let mut contracts = Vec::new(); + for handler in list(entry, handlers)? { + let handler_name = name(&handler["name"])?; + declared.insert(handler_name.to_owned()); + if seen.insert(handler_name.to_owned()) { + contracts.push( + json!({"name":handler_name,"parameters":handler.get("parameters") + .filter(|value| value.is_array()).cloned().unwrap_or_else(|| json!([]))}), + ); + } + } + result[names] = json!(declared); + result[handlers] = json!(contracts); + } + if entry.get("update_validators").is_some() { + let mut validators = BTreeSet::new(); + for value in list(entry, "update_validators")? { + validators.insert(name(value)?.to_owned()); + } + result["update_validators"] = json!(validators); + } + Ok(result) +} + +pub(super) fn normalize(value: &Value, supported: &Value) -> Result { + if value.is_null() || value.as_array().is_some_and(Vec::is_empty) { + return Ok(json!({})); + } + let entries = value.as_object().ok_or_else(invalid)?; + let mut result = json!({}); + for (workflow_type, entry) in entries { + // Validate even an unadvertised declaration before filtering it out. + // Otherwise malformed metadata could rotate an existing registration. + let entry = declaration(entry)?; + let workflow_type = workflow_type.trim_matches([' ', '\t', '\n', '\r', '\0', '\u{b}']); + if !workflow_type.is_empty() + && supported.as_array().is_some_and(|types| { + types + .iter() + .any(|value| value.as_str() == Some(workflow_type)) + }) + { + result[workflow_type] = entry; + } + } + Ok(result) +} diff --git a/rust/tests/execution.rs b/rust/tests/execution.rs index 19d45b684..bc43ee2b1 100644 --- a/rust/tests/execution.rs +++ b/rust/tests/execution.rs @@ -49,6 +49,9 @@ mod selected_run_signals; #[path = "execution/worker_incarnations.rs"] mod worker_incarnations; +#[path = "execution/worker_contracts.rs"] +mod worker_contracts; + #[path = "execution/worker_deregistration.rs"] mod worker_deregistration; @@ -61,6 +64,8 @@ mod version_markers; #[path = "execution/activity_deadlines.rs"] mod activity_deadlines; +#[path = "execution/lock_free_waiting.rs"] +mod lock_free_waiting; #[path = "execution/read_snapshots.rs"] mod read_snapshots; diff --git a/rust/tests/execution/cross_kind_poll.rs b/rust/tests/execution/cross_kind_poll.rs index 754af04ff..5986b0ce4 100644 --- a/rust/tests/execution/cross_kind_poll.rs +++ b/rust/tests/execution/cross_kind_poll.rs @@ -177,7 +177,7 @@ async fn scoped_hints_require_original_opt_in_and_a_new_ready_identity() { let result = held.await.unwrap(); assert_eq!(result.0, StatusCode::OK, "{scenario}: {}", result.1); assert_eq!( - result.1["poll_status"], "timeout", + result.1["poll_status"], "empty", "{scenario}: {}", result.1 ); @@ -216,7 +216,7 @@ async fn activity_admission_wakes_other_polls_but_same_kind_claim_wins() { json!({"lease_owner":"mixed","activity_attempt_id":activity["activity_attempt_id"],"result":envelope(Payload::Null)})).await.0,StatusCode::OK); let result = held.await.unwrap(); assert_eq!(result.0, StatusCode::OK, "{}", result.1); - assert_eq!(result.1["poll_status"], "task"); + assert_eq!(result.1["poll_status"], "leased"); assert!(result.1["task"].is_object()); runtime.close().await; database.remove().await; @@ -311,7 +311,7 @@ async fn zero_timeout_polls_keep_the_existing_empty_reply() { ) .await; assert_eq!(result.0, StatusCode::OK, "{}", result.1); - assert_eq!(result.1["poll_status"], "timeout"); + assert_eq!(result.1["poll_status"], "empty"); assert_eq!(result.1.get("task"), Some(&Value::Null)); } runtime.close().await; diff --git a/rust/tests/execution/lock_free_waiting.rs b/rust/tests/execution/lock_free_waiting.rs new file mode 100644 index 000000000..bd22759dd --- /dev/null +++ b/rust/tests/execution/lock_free_waiting.rs @@ -0,0 +1,123 @@ +//! A held long poll waits without the transition lock, so idle polls neither +//! wait behind transitions nor delay them. It still claims work, and refuses +//! a removed registration, committed by another server process. +use super::read_snapshots::hold_transition_lock; +use super::*; +use std::time::Duration; + +fn held_poll( + app: &Router, + worker: &str, + kind: &str, + seconds: u64, +) -> tokio::task::JoinHandle<(StatusCode, Value)> { + let app = app.clone(); + let path = format!("/api/worker/{kind}-tasks/poll"); + let body = json!({"worker_id":worker,"task_queue":"test","timeout_seconds":seconds}); + tokio::spawn(async move { request(&app, "POST", &path, body).await }) +} + +#[tokio::test] +async fn held_polls_reach_their_deadline_while_a_transition_holds_the_lock() { + let database = TestDatabase::new().await; + let runtime = database.open().await.unwrap(); + let app = router(runtime.clone()); + register(&app, "waiting-worker", json!(["echo"]), json!(["echo-activity"])).await; + query_worker(&app, "waiting-query-worker").await; + let polls = [ + held_poll(&app, "waiting-worker", "workflow", 2), + held_poll(&app, "waiting-worker", "activity", 2), + held_poll(&app, "waiting-query-worker", "query", 2), + ]; + // Each poll's first claim takes the lock; let them finish and wait. + tokio::time::sleep(Duration::from_millis(500)).await; + let writer = hold_transition_lock(&database).await; + for poll in polls { + // The polls end about 1.5 s from now. A probe that took the lock + // would wait for it, and the lock is held past this bound. + let (status, body) = tokio::time::timeout(Duration::from_secs(3), poll) + .await + .expect("an idle poll must reach its deadline without the transition lock") + .unwrap(); + assert_eq!(status, StatusCode::OK, "{body}"); + assert!(body["task"].is_null(), "{body}"); + } + writer.release().await; + runtime.close().await; + database.remove().await; +} + +#[tokio::test] +async fn held_polls_observe_work_and_removed_registrations_from_another_node() { + let database = TestDatabase::new().await; + let node_a = database.open().await.unwrap(); + let node_b = database.open().await.unwrap(); + let app_a = router(node_a.clone()); + let app_b = router(node_b.clone()); + register(&app_a, "observing-worker", json!(["echo"]), json!([])).await; + query_worker(&app_a, "observing-query-worker").await; + + // Node B's commits cannot wake node A's poll in process. + let waiting = held_poll(&app_a, "observing-worker", "workflow", 20); + tokio::time::sleep(Duration::from_millis(300)).await; + let started = start(&app_b, "observed-elsewhere").await; + let (status, claimed) = tokio::time::timeout(Duration::from_secs(5), waiting) + .await + .expect("a held poll claims work another node committed") + .unwrap(); + assert_eq!(status, StatusCode::OK, "{claimed}"); + assert_eq!(claimed["task"]["run_id"], started["run_id"], "{claimed}"); + let completed = request( + &app_a, + "POST", + &format!( + "/api/worker/workflow-tasks/{}/complete", + claimed["task"]["task_id"].as_str().unwrap() + ), + completion( + &claimed["task"], + json!([{"type":"complete_workflow","result":envelope(Payload::Null)}]), + ), + ) + .await; + assert_eq!(completed.0, StatusCode::OK, "{}", completed.1); + + let waiting = held_poll(&app_a, "observing-query-worker", "query", 5); + tokio::time::sleep(Duration::from_millis(300)).await; + let query = { + let app = app_b.clone(); + let path = format!( + "/api/workflows/observed-elsewhere/runs/{}/query/state", + started["run_id"].as_str().unwrap() + ); + let input = json!({"input":envelope(Payload::Array(vec![Payload::Long(1)]))}); + tokio::spawn(async move { request(&app, "POST", &path, input).await }) + }; + let (status, claimed) = tokio::time::timeout(Duration::from_secs(3), waiting) + .await + .expect("a held query poll claims a query another node admitted") + .unwrap(); + assert_eq!(status, StatusCode::OK, "{claimed}"); + assert_eq!(claimed["task"]["query_name"], "state", "{claimed}"); + query.abort(); + + let waiting = held_poll(&app_a, "observing-worker", "workflow", 20); + tokio::time::sleep(Duration::from_millis(300)).await; + let removed = request( + &app_b, + "DELETE", + "/api/worker/registrations/observing-worker", + Value::Null, + ) + .await; + assert_eq!(removed.0, StatusCode::OK, "{}", removed.1); + let (status, refused) = tokio::time::timeout(Duration::from_secs(5), waiting) + .await + .expect("a held poll refuses a registration another node removed") + .unwrap(); + assert_eq!(status, StatusCode::CONFLICT, "{refused}"); + assert_eq!(refused["reason"], "worker_not_registered", "{refused}"); + node_a.close().await; + node_b.close().await; + database.remove().await; +} diff --git a/rust/tests/execution/read_snapshots.rs b/rust/tests/execution/read_snapshots.rs index 8dcb0d814..2c0fdcfb9 100644 --- a/rust/tests/execution/read_snapshots.rs +++ b/rust/tests/execution/read_snapshots.rs @@ -3,7 +3,7 @@ use super::*; use sqlx::Connection; -enum Writer { +pub(super) enum Writer { Sqlite(sqlx::SqliteConnection), Postgres(sqlx::PgConnection), MySql(sqlx::MySqlConnection), @@ -11,7 +11,7 @@ enum Writer { // Takes the lock a native transition takes, on a connection outside the // runtime's pool, as a transition from another server process would. -async fn hold_transition_lock(database: &TestDatabase) -> Writer { +pub(super) async fn hold_transition_lock(database: &TestDatabase) -> Writer { match database { TestDatabase::Sqlite(dir) => { let mut connection = sqlx::SqliteConnection::connect(&format!( @@ -56,7 +56,7 @@ async fn hold_transition_lock(database: &TestDatabase) -> Writer { } impl Writer { - async fn release(self) { + pub(super) async fn release(self) { match self { Self::Sqlite(mut connection) => { sqlx::raw_sql("ROLLBACK") diff --git a/rust/tests/execution/worker_contracts.rs b/rust/tests/execution/worker_contracts.rs new file mode 100644 index 000000000..9dac8ad68 --- /dev/null +++ b/rust/tests/execution/worker_contracts.rs @@ -0,0 +1,173 @@ +use super::*; + +fn definition() -> Value { + json!({"worker_id":"contract-worker","task_queue":"test","runtime":"python", + "supported_workflow_types":["echo"],"supported_activity_types":[]}) +} + +async fn register_contract(app: &Router, declaration: Option) -> (StatusCode, Value) { + let mut body = definition(); + if let Some(declaration) = declaration { + body["workflow_command_contracts"] = declaration; + } + request(app, "POST", "/api/worker/register", body).await +} + +async fn snapshot(app: &Router, started: &Value) -> Value { + let history = run_history(app, &started["workflow_id"], &started["run_id"]).await; + history + .as_array() + .unwrap() + .iter() + .find(|event| event["event_type"] == "WorkflowStarted") + .unwrap()["payload"] + .clone() +} + +fn assert_empty(payload: &Value) { + for field in [ + "declared_queries", + "declared_signals", + "declared_updates", + "declared_query_contracts", + "declared_signal_contracts", + "declared_update_contracts", + ] { + assert_eq!(payload.get(field), Some(&json!([])), "{field}: {payload}"); + } +} + +#[tokio::test] +async fn explicit_empty_declarations_survive_cold_reads_and_later_registration() { + let database = TestDatabase::new().await; + let runtime = database.open().await.unwrap(); + let app = router(runtime.clone()); + let mut originals = Vec::new(); + for (index, declaration) in [ + json!({}), + json!([]), + json!({"queries":[],"signals":[], + "updates":[],"query_contracts":[],"signal_contracts":[],"update_contracts":[]}), + ] + .into_iter() + .enumerate() + { + let registered = register_contract(&app, Some(json!({"echo":declaration}))).await; + assert_eq!(registered.0, StatusCode::CREATED, "{}", registered.1); + let started = start(&app, &format!("empty-contract-{index}")).await; + let before = snapshot(&app, &started).await; + assert_empty(&before); + originals.push((started, before)); + } + assert_eq!( + register_contract( + &app, + Some(json!({"echo":{"query_contracts":[ + {"name":"state","parameters":[]}],"signals":["z","a","z"]}})) + ) + .await + .0, + StatusCode::CREATED + ); + let later = start(&app, "named-contract").await; + let named = snapshot(&app, &later).await; + assert_eq!(named["declared_queries"], json!(["state"])); + assert_eq!(named["declared_signals"], json!(["a", "z"])); + assert_eq!(register_contract(&app, None).await.0, StatusCode::CREATED); + let omitted = start(&app, "reregister-omitted").await; + assert_eq!( + snapshot(&app, &omitted).await["declared_queries"], + json!(["state"]) + ); + drop(app); + drop(runtime); + let cold = router(database.open().await.unwrap()); + for (started, before) in originals { + assert_eq!(snapshot(&cold, &started).await, before); + } + drop(cold); + database.remove().await; +} + +#[tokio::test] +async fn legacy_missing_entries_remain_unknown_instead_of_becoming_empty() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + for (index, declaration) in [ + None, + Some(Value::Null), + Some(json!({})), + Some(json!([])), + Some(json!({"other":{}})), + ] + .into_iter() + .enumerate() + { + let registered = register_contract(&app, declaration).await; + assert_eq!(registered.0, StatusCode::CREATED, "{}", registered.1); + let original = start(&app, &format!("legacy-contract-{index}")).await; + let before = snapshot(&app, &original).await; + assert!(before.get("declared_queries").is_none(), "{before}"); + assert_eq!( + register_contract(&app, Some(json!({"echo":{}}))).await.0, + StatusCode::CREATED + ); + assert_eq!(snapshot(&app, &original).await, before); + assert_eq!( + register_contract(&app, Some(json!({}))).await.0, + StatusCode::CREATED + ); + } + drop(app); + database.remove().await; +} + +#[tokio::test] +async fn invalid_declarations_refuse_before_registration_rotation_or_lease_release() { + let database = TestDatabase::new().await; + let app = router(database.open().await.unwrap()); + let accepted = register_contract(&app, Some(json!({"echo":{}}))).await; + assert_eq!(accepted.0, StatusCode::CREATED); + let started = start(&app, "contract-lease").await; + let task = poll(&app, "contract-worker", "workflow").await; + let before = snapshot(&app, &started).await; + for declaration in [ + Value::Null, + json!("bad"), + json!({"queries":null}), + json!({"queries":"state"}), + json!({"queries":[17]}), + json!({"signals":[" "]}), + json!({"updates":{"key":"set"}}), + json!({"update_validators":null}), + json!({"query_contracts":"bad"}), + json!({"signal_contracts":[null]}), + json!({"query_contracts":[{}]}), + json!({"update_contracts":[{"name":17}]}), + ] { + let refused = register_contract(&app, Some(json!({"echo":declaration}))).await; + assert_eq!(refused.0, StatusCode::UNPROCESSABLE_ENTITY, "{}", refused.1); + assert_eq!(snapshot(&app, &started).await, before); + } + let completed = finish_task( + &app, + &task, + json!([{"type":"complete_workflow", + "result":envelope(Payload::Long(9007199254740993))}]), + ) + .await; + assert_eq!(completed.0, StatusCode::OK, "{}", completed.1); + assert_eq!(completed.1["recorded"], true); + let retired = request_protocol( + &app, + "POST", + "/api/worker/registrations/contract-worker/deregister", + json!({"registration_token":accepted.1["registration_token"]}), + "1.20", + ) + .await; + assert_eq!(retired.0, StatusCode::OK, "{}", retired.1); + assert_eq!(retired.1["recovered_workflow_task_count"], 0); + drop(app); + database.remove().await; +} diff --git a/scripts/conformance/server-parity/admission-probe.php b/scripts/conformance/server-parity/admission-probe.php index 0e681a1db..337bd9564 100644 --- a/scripts/conformance/server-parity/admission-probe.php +++ b/scripts/conformance/server-parity/admission-probe.php @@ -8,6 +8,7 @@ use Workflow\V2\StartOptions; use Workflow\V2\WorkflowStub; use Workflow\WorkflowOptions; +require_once __DIR__.'/lock-pressure-middleware.php'; function parityAdmissionToken(string $auth): string { @@ -86,7 +87,7 @@ function finishHttpAdmission(Client $client, array $fixture, string $workflowId, throw $error; } finally { if (isset($definition['cleanup_auth'])) { - $cleanupClient = new Client($url, namespace: $namespace, token: parityAdmissionToken($definition['cleanup_auth'])); + $cleanupClient = parityClient($url, namespace: $namespace, token: parityAdmissionToken($definition['cleanup_auth'])); $state['cleanup_auth'] = $definition['cleanup_auth']; $state['cleanup'] = $cleanupClient->cancelWorkflow($peerId, $definition['cleanup_reason'], $peer->selectedRunId); } else { diff --git a/scripts/conformance/server-parity/cancellation-probe.php b/scripts/conformance/server-parity/cancellation-probe.php index 0292c6abc..98f31d054 100644 --- a/scripts/conformance/server-parity/cancellation-probe.php +++ b/scripts/conformance/server-parity/cancellation-probe.php @@ -17,6 +17,7 @@ use Workflow\V2\WorkflowStub; use Workflow\V2\StartOptions; use Workflow\WorkflowOptions; +require_once __DIR__.'/lock-pressure-middleware.php'; // Application callbacks use only the published APIs. These observations never // mutate engine rows or synthesize a claim, cancellation or terminal outcome. @@ -40,7 +41,7 @@ function cancellationReceipt(callable $submit): array function httpCancellationObservation(array $fixture, string $workflowId, string $namespace, string $queue, string $url): array { $polls = $completions = $activityPolls = $activityOutcomes = []; - $client = new Client($url, namespace: $namespace, token: getenv('DW_PARITY_TOKEN') ?: null, + $client = parityClient($url, namespace: $namespace, token: getenv('DW_PARITY_TOKEN') ?: null, transport: observedChildTransport($polls, $completions, $activityPolls, $activityOutcomes)); $phase = $fixture['immediate_cancellation']['phase']; $reason = $fixture['immediate_cancellation']['reason']; @@ -113,7 +114,7 @@ function httpCancellationObservation(array $fixture, string $workflowId, string $task['activity_attempt_id'], $task['lease_owner'], $fixture['input'])); } // A separate published Client reloads the same durable run and history. - $fresh = new Client($url, namespace: $namespace, token: getenv('DW_PARITY_TOKEN') ?: null); + $fresh = parityClient($url, namespace: $namespace, token: getenv('DW_PARITY_TOKEN') ?: null); $inspectionWorker = $workflowId.':inspection'; $fresh->registerWorker($inspectionWorker, $queue, [$fixture['workflow_type']], ['parity.v1.cancel_activity']); $remaining = ['workflow' => $fresh->pollWorkflowTask($inspectionWorker, $queue, 0), diff --git a/scripts/conformance/server-parity/child-cancellation-probe.php b/scripts/conformance/server-parity/child-cancellation-probe.php index b548be1bb..7b8ee4449 100644 --- a/scripts/conformance/server-parity/child-cancellation-probe.php +++ b/scripts/conformance/server-parity/child-cancellation-probe.php @@ -5,6 +5,7 @@ use DurableWorkflow\Client; use Workflow\V2\Models\WorkflowRun; use Workflow\V2\WorkflowStub; +require_once __DIR__.'/lock-pressure-middleware.php'; final class ChildCancellationProbeState { @@ -54,7 +55,7 @@ function finishHttpChildCancellation(Client $client, string $parentId, string $p $state['terminal_duplicate'] = cancellationReceipt(static fn (): array => $client->cancelWorkflow($id, 'post-parent-completion replacement', $runId)); $state['child_history_after_terminal_duplicate'] = httpHistory($client, $id, $runId); $state['parent_history_after_terminal_duplicate'] = httpHistory($client, $parentId, $parentRunId); - $fresh = new Client($url, namespace: $namespace, token: getenv('DW_PARITY_TOKEN') ?: null); + $fresh = parityClient($url, namespace: $namespace, token: getenv('DW_PARITY_TOKEN') ?: null); $state['fresh_child'] = $fresh->describeWorkflow($id, $runId)->raw; $state['fresh_child_history'] = httpHistory($fresh, $id, $runId); $state['fresh_parent'] = $fresh->describeWorkflow($parentId, $parentRunId)->raw; diff --git a/scripts/conformance/server-parity/cooperative-probe.php b/scripts/conformance/server-parity/cooperative-probe.php index cb2e19d83..3a52e3ee4 100644 --- a/scripts/conformance/server-parity/cooperative-probe.php +++ b/scripts/conformance/server-parity/cooperative-probe.php @@ -21,6 +21,7 @@ use Workflow\V2\WorkflowStub; use Workflow\V2\StartOptions; use Workflow\WorkflowOptions; +require_once __DIR__.'/lock-pressure-middleware.php'; function cooperativeObservedTransport(array &$traffic): Psr18Transport { @@ -42,6 +43,7 @@ function cooperativeObservedTransport(array &$traffic): Psr18Transport }); }; }); + $stack->push(LockPressurePollRetries::middleware(), 'lock_pressure_poll_retry'); return new Psr18Transport(client: new GuzzleClient(['http_errors' => false, 'handler' => $stack])); } @@ -75,7 +77,7 @@ function httpCooperativeObservation(array $fixture, string $workflowId, string $ { $traffic = []; $diagnostics = []; - $client = new Client($url, namespace: $namespace, token: getenv('DW_PARITY_TOKEN') ?: null, + $client = parityClient($url, namespace: $namespace, token: getenv('DW_PARITY_TOKEN') ?: null, transport: cooperativeObservedTransport($traffic), workerProtocolVersion: '1.20'); $phase = $fixture['cooperative_cancellation']['phase']; $reason = $fixture['cooperative_cancellation']['reason']; @@ -150,7 +152,7 @@ static function (string $event, array $details) use (&$handle, &$diagnostics, $c } catch (WorkflowCancelled $error) { $outcome = ['type' => $error::class, 'message' => $error->getMessage()]; } - $fresh = new Client($url, namespace: $namespace, token: getenv('DW_PARITY_TOKEN') ?: null, workerProtocolVersion: '1.20'); + $fresh = parityClient($url, namespace: $namespace, token: getenv('DW_PARITY_TOKEN') ?: null, workerProtocolVersion: '1.20'); return ['execution' => $execution->raw, 'workflow_id' => $execution->workflowId, 'run_id' => $execution->runId, 'workflow_type' => $execution->workflowType, 'namespace' => $execution->namespace, 'task_queue' => $execution->taskQueue, diff --git a/scripts/conformance/server-parity/cooperative-restart.php b/scripts/conformance/server-parity/cooperative-restart.php index 6fd9950b3..2e4b81d3b 100644 --- a/scripts/conformance/server-parity/cooperative-restart.php +++ b/scripts/conformance/server-parity/cooperative-restart.php @@ -8,6 +8,7 @@ use DurableWorkflow\Worker\CancellationDelivery; require __DIR__.'/vendor/autoload.php'; +require_once __DIR__.'/lock-pressure-middleware.php'; require __DIR__.'/observation-values.php'; require __DIR__.'/cooperative-probe.php'; @@ -56,7 +57,7 @@ function cooperativeEvent(array $events, string $type): array $workflowId = 'root-restart-'.$outcome; $queue = 'root-restart-'.$outcome; $traffic = []; - $client = new Client($url, namespace: 'default', token: getenv('DW_PARITY_TOKEN') ?: null, + $client = parityClient($url, namespace: 'default', token: getenv('DW_PARITY_TOKEN') ?: null, transport: cooperativeObservedTransport($traffic), workerProtocolVersion: '1.20'); $worker = $handle = $accepted = null; $guard = microtime(true) + 20; @@ -118,7 +119,7 @@ static function (string $event) use (&$handle, $client, $fixture, $workflowId, $ foreach (['deadline_expired', 'completed'] as $outcome) { $receipt = $receipts[$outcome]; $traffic = []; - $client = new Client($url, namespace: 'default', token: getenv('DW_PARITY_TOKEN') ?: null, + $client = parityClient($url, namespace: 'default', token: getenv('DW_PARITY_TOKEN') ?: null, transport: cooperativeObservedTransport($traffic), workerProtocolVersion: '1.20'); $handle = $client->workflowHandle($receipt['workflow_id'], $receipt['run_id']); $before = httpHistory($client, $receipt['workflow_id'], $receipt['run_id']); @@ -188,7 +189,7 @@ static function () use (&$worker, $handle, $guard): float { } $duplicate = $client->requestWorkflowCancellation($receipt['workflow_id'], 'post-terminal replacement', 3600, $receipt['run_id']); assertCooperativeRecovery($duplicate['duplicate'] === true && $duplicate['cancellation_request']['cleanup_deadline_at'] === $receipt['context']['cleanup_deadline_at'], 'Terminal duplicate changed the original budget.'); - $fresh = new Client($url, namespace: 'default', token: getenv('DW_PARITY_TOKEN') ?: null, workerProtocolVersion: '1.20'); + $fresh = parityClient($url, namespace: 'default', token: getenv('DW_PARITY_TOKEN') ?: null, workerProtocolVersion: '1.20'); assertCooperativeRecovery(sameCooperativeValue($events, httpHistory($fresh, $receipt['workflow_id'], $receipt['run_id'])), 'Late calls or fresh reads changed terminal history.'); try { $fresh->workflowHandle($receipt['workflow_id'], $receipt['run_id'])->resultOfSelectedRun(1); diff --git a/scripts/conformance/server-parity/corrected-markers/rust/src/lock_pressure.rs b/scripts/conformance/server-parity/corrected-markers/rust/src/lock_pressure.rs new file mode 100644 index 000000000..5f9f9f9b9 --- /dev/null +++ b/scripts/conformance/server-parity/corrected-markers/rust/src/lock_pressure.rs @@ -0,0 +1,94 @@ +//! The published PHP SQLite reference can refuse a worker operation with its +//! documented HTTP 503 `backend_lock_pressure` reply and `retryable: true` +//! after its transaction failed, so nothing was committed. Only that refusal is +//! retried, within 15 retries and 30 seconds of waiting, as the PHP probes' +//! transport does (`../lock-pressure-retry.php`). Every other outcome is +//! returned unchanged. +use durable_workflow::{Error, Result}; +use serde_json::Value; +use std::{future::Future, time::Duration}; + +const MAX_RETRIES: u32 = 15; +const MAX_WAIT: Duration = Duration::from_secs(30); + +fn delay_for(status: u16, body: &str, retries: u32, waited: Duration) -> Option { + if status != 503 || retries >= MAX_RETRIES { + return None; + } + let body: Value = serde_json::from_str(body).ok()?; + if body.get("reason")?.as_str()? != "backend_lock_pressure" + || body.get("retryable") != Some(&Value::Bool(true)) + { + return None; + } + let seconds = match body.get("retry_after_seconds") { + None => 1.0, + Some(value) => value.as_f64()?, + }; + let delay = Duration::from_secs_f64(seconds.clamp(0.1, 5.0)); + (waited + delay <= MAX_WAIT).then_some(delay) +} + +/// Runs `call`, repeating it only after the documented lock-pressure refusal. +pub async fn retrying(what: &str, mut call: F) -> Result +where + F: FnMut() -> Fut, + Fut: Future>, +{ + let (mut retries, mut waited) = (0, Duration::ZERO); + loop { + let error = match call().await { + Err(error) => error, + done => return done, + }; + let delay = match &error { + Error::Http { status, body } => delay_for(status.as_u16(), body, retries, waited), + _ => None, + }; + let Some(delay) = delay else { + return Err(error); + }; + eprintln!("{what}: retrying backend_lock_pressure refusal after {delay:?}"); + tokio::time::sleep(delay).await; + waited += delay; + retries += 1; + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const REFUSAL: &str = r#"{"reason":"backend_lock_pressure","operation":"destroy","outcome":"deferred","retryable":true,"retry_after_seconds":1}"#; + + #[test] + fn documented_refusal_waits_for_the_advertised_interval() { + assert_eq!(delay_for(503, REFUSAL, 0, Duration::ZERO), Some(Duration::from_secs(1))); + let long = REFUSAL.replace(r#""retry_after_seconds":1"#, r#""retry_after_seconds":60"#); + assert_eq!(delay_for(503, &long, 0, Duration::ZERO), Some(Duration::from_secs(5))); + let absent = REFUSAL.replace(r#","retry_after_seconds":1"#, ""); + assert_eq!(delay_for(503, &absent, 0, Duration::ZERO), Some(Duration::from_secs(1))); + } + + #[test] + fn other_statuses_reasons_and_bodies_are_returned() { + for (status, body) in [ + (409, REFUSAL.to_owned()), + (500, REFUSAL.to_owned()), + (503, REFUSAL.replace("backend_lock_pressure", "backend_unavailable")), + (503, REFUSAL.replace(r#""retryable":true"#, r#""retryable":false"#)), + (503, REFUSAL.replace(r#","retryable":true"#, "")), + (503, REFUSAL.replace(r#""retry_after_seconds":1"#, r#""retry_after_seconds":"soon""#)), + (503, "not json".to_owned()), + ] { + assert_eq!(delay_for(status, &body, 0, Duration::ZERO), None, "{status} {body}"); + } + } + + #[test] + fn retries_stop_at_the_attempt_and_wait_budgets() { + assert_eq!(delay_for(503, REFUSAL, 15, Duration::ZERO), None); + assert_eq!(delay_for(503, REFUSAL, 2, Duration::from_millis(29_500)), None); + assert_eq!(delay_for(503, REFUSAL, 14, Duration::from_secs(29)), Some(Duration::from_secs(1))); + } +} diff --git a/scripts/conformance/server-parity/corrected-markers/rust/src/main.rs b/scripts/conformance/server-parity/corrected-markers/rust/src/main.rs index c5f7bdb7d..3a1d863c7 100644 --- a/scripts/conformance/server-parity/corrected-markers/rust/src/main.rs +++ b/scripts/conformance/server-parity/corrected-markers/rust/src/main.rs @@ -11,6 +11,7 @@ use durable_workflow::{ use serde_json::{json, Value}; mod legacy_range; +mod lock_pressure; mod memo_author; const WORKFLOW: &str = "parity.v1.corrected_markers"; @@ -221,7 +222,10 @@ async fn main() -> Result<()> { } }) .await; - let deregistered = client.deregister_worker_registration(&worker_id).await; + let deregistered = lock_pressure::retrying("deregister", || { + client.deregister_worker_registration(&worker_id) + }) + .await; let result = match outcome { Ok(result) => result, Err(_) => Err(Error::WorkerLoop( diff --git a/scripts/conformance/server-parity/corrected-markers/rust/src/memo_author.rs b/scripts/conformance/server-parity/corrected-markers/rust/src/memo_author.rs index 8520ac901..d2c4c04c2 100644 --- a/scripts/conformance/server-parity/corrected-markers/rust/src/memo_author.rs +++ b/scripts/conformance/server-parity/corrected-markers/rust/src/memo_author.rs @@ -90,7 +90,10 @@ pub async fn run(args: &[String]) -> Result<()> { }) .await .map_err(|_| Error::WorkerLoop("Original memo phase observation expired".into()))?; - let retired = client.deregister_worker_registration(&worker_id).await; + let retired = super::lock_pressure::retrying("deregister", || { + client.deregister_worker_registration(&worker_id) + }) + .await; outcome?; retired?; println!( diff --git a/scripts/conformance/server-parity/cross-kind-poll-contract.mjs b/scripts/conformance/server-parity/cross-kind-poll-contract.mjs index 5dceb69a9..adf89f2d3 100644 --- a/scripts/conformance/server-parity/cross-kind-poll-contract.mjs +++ b/scripts/conformance/server-parity/cross-kind-poll-contract.mjs @@ -44,7 +44,7 @@ export function checkCrossKindPollWake(record, profile) { } const sweep = following[sweepIndex]; assert.equal(sweep.http, 200); - assert(['task', 'leased'].includes(sweep.poll_status)); + assert.equal(sweep.poll_status, 'leased', 'a claim reports leased, as the worker protocol requires'); assert.equal(sweep.task_is_null, false); assert.equal(typeof sweep.task_id, 'string'); assert(sweep.task_id.length > 0, 'the ordinary poll separately grants a task'); diff --git a/scripts/conformance/server-parity/lock-pressure-middleware.php b/scripts/conformance/server-parity/lock-pressure-middleware.php new file mode 100644 index 000000000..1ced25c8d --- /dev/null +++ b/scripts/conformance/server-parity/lock-pressure-middleware.php @@ -0,0 +1,97 @@ + */ + public static array $retries = []; + private static bool $reporting = false; + + public static function middleware(): callable + { + if (!self::$reporting) { + self::$reporting = true; + register_shutdown_function(static function (): void { + if (self::$retries !== []) { + fwrite(STDERR, json_encode(['lock_pressure_poll_retries' => self::$retries], JSON_THROW_ON_ERROR).PHP_EOL); + } + }); + } + + return static fn (callable $handler): callable => static function (RequestInterface $request, array $options) use ($handler) { + $started = microtime(true); + $budget = (float) ($options['timeout'] ?? 0); + $waited = 0.0; + $count = 0; + while (true) { + try { + $response = $handler($request, $options)->wait(); + } catch (\Throwable $error) { + return Create::rejectionFor($error); + } + // A request body that cannot be replayed exactly is never retried. + if (!$response instanceof ResponseInterface || $response->getStatusCode() !== 503 + || (!$request->getBody()->isSeekable() && $request->getBody()->getSize() !== 0)) { + return Create::promiseFor($response); + } + $text = (string) $response->getBody(); + $response = $response->withBody(Utils::streamFor($text)); + $decoded = json_decode($text, true); + $delay = lockPressureOperationRetryDelay($request->getMethod(), (string) $request->getUri(), 503, + is_array($decoded) ? $decoded : null, $count, $waited); + if ($delay === null || ($budget > 0 && microtime(true) - $started + $delay >= $budget)) { + return Create::promiseFor($response); + } + self::$retries[] = ['path' => $request->getUri()->getPath(), 'retry_after_seconds' => $delay]; + usleep((int) round($delay * 1_000_000)); + $waited += $delay; + $count++; + if ($request->getBody()->isSeekable()) { + $request->getBody()->rewind(); + } + } + }; + } + + /** The SDK's default transport, with this retry innermost. */ + public static function transport(): Psr18Transport + { + $stack = HandlerStack::create(); + $stack->push(self::middleware(), 'lock_pressure_poll_retry'); + + return new Psr18Transport(client: new GuzzleClient(['http_errors' => false, 'handler' => $stack])); + } +} + +/** An SDK client for a parity probe. A caller-supplied transport is used unchanged. */ +function parityClient(mixed ...$arguments): Client +{ + $arguments['transport'] ??= LockPressurePollRetries::transport(); + + return new Client(...$arguments); +} diff --git a/scripts/conformance/server-parity/lock-pressure-retry.php b/scripts/conformance/server-parity/lock-pressure-retry.php index fa7c28af4..112299d78 100644 --- a/scripts/conformance/server-parity/lock-pressure-retry.php +++ b/scripts/conformance/server-parity/lock-pressure-retry.php @@ -38,3 +38,45 @@ function lockPressurePollRetryDelay( return $waitedSeconds + $delay <= $maxWaitSeconds ? $delay : null; } + +/** + * Decide whether any other request refused for backend lock pressure may be + * retried. + * + * Server returns this refusal as HTTP 503 with `reason: backend_lock_pressure` + * and `retryable: true` only after its transaction failed with a concurrency + * error, so nothing was committed. Worker mutations remain fenced by their + * task and attempt identifiers (`outcome: deferred`), so repeating the request + * cannot record a second outcome. Polls keep lockPressurePollRetryDelay's + * stricter check that no task was handed out. + * + * Returns the delay in seconds before the next attempt, or null to return the + * refusal to the caller unchanged. + */ +function lockPressureOperationRetryDelay( + string $method, + string $uri, + ?int $status, + mixed $response, + int $retriesSoFar, + float $waitedSeconds, + int $maxRetries = 15, + float $maxWaitSeconds = 30.0, +): ?float { + $path = parse_url($uri, PHP_URL_PATH); + if (! is_string($path) || str_ends_with($path, '/poll')) { + return lockPressurePollRetryDelay($method, $uri, $status, $response, $retriesSoFar, $waitedSeconds, + $maxRetries, $maxWaitSeconds); + } + if ($status !== 503 || ! is_array($response) || ($response['reason'] ?? null) !== 'backend_lock_pressure' + || ($response['retryable'] ?? null) !== true || $retriesSoFar >= $maxRetries) { + return null; + } + $retryAfter = $response['retry_after_seconds'] ?? 1; + if (! is_int($retryAfter) && ! is_float($retryAfter)) { + return null; + } + $delay = min(max((float) $retryAfter, 0.1), 5.0); + + return $waitedSeconds + $delay <= $maxWaitSeconds ? $delay : null; +} diff --git a/scripts/conformance/server-parity/namespace-probe.php b/scripts/conformance/server-parity/namespace-probe.php index 7422ec260..f6d1e0fa1 100644 --- a/scripts/conformance/server-parity/namespace-probe.php +++ b/scripts/conformance/server-parity/namespace-probe.php @@ -20,6 +20,7 @@ use Workflow\V2\WorkflowStub; use Workflow\WorkflowOptions; use Workflow\Serializers\Serializer; +require_once __DIR__.'/lock-pressure-middleware.php'; function namespaceReceipt(string $id, callable $request): array { @@ -66,6 +67,7 @@ function namespaceTransport(array &$receipts, ?Closure $beforeCompletion = null) }); }; }); + $stack->push(LockPressurePollRetries::middleware(), 'lock_pressure_poll_retry'); return new Psr18Transport(client: new NamespaceHttp(['http_errors' => false, 'handler' => $stack, 'connect_timeout' => 1, 'timeout' => 5])); @@ -98,7 +100,7 @@ function finishHttpNamespaces(Client $admin, array $fixture, string $workflowId, 'before' => $before, 'after' => namespaceHttpRun($clients[0], $ids[0], $handles[0]->selectedRunId)]; }; foreach ($names as $index => $name) { - $clients[] = new Client($url, namespace: $name, token: getenv('DW_PARITY_TOKEN') ?: null, + $clients[] = parityClient($url, namespace: $name, token: getenv('DW_PARITY_TOKEN') ?: null, transport: namespaceTransport($state['worker_receipts'][$index], $index === 0 ? $beforeCompletion : null)); $state['registrations'][] = $clients[$index]->registerWorker($state['worker_id'], $queue, [$definition['peer_workflow_type']], ['parity.v1.echo_activity']); diff --git a/scripts/conformance/server-parity/patch-deployment-probe.php b/scripts/conformance/server-parity/patch-deployment-probe.php index 032944130..030b34773 100644 --- a/scripts/conformance/server-parity/patch-deployment-probe.php +++ b/scripts/conformance/server-parity/patch-deployment-probe.php @@ -20,6 +20,7 @@ use Workflow\V2\StartOptions; use Workflow\V2\WorkflowStub; use Workflow\WorkflowOptions; +require_once __DIR__.'/lock-pressure-middleware.php'; final class PatchDeploymentState { @@ -126,7 +127,8 @@ function patchHttpPhase(array $fixture, array $options): array }); }; }); - $client = new Client($options['url'], namespace: 'default', token: getenv('DW_PARITY_TOKEN') ?: null, + $stack->push(LockPressurePollRetries::middleware(), 'lock_pressure_poll_retry'); + $client = parityClient($options['url'], namespace: 'default', token: getenv('DW_PARITY_TOKEN') ?: null, transport: new Psr18Transport(client: new GuzzleClient(['http_errors' => false, 'handler' => $stack]))); if ($phase === 'replacement') { $handle = $client->workflowHandle($workflowId, $options['run-id']); @@ -279,7 +281,7 @@ function patchPublishedSdkPhase(array $fixture, array $options): array if ($result['language'] !== $consumer['language'] || $result['sdk_version'] !== $consumer['version']) { throw new RuntimeException('Actual published worker differs from the selected consumer.'); } - $client = new Client($options['url'], namespace: 'default', token: getenv('DW_PARITY_TOKEN') ?: null); + $client = parityClient($options['url'], namespace: 'default', token: getenv('DW_PARITY_TOKEN') ?: null); $runId = $original ? ($result['run_id'] ?? null) : $options['run-id']; if (! is_string($runId) || $runId === '') { throw new RuntimeException('Actual published author returned no original run identity.'); diff --git a/scripts/conformance/server-parity/probe.php b/scripts/conformance/server-parity/probe.php index d63bc0f8d..060d26745 100644 --- a/scripts/conformance/server-parity/probe.php +++ b/scripts/conformance/server-parity/probe.php @@ -64,6 +64,7 @@ require __DIR__.'/observation-values.php'; require __DIR__.'/probe-failure.php'; +require_once __DIR__.'/lock-pressure-middleware.php'; function observedChildTransport(array &$polls, array &$completions, array &$activityPolls, array &$activityOutcomes, ?Closure $afterCompletion = null, ?array &$scheduleReceipts = null): Psr18Transport { @@ -106,6 +107,7 @@ function observedChildTransport(array &$polls, array &$completions, array &$acti }); }; }); + $stack->push(LockPressurePollRetries::middleware(), 'lock_pressure_poll_retry'); return new Psr18Transport(client: new GuzzleClient(['http_errors' => false, 'handler' => $stack])); } @@ -114,7 +116,7 @@ function observedChildTransport(array &$polls, array &$completions, array &$acti // client in an independent PHP process so this probe's worker keeps polling. if ($mode === 'query') { try { - $client = new Client($options['url'], namespace: $namespace, token: getenv('DW_PARITY_TOKEN') ?: null); + $client = parityClient($options['url'], namespace: $namespace, token: getenv('DW_PARITY_TOKEN') ?: null); $result = $client->queryWorkflow($workflowId, $fixture['query_name'], $fixture['query_arguments'], $options['run-id']); echo json_encode(['result' => $result, 'typed_result' => typedValue($result)], JSON_THROW_ON_ERROR | JSON_UNESCAPED_UNICODE)."\n"; exit(0); @@ -142,9 +144,9 @@ function httpObservation(array $fixture, string $workflowId, string $namespace, } : null; $transport = isset($fixture['child_count']) || isset($fixture['child_cancellation']) || isset($fixture['retry_policy']) || isset($fixture['schedule']) ? observedChildTransport($workflowPolls, $workflowCompletions, $activityPolls, $activityOutcomes, $afterCompletion, $scheduleReceipts) : null; - $client = new Client($url, namespace: $namespace, transport: $transport, token: getenv('DW_PARITY_TOKEN') ?: null); + $client = parityClient($url, namespace: $namespace, transport: $transport, token: getenv('DW_PARITY_TOKEN') ?: null); $workerClient = ($fixture['auth_profile'] ?? null) === 'role_tokens' - ? new Client($url, namespace: $namespace, token: parityAdmissionToken('worker')) : $client; + ? parityClient($url, namespace: $namespace, token: parityAdmissionToken('worker')) : $client; $handle = null; $deliveries = []; $queries = []; diff --git a/scripts/conformance/server-parity/restart.php b/scripts/conformance/server-parity/restart.php index a72f94979..5ded7dc5a 100644 --- a/scripts/conformance/server-parity/restart.php +++ b/scripts/conformance/server-parity/restart.php @@ -53,6 +53,13 @@ function restartControlHistory(array $events): array 'event_type' => $event['event_type'], 'timestamp' => $event['recorded_at'], 'payload' => $event['payload']], $events); } +// Public history lifts each command's principal onto its event; worker task +// history has no principal. Compare the fields both views share. +function restartWithoutPrincipal(array $events): array +{ + return array_map(static fn (array $event): array => array_diff_key($event, ['principal' => true]), $events); +} + function restartSignalWorkflow(WorkflowContext $context, array $input): array { $context->waitCondition(fn (): bool => count($context->signals('payload')) > 0, 'restart.payload'); @@ -336,7 +343,7 @@ function restartTerminalWorkflow(WorkflowContext $context, array $input): array $terminalClaim = $client->pollWorkflowTask('restart-terminal-new', 'restart-terminal-v1', 0); assertRestart(is_array($terminalClaim) && $terminalClaim['task_id'] === $terminal['failure_receipt']['next_task_id'] && $terminalClaim['run_id'] === $terminal['run_id'] - && sameRestartValue(restartControlHistory($terminalClaim['history_events']), $terminal['history']) + && sameRestartValue(restartControlHistory($terminalClaim['history_events']), restartWithoutPrincipal($terminal['history'])) && $terminalClaim['history_events'][4]['workflow_task_id'] === $terminalActivity['task_id'], 'Recovered terminal resumption lost original task/run/history.'); $terminalReplay = (new Replayer($client->payloadCodec()))->replay(restartTerminalWorkflow(...), $terminalClaim['history_events'], $client->payloadCodec()->decodeEnvelope($terminalClaim['arguments']), 'restart-terminal-v1', $terminalClaim); diff --git a/scripts/conformance/server-parity/rust-test-failures.sh b/scripts/conformance/server-parity/rust-test-failures.sh new file mode 100755 index 000000000..6e83f9e9c --- /dev/null +++ b/scripts/conformance/server-parity/rust-test-failures.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# Print the failures in this job's Rust test output, so a failed test step is +# diagnosable from its own log. Reads parity-evidence; changes nothing. +set -uo pipefail +for output in parity-evidence/*tests*.txt; do + [[ -f $output ]] && grep -q '^test result: FAILED' "$output" || continue + echo "::group::Rust test failures in ${output#parity-evidence/}" + grep -E -A3 'panicked at' "$output" | cut -c1-600 | head -n 120 + # The closing summary names every failed test. + sed -n '/^failures:$/,$p' "$output" | grep -E '^failures:$|^ [A-Za-z0-9_:]+$|^test result:' | tail -n 60 + echo "::endgroup::" +done +exit 0 diff --git a/scripts/conformance/server-parity/sdk-direction-probe.php b/scripts/conformance/server-parity/sdk-direction-probe.php index b7fa2df5d..1a582e4a8 100644 --- a/scripts/conformance/server-parity/sdk-direction-probe.php +++ b/scripts/conformance/server-parity/sdk-direction-probe.php @@ -5,6 +5,7 @@ use DurableWorkflow\Client; use DurableWorkflow\Worker; use DurableWorkflow\Worker\WorkflowContext; +require_once __DIR__.'/lock-pressure-middleware.php'; function sdkDirectionResponseSummary(mixed $response, bool $workflowHistory = false): mixed { @@ -73,7 +74,7 @@ function sdkDirectionHttpObservation(array $fixture, array $options, ?Closure $a $spec = $fixture[$directionKey]; $childWorkflowId = $childRunId = ''; $namespace = $fixture['namespace'] ?? 'default'; - $client = new Client($options['url'], namespace: $namespace, token: getenv('DW_PARITY_TOKEN') ?: null); + $client = parityClient($options['url'], namespace: $namespace, token: getenv('DW_PARITY_TOKEN') ?: null); $deadline = $scenarioDeadline ?? microtime(true) + 35; $observerStart = null; try { @@ -81,7 +82,7 @@ function sdkDirectionHttpObservation(array $fixture, array $options, ?Closure $a if (! preg_match('#^http://127\\.0\\.0\\.1:[0-9]+$#', $url)) { throw new RuntimeException('SDK observer did not bind its local port.'); } - $startClient = $signalDirection ? new Client($url, namespace: $namespace, token: getenv('DW_PARITY_TOKEN') ?: null) : null; + $startClient = $signalDirection ? parityClient($url, namespace: $namespace, token: getenv('DW_PARITY_TOKEN') ?: null) : null; foreach ($spec['phase_order'] as $phase) { $language = $signalDirection ? $spec[in_array($phase, ['signal-1', 'signal-2'], true) ? 'client' : 'workflow'] : ($timerAuthor ? $spec['language'] : ($childDirection ? $spec[$phase === 'child' ? 'child' : 'parent'] diff --git a/scripts/conformance/server-parity/sdk-namespace-signal-probe.php b/scripts/conformance/server-parity/sdk-namespace-signal-probe.php index 082f7364a..1f7e411d5 100644 --- a/scripts/conformance/server-parity/sdk-namespace-signal-probe.php +++ b/scripts/conformance/server-parity/sdk-namespace-signal-probe.php @@ -11,6 +11,7 @@ use Workflow\V2\StartOptions; use Workflow\V2\WorkflowStub; use Workflow\WorkflowOptions; +require_once __DIR__.'/lock-pressure-middleware.php'; function namespaceSignalRunFixture(array $fixture, bool $peer): array { @@ -158,7 +159,7 @@ function namespaceSignalHttpObservation(array $fixture, array $options): array throw new RuntimeException('Namespace control observer did not bind its local port.'); } $checkpoint('client_setup'); - $admin = new Client($url, namespace: 'default', token: getenv('DW_PARITY_TOKEN') ?: null); + $admin = parityClient($url, namespace: 'default', token: getenv('DW_PARITY_TOKEN') ?: null); foreach ($fixtures as $index => $runFixture) { try { $checkpoint('namespace_describe', $runFixture['namespace']); @@ -175,7 +176,7 @@ function namespaceSignalHttpObservation(array $fixture, array $options): array $state['administration'][] = ['created' => $created, 'namespace' => $namespace->raw]; $paths[$index] = tempnam(sys_get_temp_dir(), 'parity-namespace-fixture-'); file_put_contents($paths[$index], json_encode($runFixture, JSON_THROW_ON_ERROR | JSON_UNESCAPED_UNICODE)); - $clients[$index] = new Client($options['url'], namespace: $runFixture['namespace'], token: getenv('DW_PARITY_TOKEN') ?: null); + $clients[$index] = parityClient($options['url'], namespace: $runFixture['namespace'], token: getenv('DW_PARITY_TOKEN') ?: null); $start = microtime(true); $deadline = $start + 35; $state['run_budgets'][$index] = ['started_at_unix' => $start, 'deadline_at_unix' => $deadline]; diff --git a/scripts/conformance/server-parity/sdk-query-direction-probe.php b/scripts/conformance/server-parity/sdk-query-direction-probe.php index 18dce9429..1beb8c232 100644 --- a/scripts/conformance/server-parity/sdk-query-direction-probe.php +++ b/scripts/conformance/server-parity/sdk-query-direction-probe.php @@ -4,6 +4,7 @@ use Composer\InstalledVersions; use DurableWorkflow\Client; +require_once __DIR__.'/lock-pressure-middleware.php'; function sdkQueryActorOpen(string $language, string $phase, array $fixture, array $options, string $runId, float $deadline): array { @@ -242,7 +243,7 @@ function sdkQueryDirectionHttpObservation(array $fixture, array $options): array throw new RuntimeException('SDK query direction requires its evidence directory.'); } $spec = $fixture['sdk_query_direction']; - $client = new Client($options['url'], namespace: 'default', token: getenv('DW_PARITY_TOKEN') ?: null); + $client = parityClient($options['url'], namespace: 'default', token: getenv('DW_PARITY_TOKEN') ?: null); $deadline = microtime(true) + 35; $runId = ''; $phases = $actors = []; @@ -262,7 +263,7 @@ function sdkQueryDirectionHttpObservation(array $fixture, array $options): array if ($request['method'] === 'POST' && $request['path'] === '/api/worker/register' && $request['status'] === 201 && ($request['response']['registered'] ?? null) === true && ($request['request']['worker_id'] ?? null) === $options['workflow-id'].':original') { - $startClient = new Client($author['url'], namespace: 'default', token: getenv('DW_PARITY_TOKEN') ?: null); + $startClient = parityClient($author['url'], namespace: 'default', token: getenv('DW_PARITY_TOKEN') ?: null); $runId = $startClient->startWorkflow($fixture['workflow_type'], $options['workflow-id'], $spec['task_queue'], [$fixture['input'], $fixture['signal_count']])->selectedRunId; $observerStart = ['role' => 'observer', 'language' => 'php', 'pid' => getmypid(), diff --git a/scripts/conformance/server-parity/sdk-selected-run-signal-probe.php b/scripts/conformance/server-parity/sdk-selected-run-signal-probe.php index a97a41a0f..cd11ce1c7 100644 --- a/scripts/conformance/server-parity/sdk-selected-run-signal-probe.php +++ b/scripts/conformance/server-parity/sdk-selected-run-signal-probe.php @@ -10,6 +10,7 @@ use Workflow\V2\StartOptions; use Workflow\V2\WorkflowStub; use Workflow\WorkflowOptions; +require_once __DIR__.'/lock-pressure-middleware.php'; require __DIR__.'/selected-run-signal-state.php'; @@ -70,7 +71,7 @@ function selectedRunSignalHttpObservation(array $fixture, array $options): array if (! preg_match('#^http://127\.0\.0\.1:[0-9]+$#', $url)) { throw new RuntimeException('Selected-run control observer failed to bind.'); } - $admin = new Client($url, namespace: 'default', token: getenv('DW_PARITY_TOKEN') ?: null); + $admin = parityClient($url, namespace: 'default', token: getenv('DW_PARITY_TOKEN') ?: null); foreach ($fixtures as $index => $runFixture) { $checkpoint('namespace_setup_'.$index); try { @@ -84,7 +85,7 @@ function selectedRunSignalHttpObservation(array $fixture, array $options): array $state['administration'][] = ['created' => $created, 'namespace' => $namespace->raw]; $paths[$index] = tempnam(sys_get_temp_dir(), 'parity-selected-run-'); file_put_contents($paths[$index], json_encode($runFixture, JSON_THROW_ON_ERROR | JSON_UNESCAPED_UNICODE)); - $clients[$index] = new Client($options['url'], namespace: $runFixture['namespace'], token: getenv('DW_PARITY_TOKEN') ?: null); + $clients[$index] = parityClient($options['url'], namespace: $runFixture['namespace'], token: getenv('DW_PARITY_TOKEN') ?: null); $start = microtime(true); $deadline = $start + 35; $state['run_budgets'][$index] = ['started_at_unix' => $start, 'deadline_at_unix' => $deadline]; diff --git a/scripts/conformance/server-parity/sdk-update-direction-probe.php b/scripts/conformance/server-parity/sdk-update-direction-probe.php index 9fe3f7bb3..e8ce522c0 100644 --- a/scripts/conformance/server-parity/sdk-update-direction-probe.php +++ b/scripts/conformance/server-parity/sdk-update-direction-probe.php @@ -4,6 +4,7 @@ use Composer\InstalledVersions; use DurableWorkflow\Client; +require_once __DIR__.'/lock-pressure-middleware.php'; function sdkUpdateActorOpen(string $language, string $phase, array $fixture, array $options, string $runId, float $deadline): array { @@ -78,7 +79,7 @@ function sdkUpdateDirectionHttpObservation(array $fixture, array $options): arra throw new RuntimeException('SDK update direction requires its evidence directory.'); } $spec = $fixture['sdk_update_direction']; - $client = new Client($options['url'], namespace: 'default', token: getenv('DW_PARITY_TOKEN') ?: null); + $client = parityClient($options['url'], namespace: 'default', token: getenv('DW_PARITY_TOKEN') ?: null); $deadline = microtime(true) + 35; $workflowId = $options['workflow-id']; $runId = ''; @@ -113,7 +114,7 @@ function sdkUpdateDirectionHttpObservation(array $fixture, array $options): arra if ($request['method'] === 'POST' && $request['path'] === '/api/worker/register' && $request['status'] === 201 && ($request['response']['registered'] ?? null) === true && ($request['request']['worker_id'] ?? null) === $workflowId.':original') { - $start = new Client($actor['url'], namespace: 'default', token: getenv('DW_PARITY_TOKEN') ?: null); + $start = parityClient($actor['url'], namespace: 'default', token: getenv('DW_PARITY_TOKEN') ?: null); $runId = $start->startWorkflow($fixture['workflow_type'], $workflowId, $spec['task_queue'], [$fixture['input'], $fixture['signal_count']])->selectedRunId; $observerStart = ['role' => 'observer', 'language' => 'php', 'pid' => getmypid(), diff --git a/scripts/conformance/server-parity/task-retry-probe.php b/scripts/conformance/server-parity/task-retry-probe.php index 42c81935b..fa05a5fd8 100644 --- a/scripts/conformance/server-parity/task-retry-probe.php +++ b/scripts/conformance/server-parity/task-retry-probe.php @@ -3,12 +3,13 @@ declare(strict_types=1); use DurableWorkflow\Client; +require_once __DIR__.'/lock-pressure-middleware.php'; function finishHttpTaskRetry(Client $admin, array $fixture, string $workflowId, string $queue, string $url): array { $definition = $fixture['workflow_task_retry']; $requests = []; - $client = new Client($url, namespace: 'default', token: getenv('DW_PARITY_TOKEN') ?: null, + $client = parityClient($url, namespace: 'default', token: getenv('DW_PARITY_TOKEN') ?: null, transport: namespaceTransport($requests)); $workerId = $workflowId.'-retry-worker'; $peerId = $workflowId.$definition['peer_suffix']; diff --git a/scripts/conformance/server-parity/upgrade-source-author.php b/scripts/conformance/server-parity/upgrade-source-author.php index cae370f95..fa85db118 100644 --- a/scripts/conformance/server-parity/upgrade-source-author.php +++ b/scripts/conformance/server-parity/upgrade-source-author.php @@ -42,7 +42,13 @@ $worker = $prefix.'-worker'; $expect($request('POST', '/api/worker/register', ['worker_id' => $worker, 'task_queue' => $prefix, 'runtime' => 'php', 'sdk_version' => 'raw-upgrade-inspection-author', - 'supported_workflow_types' => ['parity.upgrade_activity', 'parity.upgrade_timer'], + 'supported_workflow_types' => ['parity.upgrade_activity', 'parity.upgrade_timer', 'parity.upgrade_signal', + 'parity.upgrade_lease'], + // The signal original declares its one signal, as an SDK worker would. + 'workflow_command_contracts' => ['parity.upgrade_signal' => ['signals' => ['go'], 'signal_contracts' => [[ + 'name' => 'go', 'parameters' => [['name' => 'value', 'position' => 0, 'type' => 'array', 'required' => true, + 'variadic' => false, 'allows_null' => false, 'default_available' => false]], + ]]]], 'supported_activity_types' => ['parity.upgrade_activity'], 'capabilities' => [], 'capability_manifest' => array_fill_keys(['local_activities', 'worker_sessions', 'sticky_execution'], ['supported' => false, 'minimum_protocol_version' => '1.18', 'reason' => 'raw_protocol_author']), @@ -54,7 +60,8 @@ $timerSeconds = (int) (getenv('DW_PARITY_UPGRADE_TIMER_SECONDS') ?: 600); $check($timerSeconds >= 30 && $timerSeconds <= 600, 'Unsupported original timer delay'); $originals = []; -foreach (['activity', 'timer'] as $kind) { +// The lease original claims before the signal original makes its resume task. +foreach (['activity', 'timer', 'lease', 'signal'] as $kind) { $id = $prefix.'-'.$kind; $run = $expect($request('POST', '/api/workflows', ['workflow_id' => $id, 'workflow_type' => 'parity.upgrade_'.$kind, 'task_queue' => $prefix, 'input' => $envelope, @@ -63,24 +70,43 @@ 'started_at' => $started, 'original_deadline' => $deadline], JSON_THROW_ON_ERROR)."\n"); $task = $expect($request('POST', '/api/worker/workflow-tasks/poll', $poll), 200, $kind.' claim')['task']; $check($task['run_id'] === $run['run_id'], 'Original workflow claim changed'); - $command = $kind === 'activity' - ? ['type' => 'schedule_activity', 'activity_type' => 'parity.upgrade_activity', 'arguments' => $envelope, + $command = match ($kind) { + 'activity' => ['type' => 'schedule_activity', 'activity_type' => 'parity.upgrade_activity', 'arguments' => $envelope, 'retry_policy' => ['max_attempts' => 1, 'backoff_seconds' => []], - 'start_to_close_timeout' => 600, 'schedule_to_close_timeout' => 900] - : ['type' => 'start_timer', 'delay_seconds' => $timerSeconds]; - $expect($request('POST', '/api/worker/workflow-tasks/'.$task['task_id'].'/complete', [ - 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], - 'commands' => [$command], - ]), 200, $kind.' command'); + 'start_to_close_timeout' => 600, 'schedule_to_close_timeout' => 900], + 'timer' => ['type' => 'start_timer', 'delay_seconds' => $timerSeconds], + 'signal' => ['type' => 'open_signal_wait', 'signal_name' => 'go'], + // The lease original's worker stops after its claim. PHP's workflow + // task lease expires after the takeover, and native redelivers it. + 'lease' => null, + }; + if ($command !== null) { + $expect($request('POST', '/api/worker/workflow-tasks/'.$task['task_id'].'/complete', [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'commands' => [$command], + ]), 200, $kind.' command'); + } $activity = null; if ($kind === 'activity') { $activity = $expect($request('POST', '/api/worker/activity-tasks/poll', $poll), 200, 'activity claim')['task']; $check($activity['run_id'] === $run['run_id'] && $activity['attempt_number'] === 1, 'Original activity changed'); } + if ($kind === 'signal') { + // PHP accepts the signal and makes its resume task, which stays + // unclaimed until native resumes it after the takeover. + $signalled = $request('POST', '/api/workflows/'.$id.'/signal/go', ['input' => $envelope]); + $check(in_array($signalled[0], [200, 202], true), 'signal: '.json_encode(['status' => $signalled[0], + 'reason' => $signalled[1]['reason'] ?? null, 'errors' => $signalled[1]['errors'] ?? null])); + } $history = $expect($request('GET', '/api/workflows/'.$id.'/runs/'.$run['run_id'].'/history'), 200, 'history')['events']; - $check(array_column($history, 'event_type') === ($kind === 'activity' - ? ['StartAccepted', 'WorkflowStarted', 'ActivityScheduled', 'ActivityStarted'] - : ['StartAccepted', 'WorkflowStarted', 'TimerScheduled']), 'Original pending history differs'); + fwrite(STDERR, json_encode(['phase' => 'history', 'kind' => $kind, 'events' => array_column($history, 'event_type')], + JSON_THROW_ON_ERROR)."\n"); + $check(array_column($history, 'event_type') === match ($kind) { + 'activity' => ['StartAccepted', 'WorkflowStarted', 'ActivityScheduled', 'ActivityStarted'], + 'timer' => ['StartAccepted', 'WorkflowStarted', 'TimerScheduled'], + 'signal' => ['StartAccepted', 'WorkflowStarted', 'SignalWaitOpened', 'SignalReceived'], + 'lease' => ['StartAccepted', 'WorkflowStarted'], + }, 'Original pending history differs'); $originals[$kind] = ['workflow_id' => $id, 'run_id' => $run['run_id'], 'workflow_task' => $task, 'activity_task' => $activity, 'history' => $history]; } diff --git a/scripts/conformance/server-parity/upgrade-source-cleanup.sh b/scripts/conformance/server-parity/upgrade-source-cleanup.sh index fb8f53b31..bd2b67148 100755 --- a/scripts/conformance/server-parity/upgrade-source-cleanup.sh +++ b/scripts/conformance/server-parity/upgrade-source-cleanup.sh @@ -15,7 +15,7 @@ source=upgrade_source_ref${label:+_$label} restored=upgrade_restore_ref${label:+_$label} image=${DW_PARITY_UPGRADE_PHP_IMAGE:-ghcr.io/durable-workflow/server@sha256:cba7c4c6732529293ef6c2625a64785dcb21b4f7fe0013fdc74f1af466f534ac} [[ "$image" =~ ^ghcr.io/durable-workflow/server@sha256:[0-9a-f]{64}$ ]] || exit 2 -for role in author control inspect php takeover native resume; do +for role in author control inspect php prepare takeover native resume; do if docker inspect "$resource-$role" >/dev/null 2>&1; then docker stop --signal SIGTERM --time 5 "$resource-$role" >/dev/null docker rm "$resource-$role" >/dev/null @@ -53,6 +53,21 @@ case "$backend" in *) ;; esac fi +if [[ -f "$evidence/takeover-owned.txt" && "$(cat "$evidence/takeover-owned.txt")" == "$RESOURCE_SCOPE" ]]; then +case "$backend" in + mysql) + docker run --rm --user=1000:1000 --network "$RESOURCE_SCOPE" --entrypoint mysql "$MYSQL_IMAGE" \ + -h "${DW_PARITY_UPGRADE_DB_HOST:-parity-mysql}" -uroot -p"$password" \ + -e "DROP DATABASE IF EXISTS upgrade_takeover_ref${label:+_$label}" \ + >> "$evidence/database-cleanup.txt" 2>&1 ;; + pgsql) + docker run --rm --user=1000:1000 --network "$RESOURCE_SCOPE" -e PGPASSWORD="$password" \ + --entrypoint psql "$POSTGRES_IMAGE" -h "${DW_PARITY_UPGRADE_DB_HOST:-parity-pg}" -U parity -d postgres -v ON_ERROR_STOP=1 \ + -c "DROP DATABASE IF EXISTS upgrade_takeover_ref${label:+_$label}" \ + >> "$evidence/database-cleanup.txt" 2>&1 ;; + *) ;; +esac +fi if test -f "$evidence/roles-owned.txt" && test "$(cat "$evidence/roles-owned.txt")" = "$RESOURCE_SCOPE"; then if test "$backend" = mysql; then docker run --rm --user=1000:1000 --network "$RESOURCE_SCOPE" --entrypoint mysql "$MYSQL_IMAGE" \ diff --git a/scripts/conformance/server-parity/upgrade-source-profile.sh b/scripts/conformance/server-parity/upgrade-source-profile.sh index 06a1dd4ec..b5f340ef0 100755 --- a/scripts/conformance/server-parity/upgrade-source-profile.sh +++ b/scripts/conformance/server-parity/upgrade-source-profile.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash -# A separate published PHP cohort; inventory is not migration or SDK takeover. +# A separate published PHP cohort: inventory and refusals, a verified backup, +# and the takeover of a restored copy that native then resumes. set -euo pipefail test "$#" = 1 || exit 2 backend=$1 @@ -52,11 +53,11 @@ else chmod 0666 "$state/database.sqlite" fi printf '%s\n' "$RESOURCE_SCOPE" > "$evidence/database-owned.txt" -# SQLite also takes over a copy and resumes the originals, so its timer is short. -timer_seconds=600 -if test "$backend" = sqlite; then timer_seconds=60; fi +# Every backend also takes over a copy and resumes the originals, so the +# timer is short. +timer_seconds=60 configuration=("${admin[@]}" -e DB_DATABASE="$database" -e DW_AUTH_TOKEN=upgrade-fixture-token - -e CACHE_STORE=file -e QUEUE_CONNECTION=database -e DB_SQLITE_JOURNAL_MODE=DELETE) + -e CACHE_STORE=file -e QUEUE_CONNECTION=database) php_run() { docker run --rm --name "$resource-control" --user=1000:1000 --cpus=1 --memory=512m --memory-swap=512m \ --network "$RESOURCE_SCOPE" "${configuration[@]}" -v "$PWD/$state:/state" -v "$PWD:/source:ro" \ @@ -80,14 +81,28 @@ docker run --rm --name "$resource-author" --user=1000:1000 --cpus=1 --memory=512 docker stop --signal SIGTERM --time 10 "$resource-php" >/dev/null docker inspect --format '{{json .State}}' "$resource-php" > "$evidence/stopped-state.json" jq -e '.Running==false and .ExitCode==0 and .OOMKilled==false' "$evidence/stopped-state.json" >/dev/null +if test "$backend" = sqlite; then + # PHP Server keeps SQLite in WAL mode. Once PHP is stopped, the source is + # switched to the rollback journal the upgrade commands need, as an operator + # does first; its rows are unchanged. + [[ "$(od -An -tu1 -j18 -N2 "$state/database.sqlite" | tr -s ' ')" == " 2 2" ]] + docker run --rm --name "$resource-prepare" --user=1000:1000 --cpus=1 --memory=256m --memory-swap=256m \ + --network none -e DB_CONNECTION=sqlite -e DB_DATABASE=/state/database.sqlite -e DW_RUST_EXPERIMENTAL=1 \ + -v "$PWD/$state:/state" -v "$PWD/rust-target:/target:ro" \ + --entrypoint /target/debug/durable-workflow-server "$rust_image" upgrade-prepare-sqlite \ + > "$evidence/prepare.json" 2> "$evidence/prepare.stderr" + jq -e '.status=="rollback_journal" and .previous_journal_mode=="wal" and .source_mutated==true' \ + "$evidence/prepare.json" >/dev/null + [[ "$(od -An -tu1 -j18 -N2 "$state/database.sqlite" | tr -s ' ')" == " 1 1" ]] +fi if test "$backend" = sqlite; then docker run --rm --name "$resource-control" --user=1000:1000 --cpus=1 --memory=512m --memory-swap=512m \ --network=none -v "$PWD/$state:/state:ro" -v "$PWD:/source:ro" --entrypoint php "$image" \ /source/scripts/conformance/server-parity/upgrade-source-sqlite-representations.php upgrade-source \ > "$evidence/representations-comparison.json" jq -e '.outcome=="pass" and .source_mutated==false and .php_objects_executed==false - and (.runs|length)==2 and (.tasks|length)==4 and (.activities|length)==1 - and (.history|length)==7 and (.queue_jobs|length)==1 + and (.runs|length)==4 and (.tasks|length)==7 and (.activities|length)==1 + and (.history|length)==13 and (.queue_jobs|length)==1 and .queue_jobs[0].canonical_timer_task_match==true and all(.runs[]; .typed_arguments_match==true) and .backup_verified==false and .stored_representation_compatibility_verified==false @@ -119,7 +134,7 @@ snapshot() { } snapshot > "$evidence/before.json" inspect_source > "$evidence/inspection.json" 2> "$evidence/inspection.stderr" -jq -e --arg layout "$layout" '.status=="inspected" and .source_layout==$layout and (.table_rows|length)==50 and .table_rows.workflow_runs==2 +jq -e --arg layout "$layout" '.status=="inspected" and .source_layout==$layout and (.table_rows|length)==50 and .table_rows.workflow_runs==4 and .table_rows.activity_executions==1 and .catalog_visibility=="configured_role" and .source_mutated==false and .takeover_allowed==false and .backup_verified==false and .php_quiescence_verified==false and .stored_representation_compatibility_verified==false' "$evidence/inspection.json" >/dev/null @@ -130,10 +145,10 @@ jq -e '.status=="inspected" and .queue_inspection.status=="known_subset_bound" and .source_mutated==false and .takeover_allowed==false and .backup_verified==false and .php_quiescence_verified==false and .stored_representation_compatibility_verified==false' "$evidence/queue-inspection.json" >/dev/null inspect_source upgrade-payload-inspect > "$evidence/payload-inspection.json" 2> "$evidence/payload-inspection.stderr" -jq -e '.status=="inspected" and (.payload_inspection.columns|length)==4 +jq -e '.status=="inspected" and (.payload_inspection.columns|length)==8 and .payload_inspection.references_metadata_supported==true and ([.payload_inspection.columns[].counts.php_v1_references]|add)==0 - and ([.payload_inspection.columns[].counts.avro_inline_values]|add)==3 + and ([.payload_inspection.columns[].counts.avro_inline_values]|add)==11 and ([.payload_inspection.columns[].counts.other_unvalidated]|add)==0 and .payload_inspection.inline_values_verified==true and .payload_inspection.inline_validation_limits=={encoded_frame_bytes:65536,traversal_depth:64,traversal_visits:16384,scalar_allocation_bytes:65536} @@ -151,7 +166,7 @@ fingerprint() { } fingerprint "$database" > "$evidence/fingerprint.json" 2> "$evidence/fingerprint.stderr" jq -e --arg layout "$layout" '.status=="fingerprinted" and .source_layout==$layout and (.fingerprint|test("^sha256:[0-9a-f]{64}$")) - and (.table_rows|length)==50 and .table_rows.workflow_runs==2 and .table_rows.jobs==1 + and (.table_rows|length)==50 and .table_rows.workflow_runs==4 and .table_rows.jobs==1 and .source_mutated==false and .takeover_allowed==false and .backup_verified==false and .php_quiescence_verified==false and .stored_representation_compatibility_verified==false' "$evidence/fingerprint.json" >/dev/null source_fingerprint=$(jq -r .fingerprint "$evidence/fingerprint.json") @@ -206,29 +221,56 @@ change_restored fingerprint "$restored" > "$evidence/changed-fingerprint.json" 2>> "$evidence/fingerprint.stderr" changed_fingerprint=$(jq -r .fingerprint "$evidence/changed-fingerprint.json") [[ $changed_fingerprint =~ ^sha256:[0-9a-f]{64}$ && $changed_fingerprint != "$source_fingerprint" ]] -if test "$backend" = sqlite; then - # Backup-first takeover of a copy of the stopped source, as an operator's - # live database: its digest is the one the restored backup matched. Native - # then resumes the originals, and PHP must refuse the upgraded file. - cp "$state/database.sqlite" "$state/takeover.sqlite" - chmod 0666 "$state/takeover.sqlite" +# Backup-first takeover of a copy of the stopped source, as an operator's +# live database: its digest is the one the restored backup matched. Native +# then resumes the originals, and PHP must refuse the upgraded database. +target=() +case "$backend" in + sqlite) + cp "$state/database.sqlite" "$state/takeover.sqlite" + chmod 0666 "$state/takeover.sqlite" + target=(-e DB_CONNECTION=sqlite -e DB_DATABASE=/state/takeover.sqlite) + target_network=none ;; + pgsql) + # The same backup restored once more, taken over with the owner role. + takeover_database=upgrade_takeover_ref${label:+_$label} + printf '%s\n' "$RESOURCE_SCOPE" > "$evidence/takeover-owned.txt" + pg_admin createdb "$takeover_database" 2>> "$evidence/backup.stderr" + pg_admin pg_restore --no-owner --exit-on-error -d "$takeover_database" < "$state/backup.dump" 2>> "$evidence/backup.stderr" + target=(-e DB_CONNECTION=pgsql -e DB_HOST="$host" -e DB_DATABASE="$takeover_database" + -e DB_USERNAME=parity -e DB_PASSWORD="$password") + target_network=$RESOURCE_SCOPE ;; + mysql) + # The same backup restored once more, taken over as root, which can see + # every session (the PROCESS privilege). + takeover_database=upgrade_takeover_ref${label:+_$label} + printf '%s\n' "$RESOURCE_SCOPE" > "$evidence/takeover-owned.txt" + mysql_admin mysql -e "CREATE DATABASE $takeover_database CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci" 2>> "$evidence/backup.stderr" + mysql_admin mysql "$takeover_database" < "$state/backup.sql" 2>> "$evidence/backup.stderr" + target=(-e DB_CONNECTION=mysql -e DB_HOST="$host" -e DB_DATABASE="$takeover_database" + -e DB_USERNAME=root -e DB_PASSWORD="$password") + target_network=$RESOURCE_SCOPE ;; + *) ;; +esac +if (( ${#target[@]} )); then docker run --rm --name "$resource-takeover" --user=1000:1000 --cpus=1 --memory=256m --memory-swap=256m \ - --network none -e DB_CONNECTION=sqlite -e DB_DATABASE=/state/takeover.sqlite -e DW_RUST_EXPERIMENTAL=1 \ + --network "$target_network" "${target[@]}" -e DW_RUST_EXPERIMENTAL=1 \ -e DW_UPGRADE_BACKUP_FINGERPRINT="$source_fingerprint" \ -e DW_RUST_UPGRADE_QUEUE_CONNECTION=database -e DW_RUST_UPGRADE_QUEUE_DEFAULT_CONNECTION=database \ -e DW_RUST_UPGRADE_QUEUE_DEFAULT_QUEUE=default \ -v "$PWD/$state:/state" -v "$PWD/rust-target:/target:ro" \ --entrypoint /target/debug/durable-workflow-server "$rust_image" upgrade-takeover \ > "$evidence/takeover.json" 2> "$evidence/takeover.stderr" - jq -e --arg layout "$layout" --arg fingerprint "$source_fingerprint" '.status=="taken_over" - and .source_layout==$layout and .fingerprint==$fingerprint and .backup_fingerprint_matched==true + jq -e --arg backend "$backend" --arg layout "$layout" --arg fingerprint "$source_fingerprint" '.status=="taken_over" + and (.backend==$backend or ($backend=="mysql" and .backend=="mariadb")) + and .source_layout==$layout and .fingerprint==$fingerprint + and .backup_fingerprint_matched==true and .converted.incarnations_copied==(if $layout=="frozen" then 0 else 1 end) and .stored_representation_compatibility_verified==false' "$evidence/takeover.json" >/dev/null docker run -d --name "$resource-native" --network "$RESOURCE_SCOPE" --network-alias parity-upgrade-native \ --user=1000:1000 --cpus=1 --memory=512m --memory-swap=512m -e DW_RUST_EXPERIMENTAL=1 \ -e DW_BIND_ADDRESS=0.0.0.0:8080 -e DW_AUTH_DRIVER=token -e DW_AUTH_TOKEN=upgrade-fixture-token \ - -e DB_CONNECTION=sqlite -e DB_DATABASE=/state/takeover.sqlite \ - -v "$PWD/$state:/state" -v "$PWD/rust-target:/target:ro" \ + "${target[@]}" -v "$PWD/$state:/state" -v "$PWD/rust-target:/target:ro" \ --entrypoint /target/debug/durable-workflow-server "$rust_image" > "$evidence/native-container.txt" ready=0 for attempt in $(seq 1 30); do @@ -243,14 +285,15 @@ if test "$backend" = sqlite; then "/source/$evidence/originals.json" 150 > "$evidence/resume.json" 2> "$evidence/resume.stderr" jq -e '.outcome=="pass" and .activity.stale_attempt_status==409 and .activity.events[-1]=="WorkflowCompleted" and .timer.fired_after_seconds>=0 - and .timer.events[-1]=="WorkflowCompleted"' "$evidence/resume.json" >/dev/null + and .timer.events[-1]=="WorkflowCompleted" and .signal.events[-1]=="WorkflowCompleted" + and .lease.attempt==2 and .lease.redelivered_after_seconds>=0 and .lease.stale_attempt_status==409 + and .lease.events[-1]=="WorkflowCompleted"' "$evidence/resume.json" >/dev/null docker stop --signal SIGTERM --time 10 "$resource-native" >/dev/null if docker run --rm --name "$resource-control" --user=1000:1000 --cpus=1 --memory=512m --memory-swap=512m \ - --network none -e DB_CONNECTION=sqlite -e DB_DATABASE=/state/takeover.sqlite -e DW_AUTH_TOKEN=upgrade-fixture-token \ + --network "$target_network" "${target[@]}" -e DW_AUTH_TOKEN=upgrade-fixture-token \ -e CACHE_STORE=file -e QUEUE_CONNECTION=database -v "$PWD/$state:/state" --entrypoint php "$image" \ artisan server:assert-php-database > "$evidence/php-refusal.txt" 2>&1; then exit 1; fi grep -q php_database_refused "$evidence/php-refusal.txt" - sqlite_takeover=true fi snapshot > "$evidence/after.json" cmp "$evidence/before.json" "$evidence/after.json" diff --git a/scripts/conformance/server-parity/upgrade-source-sqlite-representations.php b/scripts/conformance/server-parity/upgrade-source-sqlite-representations.php index da17fe171..09da717d5 100644 --- a/scripts/conformance/server-parity/upgrade-source-sqlite-representations.php +++ b/scripts/conformance/server-parity/upgrade-source-sqlite-representations.php @@ -20,11 +20,11 @@ if ($pdo->query('PRAGMA journal_mode')->fetchColumn() !== 'delete') { throw new RuntimeException('Audit requires the already stopped DELETE-journal fixture'); } -$runs = $pdo->query('SELECT * FROM workflow_runs ORDER BY id LIMIT 3')->fetchAll(PDO::FETCH_ASSOC); +$runs = $pdo->query('SELECT * FROM workflow_runs ORDER BY id LIMIT 5')->fetchAll(PDO::FETCH_ASSOC); $workflowIds = array_column($runs, 'workflow_instance_id'); sort($workflowIds); -if ($workflowIds !== [$prefix.'-activity', $prefix.'-timer']) { - throw new RuntimeException('This reader accepts only the two isolated source-author originals'); +if ($workflowIds !== [$prefix.'-activity', $prefix.'-lease', $prefix.'-signal', $prefix.'-timer']) { + throw new RuntimeException('This reader accepts only the four isolated source-author originals'); } $report = ['scope' => 'published_php_stored_representation_observation', 'runs' => [], 'table_rows' => [], 'tasks' => [], 'activities' => [], 'history' => [], 'queue_jobs' => [], 'cache_files' => []]; diff --git a/scripts/conformance/server-parity/upgrade-takeover-resume.php b/scripts/conformance/server-parity/upgrade-takeover-resume.php index bddd3db37..41f0281d5 100644 --- a/scripts/conformance/server-parity/upgrade-takeover-resume.php +++ b/scripts/conformance/server-parity/upgrade-takeover-resume.php @@ -56,7 +56,9 @@ $outcome = []; $activity = $originals['activity']; $timer = $originals['timer']; -$complete = static function (array $claimed, string $operation) use ($expect, $request, $result): void { +$signal = $originals['signal']; +$lease = $originals['lease']; +$complete =static function (array $claimed, string $operation) use ($expect, $request, $result): void { $expect($request('POST', '/api/worker/workflow-tasks/'.$claimed['task_id'].'/complete', ['lease_owner' => $claimed['lease_owner'], 'workflow_task_attempt' => $claimed['workflow_task_attempt'], 'commands' => [['type' => 'complete_workflow', 'result' => $result]]]), 200, $operation); @@ -69,6 +71,12 @@ $deadline = (float) (new DateTimeImmutable($fireAt))->format('U.u'); $check($deadline - microtime(true) <= (float) $timerBudget, 'The original timer deadline is outside the wait budget'); +// The workflow task PHP leased to a stopped worker keeps its PHP expiry. +$check(is_string($lease['workflow_task']['lease_expires_at'] ?? null), 'The original lease expiry is missing'); +$leaseExpiry = (float) (new DateTimeImmutable($lease['workflow_task']['lease_expires_at']))->format('U.u'); +$check($leaseExpiry - microtime(true) <= (float) $timerBudget, 'The original lease expiry is outside the wait budget'); +$check($keep($lease['history'], $history($lease)), 'Original lease history changed'); + // The activity leased under PHP completes with its original attempt. $task = $activity['activity_task']; $completion = ['activity_attempt_id' => $task['activity_attempt_id'], 'lease_owner' => $task['lease_owner'], 'result' => $result]; @@ -77,20 +85,22 @@ $replayed = $expect($request('POST', '/api/worker/activity-tasks/'.$task['task_id'].'/complete', $completion), 200, 'activity completion replay'); $check(($replayed['recorded'] ?? null) === false, 'A replayed activity completion was recorded twice'); -// Each original workflow then gets one workflow task: the activity's at -// once and the timer's at its deadline. On a slow host the timer is already -// due, so either may be claimed first. +// Each original workflow then gets one workflow task: the activity's and the +// signal's at once, the timer's at its deadline and the lease's when PHP's +// lease expires. On a slow host some are already due, so they may be claimed +// in any order. $original = $activity['history']; $claims = []; -$until = max($deadline, microtime(true)) + 30; -while (count($claims) < 2 && microtime(true) < $until) { +$leaseOutcome = null; +$until = max($deadline, $leaseExpiry, microtime(true)) + 30; +while (count($claims) < 4 && microtime(true) < $until) { $claimed = $expect($request('POST', '/api/worker/workflow-tasks/poll', ['worker_id' => $worker, 'task_queue' => $queue, 'timeout_seconds' => 5]), 200, 'workflow task claim')['task'] ?? null; if (! is_array($claimed)) { continue; } $run = $claimed['run_id']; - $check(in_array($run, [$activity['run_id'], $timer['run_id']], true) && ! isset($claims[$run]), + $check(in_array($run, [$activity['run_id'], $timer['run_id'], $signal['run_id'], $lease['run_id']], true) && ! isset($claims[$run]), 'An unexpected workflow task was claimed'); $claims[$run] = true; if ($run === $activity['run_id']) { @@ -98,13 +108,42 @@ 'Original activity history changed in the resumed task'); $check(end($claimed['history_events'])['event_type'] === 'ActivityCompleted', 'Activity completion is missing from history'); $complete($claimed, 'resumed workflow completion'); + } elseif ($run === $signal['run_id']) { + // The signal PHP accepted is delivered from PHP's rows, with its arguments. + $check(array_slice($types($claimed['history_events']), 0, count($signal['history'])) === $types($signal['history']), + 'Original signal history changed in the resumed task'); + $received = array_values(array_filter($claimed['history_events'], + static fn (array $event): bool => $event['event_type'] === 'SignalReceived')); + $check(count($received) === 1 && json_encode($sorted($received[0]['payload']['arguments']), JSON_THROW_ON_ERROR) + === json_encode($sorted(end($signal['history'])['payload']['arguments']), JSON_THROW_ON_ERROR), + 'The PHP-accepted signal arguments differ under native'); + $complete($claimed, 'signal workflow completion'); + } elseif ($run === $lease['run_id']) { + // After PHP's lease expires, native redelivers the same task as its + // next attempt. The expired PHP attempt cannot complete over it. + $expired = $lease['workflow_task']; + $check($claimed['task_id'] === $expired['task_id'] + && $claimed['workflow_task_attempt'] === $expired['workflow_task_attempt'] + 1, + 'The expired PHP lease was not redelivered as its next attempt'); + $check(microtime(true) >= $leaseExpiry, 'Native redelivered a PHP lease before it expired'); + $check($types($claimed['history_events']) === $types($lease['history']), + 'Original lease history changed in the redelivered task'); + $stale = $request('POST', '/api/worker/workflow-tasks/'.$expired['task_id'].'/complete', ['lease_owner' => $expired['lease_owner'], + 'workflow_task_attempt' => $expired['workflow_task_attempt'], 'commands' => [['type' => 'complete_workflow', 'result' => $result]]]); + $check($stale[0] === 409, 'The expired PHP attempt completed over its redelivery'); + $leaseOutcome = ['attempt' => $claimed['workflow_task_attempt'], + 'redelivered_after_seconds' => round(microtime(true) - $leaseExpiry, 3), + 'stale_attempt_status' => $stale[0], 'stale_attempt_reason' => $stale[1]['reason'] ?? null]; + $complete($claimed, 'redelivered lease completion'); } else { $check(end($claimed['history_events'])['event_type'] === 'TimerFired', 'Timer firing is missing from history'); $complete($claimed, 'timer workflow completion'); } } $check(isset($claims[$activity['run_id']]), 'The resumed workflow task was not claimed'); +$check(isset($claims[$signal['run_id']]), 'The signal workflow task was not claimed'); $check(isset($claims[$timer['run_id']]), 'The original timer did not fire under native'); +$check(isset($claims[$lease['run_id']]), 'The expired PHP lease was not redelivered'); // The workflow task PHP already completed is a stale attempt. $stale = $activity['workflow_task']; @@ -124,6 +163,15 @@ $check($firedAt >= $deadline, 'The original timer fired before its deadline'); $check($keep($timer['history'], $timerHistory) && end($timerHistory)['event_type'] === 'WorkflowCompleted', 'The timer workflow did not complete on its original history'); +$signalHistory = $history($signal); +$check($keep($signal['history'], $signalHistory) && end($signalHistory)['event_type'] === 'WorkflowCompleted', + 'The signal workflow did not complete on its original history'); +$outcome['signal'] = ['run_id' => $signal['run_id'], 'events' => array_column($signalHistory, 'event_type')]; +$leaseHistory = $history($lease); +$check($keep($lease['history'], $leaseHistory) && end($leaseHistory)['event_type'] === 'WorkflowCompleted', + 'The redelivered lease workflow did not complete on its original history'); +$outcome['lease'] = ['run_id' => $lease['run_id'], 'lease_expires_at' => $lease['workflow_task']['lease_expires_at']] + + $leaseOutcome + ['events' => array_column($leaseHistory, 'event_type')]; $outcome['timer'] = ['run_id' => $timer['run_id'], 'fire_at' => $fireAt, 'fired_after_seconds' => round($firedAt - $deadline, 3), 'events' => array_column($timerHistory, 'event_type')]; echo json_encode(['outcome' => 'pass', 'scope' => 'native_resumes_published_php_originals_after_takeover'] + $outcome, diff --git a/scripts/conformance/server-parity/waiting-history-probe.php b/scripts/conformance/server-parity/waiting-history-probe.php index 6d21e0026..26d4efd99 100644 --- a/scripts/conformance/server-parity/waiting-history-probe.php +++ b/scripts/conformance/server-parity/waiting-history-probe.php @@ -3,12 +3,13 @@ declare(strict_types=1); use DurableWorkflow\Client; +require_once __DIR__.'/lock-pressure-middleware.php'; function finishHttpWaitingHistory(Client $admin, array $fixture, string $workflowId, string $queue, string $url): array { $definition = $fixture['waiting_for_history']; $requests = []; - $client = new Client($url, namespace: 'default', token: getenv('DW_PARITY_TOKEN') ?: null, + $client = parityClient($url, namespace: 'default', token: getenv('DW_PARITY_TOKEN') ?: null, transport: namespaceTransport($requests)); $workerId = $workflowId.'-waiting-worker'; $peerId = $workflowId.$definition['peer_suffix']; diff --git a/scripts/k8s-kind-smoke.sh b/scripts/k8s-kind-smoke.sh index b936d210e..9d49762dd 100755 --- a/scripts/k8s-kind-smoke.sh +++ b/scripts/k8s-kind-smoke.sh @@ -7,7 +7,7 @@ cluster="${K8S_SMOKE_CLUSTER:-durable-workflow-server-smoke}" image="${K8S_SMOKE_IMAGE:-durableworkflow/server:k8s-smoke}" # Generated by scripts/ci/sync-source-release.mjs so the smoke replaces the # same default shipped by the public manifests. -manifest_image="durableworkflow/server:2.9.7" +manifest_image="durableworkflow/server:2.9.9" kind_node_image="${K8S_SMOKE_KIND_NODE_IMAGE:-kindest/node:v1.29.4}" artifact_dir="${K8S_SMOKE_ARTIFACT_DIR:-/tmp/durable-workflow-k8s-kind-smoke-artifacts}" rendered_dir="${artifact_dir}/rendered-manifests" diff --git a/tests/Feature/DatabaseStorageExhaustedTest.php b/tests/Feature/DatabaseStorageExhaustedTest.php index f17749ade..6b75c7bce 100644 --- a/tests/Feature/DatabaseStorageExhaustedTest.php +++ b/tests/Feature/DatabaseStorageExhaustedTest.php @@ -93,7 +93,7 @@ public function test_database_readiness_explains_that_write_capacity_is_not_veri $this->getJson('/api/ready') ->assertOk() ->assertJsonPath('checks.database.status', 'ok') - ->assertJsonPath('checks.database.check_scope', 'connection_only') + ->assertJsonPath('checks.database.check_scope', 'bound_read_query') ->assertJsonPath('checks.database.write_capacity_verified', false); } diff --git a/tests/Feature/HealthControllerTest.php b/tests/Feature/HealthControllerTest.php index 469190843..ac6c13c62 100644 --- a/tests/Feature/HealthControllerTest.php +++ b/tests/Feature/HealthControllerTest.php @@ -10,6 +10,7 @@ use App\Support\RedisReadinessProcess; use App\Support\ServerReadiness; use App\Support\ServerTopology; +use Illuminate\Database\SQLiteConnection; use Illuminate\Foundation\Testing\RefreshDatabase; use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Schema; @@ -25,6 +26,39 @@ class HealthControllerTest extends TestCase use RefreshDatabase; use ServerTestHelpers; + public function test_readiness_refuses_a_connected_database_that_cannot_execute_bound_reads(): void + { + $originalDefault = config('database.default'); + $pdo = DB::connection()->getPdo(); + DB::extend('broken_bound_reads', static fn (array $config): SQLiteConnection => new class($pdo, ':memory:', '', $config) extends SQLiteConnection + { + public function select($query, $bindings = [], $useReadPdo = true, array $fetchUsing = []) + { + if ($bindings !== []) { + throw new \PDOException('The connected database cannot execute bound statements.'); + } + + return parent::select($query, $bindings, $useReadPdo, $fetchUsing); + } + }); + config([ + 'database.connections.broken_bound_reads' => ['driver' => 'broken_bound_reads'], + 'database.default' => 'broken_bound_reads', + ]); + + try { + $this->getJson('/api/ready') + ->assertStatus(503) + ->assertJsonPath('checks.database.status', 'unavailable') + ->assertJsonPath('checks.database.reason', 'database_query_failed'); + } finally { + // RefreshDatabase must roll back its original connection, not + // this wrapper around the same PDO. In-memory suites reuse it. + config(['database.default' => $originalDefault]); + DB::purge('broken_bound_reads'); + } + } + public function test_health_check_returns_serving_when_database_is_available(): void { $response = $this->getJson('/api/health'); @@ -122,6 +156,8 @@ public function test_readiness_check_returns_ready_when_bootstrap_state_is_avail $response->assertOk() ->assertJsonPath('status', 'ready') ->assertJsonPath('checks.database.status', 'ok') + ->assertJsonPath('checks.database.check_scope', 'bound_read_query') + ->assertJsonPath('checks.database.write_capacity_verified', false) ->assertJsonPath('checks.migrations.status', 'ok') ->assertJsonPath('checks.migrations.operator_surface.available', true) ->assertJsonPath('checks.migrations.readiness_contract.version', 1) diff --git a/tests/Feature/PostgresPoolerTest.php b/tests/Feature/PostgresPoolerTest.php new file mode 100644 index 000000000..b04fbe2b4 --- /dev/null +++ b/tests/Feature/PostgresPoolerTest.php @@ -0,0 +1,108 @@ + [getenv('DW_POOLER_TEST_HOST') ?: null, (int) (getenv('DW_POOLER_TEST_PORT') ?: 6432), getenv('DW_POOLER_TEST_EXPECT_SWITCH') === '1']]; + $prefix = getenv('DW_POOLER_TEST_PREFIX'); + if (is_string($prefix) && $prefix !== '') { + foreach (['session', 'transaction0', 'transaction200'] as $mode) { + $endpoints[$mode] = [$prefix.'-'.$mode, 5432, $mode !== 'session']; + } + } + + return $endpoints; + } + + private function configureEndpoint(?string $host, int $port): void + { + if ($host === null) { + $this->markTestSkipped('Requires the isolated DW_POOLER_TEST_HOST endpoint.'); + } + $base = config('database.connections.pgsql'); + foreach (['pooler_first', 'pooler_second'] as $name) { + config(['database.connections.'.$name => [...$base, + 'host' => $host, + 'port' => $port, + 'database' => getenv('DW_POOLER_TEST_DATABASE') ?: 'dw_pooler_probe', + 'username' => getenv('DW_POOLER_TEST_USER') ?: 'pooler_probe', + 'password' => getenv('DW_POOLER_TEST_PASSWORD') ?: '', + ]]); + } + } + + protected function tearDown(): void + { + foreach (['pooler_first', 'pooler_second'] as $name) { + DB::purge($name); + } + parent::tearDown(); + } + + #[DataProvider('endpoints')] + public function test_clients_keep_their_bound_values_and_types_when_backend_sessions_change(?string $host, int $port, bool $expectSwitch): void + { + $this->configureEndpoint($host, $port); + $first = DB::connection('pooler_first'); + $second = DB::connection('pooler_second'); + + // Occupy two server sessions using simple protocol commands first. + // The probe itself then exercises Laravel's configured PDO strategy. + $first->getPdo()->exec('BEGIN'); + $first->getPdo()->exec('SELECT 1'); + $second->getPdo()->exec('BEGIN'); + $second->getPdo()->exec('SELECT 1'); + $first->getPdo()->exec('ROLLBACK'); + $second->getPdo()->exec('ROLLBACK'); + + $backends = []; + for ($iteration = 0; $iteration < 20; $iteration++) { + $marker = 'first-'.$iteration.'-'.bin2hex(random_bytes(8)); + $row = $first->selectOne('SELECT CAST(? AS TEXT) AS marker, CAST(? AS BIGINT) AS number, CAST(? AS TEXT) AS absent, pg_backend_pid() AS backend', + [$marker, '9223372036854775807', null]); + $backends[] = $row->backend; + self::assertSame($marker, $row->marker); + self::assertSame('9223372036854775807', (string) $row->number); + self::assertNull($row->absent); + + $other = 'second-'.$iteration.'-Київ'; + $row = $second->selectOne('SELECT CAST(? AS TEXT) AS marker', [$other]); + self::assertSame($other, $row->marker); + // An odd number of commands per iteration makes the next first + // client's query use another backend in the round-robin pool. + self::assertSame(1, (int) $first->selectOne('SELECT 1 AS value')->value); + } + if ($expectSwitch) { + self::assertGreaterThan(1, count(array_unique($backends)), 'The control must change this client\'s physical backend.'); + } + } + + #[DataProvider('endpoints')] + public function test_bound_statements_do_not_abort_a_multi_query_transaction(?string $host, int $port, bool $expectSwitch): void + { + $this->configureEndpoint($host, $port); + $connection = DB::connection('pooler_first'); + $connection->beginTransaction(); + try { + foreach (['first', 'second', 'third'] as $value) { + self::assertSame($value, $connection->selectOne('SELECT CAST(? AS TEXT) AS marker', [$value])->marker); + self::assertSame(2, (int) $connection->selectOne('SELECT 1 + 1 AS result')->result); + } + } finally { + $connection->rollBack(); + } + self::assertSame('after-rollback', $connection->selectOne('SELECT CAST(? AS TEXT) AS marker', ['after-rollback'])->marker); + } +} diff --git a/tests/Feature/WorkflowDefinitionEvolutionTest.php b/tests/Feature/WorkflowDefinitionEvolutionTest.php new file mode 100644 index 000000000..52e973657 --- /dev/null +++ b/tests/Feature/WorkflowDefinitionEvolutionTest.php @@ -0,0 +1,360 @@ +createNamespace('default'); + config(['server.polling.timeout' => 0, 'workflows.v2.compatibility.pin_to_recorded_fingerprint' => true]); + } + + public function test_default_replacement_can_claim_when_the_recorded_definition_is_gone(): void + { + [$run, $task, $started] = $this->original(); + $original = [$started->getRawOriginal('payload'), $run->getRawOriginal('run_deadline_at'), $run->getRawOriginal('execution_deadline_at')]; + $this->deleteJson('/api/workers/original-worker', [], $this->apiHeaders())->assertOk(); + $this->worker('replacement-worker', 'definition-v2'); + + $this->poll('replacement-worker')->assertOk() + ->assertJsonPath('task.run_id', $run->id) + ->assertJsonPath('task.task_id', $task->id); + + self::assertSame($original, [ + $started->fresh()->getRawOriginal('payload'), $run->fresh()->getRawOriginal('run_deadline_at'), $run->fresh()->getRawOriginal('execution_deadline_at'), + ]); + self::assertSame(1, $task->fresh()->attempt_count); + self::assertSame(0, $task->fresh()->repair_count); + } + + public function test_explicit_opt_out_does_not_wait_for_a_matching_peer(): void + { + [$run, $task] = $this->original(); + $this->worker('replacement-worker', 'definition-v2'); + config(['workflows.v2.compatibility.pin_to_recorded_fingerprint' => false]); + + $this->poll('replacement-worker')->assertOk() + ->assertJsonPath('task.run_id', $run->id) + ->assertJsonPath('task.task_id', $task->id); + } + + public function test_legacy_replacement_cannot_gain_unsafe_default_fallback(): void + { + [$run, $task] = $this->original(); + $this->deleteJson('/api/workers/original-worker', [], $this->apiHeaders())->assertOk(); + $this->worker('legacy-replacement', 'definition-v2', ['capabilities' => []]); + $this->poll('legacy-replacement')->assertOk()->assertJsonPath('task', null); + self::assertSame(0, $task->fresh()->attempt_count); + $this->getJson('/api/workflows/definition-original', $this->apiHeaders())->assertOk() + ->assertJsonPath('definition_routing.status', 'entry_validation_unavailable') + ->assertJsonPath('definition_routing.blocked', true) + ->assertJsonPath('definition_routing.entry_validation_unavailable_count', 1); + $this->worker('safe-replacement', 'definition-v2'); + $this->getJson('/api/workflows/definition-original', $this->apiHeaders())->assertOk() + ->assertJsonPath('definition_routing.status', 'replay_required') + ->assertJsonPath('definition_routing.blocked', false); + $this->poll('safe-replacement')->assertOk()->assertJsonPath('task.run_id', $run->id); + } + + public function test_default_keeps_the_matching_registered_peer_preference(): void + { + [, $task] = $this->original(); + $this->worker('replacement-worker', 'definition-v2'); + + $this->poll('replacement-worker')->assertOk()->assertJsonPath('task', null); + self::assertSame(TaskStatus::Ready, $task->fresh()->status); + self::assertNull($task->fresh()->lease_owner); + self::assertSame(0, $task->fresh()->attempt_count); + + $this->poll('original-worker')->assertOk()->assertJsonPath('task.task_id', $task->id); + } + + #[DataProvider('ineligibleRecordedWorkers')] + public function test_an_ineligible_recorded_peer_does_not_strand_a_replacement(array $attributes): void + { + [, $task] = $this->original(); + WorkerRegistration::query()->where('worker_id', 'original-worker')->firstOrFail() + ->forceFill($attributes)->save(); + $this->worker('replacement-worker', 'definition-v2'); + + $this->poll('replacement-worker')->assertOk()->assertJsonPath('task.task_id', $task->id); + self::assertSame(1, $task->fresh()->attempt_count); + self::assertSame(0, $task->fresh()->repair_count); + } + + public static function ineligibleRecordedWorkers(): array + { + return [ + 'stale heartbeat' => [['last_heartbeat_at' => '2000-01-01 00:00:00']], + 'missing heartbeat' => [['last_heartbeat_at' => null]], + 'draining' => [['status' => 'draining']], + 'other namespace' => [['namespace' => 'other']], + 'other queue' => [['task_queue' => 'other']], + 'other type' => [['supported_workflow_types' => ['other']]], + 'no workflow capacity' => [['max_concurrent_workflow_tasks' => 0]], + 'missing definition' => [['workflow_definition_fingerprints' => []]], + ]; + } + + public function test_busy_matching_peer_retains_preference_without_inventing_slot_authority(): void + { + [, $task] = $this->original(); + WorkerRegistration::query()->where('worker_id', 'original-worker')->update(['available_workflow_slots' => 0]); + $this->worker('replacement-worker', 'definition-v2'); + + $this->poll('replacement-worker')->assertOk()->assertJsonPath('task', null); + self::assertSame(TaskStatus::Ready, $task->fresh()->status); + self::assertSame(0, $task->fresh()->attempt_count); + } + + public function test_fallback_does_not_remove_the_original_build_pin(): void + { + [$run, $task] = $this->original(); + $run->forceFill(['compatibility' => 'build-v1'])->save(); + $task->forceFill(['compatibility' => 'build-v1'])->save(); + $this->deleteJson('/api/workers/original-worker', [], $this->apiHeaders())->assertOk(); + $this->worker('replacement-worker', 'definition-v2', ['build_id' => 'build-v2']); + + $this->poll('replacement-worker', ['build_id' => 'build-v2'])->assertOk()->assertJsonPath('task', null); + self::assertSame(TaskStatus::Ready, $task->fresh()->status); + self::assertSame(0, $task->fresh()->attempt_count); + self::assertSame('build-v1', $run->fresh()->compatibility); + } + + public function test_a_matching_peer_on_another_build_cannot_suppress_a_pinned_replacement(): void + { + [$run, $task] = $this->original(); + $run->forceFill(['compatibility' => 'build-v1'])->save(); + $task->forceFill(['compatibility' => 'build-v1'])->save(); + WorkerRegistration::query()->where('worker_id', 'original-worker')->update(['build_id' => 'build-v2']); + $this->worker('replacement-worker', 'definition-v2', ['build_id' => 'build-v1']); + + $this->poll('replacement-worker', ['build_id' => 'build-v1'])->assertOk()->assertJsonPath('task.task_id', $task->id); + self::assertSame('build-v1', $run->fresh()->compatibility); + } + + public function test_matching_peer_must_support_the_recorded_replay_protocol(): void + { + [$run, $task] = $this->original(); + $worker = WorkerRegistration::query()->where('worker_id', 'original-worker')->firstOrFail(); + $worker->forceFill(['process_metrics' => [WorkflowReplayEligibility::PROTOCOL_METRIC => '1.16']])->save(); + WorkflowHistoryEvent::query()->create([ + 'workflow_run_id' => $run->id, + 'sequence' => WorkflowHistoryEvent::query()->where('workflow_run_id', $run->id)->max('sequence') + 1, + 'event_type' => 'TimerScheduled', + 'payload' => ['condition_wait_occurrence_id' => 'original-condition-occurrence'], + ]); + $this->worker('replacement-worker', 'definition-v2'); + + $this->poll('replacement-worker')->assertOk()->assertJsonPath('task.task_id', $task->id); + } + + public function test_client_process_metrics_cannot_spoof_the_saved_protocol(): void + { + $this->worker('original-worker', 'definition-v1', [ + 'process_metrics' => [WorkflowReplayEligibility::PROTOCOL_METRIC => '999.0', 'memory_bytes' => 123], + ]); + $worker = WorkerRegistration::query()->where('worker_id', 'original-worker')->firstOrFail(); + self::assertSame($this->workerHeaders()['X-Durable-Workflow-Protocol-Version'], WorkflowReplayEligibility::recordedProtocol($worker)); + self::assertSame(123, $worker->process_metrics['memory_bytes']); + + $this->postJson('/api/worker/heartbeat', [ + 'worker_id' => 'original-worker', 'process_metrics' => [WorkflowReplayEligibility::PROTOCOL_METRIC => '999.0', 'memory_bytes' => 456], + ], $this->workerHeaders())->assertOk(); + self::assertSame($this->workerHeaders()['X-Durable-Workflow-Protocol-Version'], WorkflowReplayEligibility::recordedProtocol($worker->fresh())); + self::assertSame(456, $worker->fresh()->process_metrics['memory_bytes']); + } + + public function test_explicit_null_clears_client_metrics_without_clearing_server_protocol(): void + { + $this->worker('original-worker', 'definition-v1', ['process_metrics' => ['memory_bytes' => 123]]); + $this->postJson('/api/worker/heartbeat', [ + 'worker_id' => 'original-worker', 'process_metrics' => null, + ], $this->workerHeaders())->assertOk(); + $worker = WorkerRegistration::query()->where('worker_id', 'original-worker')->firstOrFail(); + self::assertSame([ + WorkflowReplayEligibility::PROTOCOL_METRIC => $this->workerHeaders()['X-Durable-Workflow-Protocol-Version'], + ], $worker->process_metrics); + } + + public function test_run_debug_queue_and_repair_distinguish_fallback_from_a_real_block(): void + { + [$run, $task] = $this->original(); + $this->deleteJson('/api/workers/original-worker', [], $this->apiHeaders())->assertOk(); + $this->worker('replacement-worker', 'definition-v2'); + $this->getJson('/api/workflows/definition-original', $this->apiHeaders())->assertOk() + ->assertJsonPath('definition_routing.status', 'replay_required') + ->assertJsonPath('definition_routing.blocked', false) + ->assertJsonPath('definition_routing.recorded_fingerprint', 'definition-v1'); + $this->getJson('/api/workflows/definition-original/runs/'.$run->id, $this->apiHeaders())->assertOk() + ->assertJsonPath('definition_routing.status', 'replay_required'); + $this->getJson('/api/workflows/definition-original/debug', $this->apiHeaders())->assertOk() + ->assertJsonPath('definition_routing.blocked', false) + ->assertJsonMissing(['code' => 'workflow_definition_routing_blocked']); + + WorkerRegistration::query()->where('worker_id', 'replacement-worker')->firstOrFail() + ->forceFill(['workflow_definition_fingerprints' => []])->save(); + $this->getJson('/api/workflows/definition-original', $this->apiHeaders())->assertOk() + ->assertJsonPath('definition_routing.status', 'worker_definition_missing') + ->assertJsonPath('definition_routing.blocked', true); + $this->getJson('/api/workflows/definition-original/debug', $this->apiHeaders())->assertOk() + ->assertJsonPath('diagnostic_status', 'needs_attention') + ->assertJsonFragment(['code' => 'workflow_definition_routing_blocked']); + $this->getJson('/api/task-queues/'.self::QUEUE, $this->apiHeaders())->assertOk() + ->assertJsonPath('definition_routing.blocked_sample_count', 1) + ->assertJsonPath('definition_routing.runs.0.run_id', $run->id) + ->assertJsonPath('repair.needs_attention', true); + $repair = $this->postJson('/api/workflows/definition-original/repair', [ + 'request_id' => 'definition-routing-repair', + ], $this->apiHeaders())->assertOk() + ->assertJsonPath('definition_routing.status', 'worker_definition_missing') + ->assertJsonPath('definition_routing.blocked', true); + $command = WorkflowCommand::query()->findOrFail($repair->json('command_id')); + self::assertSame($command->commandReason(), $repair->json('reason')); + self::assertSame($command->outcome->value, $repair->json('outcome')); + self::assertSame(0, $task->fresh()->attempt_count); + self::assertSame(0, $task->fresh()->repair_count); + + $this->createNamespace('other'); + $this->getJson('/api/task-queues/'.self::QUEUE, $this->apiHeaders('other'))->assertOk() + ->assertJsonPath('definition_routing.runs', []); + } + + #[DataProvider('notRunnableTasks')] + public function test_definition_drift_is_not_a_ready_task_block_when_delivery_is_not_due(string $case): void + { + [$run, $task] = $this->original(); + $this->deleteJson('/api/workers/original-worker', [], $this->apiHeaders())->assertOk(); + if ($case === 'terminal') { + $run->forceFill(['status' => 'completed', 'closed_at' => now()])->save(); + } elseif ($case === 'leased') { + $task->forceFill(['status' => TaskStatus::Leased, 'lease_owner' => 'original-worker'])->save(); + } else { + $task->forceFill(['available_at' => now()->addMinutes(1)])->save(); + } + $this->getJson('/api/workflows/definition-original', $this->apiHeaders())->assertOk() + ->assertJsonPath('definition_routing.runnable_task', false) + ->assertJsonPath('definition_routing.blocked', false); + $this->getJson('/api/workflows/definition-original/debug', $this->apiHeaders())->assertOk() + ->assertJsonMissing(['code' => 'workflow_definition_routing_blocked']); + } + + public static function notRunnableTasks(): array + { + return [['terminal'], ['leased'], ['future']]; + } + + #[DataProvider('entryFailureTypes')] + public function test_entry_binding_failure_preserves_the_original_until_supported_repair(string $failureType): void + { + [$run, $task, $started] = $this->original(); + $original = [$started->getRawOriginal('payload'), $run->getRawOriginal('run_deadline_at'), $run->getRawOriginal('execution_deadline_at')]; + $claim = $this->poll('original-worker')->assertOk(); + $failure = [ + 'lease_owner' => $claim->json('task.lease_owner'), + 'workflow_task_attempt' => $claim->json('task.workflow_task_attempt'), + 'failure' => [ + 'type' => $failureType, + 'reason' => 'workflow_entry_arguments_mismatch', + 'message' => 'The recorded input cannot bind to the deployed entry.', + ], + ]; + + $this->postJson('/api/worker/workflow-tasks/'.$task->id.'/fail', [ + ...$failure, 'lease_owner' => 'another-worker', + ], $this->workerHeaders())->assertConflict(); + self::assertSame(TaskStatus::Leased, $task->fresh()->status); + + $this->postJson('/api/worker/workflow-tasks/'.$task->id.'/fail', $failure, $this->workerHeaders()) + ->assertOk()->assertJsonPath('recorded', true)->assertJsonPath('next_task_id', null); + self::assertSame(TaskStatus::Failed, $task->fresh()->status); + self::assertTrue($task->fresh()->payload['replay_blocked'] ?? false); + self::assertSame('workflow_entry_arguments_mismatch', $task->fresh()->payload['failure_reason']); + self::assertSame(1, WorkflowTask::query()->where('workflow_run_id', $run->id)->count()); + self::assertNotContains($run->fresh()->status->value, ['failed', 'completed']); + $this->poll('original-worker')->assertOk()->assertJsonPath('task', null); + + $this->postJson('/api/workflows/definition-original/repair', [ + 'request_id' => 'entry-compatible-repair', + ], $this->apiHeaders())->assertOk(); + $this->poll('original-worker')->assertOk()->assertJsonPath('task.task_id', $task->id); + self::assertSame(1, $task->fresh()->repair_count); + self::assertSame($original, [ + $started->fresh()->getRawOriginal('payload'), $run->fresh()->getRawOriginal('run_deadline_at'), $run->fresh()->getRawOriginal('execution_deadline_at'), + ]); + } + + public static function entryFailureTypes(): array + { + return [ + 'PHP' => ['DurableWorkflow\\Exception\\WorkflowEntryArgumentsMismatch'], + 'Python' => ['WorkflowEntryArgumentsMismatchError'], + 'Rust' => ['WorkflowEntryArgumentsMismatch'], + ]; + } + + private function original(): array + { + $this->worker('original-worker', 'definition-v1'); + $start = $this->postJson('/api/workflows', [ + 'workflow_id' => 'definition-original', 'workflow_type' => self::TYPE, 'task_queue' => self::QUEUE, + 'run_timeout_seconds' => 60, 'execution_timeout_seconds' => 120, + ], $this->apiHeaders())->assertCreated(); + $run = WorkflowRun::query()->findOrFail($start->json('run_id')); + $task = WorkflowTask::query()->where('workflow_run_id', $run->id)->firstOrFail(); + $started = WorkflowHistoryEvent::query()->where('workflow_run_id', $run->id) + ->where('event_type', 'WorkflowStarted')->firstOrFail(); + self::assertSame('worker', $started->payload['workflow_definition_fingerprint_source']); + self::assertSame('definition-v1', $started->payload['workflow_definition_fingerprint']); + + return [$run, $task, $started]; + } + + private function worker(string $id, string $fingerprint, array $overrides = []): void + { + $this->postJson('/api/worker/register', [ + 'worker_id' => $id, 'task_queue' => self::QUEUE, 'runtime' => 'python', 'sdk_version' => '2.6.1', + 'capabilities' => [WorkflowDefinitionRouting::ENTRY_VALIDATION_CAPABILITY], + 'capability_manifest' => $this->portableWorkerAffinityRefusalManifest(), + 'supported_workflow_types' => [self::TYPE], + 'workflow_definition_fingerprints' => [self::TYPE => $fingerprint], + 'workflow_command_contracts' => [self::TYPE => ['signals' => [], 'queries' => [], 'updates' => [], 'update_validators' => []]], + ...$overrides, + ], $this->workerHeaders())->assertCreated(); + } + + private function poll(string $id, array $overrides = []): TestResponse + { + return $this->postJson('/api/worker/workflow-tasks/poll', [ + 'worker_id' => $id, 'task_queue' => self::QUEUE, 'timeout_seconds' => 0, + ...$overrides, + ], $this->workerHeaders()); + } +} diff --git a/tests/Unit/ServerParityLockPressureRetryTest.php b/tests/Unit/ServerParityLockPressureRetryTest.php index e29e3beb9..849e1b310 100644 --- a/tests/Unit/ServerParityLockPressureRetryTest.php +++ b/tests/Unit/ServerParityLockPressureRetryTest.php @@ -50,6 +50,30 @@ public function test_other_requests_statuses_and_bodies_are_never_retried(): voi } } + public function test_documented_operation_refusals_are_retried_and_polls_keep_their_rules(): void + { + $deferred = ['reason' => 'backend_lock_pressure', 'outcome' => 'deferred', 'retryable' => true, 'retry_after_seconds' => 1]; + $complete = 'http://parity-server:8080/api/worker/workflow-tasks/task/complete'; + $start = 'http://parity-server:8080/api/workflows'; + self::assertSame(1.0, \lockPressureOperationRetryDelay('POST', $complete, 503, $deferred, 0, 0.0)); + self::assertSame(1.0, \lockPressureOperationRetryDelay('POST', $start, 503, + ['reason' => 'backend_lock_pressure', 'retryable' => true, 'retry_after_seconds' => 1], 0, 0.0)); + self::assertSame(1.0, \lockPressureOperationRetryDelay('POST', self::POLL, 503, $this->refusal(), 0, 0.0)); + foreach ([ + [$complete, 503, array_replace($deferred, ['retryable' => false])], + [$complete, 503, array_diff_key($deferred, ['retryable' => true])], + [$complete, 503, array_replace($deferred, ['reason' => 'backend_unavailable'])], + [$complete, 409, $deferred], + [$complete, 503, null], + // A poll refusal still needs task null, even through this entry point. + [self::POLL, 503, $deferred], + ] as $index => [$uri, $status, $response]) { + self::assertNull(\lockPressureOperationRetryDelay('POST', $uri, $status, $response, 0, 0.0), (string) $index); + } + self::assertNull(\lockPressureOperationRetryDelay('POST', $complete, 503, $deferred, 15, 0.0)); + self::assertNull(\lockPressureOperationRetryDelay('POST', $complete, 503, $deferred, 2, 29.5)); + } + public function test_retries_stop_at_the_attempt_and_wait_budgets(): void { self::assertNull(\lockPressurePollRetryDelay('POST', self::POLL, 503, $this->refusal(), 15, 0.0));