From a3f74dae89b439c26c8286795a3d7f20ba57a637 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Tue, 27 Oct 2026 21:24:50 +0000 Subject: [PATCH 1/2] Keep service-calls' SQLite maintenance pause under full-corpus load #815 recorded update-validation and service-calls in the SQLite main corpus with published PHP's maintenance loop running. In #821's merge-group run 40579733447, published 2.11.5's service-calls record stopped ("No charge with options.id for a later path") after 2.11.5 exhausted its control-plane SQLite contention retries on a workflow_service_calls insert while the loop ran its repair pass. That is one failure in five full SQLite runs since #815, and #796 is still open. The SQLite job and the native image corpus again exclude service-calls from the main corpus and record it with update-validation-sqlite.sh, which stops PHP's maintenance loop for the PHP record only. update-validation stays in the main corpus (#726 has installed acceptance). The script, php-maintenance-window.sh and their tests are restored from before #815, pointing at the 2.11.5 baseline. The port log and a checkpoint record the walk-back. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/native-image-parity.yml | 2 + .github/workflows/phpunit-feature.yml | 2 + .github/workflows/server-parity.yml | 14 +- .../2026-10-27-sqlite-service-calls-pause.md | 33 ++++ docs/rust-server-port.md | 8 +- .../server-parity/native-image-corpus.sh | 14 +- .../server-parity/php-maintenance-window.sh | 65 +++++++ .../server-parity/update-validation-sqlite.sh | 95 +++++++++ tests/Unit/PhpMaintenanceWindowTest.mjs | 100 ++++++++++ .../Unit/ServerParityFixtureSelectionTest.mjs | 37 +++- .../ServerParityUpdateValidationRetryTest.mjs | 180 ++++++++++++++++++ 11 files changed, 536 insertions(+), 14 deletions(-) create mode 100644 docs/rust-server-port-checkpoints/2026-10-27-sqlite-service-calls-pause.md create mode 100644 scripts/conformance/server-parity/php-maintenance-window.sh create mode 100644 scripts/conformance/server-parity/update-validation-sqlite.sh create mode 100644 tests/Unit/PhpMaintenanceWindowTest.mjs create mode 100644 tests/Unit/ServerParityUpdateValidationRetryTest.mjs diff --git a/.github/workflows/native-image-parity.yml b/.github/workflows/native-image-parity.yml index b16bee185..a318e260b 100644 --- a/.github/workflows/native-image-parity.yml +++ b/.github/workflows/native-image-parity.yml @@ -19,6 +19,8 @@ on: pull_request: paths: - scripts/conformance/server-parity/native-image-corpus.sh + - scripts/conformance/server-parity/update-validation-sqlite.sh + - scripts/conformance/server-parity/php-maintenance-window.sh - .github/workflows/native-image-parity.yml permissions: diff --git a/.github/workflows/phpunit-feature.yml b/.github/workflows/phpunit-feature.yml index 670a9de07..ccfc38983 100644 --- a/.github/workflows/phpunit-feature.yml +++ b/.github/workflows/phpunit-feature.yml @@ -80,6 +80,8 @@ jobs: - name: Require safe PHP image build context run: node --test tests/Unit/PhpImageContextTest.mjs + - name: Restore PHP's SQLite maintenance loop on every path + run: node --test tests/Unit/PhpMaintenanceWindowTest.mjs - name: Restore the patch cohort's maintenance loop on every path run: node --test tests/Unit/PatchCohortMaintenanceTest.mjs diff --git a/.github/workflows/server-parity.yml b/.github/workflows/server-parity.yml index 35906b9e6..a12599449 100644 --- a/.github/workflows/server-parity.yml +++ b/.github/workflows/server-parity.yml @@ -66,6 +66,7 @@ on: - tests/Unit/ServerParityWorkerSessionActivitiesContractTest.mjs - tests/Unit/ServerParityUpdateValidationContractTest.mjs - tests/Unit/ServerParityFixtureSelectionTest.mjs + - tests/Unit/ServerParityUpdateValidationRetryTest.mjs - tests/Unit/ServerParityKilledAuthorPollTest.mjs - tests/Unit/ServerParityImagePullTest.mjs - tests/Unit/ServerParityAdapterCacheTest.mjs @@ -1905,7 +1906,7 @@ jobs: bash scripts/ci/run-composer.sh --rm -v "$PWD:/source" -w /source -- \ install --working-dir=scripts/conformance/server-parity --no-interaction --no-progress docker run --rm --user=1000:1000 --entrypoint node -v "$PWD:/source:ro" -w /source \ - "$image" --test tests/Unit/ServerParityRunnerTest.mjs tests/Unit/ServerParitySdkDirectionLanesTest.mjs tests/Unit/ServerParityCodecDeadlineCellsTest.mjs tests/Unit/ServerParityCooperativeContractTest.mjs tests/Unit/ServerParityChildCancellationContractTest.mjs tests/Unit/ServerParityScheduleContractTest.mjs tests/Unit/ServerParityVisibilityContractTest.mjs tests/Unit/ServerParityAdmissionContractTest.mjs tests/Unit/ServerParityRoleTokenContractTest.mjs tests/Unit/ServerParityWorkerDeregistrationContractTest.mjs tests/Unit/ServerParitySearchAttributeContractTest.mjs tests/Unit/ServerParityWorkerListingContractTest.mjs tests/Unit/ServerParityWorkerReplacementContractTest.mjs tests/Unit/ServerParityPatchDeploymentContractTest.mjs tests/Unit/ServerParityWaitingHistoryContractTest.mjs tests/Unit/ServerParityTaskRetryContractTest.mjs tests/Unit/ServerParityPublishedPatchContractTest.mjs tests/Unit/ServerParityPublishedPatchTransportTest.mjs tests/Unit/ServerParityRecordFileTest.mjs tests/Unit/ServerParityFailureLocationTest.mjs tests/Unit/ServerParityFailureExcerptTest.mjs tests/Unit/ServerParityLateWorkflowCompletionContractTest.mjs tests/Unit/ServerParityStorageTestContractTest.mjs tests/Unit/ServerParitySchedulePoliciesContractTest.mjs tests/Unit/ServerParitySignalsBeforeWorkerContractTest.mjs tests/Unit/ServerParityScheduleCatalogContractTest.mjs tests/Unit/ServerParityOperatorMetricsContractTest.mjs tests/Unit/ServerParityActivityLeaseRenewalContractTest.mjs tests/Unit/ServerParityRecordLocalActivityContractTest.mjs tests/Unit/ServerParityActivityCancellationAckContractTest.mjs tests/Unit/ServerParityServiceCatalogContractTest.mjs tests/Unit/ServerParityServiceEndpointsContractTest.mjs tests/Unit/ServerParityServiceCallsContractTest.mjs tests/Unit/ServerParityWorkflowImportContractTest.mjs tests/Unit/ServerParityWaterlineImportContractTest.mjs tests/Unit/ServerParitySystemMetricsContractTest.mjs tests/Unit/ServerParityOperatorDashboardTypesContractTest.mjs tests/Unit/ServerParityPrometheusMetricsContractTest.mjs tests/Unit/ServerParityPayloadCleanupContractTest.mjs tests/Unit/ServerParityRepairPassContractTest.mjs tests/Unit/ServerParityClusterInfoContractTest.mjs tests/Unit/ServerParityNamespaceDeleteContractTest.mjs tests/Unit/ServerParityMessageStreamsContractTest.mjs tests/Unit/ServerParityCancellationScopesContractTest.mjs tests/Unit/ServerParityWorkflowPollRequestContractTest.mjs tests/Unit/ServerParityUpdateValidationContractTest.mjs tests/Unit/ServerParityFixtureSelectionTest.mjs tests/Unit/ServerParityKilledAuthorPollTest.mjs tests/Unit/ServerParityImagePullTest.mjs tests/Unit/ServerParityAdapterCacheTest.mjs tests/Unit/ServerParityWorkerSessionsContractTest.mjs tests/Unit/ServerParityWorkerSessionActivitiesContractTest.mjs tests/Unit/ServerParityNexusOperationsContractTest.mjs tests/Unit/ServerParityBridgeAdaptersContractTest.mjs tests/Unit/ServerParityWorkflowStreamWritesContractTest.mjs > parity-evidence/comparator-tests.txt + "$image" --test tests/Unit/ServerParityRunnerTest.mjs tests/Unit/ServerParitySdkDirectionLanesTest.mjs tests/Unit/ServerParityCodecDeadlineCellsTest.mjs tests/Unit/ServerParityCooperativeContractTest.mjs tests/Unit/ServerParityChildCancellationContractTest.mjs tests/Unit/ServerParityScheduleContractTest.mjs tests/Unit/ServerParityVisibilityContractTest.mjs tests/Unit/ServerParityAdmissionContractTest.mjs tests/Unit/ServerParityRoleTokenContractTest.mjs tests/Unit/ServerParityWorkerDeregistrationContractTest.mjs tests/Unit/ServerParitySearchAttributeContractTest.mjs tests/Unit/ServerParityWorkerListingContractTest.mjs tests/Unit/ServerParityWorkerReplacementContractTest.mjs tests/Unit/ServerParityPatchDeploymentContractTest.mjs tests/Unit/ServerParityWaitingHistoryContractTest.mjs tests/Unit/ServerParityTaskRetryContractTest.mjs tests/Unit/ServerParityPublishedPatchContractTest.mjs tests/Unit/ServerParityPublishedPatchTransportTest.mjs tests/Unit/ServerParityRecordFileTest.mjs tests/Unit/ServerParityFailureLocationTest.mjs tests/Unit/ServerParityFailureExcerptTest.mjs tests/Unit/ServerParityLateWorkflowCompletionContractTest.mjs tests/Unit/ServerParityStorageTestContractTest.mjs tests/Unit/ServerParitySchedulePoliciesContractTest.mjs tests/Unit/ServerParitySignalsBeforeWorkerContractTest.mjs tests/Unit/ServerParityScheduleCatalogContractTest.mjs tests/Unit/ServerParityOperatorMetricsContractTest.mjs tests/Unit/ServerParityActivityLeaseRenewalContractTest.mjs tests/Unit/ServerParityRecordLocalActivityContractTest.mjs tests/Unit/ServerParityActivityCancellationAckContractTest.mjs tests/Unit/ServerParityServiceCatalogContractTest.mjs tests/Unit/ServerParityServiceEndpointsContractTest.mjs tests/Unit/ServerParityServiceCallsContractTest.mjs tests/Unit/ServerParityWorkflowImportContractTest.mjs tests/Unit/ServerParityWaterlineImportContractTest.mjs tests/Unit/ServerParitySystemMetricsContractTest.mjs tests/Unit/ServerParityOperatorDashboardTypesContractTest.mjs tests/Unit/ServerParityPrometheusMetricsContractTest.mjs tests/Unit/ServerParityPayloadCleanupContractTest.mjs tests/Unit/ServerParityRepairPassContractTest.mjs tests/Unit/ServerParityClusterInfoContractTest.mjs tests/Unit/ServerParityNamespaceDeleteContractTest.mjs tests/Unit/ServerParityMessageStreamsContractTest.mjs tests/Unit/ServerParityCancellationScopesContractTest.mjs tests/Unit/ServerParityWorkflowPollRequestContractTest.mjs tests/Unit/ServerParityUpdateValidationContractTest.mjs tests/Unit/ServerParityFixtureSelectionTest.mjs tests/Unit/ServerParityUpdateValidationRetryTest.mjs tests/Unit/ServerParityKilledAuthorPollTest.mjs tests/Unit/ServerParityImagePullTest.mjs tests/Unit/ServerParityAdapterCacheTest.mjs tests/Unit/ServerParityWorkerSessionsContractTest.mjs tests/Unit/ServerParityWorkerSessionActivitiesContractTest.mjs tests/Unit/ServerParityNexusOperationsContractTest.mjs tests/Unit/ServerParityBridgeAdaptersContractTest.mjs tests/Unit/ServerParityWorkflowStreamWritesContractTest.mjs > parity-evidence/comparator-tests.txt docker run --rm --user=1000:1000 --cpus=1 --memory=256m --memory-swap=256m \ --network=none --entrypoint node -v "$PWD:/source:ro" -w /source \ "$image" --test tests/Unit/ServerParityCodecDeadlineLoadTest.mjs > parity-evidence/codec-load-tests.txt @@ -1977,6 +1978,7 @@ jobs: -v "$PWD:/source:ro" -v "$PWD/parity-evidence:/evidence" -w /source \ -e DW_PARITY_TOKEN=parity-test-token -e DW_PARITY_RUNNER_REVISION -e DW_PARITY_CLI_PHAR "$image" scripts/conformance/server-parity.mjs record \ --mode http --url http://parity-server:8080 --target php --output /evidence/php.json \ + --exclude-fixture tests/Fixtures/ServerParity/service-calls.json \ --include-fixture tests/Fixtures/ServerParityPending/patch-insertion-activity-pending.json \ --include-fixture tests/Fixtures/ServerParityPending/patch-insertion-activity-completed.json \ --include-fixture tests/Fixtures/ServerParityPending/patch-created-activity-pending.json \ @@ -1989,6 +1991,7 @@ jobs: -e DB_CONNECTION=sqlite -e DB_DATABASE=/state/embedded.sqlite -e DW_MODE=embedded \ -e CACHE_STORE=file -e QUEUE_CONNECTION=database -e DW_PARITY_RUNNER_REVISION "$image" scripts/conformance/server-parity.mjs record \ --mode embedded --application-root /app --target embedded --output /evidence/embedded.json \ + --exclude-fixture tests/Fixtures/ServerParity/service-calls.json \ --include-fixture tests/Fixtures/ServerParityPending/patch-insertion-activity-pending.json \ --include-fixture tests/Fixtures/ServerParityPending/patch-insertion-activity-completed.json \ --include-fixture tests/Fixtures/ServerParityPending/patch-created-activity-pending.json \ @@ -2001,6 +2004,7 @@ jobs: -v "$PWD:/source:ro" -v "$PWD/parity-evidence:/evidence" -w /source \ -e DW_PARITY_TOKEN=parity-test-token -e DW_PARITY_RUNNER_REVISION -e DW_PARITY_CLI_PHAR "$image" scripts/conformance/server-parity.mjs record \ --mode http --url http://parity-rust:8080 --target rust --output /evidence/rust.json \ + --exclude-fixture tests/Fixtures/ServerParity/service-calls.json \ --include-fixture tests/Fixtures/ServerParityPending/patch-insertion-activity-pending.json \ --include-fixture tests/Fixtures/ServerParityPending/patch-insertion-activity-completed.json \ --include-fixture tests/Fixtures/ServerParityPending/patch-created-activity-pending.json \ @@ -2012,6 +2016,7 @@ jobs: docker run --rm --user=1000:1000 --entrypoint node \ -v "$PWD:/source:ro" -v "$PWD/parity-evidence:/evidence" -w /source "$image" \ scripts/conformance/server-parity.mjs compare /evidence/php.json /evidence/rust.json /evidence/embedded.json \ + --exclude-fixture tests/Fixtures/ServerParity/service-calls.json \ --include-fixture tests/Fixtures/ServerParityPending/patch-insertion-activity-pending.json \ --include-fixture tests/Fixtures/ServerParityPending/patch-insertion-activity-completed.json \ --include-fixture tests/Fixtures/ServerParityPending/patch-created-activity-pending.json \ @@ -2019,10 +2024,17 @@ jobs: --include-fixture tests/Fixtures/ServerParityPending/workflow-waiting-history-type.json \ --include-fixture tests/Fixtures/ServerParityPending/workflow-waiting-history-message.json \ --include-fixture tests/Fixtures/ServerParityPending/workflow-task-transient-retry.json > parity-evidence/comparison.json || compare_status=$? + # Server #796: service-calls records apart from the main corpus, with + # published PHP's maintenance loop stopped. Under full-corpus load the + # loop's SQLite writer contention can still exhaust 2.11.5's + # control-plane retries on a service-call insert. + service_calls_status=0 + bash scripts/conformance/server-parity/update-validation-sqlite.sh http://parity-server:8080 http://parity-rust:8080 service-calls || service_calls_status=$? # A native fixture difference fails the step only after the # comparison has run, so one difference cannot hide the rest. test "$native_record_status" -eq 0 test "$compare_status" -eq 0 + test "$service_calls_status" -eq 0 - name: Compare published patch, legacy-marker and range-repair profiles if: ${{ matrix.part == 'restarts' }} env: diff --git a/docs/rust-server-port-checkpoints/2026-10-27-sqlite-service-calls-pause.md b/docs/rust-server-port-checkpoints/2026-10-27-sqlite-service-calls-pause.md new file mode 100644 index 000000000..8c8a1083a --- /dev/null +++ b/docs/rust-server-port-checkpoints/2026-10-27-sqlite-service-calls-pause.md @@ -0,0 +1,33 @@ +# October 27, `service-calls` keeps its SQLite maintenance pause + +#815 moved the reference to published 2.11.5 and recorded both +`update-validation` and `service-calls` in the SQLite main corpus with PHP's +maintenance loop running. That was based on Cloud's installed #726 acceptance +(15 of 15) and General's isolated #796 check (10 of 10). This walks part of it +back. + +- **What failed.** #821's merge-group run + [40579733447](https://github.com/durable-workflow/server/actions/runs/40579733447), + `execution-and-codec (corpus)`. Published 2.11.5's `service-calls` record + stopped with `RuntimeException: No charge with options.id for a later path.` + - Its server log shows `Control-plane storage contention exhausted retries` + and SQLite `database is locked` on an insert into `workflow_service_calls` + while the maintenance loop ran its repair pass. + - One earlier execute got the typed 503 `backend_lock_pressure`, and the + probe retried it; this refusal was not that reply, so it was not retried. + - Native and embedded were not involved. +- **Frequency.** One failure in five full SQLite runs since #815: #815's and + #816's merge groups and two local runs passed. That is too flaky for a + required check. #796 is still open. +- **Change.** The SQLite job and the native image corpus exclude + `service-calls` from the main corpus again. They record it with + `update-validation-sqlite.sh … service-calls`, which stops PHP's + maintenance loop for the PHP record only, as before #815. + - `update-validation` stays in the main corpus: #726 is closed with + installed acceptance, and it has not failed since. + - The script, `php-maintenance-window.sh` and their tests are restored + from before #815, now pointing at the 2.11.5 baseline. + - The MySQL, MariaDB and PostgreSQL jobs keep `service-calls` in their + main corpus with the loop running. +- **Next.** Remove the pause again once a published reference no longer + exhausts its retries under full-corpus SQLite load (#796). diff --git a/docs/rust-server-port.md b/docs/rust-server-port.md index 0d4aa3496..a991dfc20 100644 --- a/docs/rust-server-port.md +++ b/docs/rust-server-port.md @@ -41,7 +41,8 @@ Updated October 27, 2026. the takeover runbook's image commands. - The parity reference moved to published 2.11.4 (#805, [checkpoint](rust-server-port-checkpoints/2026-10-27-reference-2114.md)), - and moves to published 2.11.5, without the SQLite maintenance pause + and moved to published 2.11.5 (#815); the SQLite maintenance pause stays + for `service-calls` only ([checkpoint](rust-server-port-checkpoints/2026-10-27-reference-2115.md)). - [#814](https://github.com/durable-workflow/server/pull/814): the shared fixtures accept published PHP's typed `429 long_poll_capacity_exhausted` @@ -68,7 +69,10 @@ Updated October 27, 2026. Closed by 2.11.5: Workflow #931 (`service-calls` repeats a key from another caller again, now refused 403), and the SQLite maintenance pause - for `update-validation` and `service-calls` (#723, #726, #796). + for `update-validation` (#723, #726). `service-calls` still records apart + with the loop stopped: under full-corpus load 2.11.5 can exhaust its SQLite + contention retries on a service-call insert (#796, open; + [checkpoint](rust-server-port-checkpoints/2026-10-27-sqlite-service-calls-pause.md)). No expectation is edited to make native pass. - **Parity surface.** Native answers 174 of the 175 API route methods in diff --git a/scripts/conformance/server-parity/native-image-corpus.sh b/scripts/conformance/server-parity/native-image-corpus.sh index 2d8466043..7a340f120 100755 --- a/scripts/conformance/server-parity/native-image-corpus.sh +++ b/scripts/conformance/server-parity/native-image-corpus.sh @@ -29,9 +29,10 @@ export DW_PARITY_RUNNER_REVISION DW_PARITY_RUNNER_REVISION=$(git rev-parse HEAD) image=$(jq -r '.server.images["linux/amd64"] | sub("^durableworkflow/server@"; "ghcr.io/durable-workflow/server@")' tests/Fixtures/ServerParityBaselines/php-2.11.5.json) -# CI's corpus selection: the reviewed corpus plus the pending candidates it -# carries. -selection=() +# CI's corpus selection: the reviewed corpus without service-calls, which +# records separately below with PHP's maintenance loop stopped (Server #796), +# plus the pending candidates it carries. +selection=(--exclude-fixture tests/Fixtures/ServerParity/service-calls.json) for pending in patch-insertion-activity-pending patch-insertion-activity-completed patch-created-activity-pending \ patch-created-activity-completed workflow-waiting-history-type workflow-waiting-history-message workflow-task-transient-retry; do selection+=(--include-fixture "tests/Fixtures/ServerParityPending/$pending.json") @@ -111,15 +112,20 @@ compare_status=0 docker run --rm --user=1000:1000 --entrypoint node -v "$PWD:/source:ro" -v "$PWD/parity-evidence:/evidence" -w /source "$image" \ scripts/conformance/server-parity.mjs compare /evidence/php.json /evidence/rust.json /evidence/embedded.json "${selection[@]}" \ > parity-evidence/comparison.json || compare_status=$? +service_calls_status=0 +bash scripts/conformance/server-parity/update-validation-sqlite.sh http://parity-server:8080 http://parity-rust:8080 service-calls \ + || service_calls_status=$? native_id=$(docker image inspect --format '{{.Id}}' "$native") jq -n --arg native "$native" --arg id "$native_id" --arg php "$image" --arg revision "$DW_PARITY_RUNNER_REVISION" \ --argjson reference "$status" --argjson nativeStatus "$native_status" --argjson compare "$compare_status" \ + --argjson serviceCalls "$service_calls_status" \ '{schema:"durable-workflow.native-image-corpus.v1", native_image:$native, native_image_id:$id, php_reference:$php, runner_revision:$revision, backend:"sqlite", - status:{reference_records:$reference, native_record:$nativeStatus, compare:$compare}}' \ + status:{reference_records:$reference, native_record:$nativeStatus, compare:$compare, service_calls:$serviceCalls}}' \ > parity-evidence/native-image-corpus.json cat parity-evidence/native-image-corpus.json test "$status" -eq 0 test "$native_status" -eq 0 test "$compare_status" -eq 0 +test "$service_calls_status" -eq 0 diff --git a/scripts/conformance/server-parity/php-maintenance-window.sh b/scripts/conformance/server-parity/php-maintenance-window.sh new file mode 100644 index 000000000..ec55e95a7 --- /dev/null +++ b/scripts/conformance/server-parity/php-maintenance-window.sh @@ -0,0 +1,65 @@ +# Sourced by SQLite-job scripts that record with PHP's maintenance loop stopped +# (Server #721, #723). +# +# run_without_maintenance SCHEDULE -- COMMAND... +# +# Runs COMMAND while the SQLite job's PHP maintenance container is stopped, +# then restarts it. Restoration is armed before the stop is attempted and +# stays armed until the container has stayed running for a sustained check, +# so a failed stop, a failed COMMAND or a restart that exits at once never +# leaves it stopped. SCHEDULE records the container's state before, during +# and after. + +maintenance_poll_seconds="${DW_PARITY_MAINTENANCE_POLL_SECONDS:-1}" +# The SQLite job's container, named from this job's identity. +maintenance_container="dw-parity-${GITHUB_RUN_ID:-local}-${GITHUB_RUN_ATTEMPT:-1}-${GITHUB_JOB:-parity}-php-maintenance" + +maintenance_state() { + docker inspect -f '{{.State.Status}}' "$maintenance_container" 2>/dev/null || echo unknown + return 0 +} + +# Running at each of three checks, one poll interval apart. +maintenance_stays_running() { + local check + for check in 1 2 3; do + [[ "$(maintenance_state)" == running ]] || return 1 + sleep "$maintenance_poll_seconds" + done + [[ "$(maintenance_state)" == running ]] +} + +# Up to three starts, each followed by the sustained check. +maintenance_restore() { + local attempt + for attempt in 1 2 3; do + docker start "$maintenance_container" >/dev/null 2>&1 || true + if maintenance_stays_running; then + return 0 + fi + done + echo "maintenance container $maintenance_container did not stay running: $(maintenance_state)" >&2 + return 1 +} + +run_without_maintenance() { + local schedule="$1" + shift + local separator="$1" + [[ "$separator" == -- ]] && shift + # The job started this container as "$RESOURCE_SCOPE-php-maintenance". + [[ "$maintenance_container" == "$RESOURCE_SCOPE-php-maintenance" ]] + echo "before: $(maintenance_state)" > "$schedule" + grep -qx 'before: running' "$schedule" + trap 'maintenance_restore || true' EXIT + docker stop --time 2 "$maintenance_container" >/dev/null + echo "during php record: $(maintenance_state)" >> "$schedule" + grep -qx 'during php record: exited' "$schedule" + "$@" + if ! maintenance_restore; then + echo "after php record: $(maintenance_state)" >> "$schedule" + return 1 + fi + trap - EXIT + echo "after php record: running for ${maintenance_poll_seconds}s x3" >> "$schedule" +} diff --git a/scripts/conformance/server-parity/update-validation-sqlite.sh b/scripts/conformance/server-parity/update-validation-sqlite.sh new file mode 100644 index 000000000..0129588bc --- /dev/null +++ b/scripts/conformance/server-parity/update-validation-sqlite.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash +# update-validation-sqlite.sh [fixture name] +# +# Server #723. The SQLite job records the update-validation fixture apart from +# its main corpus, for all three targets, and compares the three records. The +# optional third argument names another fixture recorded the same way: +# service-calls, whose published PHP record under SQLite writer contention has +# refused the first call to a limited operation with 429 (Server #796). +# +# The published PHP record runs with PHP's maintenance loop stopped. On SQLite +# the API server and that loop share one database file. Under the loop's writer +# contention, published PHP 2.11.1 loses its own validation-timeout and +# lease-reclaim writes to "database is locked", and its validation poll answers +# without the typed backend_lock_pressure refusal. Locally 10 of 15 runs failed +# with the loop running and 5 of 5 passed with it stopped. This fixture tests +# update validation, not writer contention, and needs no repair pass or +# schedule. The loop restarts right after the PHP record. The MySQL and +# PostgreSQL jobs record this fixture in their main corpus with their loops +# running. +# +# Server #754. Even with the loop stopped, the probe's own concurrent updates +# can cost published PHP a write: 2.11.1 claims a validation task with a read +# and then a write in one deferred SQLite transaction, and when another writer +# commits in between, SQLite refuses the upgrade at once ("database is +# locked"), whatever the busy timeout. Current PHP retries that write; the +# published reference cannot. So when the PHP record fails and the PHP server +# logged that error during it, the PHP record runs once more under a fresh +# workflow prefix, and embedded and Rust record under that prefix too, as +# compare requires. Any other PHP failure, and any embedded or Rust failure, +# fails at once. A retry leaves a warning and the first attempt's files. +set -euo pipefail +php_url="$1" +native_url="$2" +name="${3:-update-validation}" +[[ "$name" =~ ^[a-z0-9-]+$ ]] +[[ -n "$RESOURCE_SCOPE" ]] +[[ -n "$DW_PARITY_RUNNER_REVISION" ]] +fixture="tests/Fixtures/ServerParity/$name.json" +test -f "$fixture" +image=$(jq -r '.server.images["linux/amd64"] | sub("^durableworkflow/server@"; "ghcr.io/durable-workflow/server@")' tests/Fixtures/ServerParityBaselines/php-2.11.5.json) +. scripts/conformance/server-parity/php-maintenance-window.sh + +prefix=parity-v1 +http_record() { + local url="$1" target="$2" + docker run --rm --user=1000:1000 --entrypoint node --network "$RESOURCE_SCOPE" \ + -v "$PWD:/source:ro" -v "$PWD/parity-evidence:/evidence" -w /source \ + -e DW_PARITY_TOKEN=parity-test-token -e DW_PARITY_RUNNER_REVISION "$image" scripts/conformance/server-parity.mjs record \ + --mode http --url "$url" --target "$target" --output "/evidence/$name-$target.json" --fixture "$fixture" \ + --prefix "$prefix" +} + +# run_without_maintenance runs its command under errexit's exemption, so the +# command keeps the record's status instead of returning it. The record's own +# start and end bound the PHP log lines that can justify a retry. +php_status=0 +php_record() { + php_status=0 + record_started=$(date -u +%Y-%m-%dT%H:%M:%S.%NZ) + http_record "$php_url" php || php_status=$? + record_finished=$(date -u +%Y-%m-%dT%H:%M:%S.%NZ) +} + +run_without_maintenance "parity-evidence/$name-maintenance-schedule.txt" -- php_record +if ((php_status != 0)); then + locked="parity-evidence/$name-php.attempt-1.locked.txt" + { docker logs --since "$record_started" --until "$record_finished" "$RESOURCE_SCOPE-server" 2>&1 || true; } \ + | grep -m 5 'database is locked' | cut -c 1-400 > "$locked" || true + if [[ ! -s "$locked" ]]; then + rm -f "$locked" + exit "$php_status" + fi + for file in "$name-php.json" "$name-php.failure-excerpt.txt"; do + if [[ -e "parity-evidence/$file" ]]; then + mv "parity-evidence/$file" "parity-evidence/${file/$name-php./$name-php.attempt-1.}" + fi + done + excerpt=$(tr '\n' ' ' < "parity-evidence/$name-php.attempt-1.failure-excerpt.txt" 2>/dev/null || true) + echo "::warning title=Published PHP $name record retried::The PHP record failed (${excerpt:-no excerpt}) after published PHP logged 'database is locked'; recording all three targets under a fresh prefix (Server #754)." + prefix=parity-v1-retry + run_without_maintenance "parity-evidence/$name-maintenance-schedule-retry.txt" -- php_record + ((php_status == 0)) || exit "$php_status" +fi +docker run --rm --user=1000:1000 --entrypoint node \ + -v "$PWD:/source:ro" -v "$PWD/parity-state:/state" -v "$PWD/parity-evidence:/evidence" -w /source \ + -e DB_CONNECTION=sqlite -e DB_DATABASE=/state/embedded.sqlite -e DW_MODE=embedded \ + -e CACHE_STORE=file -e QUEUE_CONNECTION=database -e DW_PARITY_RUNNER_REVISION "$image" scripts/conformance/server-parity.mjs record \ + --mode embedded --application-root /app --target embedded --output "/evidence/$name-embedded.json" --fixture "$fixture" \ + --prefix "$prefix" +http_record "$native_url" rust +docker run --rm --user=1000:1000 --entrypoint node \ + -v "$PWD:/source:ro" -v "$PWD/parity-evidence:/evidence" -w /source "$image" \ + scripts/conformance/server-parity.mjs compare \ + "/evidence/$name-php.json" "/evidence/$name-rust.json" "/evidence/$name-embedded.json" \ + --fixture "$fixture" > "parity-evidence/$name-comparison.json" diff --git a/tests/Unit/PhpMaintenanceWindowTest.mjs b/tests/Unit/PhpMaintenanceWindowTest.mjs new file mode 100644 index 000000000..207dd0b1a --- /dev/null +++ b/tests/Unit/PhpMaintenanceWindowTest.mjs @@ -0,0 +1,100 @@ +// Server #721, #723: SQLite-job records that stop PHP's maintenance loop for +// their PHP record. Every failure path must leave the loop restored or fail +// loudly, and never record a restart that did not stay running. +import test from 'node:test'; +import assert from 'node:assert/strict'; +import {chmodSync, existsSync, mkdtempSync, readFileSync, writeFileSync} from 'node:fs'; +import {tmpdir} from 'node:os'; +import {join, resolve} from 'node:path'; +import {spawnSync} from 'node:child_process'; + +const helper = resolve('scripts/conformance/server-parity/php-maintenance-window.sh'); + +// One container's state in a file; every call is logged. +// SCENARIO=stop-nonzero: the stop takes effect but exits 1. +// SCENARIO=immediate-exit: start exits 0, the container runs for one +// inspection and then exits. +const fakeDocker = `#!/usr/bin/env bash +set -u +dir=$(dirname "$0") +echo "$*" >> "$dir/calls" +test "\${!#}" = dw-parity-7-2-sqlite-php-maintenance || exit 3 +case "$1" in + inspect) + state=$(cat "$dir/state") + if test "$state" = running-once; then echo exited > "$dir/state"; state=running; fi + echo "$state" ;; + stop) echo exited > "$dir/state"; test "$SCENARIO" != stop-nonzero ;; + start) + if test "$SCENARIO" = immediate-exit; then echo running-once > "$dir/state"; else echo running > "$dir/state"; fi ;; + *) exit 2 ;; +esac +`; + +function run(scenario, command, scope = 'dw-parity-7-2-sqlite') { + const dir = mkdtempSync(join(tmpdir(), 'php-maintenance-window-')); + writeFileSync(join(dir, 'docker'), fakeDocker); + chmodSync(join(dir, 'docker'), 0o755); + writeFileSync(join(dir, 'state'), 'running\n'); + writeFileSync(join(dir, 'calls'), ''); + const result = spawnSync('bash', ['-c', + `set -euo pipefail; . "$HELPER"; run_without_maintenance "$SCHEDULE" -- ${command}`], { + encoding: 'utf8', + env: {...process.env, PATH: `${dir}:${process.env.PATH}`, HELPER: helper, + SCHEDULE: join(dir, 'schedule.txt'), SCENARIO: scenario, DW_PARITY_MAINTENANCE_POLL_SECONDS: '0.05', + GITHUB_RUN_ID: '7', GITHUB_RUN_ATTEMPT: '2', GITHUB_JOB: 'sqlite', RESOURCE_SCOPE: scope}, + }); + const read = name => (existsSync(join(dir, name)) ? readFileSync(join(dir, name), 'utf8') : ''); + const calls = read('calls').trim().split('\n'); + return { + status: result.status, + stderr: result.stderr, + schedule: read('schedule.txt').trim().split('\n'), + starts: calls.filter(call => call.startsWith('start ')).length, + stopIndex: calls.findIndex(call => call.startsWith('stop ')), + lastStart: calls.findLastIndex(call => call.startsWith('start ')), + state: read('state').trim(), + }; +} + +test('a normal window stops, runs, and restores a loop that stays running', () => { + const result = run('normal', 'true'); + assert.equal(result.status, 0, result.stderr); + assert.deepEqual(result.schedule, ['before: running', 'during php record: exited', 'after php record: running for 0.05s x3']); + assert.equal(result.starts, 1, 'restoration is disarmed after the sustained check'); + assert.equal(result.state, 'running'); +}); + +test('a stop that takes effect but exits nonzero still restores', () => { + const result = run('stop-nonzero', 'true'); + assert.notEqual(result.status, 0); + assert.ok(result.lastStart > result.stopIndex, 'a start follows the stop'); + assert.equal(result.state, 'running'); + assert.deepEqual(result.schedule, ['before: running']); +}); + +test('a failed PHP record restores the loop', () => { + const result = run('normal', 'false'); + assert.notEqual(result.status, 0); + assert.equal(result.state, 'running'); + assert.ok(result.starts >= 1); + assert.ok(!result.schedule.some(line => line.startsWith('after php record'))); +}); + +test('a restart that exits at once stays armed and fails without recording running', () => { + const result = run('immediate-exit', 'true'); + assert.notEqual(result.status, 0); + assert.ok(result.schedule.includes('after php record: exited'), result.schedule.join('\n')); + assert.ok(!result.schedule.some(line => line.startsWith('after php record: running'))); + // Three attempts after the record, then three more from the still-armed trap. + assert.equal(result.starts, 6); + assert.match(result.stderr, /did not stay running/); +}); + +test('a container outside the job scope is never touched', () => { + const result = run('normal', 'true', 'dw-parity-other-job'); + assert.notEqual(result.status, 0); + assert.equal(result.starts, 0); + assert.equal(result.stopIndex, -1); + assert.equal(result.state, 'running'); +}); diff --git a/tests/Unit/ServerParityFixtureSelectionTest.mjs b/tests/Unit/ServerParityFixtureSelectionTest.mjs index d518cb481..9e91d2048 100644 --- a/tests/Unit/ServerParityFixtureSelectionTest.mjs +++ b/tests/Unit/ServerParityFixtureSelectionTest.mjs @@ -1,7 +1,6 @@ -// Fixture selection for parity records and comparisons. Every database job -// records update-validation and service-calls in its main corpus, with PHP's -// maintenance loop running: published 2.11.5 carries the #726 and #796 fixes -// that the SQLite job's former separate records worked around. +// Fixture selection for parity records and comparisons, and the SQLite job's +// separate service-calls record (Server #796). update-validation records in +// every job's main corpus: published 2.11.5 carries the #726 fixes. import test from 'node:test'; import assert from 'node:assert/strict'; import {readFileSync, readdirSync} from 'node:fs'; @@ -58,14 +57,18 @@ function mainCorpusCommands(job) { return job.split(/(?=docker run )/).filter(command => /server-parity\.mjs (record|compare)/.test(command)); } -test('the SQLite job keeps update-validation and service-calls in its main corpus', () => { +test('the SQLite job keeps update-validation in its main corpus and records service-calls separately', () => { const sqlite = jobs()['execution-and-codec']; const commands = mainCorpusCommands(sqlite); assert.equal(commands.length, 4, 'php, embedded and rust records plus their comparison'); - for (const command of commands) assert.doesNotMatch(command, /--exclude-fixture/); + for (const command of commands) { + assert.deepEqual(command.match(/--exclude-fixture \S+/g), ['--exclude-fixture tests/Fixtures/ServerParity/service-calls.json']); + } assert.ok(corpus.includes('update-validation.json')); assert.ok(corpus.includes('service-calls.json')); - assert.doesNotMatch(sqlite, /update-validation-sqlite|run_without_maintenance/); + assert.match(sqlite, /service_calls_status=0\n\s+bash scripts\/conformance\/server-parity\/update-validation-sqlite\.sh http:\/\/parity-server:8080 http:\/\/parity-rust:8080 service-calls \|\| service_calls_status=\$\?/); + assert.match(sqlite, /test "\$service_calls_status" -eq 0/); + assert.doesNotMatch(sqlite, /update-validation-sqlite\.sh http:\/\/parity-server:8080 http:\/\/parity-rust:8080 \|\|/); }); test('the MySQL and PostgreSQL jobs keep update-validation and service-calls in their main corpus', () => { @@ -75,3 +78,23 @@ test('the MySQL and PostgreSQL jobs keep update-validation and service-calls in assert.doesNotMatch(all[name], /--exclude-fixture/, name); } }); + +test('the separate record stops maintenance for the PHP record only and compares all three targets', () => { + const script = readFileSync(resolve(root, 'scripts/conformance/server-parity/update-validation-sqlite.sh'), 'utf8'); + // update-validation by default, or the named reviewed fixture. + assert.match(script, /^name="\$\{3:-update-validation\}"\n\[\[ "\$name" =~ \^\[a-z0-9-\]\+\$ \]\]$/m); + assert.match(script, /^fixture="tests\/Fixtures\/ServerParity\/\$name\.json"\ntest -f "\$fixture"$/m); + assert.match(script, /^\. scripts\/conformance\/server-parity\/php-maintenance-window\.sh$/m); + // The PHP record, and its one retry after a lost SQLite write (#754). + const windows = script.match(/^ *run_without_maintenance .*$/gm).map(line => line.trim()); + assert.deepEqual(windows, [ + 'run_without_maintenance "parity-evidence/$name-maintenance-schedule.txt" -- php_record', + 'run_without_maintenance "parity-evidence/$name-maintenance-schedule-retry.txt" -- php_record', + ]); + assert.match(script, /^php_record\(\) \{\n php_status=0\n record_started=\$\(date -u \+%Y-%m-%dT%H:%M:%S\.%NZ\)\n http_record "\$php_url" php \|\| php_status=\$\?\n record_finished=\$\(date -u \+%Y-%m-%dT%H:%M:%S\.%NZ\)\n\}$/m); + assert.match(script, /docker logs --since "\$record_started" --until "\$record_finished" "\$RESOURCE_SCOPE-server"/); + assert.match(script, /^http_record "\$native_url" rust$/m); + assert.match(script, /--mode embedded --application-root \/app --target embedded --output "\/evidence\/\$name-embedded\.json" --fixture "\$fixture"/); + assert.match(script, /"\/evidence\/\$name-php\.json" "\/evidence\/\$name-rust\.json" "\/evidence\/\$name-embedded\.json" \\\n\s+--fixture "\$fixture" > "parity-evidence\/\$name-comparison\.json"/); + assert.match(script, /^set -euo pipefail$/m); +}); diff --git a/tests/Unit/ServerParityUpdateValidationRetryTest.mjs b/tests/Unit/ServerParityUpdateValidationRetryTest.mjs new file mode 100644 index 000000000..d000b4792 --- /dev/null +++ b/tests/Unit/ServerParityUpdateValidationRetryTest.mjs @@ -0,0 +1,180 @@ +// Server #754: the SQLite job's separate update-validation record retries the +// published PHP record once, and only when that record failed after PHP +// logged "database is locked". Native and embedded records never retry. The +// separate service-calls record (Server #796) runs the same script by name. +import test from 'node:test'; +import assert from 'node:assert/strict'; +import {chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, symlinkSync, writeFileSync} from 'node:fs'; +import {tmpdir} from 'node:os'; +import {join, resolve} from 'node:path'; +import {spawnSync} from 'node:child_process'; + +const root = resolve('.'); + +// Records follow OUTCOMES, one "=" per record +// (default pass). fail fails without a lock error; fail-locked fails after PHP +// logs one during the record; fail-lockbefore and fail-lockafter fail with +// one logged 5 s before or after it. `docker logs` honors --since and --until. +const fakeDocker = `#!/usr/bin/env bash +set -u +dir=$(cd "$(dirname "$0")/.." && pwd) +echo "$*" >> "$dir/calls" +case "$1" in + inspect) cat "$dir/state" ;; + stop) echo exited > "$dir/state" ;; + start) echo running > "$dir/state" ;; + logs) + test "\${!#}" = dw-parity-7-2-sqlite-server || exit 3 + since=0 until=99999999999999999999 + while (($# > 1)); do + case "$1" in + --since) since=$(date -d "$2" +%s%N); shift ;; + --until) until=$(date -d "$2" +%s%N); shift ;; + esac + shift + done + touch "$dir/serverlog" + while read -r at line; do + if ((at >= since && at <= until)); then echo "$line"; fi + done < "$dir/serverlog" ;; + run) + command= target= output= prefix= + while (($#)); do + case "$1" in + record | compare) command=$1 ;; + --target) target=$2; shift ;; + --output) output=$2; shift ;; + --prefix) prefix=$2; shift ;; + esac + shift + done + if test "$command" = compare; then echo '{"outcome":"pass"}'; exit 0; fi + attempt=$(( $(cat "$dir/count-$target" 2>/dev/null || echo 0) + 1 )) + echo "$attempt" > "$dir/count-$target" + echo "$target $prefix" >> "$dir/records" + outcome=pass + for entry in \${OUTCOMES:-}; do + if test "\${entry%%=*}" = "$target$attempt"; then outcome=\${entry#*=}; fi + done + file="$dir/parity-evidence/\${output##*/}" + lock="production.ERROR: SQLSTATE[HY000]: General error: 5 database is locked" + echo "$(date +%s%N) 172.18.0.4 - - POST /api/worker/update-validation-tasks/poll 200" >> "$dir/serverlog" + case "$outcome" in + pass) echo '{"outcome":"pass"}' > "$file"; exit 0 ;; + fail-locked) echo "$(date +%s%N) $lock" >> "$dir/serverlog" ;; + fail-lockbefore) echo "$(( $(date +%s%N) - 5000000000 )) $lock" >> "$dir/serverlog" ;; + fail-lockafter) echo "$(( $(date +%s%N) + 5000000000 )) $lock" >> "$dir/serverlog" ;; + esac + echo '{"outcome":"product-fail"}' > "$file" + echo "== update-validation (product-fail): update validation lost_approve" > "\${file%.json}.failure-excerpt.txt" + exit 1 ;; + *) exit 2 ;; +esac +`; + +function run(outcomes, ...name) { + const dir = mkdtempSync(join(tmpdir(), 'update-validation-retry-')); + for (const name of ['scripts', 'tests']) symlinkSync(join(root, name), join(dir, name)); + for (const name of ['bin', 'parity-evidence', 'parity-state']) mkdirSync(join(dir, name)); + writeFileSync(join(dir, 'bin', 'docker'), fakeDocker); + writeFileSync(join(dir, 'bin', 'jq'), '#!/bin/sh\necho fake-image\n'); + for (const tool of ['docker', 'jq']) chmodSync(join(dir, 'bin', tool), 0o755); + writeFileSync(join(dir, 'state'), 'running\n'); + writeFileSync(join(dir, 'records'), ''); + const result = spawnSync('bash', ['scripts/conformance/server-parity/update-validation-sqlite.sh', 'http://parity-server:8080', 'http://parity-rust:8080', ...name], { + cwd: dir, encoding: 'utf8', + env: {...process.env, PATH: `${join(dir, 'bin')}:${process.env.PATH}`, OUTCOMES: outcomes, + RESOURCE_SCOPE: 'dw-parity-7-2-sqlite', GITHUB_RUN_ID: '7', GITHUB_RUN_ATTEMPT: '2', GITHUB_JOB: 'sqlite', + DW_PARITY_RUNNER_REVISION: 'a'.repeat(40), DW_PARITY_MAINTENANCE_POLL_SECONDS: '0'}, + }); + const read = name => readFileSync(join(dir, name), 'utf8'); + return { + status: result.status, stdout: result.stdout, stderr: result.stderr, + records: read('records').trim().split('\n').filter(Boolean), + state: read('state').trim(), + evidence: name => existsSync(join(dir, 'parity-evidence', name)), + read: name => read(join('parity-evidence', name)), + }; +} + +test('a passing PHP record runs every target once under the usual prefix', () => { + const result = run(''); + assert.equal(result.status, 0, result.stderr); + assert.deepEqual(result.records, ['php parity-v1', 'embedded parity-v1', 'rust parity-v1']); + assert.doesNotMatch(result.stdout, /::warning/); + assert.equal(result.evidence('update-validation-php.attempt-1.json'), false); + assert.equal(result.evidence('update-validation-maintenance-schedule-retry.txt'), false); + assert.equal(result.state, 'running'); +}); + +test('a PHP record that lost a write to the SQLite lock runs once more, and every target takes the new prefix', () => { + const result = run('php1=fail-locked'); + assert.equal(result.status, 0, result.stderr); + assert.deepEqual(result.records, ['php parity-v1', 'php parity-v1-retry', 'embedded parity-v1-retry', 'rust parity-v1-retry']); + assert.match(result.stdout, /^::warning title=Published PHP update-validation record retried::The PHP record failed \(== update-validation \(product-fail\): update validation lost_approve \) after published PHP logged 'database is locked'/m); + // The first attempt's record, excerpt and lock lines stay as evidence. + assert.match(result.read('update-validation-php.attempt-1.json'), /product-fail/); + assert.match(result.read('update-validation-php.attempt-1.failure-excerpt.txt'), /lost_approve/); + assert.match(result.read('update-validation-php.attempt-1.locked.txt'), /database is locked/); + assert.match(result.read('update-validation-php.json'), /"pass"/); + assert.equal(result.evidence('update-validation-php.failure-excerpt.txt'), false); + // Both PHP records ran with maintenance stopped, and it is running again. + for (const schedule of ['update-validation-maintenance-schedule.txt', 'update-validation-maintenance-schedule-retry.txt']) { + assert.match(result.read(schedule), /^during php record: exited$/m, schedule); + assert.match(result.read(schedule), /^after php record: running/m, schedule); + } + assert.equal(result.state, 'running'); +}); + +test('a PHP failure without the lock error during the record fails at once', () => { + // No lock error at all, or one PHP logged only before or after the record. + for (const outcome of ['fail', 'fail-lockbefore', 'fail-lockafter']) { + const result = run(`php1=${outcome}`); + assert.equal(result.status, 1, outcome); + assert.deepEqual(result.records, ['php parity-v1'], outcome); + assert.doesNotMatch(result.stdout, /::warning/, outcome); + assert.equal(result.evidence('update-validation-php.attempt-1.locked.txt'), false, outcome); + assert.match(result.read('update-validation-php.failure-excerpt.txt'), /lost_approve/, outcome); + assert.equal(result.state, 'running', outcome); + } +}); + +test('a PHP retry that fails again fails the record', () => { + const result = run('php1=fail-locked php2=fail-locked'); + assert.equal(result.status, 1); + assert.deepEqual(result.records, ['php parity-v1', 'php parity-v1-retry']); + assert.equal(result.state, 'running'); +}); + +test('a native or embedded failure is never retried, even after a PHP retry', () => { + for (const [outcomes, records] of [ + ['rust1=fail-locked', ['php parity-v1', 'embedded parity-v1', 'rust parity-v1']], + ['embedded1=fail', ['php parity-v1', 'embedded parity-v1']], + ['php1=fail-locked rust1=fail', ['php parity-v1', 'php parity-v1-retry', 'embedded parity-v1-retry', 'rust parity-v1-retry']], + ]) { + const result = run(outcomes); + assert.equal(result.status, 1, outcomes); + assert.deepEqual(result.records, records, outcomes); + } +}); + +test('a named fixture records and retries under its own evidence names', () => { + const result = run('php1=fail-locked', 'service-calls'); + assert.equal(result.status, 0, result.stderr); + assert.deepEqual(result.records, ['php parity-v1', 'php parity-v1-retry', 'embedded parity-v1-retry', 'rust parity-v1-retry']); + assert.match(result.stdout, /^::warning title=Published PHP service-calls record retried::/m); + for (const name of ['service-calls-php.json', 'service-calls-embedded.json', 'service-calls-rust.json', 'service-calls-comparison.json', + 'service-calls-php.attempt-1.json', 'service-calls-php.attempt-1.locked.txt', 'service-calls-maintenance-schedule-retry.txt']) { + assert.equal(result.evidence(name), true, name); + } + assert.equal(result.evidence('update-validation-php.json'), false); + assert.equal(result.state, 'running'); +}); + +test('a fixture name outside the reviewed corpus records nothing', () => { + for (const name of ['no-such-fixture', '../ServerParityPending/workflow-task-transient-retry', 'Service-Calls']) { + const result = run('', name); + assert.notEqual(result.status, 0, name); + assert.deepEqual(result.records, [], name); + } +}); From 85508474c4a01ed210ee59715c32b673b2ddbbb0 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Tue, 27 Oct 2026 21:30:21 +0000 Subject: [PATCH 2/2] Rust #325: give parallel-group timer waits a CI-load bound parallel_all's wait_for_timer gave a 3-second timer 3 seconds from the call to fire. In #821's merge-group run 40579733447 the PostgreSQL native-tests job (631 s for the suite) failed nested_mixed_barrier_retains_all_paths_and_original_timer_after_cold_reopen with "original all-group timer did not fire". The waits are positive: they return as soon as TimerFired is recorded, so a longer bound only changes how long a real failure takes to report. The three timer-fire waits in the file now share a 15-second bound. parallel_all passes 11/11 on SQLite. Co-Authored-By: Claude Opus 5.5 --- rust/tests/execution/parallel_all.rs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/rust/tests/execution/parallel_all.rs b/rust/tests/execution/parallel_all.rs index 9c13882c4..6f87fe173 100644 --- a/rust/tests/execution/parallel_all.rs +++ b/rust/tests/execution/parallel_all.rs @@ -155,8 +155,13 @@ async fn complete_activity(app: &Router, task: &Value, value: i64) -> (StatusCod .await } +/// A positive wait for a timer: it returns as soon as TimerFired is recorded. +/// The bound only limits how long a real failure takes to report, and allows +/// for the timer loop under a loaded CI database (Server #325). +const TIMER_FIRE_BOUND: std::time::Duration = std::time::Duration::from_secs(15); + async fn wait_for_timer(app: &Router, run: &Value) -> Value { - let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(3); + let deadline = tokio::time::Instant::now() + TIMER_FIRE_BOUND; loop { let history = run_history(app, &run["workflow_id"], &run["run_id"]).await; if history @@ -737,7 +742,7 @@ async fn timer_barrier_fires_zero_and_positive_members_once_on_competing_cold_no let (one, two) = tokio::join!(database.open(), database.open()); let (one, two) = (one.unwrap(), two.unwrap()); let (app_a, app_b) = (router(one.clone()), router(two.clone())); - let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(3); + let deadline = tokio::time::Instant::now() + TIMER_FIRE_BOUND; let history = loop { let history = run_history(&app_b, &run["workflow_id"], &run["run_id"]).await; if history @@ -1152,7 +1157,7 @@ async fn a_timer_group_wakes_the_workflow_only_after_its_last_member_fires() { let waiting = request(&app, "POST", "/api/worker/workflow-tasks/poll", json!({"worker_id":"staggered","task_queue":"test","timeout_seconds":1})).await; assert!(waiting.1["task"].is_null(), "{}", waiting.1); - let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(5); + let deadline = tokio::time::Instant::now() + TIMER_FIRE_BOUND; while fired(&run_history(&app, &run["workflow_id"], &run["run_id"]).await) < 2 { assert!(tokio::time::Instant::now() < deadline, "the second timer did not fire"); tokio::time::sleep(std::time::Duration::from_millis(50)).await;