Shared fixtures and native tests: keep service-calls' SQLite pause; bound parallel timer waits

.github/workflows/native-image-parity.yml

@@ -19,6 +19,8 @@ on:
   pull_request:
     paths:
       - scripts/conformance/server-parity/native-image-corpus.sh
+      - scripts/conformance/server-parity/update-validation-sqlite.sh
+      - scripts/conformance/server-parity/php-maintenance-window.sh
       - .github/workflows/native-image-parity.yml
 
 permissions:

.github/workflows/phpunit-feature.yml

@@ -80,6 +80,8 @@ jobs:
       - name: Require safe PHP image build context
         run: node --test tests/Unit/PhpImageContextTest.mjs
 
+      - name: Restore PHP's SQLite maintenance loop on every path
+        run: node --test tests/Unit/PhpMaintenanceWindowTest.mjs
       - name: Restore the patch cohort's maintenance loop on every path
         run: node --test tests/Unit/PatchCohortMaintenanceTest.mjs
 

.github/workflows/server-parity.yml

@@ -66,6 +66,7 @@ on:
       - tests/Unit/ServerParityWorkerSessionActivitiesContractTest.mjs
       - tests/Unit/ServerParityUpdateValidationContractTest.mjs
       - tests/Unit/ServerParityFixtureSelectionTest.mjs
+      - tests/Unit/ServerParityUpdateValidationRetryTest.mjs
       - tests/Unit/ServerParityKilledAuthorPollTest.mjs
       - tests/Unit/ServerParityImagePullTest.mjs
       - tests/Unit/ServerParityAdapterCacheTest.mjs
@@ -1905,7 +1906,7 @@ jobs:
           bash scripts/ci/run-composer.sh --rm -v "$PWD:/source" -w /source -- \
             install --working-dir=scripts/conformance/server-parity --no-interaction --no-progress
           docker run --rm --user=1000:1000 --entrypoint node -v "$PWD:/source:ro" -w /source \
-            "$image" --test tests/Unit/ServerParityRunnerTest.mjs tests/Unit/ServerParitySdkDirectionLanesTest.mjs tests/Unit/ServerParityCodecDeadlineCellsTest.mjs tests/Unit/ServerParityCooperativeContractTest.mjs tests/Unit/ServerParityChildCancellationContractTest.mjs tests/Unit/ServerParityScheduleContractTest.mjs tests/Unit/ServerParityVisibilityContractTest.mjs tests/Unit/ServerParityAdmissionContractTest.mjs tests/Unit/ServerParityRoleTokenContractTest.mjs tests/Unit/ServerParityWorkerDeregistrationContractTest.mjs tests/Unit/ServerParitySearchAttributeContractTest.mjs tests/Unit/ServerParityWorkerListingContractTest.mjs tests/Unit/ServerParityWorkerReplacementContractTest.mjs tests/Unit/ServerParityPatchDeploymentContractTest.mjs tests/Unit/ServerParityWaitingHistoryContractTest.mjs tests/Unit/ServerParityTaskRetryContractTest.mjs tests/Unit/ServerParityPublishedPatchContractTest.mjs tests/Unit/ServerParityPublishedPatchTransportTest.mjs tests/Unit/ServerParityRecordFileTest.mjs tests/Unit/ServerParityFailureLocationTest.mjs tests/Unit/ServerParityFailureExcerptTest.mjs tests/Unit/ServerParityLateWorkflowCompletionContractTest.mjs tests/Unit/ServerParityStorageTestContractTest.mjs tests/Unit/ServerParitySchedulePoliciesContractTest.mjs tests/Unit/ServerParitySignalsBeforeWorkerContractTest.mjs tests/Unit/ServerParityScheduleCatalogContractTest.mjs tests/Unit/ServerParityOperatorMetricsContractTest.mjs tests/Unit/ServerParityActivityLeaseRenewalContractTest.mjs tests/Unit/ServerParityRecordLocalActivityContractTest.mjs tests/Unit/ServerParityActivityCancellationAckContractTest.mjs tests/Unit/ServerParityServiceCatalogContractTest.mjs tests/Unit/ServerParityServiceEndpointsContractTest.mjs tests/Unit/ServerParityServiceCallsContractTest.mjs tests/Unit/ServerParityWorkflowImportContractTest.mjs tests/Unit/ServerParityWaterlineImportContractTest.mjs tests/Unit/ServerParitySystemMetricsContractTest.mjs tests/Unit/ServerParityOperatorDashboardTypesContractTest.mjs tests/Unit/ServerParityPrometheusMetricsContractTest.mjs tests/Unit/ServerParityPayloadCleanupContractTest.mjs tests/Unit/ServerParityRepairPassContractTest.mjs tests/Unit/ServerParityClusterInfoContractTest.mjs tests/Unit/ServerParityNamespaceDeleteContractTest.mjs tests/Unit/ServerParityMessageStreamsContractTest.mjs tests/Unit/ServerParityCancellationScopesContractTest.mjs tests/Unit/ServerParityWorkflowPollRequestContractTest.mjs tests/Unit/ServerParityUpdateValidationContractTest.mjs tests/Unit/ServerParityFixtureSelectionTest.mjs tests/Unit/ServerParityKilledAuthorPollTest.mjs tests/Unit/ServerParityImagePullTest.mjs tests/Unit/ServerParityAdapterCacheTest.mjs tests/Unit/ServerParityWorkerSessionsContractTest.mjs tests/Unit/ServerParityWorkerSessionActivitiesContractTest.mjs tests/Unit/ServerParityNexusOperationsContractTest.mjs tests/Unit/ServerParityBridgeAdaptersContractTest.mjs tests/Unit/ServerParityWorkflowStreamWritesContractTest.mjs > parity-evidence/comparator-tests.txt
+            "$image" --test tests/Unit/ServerParityRunnerTest.mjs tests/Unit/ServerParitySdkDirectionLanesTest.mjs tests/Unit/ServerParityCodecDeadlineCellsTest.mjs tests/Unit/ServerParityCooperativeContractTest.mjs tests/Unit/ServerParityChildCancellationContractTest.mjs tests/Unit/ServerParityScheduleContractTest.mjs tests/Unit/ServerParityVisibilityContractTest.mjs tests/Unit/ServerParityAdmissionContractTest.mjs tests/Unit/ServerParityRoleTokenContractTest.mjs tests/Unit/ServerParityWorkerDeregistrationContractTest.mjs tests/Unit/ServerParitySearchAttributeContractTest.mjs tests/Unit/ServerParityWorkerListingContractTest.mjs tests/Unit/ServerParityWorkerReplacementContractTest.mjs tests/Unit/ServerParityPatchDeploymentContractTest.mjs tests/Unit/ServerParityWaitingHistoryContractTest.mjs tests/Unit/ServerParityTaskRetryContractTest.mjs tests/Unit/ServerParityPublishedPatchContractTest.mjs tests/Unit/ServerParityPublishedPatchTransportTest.mjs tests/Unit/ServerParityRecordFileTest.mjs tests/Unit/ServerParityFailureLocationTest.mjs tests/Unit/ServerParityFailureExcerptTest.mjs tests/Unit/ServerParityLateWorkflowCompletionContractTest.mjs tests/Unit/ServerParityStorageTestContractTest.mjs tests/Unit/ServerParitySchedulePoliciesContractTest.mjs tests/Unit/ServerParitySignalsBeforeWorkerContractTest.mjs tests/Unit/ServerParityScheduleCatalogContractTest.mjs tests/Unit/ServerParityOperatorMetricsContractTest.mjs tests/Unit/ServerParityActivityLeaseRenewalContractTest.mjs tests/Unit/ServerParityRecordLocalActivityContractTest.mjs tests/Unit/ServerParityActivityCancellationAckContractTest.mjs tests/Unit/ServerParityServiceCatalogContractTest.mjs tests/Unit/ServerParityServiceEndpointsContractTest.mjs tests/Unit/ServerParityServiceCallsContractTest.mjs tests/Unit/ServerParityWorkflowImportContractTest.mjs tests/Unit/ServerParityWaterlineImportContractTest.mjs tests/Unit/ServerParitySystemMetricsContractTest.mjs tests/Unit/ServerParityOperatorDashboardTypesContractTest.mjs tests/Unit/ServerParityPrometheusMetricsContractTest.mjs tests/Unit/ServerParityPayloadCleanupContractTest.mjs tests/Unit/ServerParityRepairPassContractTest.mjs tests/Unit/ServerParityClusterInfoContractTest.mjs tests/Unit/ServerParityNamespaceDeleteContractTest.mjs tests/Unit/ServerParityMessageStreamsContractTest.mjs tests/Unit/ServerParityCancellationScopesContractTest.mjs tests/Unit/ServerParityWorkflowPollRequestContractTest.mjs tests/Unit/ServerParityUpdateValidationContractTest.mjs tests/Unit/ServerParityFixtureSelectionTest.mjs tests/Unit/ServerParityUpdateValidationRetryTest.mjs tests/Unit/ServerParityKilledAuthorPollTest.mjs tests/Unit/ServerParityImagePullTest.mjs tests/Unit/ServerParityAdapterCacheTest.mjs tests/Unit/ServerParityWorkerSessionsContractTest.mjs tests/Unit/ServerParityWorkerSessionActivitiesContractTest.mjs tests/Unit/ServerParityNexusOperationsContractTest.mjs tests/Unit/ServerParityBridgeAdaptersContractTest.mjs tests/Unit/ServerParityWorkflowStreamWritesContractTest.mjs > parity-evidence/comparator-tests.txt
           docker run --rm --user=1000:1000 --cpus=1 --memory=256m --memory-swap=256m \
             --network=none --entrypoint node -v "$PWD:/source:ro" -w /source \
             "$image" --test tests/Unit/ServerParityCodecDeadlineLoadTest.mjs > parity-evidence/codec-load-tests.txt
@@ -1977,6 +1978,7 @@ jobs:
             -v "$PWD:/source:ro" -v "$PWD/parity-evidence:/evidence" -w /source \
             -e DW_PARITY_TOKEN=parity-test-token -e DW_PARITY_RUNNER_REVISION -e DW_PARITY_CLI_PHAR "$image" scripts/conformance/server-parity.mjs record \
             --mode http --url http://parity-server:8080 --target php --output /evidence/php.json \
+            --exclude-fixture tests/Fixtures/ServerParity/service-calls.json \
             --include-fixture tests/Fixtures/ServerParityPending/patch-insertion-activity-pending.json \
             --include-fixture tests/Fixtures/ServerParityPending/patch-insertion-activity-completed.json \
             --include-fixture tests/Fixtures/ServerParityPending/patch-created-activity-pending.json \
@@ -1989,6 +1991,7 @@ jobs:
             -e DB_CONNECTION=sqlite -e DB_DATABASE=/state/embedded.sqlite -e DW_MODE=embedded \
             -e CACHE_STORE=file -e QUEUE_CONNECTION=database -e DW_PARITY_RUNNER_REVISION "$image" scripts/conformance/server-parity.mjs record \
             --mode embedded --application-root /app --target embedded --output /evidence/embedded.json \
+            --exclude-fixture tests/Fixtures/ServerParity/service-calls.json \
             --include-fixture tests/Fixtures/ServerParityPending/patch-insertion-activity-pending.json \
             --include-fixture tests/Fixtures/ServerParityPending/patch-insertion-activity-completed.json \
             --include-fixture tests/Fixtures/ServerParityPending/patch-created-activity-pending.json \
@@ -2001,6 +2004,7 @@ jobs:
             -v "$PWD:/source:ro" -v "$PWD/parity-evidence:/evidence" -w /source \
             -e DW_PARITY_TOKEN=parity-test-token -e DW_PARITY_RUNNER_REVISION -e DW_PARITY_CLI_PHAR "$image" scripts/conformance/server-parity.mjs record \
             --mode http --url http://parity-rust:8080 --target rust --output /evidence/rust.json \
+            --exclude-fixture tests/Fixtures/ServerParity/service-calls.json \
             --include-fixture tests/Fixtures/ServerParityPending/patch-insertion-activity-pending.json \
             --include-fixture tests/Fixtures/ServerParityPending/patch-insertion-activity-completed.json \
             --include-fixture tests/Fixtures/ServerParityPending/patch-created-activity-pending.json \
@@ -2012,17 +2016,25 @@ jobs:
           docker run --rm --user=1000:1000 --entrypoint node \
             -v "$PWD:/source:ro" -v "$PWD/parity-evidence:/evidence" -w /source "$image" \
             scripts/conformance/server-parity.mjs compare /evidence/php.json /evidence/rust.json /evidence/embedded.json \
+            --exclude-fixture tests/Fixtures/ServerParity/service-calls.json \
             --include-fixture tests/Fixtures/ServerParityPending/patch-insertion-activity-pending.json \
             --include-fixture tests/Fixtures/ServerParityPending/patch-insertion-activity-completed.json \
             --include-fixture tests/Fixtures/ServerParityPending/patch-created-activity-pending.json \
             --include-fixture tests/Fixtures/ServerParityPending/patch-created-activity-completed.json \
             --include-fixture tests/Fixtures/ServerParityPending/workflow-waiting-history-type.json \
             --include-fixture tests/Fixtures/ServerParityPending/workflow-waiting-history-message.json \
             --include-fixture tests/Fixtures/ServerParityPending/workflow-task-transient-retry.json > parity-evidence/comparison.json || compare_status=$?
+          # Server #796: service-calls records apart from the main corpus, with
+          # published PHP's maintenance loop stopped. Under full-corpus load the
+          # loop's SQLite writer contention can still exhaust 2.11.5's
+          # control-plane retries on a service-call insert.
+          service_calls_status=0
+          bash scripts/conformance/server-parity/update-validation-sqlite.sh http://parity-server:8080 http://parity-rust:8080 service-calls || service_calls_status=$?
           # A native fixture difference fails the step only after the
           # comparison has run, so one difference cannot hide the rest.
           test "$native_record_status" -eq 0
           test "$compare_status" -eq 0
+          test "$service_calls_status" -eq 0
       - name: Compare published patch, legacy-marker and range-repair profiles
         if: ${{ matrix.part == 'restarts' }}
         env:

docs/rust-server-port-checkpoints/2026-10-27-sqlite-service-calls-pause.md

@@ -0,0 +1,33 @@
+# October 27, `service-calls` keeps its SQLite maintenance pause
+
+#815 moved the reference to published 2.11.5 and recorded both
+`update-validation` and `service-calls` in the SQLite main corpus with PHP's
+maintenance loop running. That was based on Cloud's installed #726 acceptance
+(15 of 15) and General's isolated #796 check (10 of 10). This walks part of it
+back.
+
+- **What failed.** #821's merge-group run
+  [40579733447](https://github.com/durable-workflow/server/actions/runs/40579733447),
+  `execution-and-codec (corpus)`. Published 2.11.5's `service-calls` record
+  stopped with `RuntimeException: No charge with options.id for a later path.`
+  - Its server log shows `Control-plane storage contention exhausted retries`
+    and SQLite `database is locked` on an insert into `workflow_service_calls`
+    while the maintenance loop ran its repair pass.
+  - One earlier execute got the typed 503 `backend_lock_pressure`, and the
+    probe retried it; this refusal was not that reply, so it was not retried.
+  - Native and embedded were not involved.
+- **Frequency.** One failure in five full SQLite runs since #815: #815's and
+  #816's merge groups and two local runs passed. That is too flaky for a
+  required check. #796 is still open.
+- **Change.** The SQLite job and the native image corpus exclude
+  `service-calls` from the main corpus again. They record it with
+  `update-validation-sqlite.sh … service-calls`, which stops PHP's
+  maintenance loop for the PHP record only, as before #815.
+  - `update-validation` stays in the main corpus: #726 is closed with
+    installed acceptance, and it has not failed since.
+  - The script, `php-maintenance-window.sh` and their tests are restored
+    from before #815, now pointing at the 2.11.5 baseline.
+  - The MySQL, MariaDB and PostgreSQL jobs keep `service-calls` in their
+    main corpus with the loop running.
+- **Next.** Remove the pause again once a published reference no longer
+  exhausts its retries under full-corpus SQLite load (#796).

docs/rust-server-port.md

@@ -41,7 +41,8 @@ Updated October 27, 2026.
     the takeover runbook's image commands.
   - The parity reference moved to published 2.11.4 (#805,
     [checkpoint](rust-server-port-checkpoints/2026-10-27-reference-2114.md)),
-    and moves to published 2.11.5, without the SQLite maintenance pause
+    and moved to published 2.11.5 (#815); the SQLite maintenance pause stays
+    for `service-calls` only
     ([checkpoint](rust-server-port-checkpoints/2026-10-27-reference-2115.md)).
   - [#814](https://github.com/durable-workflow/server/pull/814): the shared
     fixtures accept published PHP's typed `429 long_poll_capacity_exhausted`
@@ -68,7 +69,10 @@ Updated October 27, 2026.
 
   Closed by 2.11.5: Workflow #931 (`service-calls` repeats a key from
   another caller again, now refused 403), and the SQLite maintenance pause
-  for `update-validation` and `service-calls` (#723, #726, #796).
+  for `update-validation` (#723, #726). `service-calls` still records apart
+  with the loop stopped: under full-corpus load 2.11.5 can exhaust its SQLite
+  contention retries on a service-call insert (#796, open;
+  [checkpoint](rust-server-port-checkpoints/2026-10-27-sqlite-service-calls-pause.md)).
 
   No expectation is edited to make native pass.
 - **Parity surface.** Native answers 174 of the 175 API route methods in

rust/tests/execution/parallel_all.rs

@@ -155,8 +155,13 @@ async fn complete_activity(app: &Router, task: &Value, value: i64) -> (StatusCod
     .await
 }
 
+/// A positive wait for a timer: it returns as soon as TimerFired is recorded.
+/// The bound only limits how long a real failure takes to report, and allows
+/// for the timer loop under a loaded CI database (Server #325).
+const TIMER_FIRE_BOUND: std::time::Duration = std::time::Duration::from_secs(15);
+
 async fn wait_for_timer(app: &Router, run: &Value) -> Value {
-    let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(3);
+    let deadline = tokio::time::Instant::now() + TIMER_FIRE_BOUND;
     loop {
         let history = run_history(app, &run["workflow_id"], &run["run_id"]).await;
         if history
@@ -737,7 +742,7 @@ async fn timer_barrier_fires_zero_and_positive_members_once_on_competing_cold_no
     let (one, two) = tokio::join!(database.open(), database.open());
     let (one, two) = (one.unwrap(), two.unwrap());
     let (app_a, app_b) = (router(one.clone()), router(two.clone()));
-    let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(3);
+    let deadline = tokio::time::Instant::now() + TIMER_FIRE_BOUND;
     let history = loop {
         let history = run_history(&app_b, &run["workflow_id"], &run["run_id"]).await;
         if history
@@ -1152,7 +1157,7 @@ async fn a_timer_group_wakes_the_workflow_only_after_its_last_member_fires() {
     let waiting = request(&app, "POST", "/api/worker/workflow-tasks/poll",
         json!({"worker_id":"staggered","task_queue":"test","timeout_seconds":1})).await;
     assert!(waiting.1["task"].is_null(), "{}", waiting.1);
-    let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(5);
+    let deadline = tokio::time::Instant::now() + TIMER_FIRE_BOUND;
     while fired(&run_history(&app, &run["workflow_id"], &run["run_id"]).await) < 2 {
         assert!(tokio::time::Instant::now() < deadline, "the second timer did not fire");
         tokio::time::sleep(std::time::Duration::from_millis(50)).await;

scripts/conformance/server-parity/native-image-corpus.sh

@@ -29,9 +29,10 @@
 DW_PARITY_RUNNER_REVISION=$(git rev-parse HEAD)
 image=$(jq -r '.server.images["linux/amd64"] | sub("^durableworkflow/server@"; "ghcr.io/durable-workflow/server@")' tests/Fixtures/ServerParityBaselines/php-2.11.5.json)
 
-# CI's corpus selection: the reviewed corpus plus the pending candidates it
-# carries.
-selection=()
+# CI's corpus selection: the reviewed corpus without service-calls, which
+# records separately below with PHP's maintenance loop stopped (Server #796),
+# plus the pending candidates it carries.
+selection=(--exclude-fixture tests/Fixtures/ServerParity/service-calls.json)
 for pending in patch-insertion-activity-pending patch-insertion-activity-completed patch-created-activity-pending \
   patch-created-activity-completed workflow-waiting-history-type workflow-waiting-history-message workflow-task-transient-retry; do
   selection+=(--include-fixture "tests/Fixtures/ServerParityPending/$pending.json")
@@ -111,15 +112,20 @@
 docker run --rm --user=1000:1000 --entrypoint node -v "$PWD:/source:ro" -v "$PWD/parity-evidence:/evidence" -w /source "$image" \
   scripts/conformance/server-parity.mjs compare /evidence/php.json /evidence/rust.json /evidence/embedded.json "${selection[@]}" \
   > parity-evidence/comparison.json || compare_status=$?
+service_calls_status=0
+bash scripts/conformance/server-parity/update-validation-sqlite.sh http://parity-server:8080 http://parity-rust:8080 service-calls \
+  || service_calls_status=$?
 
 native_id=$(docker image inspect --format '{{.Id}}' "$native")
 jq -n --arg native "$native" --arg id "$native_id" --arg php "$image" --arg revision "$DW_PARITY_RUNNER_REVISION" \
   --argjson reference "$status" --argjson nativeStatus "$native_status" --argjson compare "$compare_status" \
+  --argjson serviceCalls "$service_calls_status" \
   '{schema:"durable-workflow.native-image-corpus.v1", native_image:$native, native_image_id:$id, php_reference:$php,
     runner_revision:$revision, backend:"sqlite",
-    status:{reference_records:$reference, native_record:$nativeStatus, compare:$compare}}' \
+    status:{reference_records:$reference, native_record:$nativeStatus, compare:$compare, service_calls:$serviceCalls}}' \
   > parity-evidence/native-image-corpus.json
 cat parity-evidence/native-image-corpus.json
 test "$status" -eq 0
 test "$native_status" -eq 0
 test "$compare_status" -eq 0
+test "$service_calls_status" -eq 0