CI: run the six deadline-health cells in turn on one runner

.github/workflows/server-parity.yml

@@ -15,6 +15,7 @@ on:
       - tests/Fixtures/ServerParityNormalization/**
       - tests/Unit/ServerParityRunnerTest.mjs
       - tests/Unit/ServerParitySdkDirectionLanesTest.mjs
+      - tests/Unit/ServerParityCodecDeadlineCellsTest.mjs
       - tests/Unit/ServerParityArtifactSummaryTest.mjs
       - tests/Unit/ServerParityQueryObserverTest.mjs
       - tests/Unit/ServerParityCodecDeadlineLoadTest.mjs
@@ -237,16 +238,15 @@ jobs:
           build_container="$RESOURCE_SCOPE-build"
           docker rm -f "$build_container" >/dev/null 2>&1 || true
 
+  # The six deadline-health cells take about a minute each, beside database
+  # parts that take half an hour, so one runner runs them in turn: a failed
+  # cell still lets the others run, and the job fails after all of them.
   codec-deadline-health:
     needs: [published-sdk-adapters, codec-native-build]
     runs-on: ubuntu-latest
-    timeout-minutes: 10
-    strategy:
-      fail-fast: false
-      matrix:
-        database: [sqlite, postgres-16, postgres-17, mysql-8.0, mysql-8.4, mariadb-10.11]
+    timeout-minutes: 25
     env:
-      RESOURCE_SCOPE: dw-codec-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }}-${{ matrix.database }}
+      RESOURCE_SCOPE: dw-codec-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }}
     steps:
       - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
       - name: Fetch this run's optimized native binary
@@ -266,47 +266,46 @@ jobs:
           set -euo pipefail
           (cd codec-native-build; sha256sum -c SHA256SUMS)
           chmod +x codec-native-build/server
-          bash scripts/conformance/server-parity/codec-operations-cell.sh \
-            '${{ matrix.database }}' cpu-cell codec-native-build corrected-marker-adapters/php codec-evidence
-      - name: Retire this cell's exact owned resources
+          bash scripts/conformance/server-parity/codec-deadline-cells.sh
+      - name: Retire every cell's exact owned resources
         if: always()
         run: |
-          scope="$RESOURCE_SCOPE"
-          native_server="$scope-native"
-          db="$scope-db"
-          probe="$scope-ready"
-          observer="$RESOURCE_SCOPE-native-codec-observer"
-          docker rm -f "$observer" "$probe" "$native_server" >/dev/null 2>&1 || true
-          docker stop --time 10 "$db" >/dev/null 2>&1 || true
-          docker rm "$db" >/dev/null 2>&1 || true
-          docker network rm "$scope" >/dev/null 2>&1 || true
-          if test -d codec-evidence/state; then rm -r -- codec-evidence/state; fi
-      - name: Publish the bounded outcome
+          for database in sqlite postgres-16 postgres-17 mysql-8.0 mysql-8.4 mariadb-10.11; do
+            scope="$RESOURCE_SCOPE-$database"
+            docker rm -f "$scope-native-codec-observer" "$scope-ready" "$scope-native" >/dev/null 2>&1 || true
+            docker stop --time 10 "$scope-db" >/dev/null 2>&1 || true
+            docker rm "$scope-db" >/dev/null 2>&1 || true
+            docker network rm "$scope" >/dev/null 2>&1 || true
+          done
+      - name: Publish the bounded outcomes
         if: always()
         run: |
-          if test -f codec-evidence/database-audit.json; then
-            printf 'Database: %s\n\n' '${{ matrix.database }}' >> "$GITHUB_STEP_SUMMARY"
-            cat codec-evidence/database-audit.json >> "$GITHUB_STEP_SUMMARY"
-          else
-            printf 'CPU/deadline original did not qualify. See this job diagnostics.\n' >> "$GITHUB_STEP_SUMMARY"
-          fi
+          for database in sqlite postgres-16 postgres-17 mysql-8.0 mysql-8.4 mariadb-10.11; do
+            if test -f "codec-evidence/$database/database-audit.json"; then
+              printf 'Database: %s\n\n' "$database" >> "$GITHUB_STEP_SUMMARY"
+              cat "codec-evidence/$database/database-audit.json" >> "$GITHUB_STEP_SUMMARY"
+            else
+              printf 'Database: %s: the CPU/deadline original did not qualify. See this job diagnostics.\n\n' "$database" >> "$GITHUB_STEP_SUMMARY"
+            fi
+          done
       - name: Keep small outcomes and failure diagnostics
         if: always()
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
         with:
-          name: codec-deadline-health-${{ matrix.database }}
+          name: codec-deadline-health
           path: |
-            codec-evidence/build.json
-            codec-evidence/database*.json
-            codec-evidence/database-version.txt
-            codec-evidence/codec-*-contract.txt
-            codec-evidence/codec-*-controls.txt
-            codec-evidence/limits.json
-            codec-evidence/*state.json
-            codec-evidence/server-log.txt
-            codec-evidence/server-ready-errors.txt
-            codec-evidence/profile.txt
-            codec-evidence/failed.json
+            codec-evidence/*.status
+            codec-evidence/*/build.json
+            codec-evidence/*/database*.json
+            codec-evidence/*/database-version.txt
+            codec-evidence/*/codec-*-contract.txt
+            codec-evidence/*/codec-*-controls.txt
+            codec-evidence/*/limits.json
+            codec-evidence/*/*state.json
+            codec-evidence/*/server-log.txt
+            codec-evidence/*/server-ready-errors.txt
+            codec-evidence/*/profile.txt
+            codec-evidence/*/failed.json
           retention-days: 1
           if-no-files-found: warn
 
@@ -1529,7 +1528,7 @@ jobs:
             -e COMPOSER_HOME=/tmp/composer -v "$PWD:/source" -w /source "$image" \
             install --working-dir=scripts/conformance/server-parity --no-interaction --no-progress
           docker run --rm --user=1000:1000 --entrypoint node -v "$PWD:/source:ro" -w /source \
-            "$image" --test tests/Unit/ServerParityRunnerTest.mjs tests/Unit/ServerParitySdkDirectionLanesTest.mjs tests/Unit/ServerParityCooperativeContractTest.mjs tests/Unit/ServerParityChildCancellationContractTest.mjs tests/Unit/ServerParityScheduleContractTest.mjs tests/Unit/ServerParityVisibilityContractTest.mjs tests/Unit/ServerParityAdmissionContractTest.mjs tests/Unit/ServerParityRoleTokenContractTest.mjs tests/Unit/ServerParityWorkerDeregistrationContractTest.mjs tests/Unit/ServerParitySearchAttributeContractTest.mjs tests/Unit/ServerParityWorkerListingContractTest.mjs tests/Unit/ServerParityWorkerReplacementContractTest.mjs tests/Unit/ServerParityPatchDeploymentContractTest.mjs tests/Unit/ServerParityWaitingHistoryContractTest.mjs tests/Unit/ServerParityTaskRetryContractTest.mjs tests/Unit/ServerParityPublishedPatchContractTest.mjs tests/Unit/ServerParityPublishedPatchTransportTest.mjs tests/Unit/ServerParityRecordFileTest.mjs tests/Unit/ServerParityLateWorkflowCompletionContractTest.mjs tests/Unit/ServerParityStorageTestContractTest.mjs tests/Unit/ServerParitySchedulePoliciesContractTest.mjs tests/Unit/ServerParitySignalsBeforeWorkerContractTest.mjs tests/Unit/ServerParityScheduleCatalogContractTest.mjs tests/Unit/ServerParityOperatorMetricsContractTest.mjs tests/Unit/ServerParityActivityLeaseRenewalContractTest.mjs tests/Unit/ServerParityServiceCatalogContractTest.mjs tests/Unit/ServerParitySystemMetricsContractTest.mjs tests/Unit/ServerParityOperatorDashboardTypesContractTest.mjs tests/Unit/ServerParityPrometheusMetricsContractTest.mjs tests/Unit/ServerParityPayloadCleanupContractTest.mjs tests/Unit/ServerParityRepairPassContractTest.mjs tests/Unit/ServerParityClusterInfoContractTest.mjs tests/Unit/ServerParityNamespaceDeleteContractTest.mjs > parity-evidence/comparator-tests.txt
+            "$image" --test tests/Unit/ServerParityRunnerTest.mjs tests/Unit/ServerParitySdkDirectionLanesTest.mjs tests/Unit/ServerParityCodecDeadlineCellsTest.mjs tests/Unit/ServerParityCooperativeContractTest.mjs tests/Unit/ServerParityChildCancellationContractTest.mjs tests/Unit/ServerParityScheduleContractTest.mjs tests/Unit/ServerParityVisibilityContractTest.mjs tests/Unit/ServerParityAdmissionContractTest.mjs tests/Unit/ServerParityRoleTokenContractTest.mjs tests/Unit/ServerParityWorkerDeregistrationContractTest.mjs tests/Unit/ServerParitySearchAttributeContractTest.mjs tests/Unit/ServerParityWorkerListingContractTest.mjs tests/Unit/ServerParityWorkerReplacementContractTest.mjs tests/Unit/ServerParityPatchDeploymentContractTest.mjs tests/Unit/ServerParityWaitingHistoryContractTest.mjs tests/Unit/ServerParityTaskRetryContractTest.mjs tests/Unit/ServerParityPublishedPatchContractTest.mjs tests/Unit/ServerParityPublishedPatchTransportTest.mjs tests/Unit/ServerParityRecordFileTest.mjs tests/Unit/ServerParityLateWorkflowCompletionContractTest.mjs tests/Unit/ServerParityStorageTestContractTest.mjs tests/Unit/ServerParitySchedulePoliciesContractTest.mjs tests/Unit/ServerParitySignalsBeforeWorkerContractTest.mjs tests/Unit/ServerParityScheduleCatalogContractTest.mjs tests/Unit/ServerParityOperatorMetricsContractTest.mjs tests/Unit/ServerParityActivityLeaseRenewalContractTest.mjs tests/Unit/ServerParityServiceCatalogContractTest.mjs tests/Unit/ServerParitySystemMetricsContractTest.mjs tests/Unit/ServerParityOperatorDashboardTypesContractTest.mjs tests/Unit/ServerParityPrometheusMetricsContractTest.mjs tests/Unit/ServerParityPayloadCleanupContractTest.mjs tests/Unit/ServerParityRepairPassContractTest.mjs tests/Unit/ServerParityClusterInfoContractTest.mjs tests/Unit/ServerParityNamespaceDeleteContractTest.mjs > parity-evidence/comparator-tests.txt
           docker run --rm --user=1000:1000 --cpus=1 --memory=256m --memory-swap=256m \
             --network=none --entrypoint node -v "$PWD:/source:ro" -w /source \
             "$image" --test tests/Unit/ServerParityCodecDeadlineLoadTest.mjs > parity-evidence/codec-load-tests.txt

scripts/conformance/server-parity/codec-deadline-cells.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+# Run every deadline-health cell in turn on this runner. Each cell owns its
+# resources under "$RESOURCE_SCOPE-<database>" and stops its database and
+# server itself; the job's always() step removes them. A failed cell still
+# lets the others run, and the script fails after all of them.
+set -euo pipefail
+if [[ -z "${RESOURCE_SCOPE:-}" ]]; then
+  echo 'Set the owning isolated resource scope' >&2
+  exit 2
+fi
+mkdir -p codec-evidence
+failed=0
+for database in sqlite postgres-16 postgres-17 mysql-8.0 mysql-8.4 mariadb-10.11; do
+  status=0
+  RESOURCE_SCOPE="$RESOURCE_SCOPE-$database" bash scripts/conformance/server-parity/codec-operations-cell.sh \
+    "$database" "cpu-$database" codec-native-build corrected-marker-adapters/php "codec-evidence/$database" || status=$?
+  if [[ -d "codec-evidence/$database/state" ]]; then
+    rm -r -- "codec-evidence/$database/state"
+  fi
+  printf '%s\n' "$status" > "codec-evidence/$database.status"
+  if [[ "$status" != 0 ]]; then
+    echo "::error::Deadline-health cell $database exited $status" >&2
+    failed=1
+  fi
+done
+exit "$failed"

tests/Unit/ServerParityCodecDeadlineCellsTest.mjs

@@ -0,0 +1,65 @@
+import assert from 'node:assert/strict';
+import {spawnSync} from 'node:child_process';
+import {copyFileSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync} from 'node:fs';
+import {tmpdir} from 'node:os';
+import {join} from 'node:path';
+import test from 'node:test';
+
+const CELLS = new URL('../../scripts/conformance/server-parity/codec-deadline-cells.sh', import.meta.url).pathname;
+const DATABASES = ['sqlite', 'postgres-16', 'postgres-17', 'mysql-8.0', 'mysql-8.4', 'mariadb-10.11'];
+
+// Runs the cell runner in a scratch tree whose cell script is a stub: it
+// records its scope and arguments, leaves a state directory, and exits
+// FAIL_<database> (default 0).
+function runCells(environment = {}) {
+  const root = mkdtempSync(join(tmpdir(), 'dw-codec-cells-'));
+  try {
+    const scripts = join(root, 'scripts/conformance/server-parity');
+    mkdirSync(scripts, {recursive: true});
+    copyFileSync(CELLS, join(scripts, 'codec-deadline-cells.sh'));
+    writeFileSync(join(scripts, 'codec-operations-cell.sh'), `#!/usr/bin/env bash
+set -euo pipefail
+echo "$RESOURCE_SCOPE $*" >> "$EVENTS"
+mkdir -p "$5/state"
+fail="FAIL_\${1//[.-]/_}"
+exit "\${!fail:-0}"
+`);
+    const events = join(root, 'events.log');
+    const child = spawnSync('bash', ['scripts/conformance/server-parity/codec-deadline-cells.sh'], {
+      cwd: root, encoding: 'utf8', timeout: 20000,
+      env: {...process.env, ...environment, EVENTS: events, RESOURCE_SCOPE: 'dw-codec-1-1-job'},
+    });
+    const statuses = Object.fromEntries(DATABASES.map(database => [database,
+      readFileSync(join(root, 'codec-evidence', `${database}.status`), 'utf8').trim()]));
+    const states = DATABASES.filter(database => {
+      try { readFileSync(join(root, 'codec-evidence', database, 'state')); return true; } catch (error) { return error.code !== 'ENOENT'; }
+    });
+    return {child, statuses, states, events: readFileSync(events, 'utf8').trim().split('\n')};
+  } finally {
+    rmSync(root, {recursive: true, force: true});
+  }
+}
+
+test('every cell runs once, in its own scope, and leaves no state behind', () => {
+  const {child, statuses, states, events} = runCells();
+  assert.equal(child.status, 0, child.stderr);
+  assert.deepEqual(events, DATABASES.map(database =>
+    `dw-codec-1-1-job-${database} ${database} cpu-${database} codec-native-build corrected-marker-adapters/php codec-evidence/${database}`));
+  assert.deepEqual(new Set(Object.values(statuses)), new Set(['0']));
+  assert.deepEqual(states, []);
+});
+
+test('a failed cell still lets every later cell run, and fails the runner after all of them', () => {
+  const {child, statuses, events} = runCells({FAIL_postgres_16: '3'});
+  assert.equal(child.status, 1);
+  assert.equal(events.length, DATABASES.length);
+  assert.equal(statuses['postgres-16'], '3');
+  assert.match(child.stderr, /^::error::Deadline-health cell postgres-16 exited 3$/m);
+  assert.equal(child.stderr.match(/::error::/g).length, 1);
+});
+
+test('the runner refuses to start without an owning resource scope', () => {
+  const child = spawnSync('bash', [CELLS], {encoding: 'utf8', cwd: tmpdir(), env: {...process.env, RESOURCE_SCOPE: ''}});
+  assert.equal(child.status, 2);
+  assert.match(child.stderr, /Set the owning isolated resource scope/);
+});