From 4a6d2a440ca79e1a02a2af5f4121e2b2112dca8b Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Sat, 24 Oct 2026 14:56:36 +0000 Subject: [PATCH 1/3] CI: run the six deadline-health cells in turn on one runner Each cell takes about a minute beside database parts that take half an hour, so the matrix only held five extra runners per run. One job now runs every cell in its own scope; a failed cell still lets the others run, and the job fails after all of them, as the matrix did. A shell test pins that. --- .github/workflows/server-parity.yml | 77 +++++++++---------- .../server-parity/codec-deadline-cells.sh | 26 +++++++ .../ServerParityCodecDeadlineCellsTest.mjs | 65 ++++++++++++++++ 3 files changed, 129 insertions(+), 39 deletions(-) create mode 100644 scripts/conformance/server-parity/codec-deadline-cells.sh create mode 100644 tests/Unit/ServerParityCodecDeadlineCellsTest.mjs diff --git a/.github/workflows/server-parity.yml b/.github/workflows/server-parity.yml index 039800628..52fe7e406 100644 --- a/.github/workflows/server-parity.yml +++ b/.github/workflows/server-parity.yml @@ -15,6 +15,7 @@ on: - tests/Fixtures/ServerParityNormalization/** - tests/Unit/ServerParityRunnerTest.mjs - tests/Unit/ServerParitySdkDirectionLanesTest.mjs + - tests/Unit/ServerParityCodecDeadlineCellsTest.mjs - tests/Unit/ServerParityArtifactSummaryTest.mjs - tests/Unit/ServerParityQueryObserverTest.mjs - tests/Unit/ServerParityCodecDeadlineLoadTest.mjs @@ -237,16 +238,15 @@ jobs: build_container="$RESOURCE_SCOPE-build" docker rm -f "$build_container" >/dev/null 2>&1 || true + # The six deadline-health cells take about a minute each, beside database + # parts that take half an hour, so one runner runs them in turn: a failed + # cell still lets the others run, and the job fails after all of them. codec-deadline-health: needs: [published-sdk-adapters, codec-native-build] runs-on: ubuntu-latest - timeout-minutes: 10 - strategy: - fail-fast: false - matrix: - database: [sqlite, postgres-16, postgres-17, mysql-8.0, mysql-8.4, mariadb-10.11] + timeout-minutes: 25 env: - RESOURCE_SCOPE: dw-codec-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }}-${{ matrix.database }} + RESOURCE_SCOPE: dw-codec-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }} steps: - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 - name: Fetch this run's optimized native binary @@ -266,47 +266,46 @@ jobs: set -euo pipefail (cd codec-native-build; sha256sum -c SHA256SUMS) chmod +x codec-native-build/server - bash scripts/conformance/server-parity/codec-operations-cell.sh \ - '${{ matrix.database }}' cpu-cell codec-native-build corrected-marker-adapters/php codec-evidence - - name: Retire this cell's exact owned resources + bash scripts/conformance/server-parity/codec-deadline-cells.sh + - name: Retire every cell's exact owned resources if: always() run: | - scope="$RESOURCE_SCOPE" - native_server="$scope-native" - db="$scope-db" - probe="$scope-ready" - observer="$RESOURCE_SCOPE-native-codec-observer" - docker rm -f "$observer" "$probe" "$native_server" >/dev/null 2>&1 || true - docker stop --time 10 "$db" >/dev/null 2>&1 || true - docker rm "$db" >/dev/null 2>&1 || true - docker network rm "$scope" >/dev/null 2>&1 || true - if test -d codec-evidence/state; then rm -r -- codec-evidence/state; fi - - name: Publish the bounded outcome + for database in sqlite postgres-16 postgres-17 mysql-8.0 mysql-8.4 mariadb-10.11; do + scope="$RESOURCE_SCOPE-$database" + docker rm -f "$scope-native-codec-observer" "$scope-ready" "$scope-native" >/dev/null 2>&1 || true + docker stop --time 10 "$scope-db" >/dev/null 2>&1 || true + docker rm "$scope-db" >/dev/null 2>&1 || true + docker network rm "$scope" >/dev/null 2>&1 || true + done + - name: Publish the bounded outcomes if: always() run: | - if test -f codec-evidence/database-audit.json; then - printf 'Database: %s\n\n' '${{ matrix.database }}' >> "$GITHUB_STEP_SUMMARY" - cat codec-evidence/database-audit.json >> "$GITHUB_STEP_SUMMARY" - else - printf 'CPU/deadline original did not qualify. See this job diagnostics.\n' >> "$GITHUB_STEP_SUMMARY" - fi + for database in sqlite postgres-16 postgres-17 mysql-8.0 mysql-8.4 mariadb-10.11; do + if test -f "codec-evidence/$database/database-audit.json"; then + printf 'Database: %s\n\n' "$database" >> "$GITHUB_STEP_SUMMARY" + cat "codec-evidence/$database/database-audit.json" >> "$GITHUB_STEP_SUMMARY" + else + printf 'Database: %s: the CPU/deadline original did not qualify. See this job diagnostics.\n\n' "$database" >> "$GITHUB_STEP_SUMMARY" + fi + done - name: Keep small outcomes and failure diagnostics if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: - name: codec-deadline-health-${{ matrix.database }} + name: codec-deadline-health path: | - codec-evidence/build.json - codec-evidence/database*.json - codec-evidence/database-version.txt - codec-evidence/codec-*-contract.txt - codec-evidence/codec-*-controls.txt - codec-evidence/limits.json - codec-evidence/*state.json - codec-evidence/server-log.txt - codec-evidence/server-ready-errors.txt - codec-evidence/profile.txt - codec-evidence/failed.json + codec-evidence/*.status + codec-evidence/*/build.json + codec-evidence/*/database*.json + codec-evidence/*/database-version.txt + codec-evidence/*/codec-*-contract.txt + codec-evidence/*/codec-*-controls.txt + codec-evidence/*/limits.json + codec-evidence/*/*state.json + codec-evidence/*/server-log.txt + codec-evidence/*/server-ready-errors.txt + codec-evidence/*/profile.txt + codec-evidence/*/failed.json retention-days: 1 if-no-files-found: warn @@ -1529,7 +1528,7 @@ jobs: -e COMPOSER_HOME=/tmp/composer -v "$PWD:/source" -w /source "$image" \ install --working-dir=scripts/conformance/server-parity --no-interaction --no-progress docker run --rm --user=1000:1000 --entrypoint node -v "$PWD:/source:ro" -w /source \ - "$image" --test tests/Unit/ServerParityRunnerTest.mjs tests/Unit/ServerParitySdkDirectionLanesTest.mjs tests/Unit/ServerParityCooperativeContractTest.mjs tests/Unit/ServerParityChildCancellationContractTest.mjs tests/Unit/ServerParityScheduleContractTest.mjs tests/Unit/ServerParityVisibilityContractTest.mjs tests/Unit/ServerParityAdmissionContractTest.mjs tests/Unit/ServerParityRoleTokenContractTest.mjs tests/Unit/ServerParityWorkerDeregistrationContractTest.mjs tests/Unit/ServerParitySearchAttributeContractTest.mjs tests/Unit/ServerParityWorkerListingContractTest.mjs tests/Unit/ServerParityWorkerReplacementContractTest.mjs tests/Unit/ServerParityPatchDeploymentContractTest.mjs tests/Unit/ServerParityWaitingHistoryContractTest.mjs tests/Unit/ServerParityTaskRetryContractTest.mjs tests/Unit/ServerParityPublishedPatchContractTest.mjs tests/Unit/ServerParityPublishedPatchTransportTest.mjs tests/Unit/ServerParityRecordFileTest.mjs tests/Unit/ServerParityLateWorkflowCompletionContractTest.mjs tests/Unit/ServerParityStorageTestContractTest.mjs tests/Unit/ServerParitySchedulePoliciesContractTest.mjs tests/Unit/ServerParitySignalsBeforeWorkerContractTest.mjs tests/Unit/ServerParityScheduleCatalogContractTest.mjs tests/Unit/ServerParityOperatorMetricsContractTest.mjs tests/Unit/ServerParityActivityLeaseRenewalContractTest.mjs tests/Unit/ServerParityServiceCatalogContractTest.mjs tests/Unit/ServerParitySystemMetricsContractTest.mjs tests/Unit/ServerParityOperatorDashboardTypesContractTest.mjs tests/Unit/ServerParityPrometheusMetricsContractTest.mjs tests/Unit/ServerParityPayloadCleanupContractTest.mjs tests/Unit/ServerParityRepairPassContractTest.mjs tests/Unit/ServerParityClusterInfoContractTest.mjs tests/Unit/ServerParityNamespaceDeleteContractTest.mjs > parity-evidence/comparator-tests.txt + "$image" --test tests/Unit/ServerParityRunnerTest.mjs tests/Unit/ServerParitySdkDirectionLanesTest.mjs tests/Unit/ServerParityCodecDeadlineCellsTest.mjs tests/Unit/ServerParityCooperativeContractTest.mjs tests/Unit/ServerParityChildCancellationContractTest.mjs tests/Unit/ServerParityScheduleContractTest.mjs tests/Unit/ServerParityVisibilityContractTest.mjs tests/Unit/ServerParityAdmissionContractTest.mjs tests/Unit/ServerParityRoleTokenContractTest.mjs tests/Unit/ServerParityWorkerDeregistrationContractTest.mjs tests/Unit/ServerParitySearchAttributeContractTest.mjs tests/Unit/ServerParityWorkerListingContractTest.mjs tests/Unit/ServerParityWorkerReplacementContractTest.mjs tests/Unit/ServerParityPatchDeploymentContractTest.mjs tests/Unit/ServerParityWaitingHistoryContractTest.mjs tests/Unit/ServerParityTaskRetryContractTest.mjs tests/Unit/ServerParityPublishedPatchContractTest.mjs tests/Unit/ServerParityPublishedPatchTransportTest.mjs tests/Unit/ServerParityRecordFileTest.mjs tests/Unit/ServerParityLateWorkflowCompletionContractTest.mjs tests/Unit/ServerParityStorageTestContractTest.mjs tests/Unit/ServerParitySchedulePoliciesContractTest.mjs tests/Unit/ServerParitySignalsBeforeWorkerContractTest.mjs tests/Unit/ServerParityScheduleCatalogContractTest.mjs tests/Unit/ServerParityOperatorMetricsContractTest.mjs tests/Unit/ServerParityActivityLeaseRenewalContractTest.mjs tests/Unit/ServerParityServiceCatalogContractTest.mjs tests/Unit/ServerParitySystemMetricsContractTest.mjs tests/Unit/ServerParityOperatorDashboardTypesContractTest.mjs tests/Unit/ServerParityPrometheusMetricsContractTest.mjs tests/Unit/ServerParityPayloadCleanupContractTest.mjs tests/Unit/ServerParityRepairPassContractTest.mjs tests/Unit/ServerParityClusterInfoContractTest.mjs tests/Unit/ServerParityNamespaceDeleteContractTest.mjs > parity-evidence/comparator-tests.txt docker run --rm --user=1000:1000 --cpus=1 --memory=256m --memory-swap=256m \ --network=none --entrypoint node -v "$PWD:/source:ro" -w /source \ "$image" --test tests/Unit/ServerParityCodecDeadlineLoadTest.mjs > parity-evidence/codec-load-tests.txt diff --git a/scripts/conformance/server-parity/codec-deadline-cells.sh b/scripts/conformance/server-parity/codec-deadline-cells.sh new file mode 100644 index 000000000..4a6d3f154 --- /dev/null +++ b/scripts/conformance/server-parity/codec-deadline-cells.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# Run every deadline-health cell in turn on this runner. Each cell owns its +# resources under "$RESOURCE_SCOPE-" and stops its database and +# server itself; the job's always() step removes them. A failed cell still +# lets the others run, and the script fails after all of them. +set -euo pipefail +if [[ -z "${RESOURCE_SCOPE:-}" ]]; then + echo 'Set the owning isolated resource scope' >&2 + exit 2 +fi +mkdir -p codec-evidence +failed=0 +for database in sqlite postgres-16 postgres-17 mysql-8.0 mysql-8.4 mariadb-10.11; do + status=0 + RESOURCE_SCOPE="$RESOURCE_SCOPE-$database" bash scripts/conformance/server-parity/codec-operations-cell.sh \ + "$database" "cpu-$database" codec-native-build corrected-marker-adapters/php "codec-evidence/$database" || status=$? + if [[ -d "codec-evidence/$database/state" ]]; then + rm -r -- "codec-evidence/$database/state" + fi + printf '%s\n' "$status" > "codec-evidence/$database.status" + if [[ "$status" != 0 ]]; then + echo "::error::Deadline-health cell $database exited $status" >&2 + failed=1 + fi +done +exit "$failed" diff --git a/tests/Unit/ServerParityCodecDeadlineCellsTest.mjs b/tests/Unit/ServerParityCodecDeadlineCellsTest.mjs new file mode 100644 index 000000000..3f537f4eb --- /dev/null +++ b/tests/Unit/ServerParityCodecDeadlineCellsTest.mjs @@ -0,0 +1,65 @@ +import assert from 'node:assert/strict'; +import {spawnSync} from 'node:child_process'; +import {copyFileSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync} from 'node:fs'; +import {tmpdir} from 'node:os'; +import {join} from 'node:path'; +import test from 'node:test'; + +const CELLS = new URL('../../scripts/conformance/server-parity/codec-deadline-cells.sh', import.meta.url).pathname; +const DATABASES = ['sqlite', 'postgres-16', 'postgres-17', 'mysql-8.0', 'mysql-8.4', 'mariadb-10.11']; + +// Runs the cell runner in a scratch tree whose cell script is a stub: it +// records its scope and arguments, leaves a state directory, and exits +// FAIL_ (default 0). +function runCells(environment = {}) { + const root = mkdtempSync(join(tmpdir(), 'dw-codec-cells-')); + try { + const scripts = join(root, 'scripts/conformance/server-parity'); + mkdirSync(scripts, {recursive: true}); + copyFileSync(CELLS, join(scripts, 'codec-deadline-cells.sh')); + writeFileSync(join(scripts, 'codec-operations-cell.sh'), `#!/usr/bin/env bash +set -euo pipefail +echo "$RESOURCE_SCOPE $*" >> "$EVENTS" +mkdir -p "$5/state" +fail="FAIL_\${1//[.-]/_}" +exit "\${!fail:-0}" +`); + const events = join(root, 'events.log'); + const child = spawnSync('bash', ['scripts/conformance/server-parity/codec-deadline-cells.sh'], { + cwd: root, encoding: 'utf8', timeout: 20000, + env: {...process.env, ...environment, EVENTS: events, RESOURCE_SCOPE: 'dw-codec-1-1-job'}, + }); + const statuses = Object.fromEntries(DATABASES.map(database => [database, + readFileSync(join(root, 'codec-evidence', `${database}.status`), 'utf8').trim()])); + const states = DATABASES.filter(database => { + try { readFileSync(join(root, 'codec-evidence', database, 'state')); return true; } catch (error) { return error.code !== 'ENOENT'; } + }); + return {child, statuses, states, events: readFileSync(events, 'utf8').trim().split('\n')}; + } finally { + rmSync(root, {recursive: true, force: true}); + } +} + +test('every cell runs once, in its own scope, and leaves no state behind', () => { + const {child, statuses, states, events} = runCells(); + assert.equal(child.status, 0, child.stderr); + assert.deepEqual(events, DATABASES.map(database => + `dw-codec-1-1-job-${database} ${database} cpu-${database} codec-native-build corrected-marker-adapters/php codec-evidence/${database}`)); + assert.deepEqual(new Set(Object.values(statuses)), new Set(['0'])); + assert.deepEqual(states, []); +}); + +test('a failed cell still lets every later cell run, and fails the runner after all of them', () => { + const {child, statuses, events} = runCells({FAIL_postgres_16: '3'}); + assert.equal(child.status, 1); + assert.equal(events.length, DATABASES.length); + assert.equal(statuses['postgres-16'], '3'); + assert.match(child.stderr, /^::error::Deadline-health cell postgres-16 exited 3$/m); + assert.equal(child.stderr.match(/::error::/g).length, 1); +}); + +test('the runner refuses to start without an owning resource scope', () => { + const child = spawnSync('bash', [CELLS], {encoding: 'utf8', cwd: tmpdir(), env: {...process.env, RESOURCE_SCOPE: ''}}); + assert.equal(child.status, 2); + assert.match(child.stderr, /Set the owning isolated resource scope/); +}); From 42e2d06780f8af28e286b382ee939d0dde79c650 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Sun, 25 Oct 2026 02:32:16 +0000 Subject: [PATCH 2/3] CI: give every deadline-health cell an accepted scope and stop it before the next Run 40110569900 failed codec-deadline-health because the runner passed cpu-mysql-8.0, cpu-mysql-8.4 and cpu-mariadb-10.11 as the cells' unique scopes. The cell admits only lowercase letters, digits and dashes, so those three exited 2 before starting. Dots now become dashes. Cloud's review (6389755191) found that a failed cell left its database and server running while later cells qualified on the same runner: the cell's exit trap only captured logs, and the job's always() cleanup waited for all six cells. Now: - The cell's exit trap stops its server and database by their exact names, on every exit, failure included, and harmlessly when they never started. - The runner removes the cell's state only after the cell has exited, so the next cell starts with neither actor running. - Container and network removal stays in the job's always() step, which the Docker isolation policy requires. Calling the removal script from the runner would break that policy. The runner test's stub cell now has the real cell's lifecycle: it refuses to start while another cell's actors are live, and stops its own in its exit trap. Cases cover success, a failed cell, and the counterfactual of a failed cell that kept its actors live, which the next cell catches. A structural case checks that the real cell's trap stops both actors and is installed before its first container. The isolation policy and its unit tests pass. Co-Authored-By: Claude Opus 5.5 --- .../server-parity/codec-deadline-cells.sh | 11 ++-- .../server-parity/codec-operations-cell.sh | 5 ++ .../ServerParityCodecDeadlineCellsTest.mjs | 64 ++++++++++++++++--- 3 files changed, 66 insertions(+), 14 deletions(-) diff --git a/scripts/conformance/server-parity/codec-deadline-cells.sh b/scripts/conformance/server-parity/codec-deadline-cells.sh index 4a6d3f154..d20f070fb 100644 --- a/scripts/conformance/server-parity/codec-deadline-cells.sh +++ b/scripts/conformance/server-parity/codec-deadline-cells.sh @@ -1,8 +1,10 @@ #!/usr/bin/env bash # Run every deadline-health cell in turn on this runner. Each cell owns its -# resources under "$RESOURCE_SCOPE-" and stops its database and -# server itself; the job's always() step removes them. A failed cell still -# lets the others run, and the script fails after all of them. +# resources under "$RESOURCE_SCOPE-" and stops its server and +# database when it exits, failed or not, so the next cell starts with none of +# them running; its state is removed after that. The job's always() step +# removes every cell's containers and networks. A failed cell still lets the +# others run, and the script fails after all of them. set -euo pipefail if [[ -z "${RESOURCE_SCOPE:-}" ]]; then echo 'Set the owning isolated resource scope' >&2 @@ -12,8 +14,9 @@ mkdir -p codec-evidence failed=0 for database in sqlite postgres-16 postgres-17 mysql-8.0 mysql-8.4 mariadb-10.11; do status=0 + # The cell's unique scope admits only lowercase letters, digits and dashes. RESOURCE_SCOPE="$RESOURCE_SCOPE-$database" bash scripts/conformance/server-parity/codec-operations-cell.sh \ - "$database" "cpu-$database" codec-native-build corrected-marker-adapters/php "codec-evidence/$database" || status=$? + "$database" "cpu-${database//./-}" codec-native-build corrected-marker-adapters/php "codec-evidence/$database" || status=$? if [[ -d "codec-evidence/$database/state" ]]; then rm -r -- "codec-evidence/$database/state" fi diff --git a/scripts/conformance/server-parity/codec-operations-cell.sh b/scripts/conformance/server-parity/codec-operations-cell.sh index 42758421f..effe0a0ac 100644 --- a/scripts/conformance/server-parity/codec-operations-cell.sh +++ b/scripts/conformance/server-parity/codec-operations-cell.sh @@ -39,6 +39,11 @@ db="$scope-db" probe="$scope-ready" cleanup() { docker logs --tail 40 "$native_server" > "$output/server-log.txt" 2>&1 || true + # Stop this cell's server and database on every exit, failure included, so + # a failed cell never keeps them running while later cells qualify. The + # job's always() step removes them. + docker stop --time 10 "$native_server" >/dev/null 2>&1 || true + docker stop --time 10 "$db" >/dev/null 2>&1 || true } trap cleanup EXIT docker network create "$scope" >/dev/null diff --git a/tests/Unit/ServerParityCodecDeadlineCellsTest.mjs b/tests/Unit/ServerParityCodecDeadlineCellsTest.mjs index 3f537f4eb..fccf70188 100644 --- a/tests/Unit/ServerParityCodecDeadlineCellsTest.mjs +++ b/tests/Unit/ServerParityCodecDeadlineCellsTest.mjs @@ -8,26 +8,39 @@ import test from 'node:test'; const CELLS = new URL('../../scripts/conformance/server-parity/codec-deadline-cells.sh', import.meta.url).pathname; const DATABASES = ['sqlite', 'postgres-16', 'postgres-17', 'mysql-8.0', 'mysql-8.4', 'mariadb-10.11']; -// Runs the cell runner in a scratch tree whose cell script is a stub: it -// records its scope and arguments, leaves a state directory, and exits -// FAIL_ (default 0). +// Runs the cell runner in a scratch tree whose cell script is a stub that +// behaves like the real cell's lifecycle. It refuses to start while another +// cell's actors are live (exit 9), marks its own actors live, records its +// scope and arguments, leaves a state directory, and exits FAIL_ +// (default 0). Its EXIT trap stops its actors, unless KEEP_LIVE_ +// simulates a cell that leaves them running. +const CELL = new URL('../../scripts/conformance/server-parity/codec-operations-cell.sh', import.meta.url).pathname; + function runCells(environment = {}) { const root = mkdtempSync(join(tmpdir(), 'dw-codec-cells-')); try { const scripts = join(root, 'scripts/conformance/server-parity'); mkdirSync(scripts, {recursive: true}); + mkdirSync(join(root, 'live')); copyFileSync(CELLS, join(scripts, 'codec-deadline-cells.sh')); writeFileSync(join(scripts, 'codec-operations-cell.sh'), `#!/usr/bin/env bash set -euo pipefail +key="\${1//[.-]/_}" +if compgen -G "$LIVE/*" > /dev/null; then echo "overlap $1" >> "$EVENTS"; exit 9; fi +keep="KEEP_LIVE_$key" +stop() { [[ -n "\${!keep:-}" ]] || rm -f "$LIVE/$1"; } +trap 'stop "$1"' EXIT +touch "$LIVE/$1" echo "$RESOURCE_SCOPE $*" >> "$EVENTS" +[[ "$2" =~ ^[a-z0-9][a-z0-9-]{1,54}$ ]] || exit 2 mkdir -p "$5/state" -fail="FAIL_\${1//[.-]/_}" +fail="FAIL_$key" exit "\${!fail:-0}" `); const events = join(root, 'events.log'); const child = spawnSync('bash', ['scripts/conformance/server-parity/codec-deadline-cells.sh'], { cwd: root, encoding: 'utf8', timeout: 20000, - env: {...process.env, ...environment, EVENTS: events, RESOURCE_SCOPE: 'dw-codec-1-1-job'}, + env: {...process.env, ...environment, EVENTS: events, LIVE: join(root, 'live'), RESOURCE_SCOPE: 'dw-codec-1-1-job'}, }); const statuses = Object.fromEntries(DATABASES.map(database => [database, readFileSync(join(root, 'codec-evidence', `${database}.status`), 'utf8').trim()])); @@ -40,24 +53,55 @@ exit "\${!fail:-0}" } } -test('every cell runs once, in its own scope, and leaves no state behind', () => { +const started = database => + `dw-codec-1-1-job-${database} ${database} cpu-${database.replaceAll('.', '-')} codec-native-build corrected-marker-adapters/php codec-evidence/${database}`; + +test('every cell runs once, in its own scope, after the previous one stopped, and leaves no state behind', () => { const {child, statuses, states, events} = runCells(); assert.equal(child.status, 0, child.stderr); - assert.deepEqual(events, DATABASES.map(database => - `dw-codec-1-1-job-${database} ${database} cpu-${database} codec-native-build corrected-marker-adapters/php codec-evidence/${database}`)); + assert.deepEqual(events, DATABASES.map(started)); assert.deepEqual(new Set(Object.values(statuses)), new Set(['0'])); assert.deepEqual(states, []); }); -test('a failed cell still lets every later cell run, and fails the runner after all of them', () => { +// Run 40110569900 on #748: the MySQL-family cells exited 2 because their +// unique scopes (cpu-mysql-8.0, ...) held a dot the cell refuses. +test('every cell gets a unique scope the cell accepts', () => { + const {statuses} = runCells(); + assert.deepEqual(statuses, Object.fromEntries(DATABASES.map(database => [database, '0']))); +}); + +test('a failed cell stops its actors, still lets every later cell run, and fails the runner after all of them', () => { const {child, statuses, events} = runCells({FAIL_postgres_16: '3'}); assert.equal(child.status, 1); - assert.equal(events.length, DATABASES.length); + assert.deepEqual(events, DATABASES.map(started)); assert.equal(statuses['postgres-16'], '3'); assert.match(child.stderr, /^::error::Deadline-health cell postgres-16 exited 3$/m); assert.equal(child.stderr.match(/::error::/g).length, 1); }); +// Counterfactual: a failed cell that left its actors running would overlap +// the next cell, which refuses to start. +test('a failed cell that kept its actors live is caught by the next cell', () => { + const {child, statuses, events} = runCells({FAIL_postgres_16: '3', KEEP_LIVE_postgres_16: '1'}); + assert.equal(child.status, 1); + assert.equal(statuses['postgres-17'], '9'); + assert.ok(events.includes('overlap postgres-17'), events.join('\n')); +}); + +// The real cell's exit trap stops both long-lived actors by their exact +// names, harmlessly when they never started, and is installed before the +// first container starts. +test('the real cell stops its server and database on every exit', () => { + const source = readFileSync(CELL, 'utf8'); + const trap = /^cleanup\(\) \{\n([\s\S]*?)^\}\ntrap cleanup EXIT$/m.exec(source); + assert.ok(trap, 'cleanup trap'); + assert.match(trap[1], /^ {2}docker stop --time 10 "\$native_server" >\/dev\/null 2>&1 \|\| true$/m); + assert.match(trap[1], /^ {2}docker stop --time 10 "\$db" >\/dev\/null 2>&1 \|\| true$/m); + assert.ok(source.indexOf('trap cleanup EXIT') < source.indexOf('docker run'), 'trap before the first container'); + assert.doesNotMatch(source, /codec-operations-cleanup\.sh/); +}); + test('the runner refuses to start without an owning resource scope', () => { const child = spawnSync('bash', [CELLS], {encoding: 'utf8', cwd: tmpdir(), env: {...process.env, RESOURCE_SCOPE: ''}}); assert.equal(child.status, 2); From c0276a1f07a41e6ca94d85b03f890421f2b8c31c Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Sun, 25 Oct 2026 17:16:57 +0000 Subject: [PATCH 3/3] CI: confirm each deadline-health cell's actors stopped before the next Cloud's review of #748 (6400346961): the real cell's EXIT trap ends each docker stop with `|| true`, so a refused stop left an actor running while the runner removed the cell's state and started the next cell. After each cell the runner now inspects that cell's exact native server and database containers. Only a container that is absent or not running counts as stopped. When either is still running, or inspection fails, it keeps the cell's state for the job's always() cleanup, marks every later cell skipped and fails. A cell that failed with confirmed cleanup still lets the others run. The runner test's stub cell now uses the real cell's cleanup() verbatim against a fake docker whose stop can refuse (exit 42) and keep the actor. New cases: a refused stop after a failed and after a passing workload, and an inspection that cannot answer. All three fail with the previous runner. Co-Authored-By: Claude Opus 5.5 --- .../server-parity/codec-deadline-cells.sh | 50 +++++++++-- .../ServerParityCodecDeadlineCellsTest.mjs | 86 ++++++++++++++----- 2 files changed, 106 insertions(+), 30 deletions(-) diff --git a/scripts/conformance/server-parity/codec-deadline-cells.sh b/scripts/conformance/server-parity/codec-deadline-cells.sh index d20f070fb..1665456d8 100644 --- a/scripts/conformance/server-parity/codec-deadline-cells.sh +++ b/scripts/conformance/server-parity/codec-deadline-cells.sh @@ -1,26 +1,60 @@ #!/usr/bin/env bash # Run every deadline-health cell in turn on this runner. Each cell owns its -# resources under "$RESOURCE_SCOPE-" and stops its server and -# database when it exits, failed or not, so the next cell starts with none of -# them running; its state is removed after that. The job's always() step -# removes every cell's containers and networks. A failed cell still lets the -# others run, and the script fails after all of them. +# resources under "$RESOURCE_SCOPE-" and tries to stop its server +# and database when it exits, failed or not. Before removing a cell's state +# and starting the next cell, the runner confirms that both of those exact +# containers are stopped or absent. If it cannot, it keeps the state, runs no +# later cell and fails; the job's always() step removes every cell's +# containers and networks. A cell that failed with confirmed cleanup still +# lets the others run, and the script fails after all of them. set -euo pipefail if [[ -z "${RESOURCE_SCOPE:-}" ]]; then echo 'Set the owning isolated resource scope' >&2 exit 2 fi +databases=(sqlite postgres-16 postgres-17 mysql-8.0 mysql-8.4 mariadb-10.11) + +# Each named container is absent, or present and not running. +actors_stopped() { + local name state + for name in "$@"; do + if ! state=$(docker inspect --format '{{.State.Running}}' "$name" 2>&1); then + if [[ "$state" == *"No such object"* || "$state" == *"No such container"* ]]; then + continue + fi + echo "::error::Could not confirm that $name stopped: $state" >&2 + return 1 + fi + if [[ "$state" != false ]]; then + echo "::error::$name is still running" >&2 + return 1 + fi + done +} + mkdir -p codec-evidence failed=0 -for database in sqlite postgres-16 postgres-17 mysql-8.0 mysql-8.4 mariadb-10.11; do +halted= +for database in "${databases[@]}"; do + if [[ -n "$halted" ]]; then + printf 'skipped after %s\n' "$halted" > "codec-evidence/$database.status" + continue + fi + scope="$RESOURCE_SCOPE-$database" status=0 # The cell's unique scope admits only lowercase letters, digits and dashes. - RESOURCE_SCOPE="$RESOURCE_SCOPE-$database" bash scripts/conformance/server-parity/codec-operations-cell.sh \ + RESOURCE_SCOPE="$scope" bash scripts/conformance/server-parity/codec-operations-cell.sh \ "$database" "cpu-${database//./-}" codec-native-build corrected-marker-adapters/php "codec-evidence/$database" || status=$? + printf '%s\n' "$status" > "codec-evidence/$database.status" + if ! actors_stopped "$scope-native" "$scope-db"; then + echo "::error::Deadline-health cell $database left its cleanup unconfirmed; keeping its state and running no later cell" >&2 + halted="$database" + failed=1 + continue + fi if [[ -d "codec-evidence/$database/state" ]]; then rm -r -- "codec-evidence/$database/state" fi - printf '%s\n' "$status" > "codec-evidence/$database.status" if [[ "$status" != 0 ]]; then echo "::error::Deadline-health cell $database exited $status" >&2 failed=1 diff --git a/tests/Unit/ServerParityCodecDeadlineCellsTest.mjs b/tests/Unit/ServerParityCodecDeadlineCellsTest.mjs index fccf70188..7d49b36b0 100644 --- a/tests/Unit/ServerParityCodecDeadlineCellsTest.mjs +++ b/tests/Unit/ServerParityCodecDeadlineCellsTest.mjs @@ -1,6 +1,6 @@ import assert from 'node:assert/strict'; import {spawnSync} from 'node:child_process'; -import {copyFileSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync} from 'node:fs'; +import {chmodSync, copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync} from 'node:fs'; import {tmpdir} from 'node:os'; import {join} from 'node:path'; import test from 'node:test'; @@ -8,13 +8,17 @@ import test from 'node:test'; const CELLS = new URL('../../scripts/conformance/server-parity/codec-deadline-cells.sh', import.meta.url).pathname; const DATABASES = ['sqlite', 'postgres-16', 'postgres-17', 'mysql-8.0', 'mysql-8.4', 'mariadb-10.11']; -// Runs the cell runner in a scratch tree whose cell script is a stub that -// behaves like the real cell's lifecycle. It refuses to start while another -// cell's actors are live (exit 9), marks its own actors live, records its -// scope and arguments, leaves a state directory, and exits FAIL_ -// (default 0). Its EXIT trap stops its actors, unless KEEP_LIVE_ -// simulates a cell that leaves them running. +// Runs the cell runner in a scratch tree whose cell script is a stub with the +// real cell's lifecycle. It refuses to start while any cell's actors are live +// (exit 9), marks its server and database live, records its scope and +// arguments, leaves a state directory, and exits FAIL_ (default 0). +// Its EXIT trap is the real cell's cleanup() function, verbatim, so a refused +// stop behaves exactly as it would in the real cell. A fake docker keeps live +// actors as marker files: `stop` removes one, or exits 42 and keeps it when +// STOP_REFUSED names it; `inspect` reports a live actor running, a stopped one +// absent, and fails outright when INSPECT_FAILS is set. const CELL = new URL('../../scripts/conformance/server-parity/codec-operations-cell.sh', import.meta.url).pathname; +const CLEANUP = /^cleanup\(\) \{\n[\s\S]*?^\}\n/m.exec(readFileSync(CELL, 'utf8'))[0]; function runCells(environment = {}) { const root = mkdtempSync(join(tmpdir(), 'dw-codec-cells-')); @@ -22,31 +26,47 @@ function runCells(environment = {}) { const scripts = join(root, 'scripts/conformance/server-parity'); mkdirSync(scripts, {recursive: true}); mkdirSync(join(root, 'live')); + mkdirSync(join(root, 'bin')); copyFileSync(CELLS, join(scripts, 'codec-deadline-cells.sh')); + writeFileSync(join(root, 'bin', 'docker'), `#!/usr/bin/env bash +command=$1; shift +name=\${!#} +case "$command" in + logs) exit 0 ;; + stop) + if [[ " \${STOP_REFUSED:-} " == *" $name "* ]]; then exit 42; fi + rm -f "$LIVE/$name" ;; + inspect) + if [[ -n "\${INSPECT_FAILS:-}" ]]; then echo "Cannot connect to the Docker daemon" >&2; exit 1; fi + if [[ -e "$LIVE/$name" ]]; then echo true; else echo "Error: No such object: $name" >&2; exit 1; fi ;; + *) exit 2 ;; +esac +`); + chmodSync(join(root, 'bin', 'docker'), 0o755); writeFileSync(join(scripts, 'codec-operations-cell.sh'), `#!/usr/bin/env bash set -euo pipefail key="\${1//[.-]/_}" if compgen -G "$LIVE/*" > /dev/null; then echo "overlap $1" >> "$EVENTS"; exit 9; fi -keep="KEEP_LIVE_$key" -stop() { [[ -n "\${!keep:-}" ]] || rm -f "$LIVE/$1"; } -trap 'stop "$1"' EXIT -touch "$LIVE/$1" +native_server="$RESOURCE_SCOPE-native" +db="$RESOURCE_SCOPE-db" +output="$5" +mkdir -p "$output/state" +${CLEANUP}trap cleanup EXIT +touch "$LIVE/$native_server" "$LIVE/$db" echo "$RESOURCE_SCOPE $*" >> "$EVENTS" [[ "$2" =~ ^[a-z0-9][a-z0-9-]{1,54}$ ]] || exit 2 -mkdir -p "$5/state" fail="FAIL_$key" exit "\${!fail:-0}" `); const events = join(root, 'events.log'); const child = spawnSync('bash', ['scripts/conformance/server-parity/codec-deadline-cells.sh'], { cwd: root, encoding: 'utf8', timeout: 20000, - env: {...process.env, ...environment, EVENTS: events, LIVE: join(root, 'live'), RESOURCE_SCOPE: 'dw-codec-1-1-job'}, + env: {...process.env, ...environment, PATH: `${join(root, 'bin')}:${process.env.PATH}`, + EVENTS: events, LIVE: join(root, 'live'), RESOURCE_SCOPE: 'dw-codec-1-1-job'}, }); const statuses = Object.fromEntries(DATABASES.map(database => [database, readFileSync(join(root, 'codec-evidence', `${database}.status`), 'utf8').trim()])); - const states = DATABASES.filter(database => { - try { readFileSync(join(root, 'codec-evidence', database, 'state')); return true; } catch (error) { return error.code !== 'ENOENT'; } - }); + const states = DATABASES.filter(database => existsSync(join(root, 'codec-evidence', database, 'state'))); return {child, statuses, states, events: readFileSync(events, 'utf8').trim().split('\n')}; } finally { rmSync(root, {recursive: true, force: true}); @@ -80,13 +100,35 @@ test('a failed cell stops its actors, still lets every later cell run, and fails assert.equal(child.stderr.match(/::error::/g).length, 1); }); -// Counterfactual: a failed cell that left its actors running would overlap -// the next cell, which refuses to start. -test('a failed cell that kept its actors live is caught by the next cell', () => { - const {child, statuses, events} = runCells({FAIL_postgres_16: '3', KEEP_LIVE_postgres_16: '1'}); +// Cloud's review of #748: the real cleanup() ignores a refused `docker stop`. +// The runner must confirm the actors stopped before it removes the state or +// starts another cell; when it cannot, it keeps the state and halts. +test("a refused stop in the real cleanup keeps the cell's state and runs no later cell", () => { + const {child, statuses, states, events} = runCells({FAIL_postgres_16: '3', STOP_REFUSED: 'dw-codec-1-1-job-postgres-16-db'}); + assert.equal(child.status, 1); + assert.deepEqual(events, DATABASES.slice(0, 2).map(started), 'no later cell started, so none overlapped'); + assert.equal(statuses['postgres-16'], '3'); + for (const database of DATABASES.slice(2)) assert.equal(statuses[database], 'skipped after postgres-16', database); + assert.deepEqual(states, ['postgres-16']); + assert.match(child.stderr, /^::error::dw-codec-1-1-job-postgres-16-db is still running$/m); + assert.match(child.stderr, /^::error::Deadline-health cell postgres-16 left its cleanup unconfirmed; keeping its state and running no later cell$/m); +}); + +test('a refused stop after a successful workload halts the same way', () => { + const {child, statuses, states, events} = runCells({STOP_REFUSED: 'dw-codec-1-1-job-sqlite-native'}); + assert.equal(child.status, 1); + assert.deepEqual(events, [started('sqlite')]); + assert.equal(statuses.sqlite, '0'); + assert.deepEqual(states, ['sqlite']); +}); + +test('a cleanup the runner cannot inspect halts rather than assuming it stopped', () => { + const {child, statuses, states, events} = runCells({INSPECT_FAILS: '1'}); assert.equal(child.status, 1); - assert.equal(statuses['postgres-17'], '9'); - assert.ok(events.includes('overlap postgres-17'), events.join('\n')); + assert.deepEqual(events, [started('sqlite')]); + assert.deepEqual(states, ['sqlite']); + assert.equal(statuses['postgres-16'], 'skipped after sqlite'); + assert.match(child.stderr, /^::error::Could not confirm that dw-codec-1-1-job-sqlite-native stopped: Cannot connect to the Docker daemon$/m); }); // The real cell's exit trap stops both long-lived actors by their exact