diff --git a/.github/workflows/server-parity.yml b/.github/workflows/server-parity.yml index 03ae6484f..27a0eba1e 100644 --- a/.github/workflows/server-parity.yml +++ b/.github/workflows/server-parity.yml @@ -15,6 +15,7 @@ on: - tests/Fixtures/ServerParityNormalization/** - tests/Unit/ServerParityRunnerTest.mjs - tests/Unit/ServerParitySdkDirectionLanesTest.mjs + - tests/Unit/ServerParityCodecDeadlineCellsTest.mjs - tests/Unit/ServerParityArtifactSummaryTest.mjs - tests/Unit/ServerParityQueryObserverTest.mjs - tests/Unit/ServerParityCodecDeadlineLoadTest.mjs @@ -237,16 +238,15 @@ jobs: build_container="$RESOURCE_SCOPE-build" docker rm -f "$build_container" >/dev/null 2>&1 || true + # The six deadline-health cells take about a minute each, beside database + # parts that take half an hour, so one runner runs them in turn: a failed + # cell still lets the others run, and the job fails after all of them. codec-deadline-health: needs: [published-sdk-adapters, codec-native-build] runs-on: ubuntu-latest - timeout-minutes: 10 - strategy: - fail-fast: false - matrix: - database: [sqlite, postgres-16, postgres-17, mysql-8.0, mysql-8.4, mariadb-10.11] + timeout-minutes: 25 env: - RESOURCE_SCOPE: dw-codec-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }}-${{ matrix.database }} + RESOURCE_SCOPE: dw-codec-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }} steps: - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 - name: Fetch this run's optimized native binary @@ -266,47 +266,46 @@ jobs: set -euo pipefail (cd codec-native-build; sha256sum -c SHA256SUMS) chmod +x codec-native-build/server - bash scripts/conformance/server-parity/codec-operations-cell.sh \ - '${{ matrix.database }}' cpu-cell codec-native-build corrected-marker-adapters/php codec-evidence - - name: Retire this cell's exact owned resources + bash scripts/conformance/server-parity/codec-deadline-cells.sh + - name: Retire every cell's exact owned resources if: always() run: | - scope="$RESOURCE_SCOPE" - native_server="$scope-native" - db="$scope-db" - probe="$scope-ready" - observer="$RESOURCE_SCOPE-native-codec-observer" - docker rm -f "$observer" "$probe" "$native_server" >/dev/null 2>&1 || true - docker stop --time 10 "$db" >/dev/null 2>&1 || true - docker rm "$db" >/dev/null 2>&1 || true - docker network rm "$scope" >/dev/null 2>&1 || true - if test -d codec-evidence/state; then rm -r -- codec-evidence/state; fi - - name: Publish the bounded outcome + for database in sqlite postgres-16 postgres-17 mysql-8.0 mysql-8.4 mariadb-10.11; do + scope="$RESOURCE_SCOPE-$database" + docker rm -f "$scope-native-codec-observer" "$scope-ready" "$scope-native" >/dev/null 2>&1 || true + docker stop --time 10 "$scope-db" >/dev/null 2>&1 || true + docker rm "$scope-db" >/dev/null 2>&1 || true + docker network rm "$scope" >/dev/null 2>&1 || true + done + - name: Publish the bounded outcomes if: always() run: | - if test -f codec-evidence/database-audit.json; then - printf 'Database: %s\n\n' '${{ matrix.database }}' >> "$GITHUB_STEP_SUMMARY" - cat codec-evidence/database-audit.json >> "$GITHUB_STEP_SUMMARY" - else - printf 'CPU/deadline original did not qualify. See this job diagnostics.\n' >> "$GITHUB_STEP_SUMMARY" - fi + for database in sqlite postgres-16 postgres-17 mysql-8.0 mysql-8.4 mariadb-10.11; do + if test -f "codec-evidence/$database/database-audit.json"; then + printf 'Database: %s\n\n' "$database" >> "$GITHUB_STEP_SUMMARY" + cat "codec-evidence/$database/database-audit.json" >> "$GITHUB_STEP_SUMMARY" + else + printf 'Database: %s: the CPU/deadline original did not qualify. See this job diagnostics.\n\n' "$database" >> "$GITHUB_STEP_SUMMARY" + fi + done - name: Keep small outcomes and failure diagnostics if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: - name: codec-deadline-health-${{ matrix.database }} + name: codec-deadline-health path: | - codec-evidence/build.json - codec-evidence/database*.json - codec-evidence/database-version.txt - codec-evidence/codec-*-contract.txt - codec-evidence/codec-*-controls.txt - codec-evidence/limits.json - codec-evidence/*state.json - codec-evidence/server-log.txt - codec-evidence/server-ready-errors.txt - codec-evidence/profile.txt - codec-evidence/failed.json + codec-evidence/*.status + codec-evidence/*/build.json + codec-evidence/*/database*.json + codec-evidence/*/database-version.txt + codec-evidence/*/codec-*-contract.txt + codec-evidence/*/codec-*-controls.txt + codec-evidence/*/limits.json + codec-evidence/*/*state.json + codec-evidence/*/server-log.txt + codec-evidence/*/server-ready-errors.txt + codec-evidence/*/profile.txt + codec-evidence/*/failed.json retention-days: 1 if-no-files-found: warn @@ -1529,7 +1528,7 @@ jobs: -e COMPOSER_HOME=/tmp/composer -v "$PWD:/source" -w /source "$image" \ install --working-dir=scripts/conformance/server-parity --no-interaction --no-progress docker run --rm --user=1000:1000 --entrypoint node -v "$PWD:/source:ro" -w /source \ - "$image" --test tests/Unit/ServerParityRunnerTest.mjs tests/Unit/ServerParitySdkDirectionLanesTest.mjs tests/Unit/ServerParityCooperativeContractTest.mjs tests/Unit/ServerParityChildCancellationContractTest.mjs tests/Unit/ServerParityScheduleContractTest.mjs tests/Unit/ServerParityVisibilityContractTest.mjs tests/Unit/ServerParityAdmissionContractTest.mjs tests/Unit/ServerParityRoleTokenContractTest.mjs tests/Unit/ServerParityWorkerDeregistrationContractTest.mjs tests/Unit/ServerParitySearchAttributeContractTest.mjs tests/Unit/ServerParityWorkerListingContractTest.mjs tests/Unit/ServerParityWorkerReplacementContractTest.mjs tests/Unit/ServerParityPatchDeploymentContractTest.mjs tests/Unit/ServerParityWaitingHistoryContractTest.mjs tests/Unit/ServerParityTaskRetryContractTest.mjs tests/Unit/ServerParityPublishedPatchContractTest.mjs tests/Unit/ServerParityPublishedPatchTransportTest.mjs tests/Unit/ServerParityRecordFileTest.mjs tests/Unit/ServerParityLateWorkflowCompletionContractTest.mjs tests/Unit/ServerParityStorageTestContractTest.mjs tests/Unit/ServerParitySchedulePoliciesContractTest.mjs tests/Unit/ServerParitySignalsBeforeWorkerContractTest.mjs tests/Unit/ServerParityScheduleCatalogContractTest.mjs tests/Unit/ServerParityOperatorMetricsContractTest.mjs tests/Unit/ServerParityActivityLeaseRenewalContractTest.mjs tests/Unit/ServerParityServiceCatalogContractTest.mjs tests/Unit/ServerParitySystemMetricsContractTest.mjs tests/Unit/ServerParityOperatorDashboardTypesContractTest.mjs tests/Unit/ServerParityPrometheusMetricsContractTest.mjs tests/Unit/ServerParityPayloadCleanupContractTest.mjs tests/Unit/ServerParityRepairPassContractTest.mjs tests/Unit/ServerParityClusterInfoContractTest.mjs tests/Unit/ServerParityNamespaceDeleteContractTest.mjs > parity-evidence/comparator-tests.txt + "$image" --test tests/Unit/ServerParityRunnerTest.mjs tests/Unit/ServerParitySdkDirectionLanesTest.mjs tests/Unit/ServerParityCodecDeadlineCellsTest.mjs tests/Unit/ServerParityCooperativeContractTest.mjs tests/Unit/ServerParityChildCancellationContractTest.mjs tests/Unit/ServerParityScheduleContractTest.mjs tests/Unit/ServerParityVisibilityContractTest.mjs tests/Unit/ServerParityAdmissionContractTest.mjs tests/Unit/ServerParityRoleTokenContractTest.mjs tests/Unit/ServerParityWorkerDeregistrationContractTest.mjs tests/Unit/ServerParitySearchAttributeContractTest.mjs tests/Unit/ServerParityWorkerListingContractTest.mjs tests/Unit/ServerParityWorkerReplacementContractTest.mjs tests/Unit/ServerParityPatchDeploymentContractTest.mjs tests/Unit/ServerParityWaitingHistoryContractTest.mjs tests/Unit/ServerParityTaskRetryContractTest.mjs tests/Unit/ServerParityPublishedPatchContractTest.mjs tests/Unit/ServerParityPublishedPatchTransportTest.mjs tests/Unit/ServerParityRecordFileTest.mjs tests/Unit/ServerParityLateWorkflowCompletionContractTest.mjs tests/Unit/ServerParityStorageTestContractTest.mjs tests/Unit/ServerParitySchedulePoliciesContractTest.mjs tests/Unit/ServerParitySignalsBeforeWorkerContractTest.mjs tests/Unit/ServerParityScheduleCatalogContractTest.mjs tests/Unit/ServerParityOperatorMetricsContractTest.mjs tests/Unit/ServerParityActivityLeaseRenewalContractTest.mjs tests/Unit/ServerParityServiceCatalogContractTest.mjs tests/Unit/ServerParitySystemMetricsContractTest.mjs tests/Unit/ServerParityOperatorDashboardTypesContractTest.mjs tests/Unit/ServerParityPrometheusMetricsContractTest.mjs tests/Unit/ServerParityPayloadCleanupContractTest.mjs tests/Unit/ServerParityRepairPassContractTest.mjs tests/Unit/ServerParityClusterInfoContractTest.mjs tests/Unit/ServerParityNamespaceDeleteContractTest.mjs > parity-evidence/comparator-tests.txt docker run --rm --user=1000:1000 --cpus=1 --memory=256m --memory-swap=256m \ --network=none --entrypoint node -v "$PWD:/source:ro" -w /source \ "$image" --test tests/Unit/ServerParityCodecDeadlineLoadTest.mjs > parity-evidence/codec-load-tests.txt diff --git a/scripts/conformance/server-parity/codec-deadline-cells.sh b/scripts/conformance/server-parity/codec-deadline-cells.sh new file mode 100644 index 000000000..4a6d3f154 --- /dev/null +++ b/scripts/conformance/server-parity/codec-deadline-cells.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# Run every deadline-health cell in turn on this runner. Each cell owns its +# resources under "$RESOURCE_SCOPE-" and stops its database and +# server itself; the job's always() step removes them. A failed cell still +# lets the others run, and the script fails after all of them. +set -euo pipefail +if [[ -z "${RESOURCE_SCOPE:-}" ]]; then + echo 'Set the owning isolated resource scope' >&2 + exit 2 +fi +mkdir -p codec-evidence +failed=0 +for database in sqlite postgres-16 postgres-17 mysql-8.0 mysql-8.4 mariadb-10.11; do + status=0 + RESOURCE_SCOPE="$RESOURCE_SCOPE-$database" bash scripts/conformance/server-parity/codec-operations-cell.sh \ + "$database" "cpu-$database" codec-native-build corrected-marker-adapters/php "codec-evidence/$database" || status=$? + if [[ -d "codec-evidence/$database/state" ]]; then + rm -r -- "codec-evidence/$database/state" + fi + printf '%s\n' "$status" > "codec-evidence/$database.status" + if [[ "$status" != 0 ]]; then + echo "::error::Deadline-health cell $database exited $status" >&2 + failed=1 + fi +done +exit "$failed" diff --git a/tests/Unit/ServerParityCodecDeadlineCellsTest.mjs b/tests/Unit/ServerParityCodecDeadlineCellsTest.mjs new file mode 100644 index 000000000..3f537f4eb --- /dev/null +++ b/tests/Unit/ServerParityCodecDeadlineCellsTest.mjs @@ -0,0 +1,65 @@ +import assert from 'node:assert/strict'; +import {spawnSync} from 'node:child_process'; +import {copyFileSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync} from 'node:fs'; +import {tmpdir} from 'node:os'; +import {join} from 'node:path'; +import test from 'node:test'; + +const CELLS = new URL('../../scripts/conformance/server-parity/codec-deadline-cells.sh', import.meta.url).pathname; +const DATABASES = ['sqlite', 'postgres-16', 'postgres-17', 'mysql-8.0', 'mysql-8.4', 'mariadb-10.11']; + +// Runs the cell runner in a scratch tree whose cell script is a stub: it +// records its scope and arguments, leaves a state directory, and exits +// FAIL_ (default 0). +function runCells(environment = {}) { + const root = mkdtempSync(join(tmpdir(), 'dw-codec-cells-')); + try { + const scripts = join(root, 'scripts/conformance/server-parity'); + mkdirSync(scripts, {recursive: true}); + copyFileSync(CELLS, join(scripts, 'codec-deadline-cells.sh')); + writeFileSync(join(scripts, 'codec-operations-cell.sh'), `#!/usr/bin/env bash +set -euo pipefail +echo "$RESOURCE_SCOPE $*" >> "$EVENTS" +mkdir -p "$5/state" +fail="FAIL_\${1//[.-]/_}" +exit "\${!fail:-0}" +`); + const events = join(root, 'events.log'); + const child = spawnSync('bash', ['scripts/conformance/server-parity/codec-deadline-cells.sh'], { + cwd: root, encoding: 'utf8', timeout: 20000, + env: {...process.env, ...environment, EVENTS: events, RESOURCE_SCOPE: 'dw-codec-1-1-job'}, + }); + const statuses = Object.fromEntries(DATABASES.map(database => [database, + readFileSync(join(root, 'codec-evidence', `${database}.status`), 'utf8').trim()])); + const states = DATABASES.filter(database => { + try { readFileSync(join(root, 'codec-evidence', database, 'state')); return true; } catch (error) { return error.code !== 'ENOENT'; } + }); + return {child, statuses, states, events: readFileSync(events, 'utf8').trim().split('\n')}; + } finally { + rmSync(root, {recursive: true, force: true}); + } +} + +test('every cell runs once, in its own scope, and leaves no state behind', () => { + const {child, statuses, states, events} = runCells(); + assert.equal(child.status, 0, child.stderr); + assert.deepEqual(events, DATABASES.map(database => + `dw-codec-1-1-job-${database} ${database} cpu-${database} codec-native-build corrected-marker-adapters/php codec-evidence/${database}`)); + assert.deepEqual(new Set(Object.values(statuses)), new Set(['0'])); + assert.deepEqual(states, []); +}); + +test('a failed cell still lets every later cell run, and fails the runner after all of them', () => { + const {child, statuses, events} = runCells({FAIL_postgres_16: '3'}); + assert.equal(child.status, 1); + assert.equal(events.length, DATABASES.length); + assert.equal(statuses['postgres-16'], '3'); + assert.match(child.stderr, /^::error::Deadline-health cell postgres-16 exited 3$/m); + assert.equal(child.stderr.match(/::error::/g).length, 1); +}); + +test('the runner refuses to start without an owning resource scope', () => { + const child = spawnSync('bash', [CELLS], {encoding: 'utf8', cwd: tmpdir(), env: {...process.env, RESOURCE_SCOPE: ''}}); + assert.equal(child.status, 2); + assert.match(child.stderr, /Set the owning isolated resource scope/); +});