From ad17872a018df53967f7a8e8096c90113b53c915 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Sun, 25 Oct 2026 12:56:43 +0000 Subject: [PATCH] Pull each fixture job's pinned images with bounded retries first Main's push run 40164159608 lost 24 fixture jobs at their first container: mirror.gcr.io answered "unknown blob" for the pinned Rust image, and the implicit pull inside `docker run` does not retry. scripts/ci/pull-images.sh pulls each named image up to DW_PULL_ATTEMPTS (4) times, doubling a DW_PULL_BACKOFF_SECONDS (15) wait, and fails the step only after the last attempt. published-sdk-adapters, mysql-execution, postgres-storage and execution-and-codec run it right after checkout for the images every part uses: their Rust image, the matrix database image, the published 2.11.1 Server image and, where bootstrapped, the 2.5.14 layout image. The new test pins that order, that every Rust digest a job runs is in its pull list, and the script's retry, bound, skip and refusal behavior. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/server-parity.yml | 58 +++++++++++++- scripts/ci/pull-images.sh | 35 +++++++++ tests/Unit/ServerParityImagePullTest.mjs | 96 ++++++++++++++++++++++++ tests/Unit/ServerParityPartsTest.mjs | 4 +- 4 files changed, 190 insertions(+), 3 deletions(-) create mode 100755 scripts/ci/pull-images.sh create mode 100644 tests/Unit/ServerParityImagePullTest.mjs diff --git a/.github/workflows/server-parity.yml b/.github/workflows/server-parity.yml index 3541956bb..e344971a5 100644 --- a/.github/workflows/server-parity.yml +++ b/.github/workflows/server-parity.yml @@ -10,6 +10,7 @@ on: - .github/workflows/server-parity.yml - scripts/conformance/server-parity* - scripts/conformance/server-parity/** + - scripts/ci/pull-images.sh - tests/Fixtures/ServerParity/** - tests/Fixtures/ServerParityBaselines/** - tests/Fixtures/ServerParityNormalization/** @@ -55,6 +56,7 @@ on: - tests/Unit/ServerParityUpdateValidationContractTest.mjs - tests/Unit/ServerParityFixtureSelectionTest.mjs - tests/Unit/ServerParityUpdateValidationRetryTest.mjs + - tests/Unit/ServerParityImagePullTest.mjs - tests/Unit/ServerParityMessageStreamsContractTest.mjs - tests/Unit/ServerParityWaitingHistoryContractTest.mjs - tests/Unit/ServerParityTaskRetryContractTest.mjs @@ -116,6 +118,19 @@ jobs: RESOURCE_SCOPE: dw-codec-build-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }} steps: - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + # A registry briefly answering "unknown blob" once failed every job at its + # first container; pull this job's common images with bounded retries. + - name: Pull the pinned images, retrying registry errors + run: | + set -euo pipefail + baseline() { + jq -r '.server.images["linux/amd64"] | sub("^durableworkflow/server@"; "ghcr.io/durable-workflow/server@")' \ + "tests/Fixtures/ServerParityBaselines/$1.json" + } + bash scripts/ci/pull-images.sh \ + mirror.gcr.io/library/rust@sha256:300ec56abce8cc9448ddea2172747d048ed902a3090e6b57babb2bf19f754081 \ + mirror.gcr.io/library/python@sha256:bf44cdfcb76cd3b41e879bc058fc37ec5872002ccfde7fcb765e218cde0cd79c \ + "$(baseline php-2.11.1)" - name: Build the locked published SDK adapter once run: | set -euo pipefail @@ -378,6 +393,20 @@ jobs: DB_MEMORY: ${{ matrix.memory }} steps: - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + # A registry briefly answering "unknown blob" once failed every job at its + # first container; pull this job's common images with bounded retries. + - name: Pull the pinned images, retrying registry errors + run: | + set -euo pipefail + baseline() { + jq -r '.server.images["linux/amd64"] | sub("^durableworkflow/server@"; "ghcr.io/durable-workflow/server@")' \ + "tests/Fixtures/ServerParityBaselines/$1.json" + } + bash scripts/ci/pull-images.sh \ + mirror.gcr.io/library/rust@sha256:ba81bc3eaa4422af576c0262515d96b0111a628a6ccc2c86557cf55c9a4bbee0 \ + "$MYSQL_IMAGE" \ + "$(baseline php-2.11.1)" \ + "$(baseline php-2.5.14)" - name: Fetch this run's published SDK adapters uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: @@ -1020,6 +1049,20 @@ jobs: POSTGRES_IMAGE: ${{ matrix.image }} steps: - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + # A registry briefly answering "unknown blob" once failed every job at its + # first container; pull this job's common images with bounded retries. + - name: Pull the pinned images, retrying registry errors + run: | + set -euo pipefail + baseline() { + jq -r '.server.images["linux/amd64"] | sub("^durableworkflow/server@"; "ghcr.io/durable-workflow/server@")' \ + "tests/Fixtures/ServerParityBaselines/$1.json" + } + bash scripts/ci/pull-images.sh \ + mirror.gcr.io/library/rust@sha256:ba81bc3eaa4422af576c0262515d96b0111a628a6ccc2c86557cf55c9a4bbee0 \ + "$POSTGRES_IMAGE" \ + "$(baseline php-2.11.1)" \ + "$(baseline php-2.5.14)" - name: Fetch this run's published SDK adapters uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: @@ -1677,6 +1720,19 @@ jobs: DW_PARITY_PART: ${{ matrix.part }} steps: - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + # A registry briefly answering "unknown blob" once failed every job at its + # first container; pull this job's common images with bounded retries. + - name: Pull the pinned images, retrying registry errors + run: | + set -euo pipefail + baseline() { + jq -r '.server.images["linux/amd64"] | sub("^durableworkflow/server@"; "ghcr.io/durable-workflow/server@")' \ + "tests/Fixtures/ServerParityBaselines/$1.json" + } + bash scripts/ci/pull-images.sh \ + mirror.gcr.io/library/rust@sha256:ba81bc3eaa4422af576c0262515d96b0111a628a6ccc2c86557cf55c9a4bbee0 \ + "$(baseline php-2.11.1)" \ + "$(baseline php-2.5.14)" - name: Fetch this run's published SDK adapters uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: @@ -1779,7 +1835,7 @@ jobs: -e COMPOSER_HOME=/tmp/composer -v "$PWD:/source" -w /source "$image" \ install --working-dir=scripts/conformance/server-parity --no-interaction --no-progress docker run --rm --user=1000:1000 --entrypoint node -v "$PWD:/source:ro" -w /source \ - "$image" --test tests/Unit/ServerParityRunnerTest.mjs tests/Unit/ServerParitySdkDirectionLanesTest.mjs tests/Unit/ServerParityCooperativeContractTest.mjs tests/Unit/ServerParityChildCancellationContractTest.mjs tests/Unit/ServerParityScheduleContractTest.mjs tests/Unit/ServerParityVisibilityContractTest.mjs tests/Unit/ServerParityAdmissionContractTest.mjs tests/Unit/ServerParityRoleTokenContractTest.mjs tests/Unit/ServerParityWorkerDeregistrationContractTest.mjs tests/Unit/ServerParitySearchAttributeContractTest.mjs tests/Unit/ServerParityWorkerListingContractTest.mjs tests/Unit/ServerParityWorkerReplacementContractTest.mjs tests/Unit/ServerParityPatchDeploymentContractTest.mjs tests/Unit/ServerParityWaitingHistoryContractTest.mjs tests/Unit/ServerParityTaskRetryContractTest.mjs tests/Unit/ServerParityPublishedPatchContractTest.mjs tests/Unit/ServerParityPublishedPatchTransportTest.mjs tests/Unit/ServerParityRecordFileTest.mjs tests/Unit/ServerParityFailureLocationTest.mjs tests/Unit/ServerParityFailureExcerptTest.mjs tests/Unit/ServerParityLateWorkflowCompletionContractTest.mjs tests/Unit/ServerParityStorageTestContractTest.mjs tests/Unit/ServerParitySchedulePoliciesContractTest.mjs tests/Unit/ServerParitySignalsBeforeWorkerContractTest.mjs tests/Unit/ServerParityScheduleCatalogContractTest.mjs tests/Unit/ServerParityOperatorMetricsContractTest.mjs tests/Unit/ServerParityActivityLeaseRenewalContractTest.mjs tests/Unit/ServerParityActivityCancellationAckContractTest.mjs tests/Unit/ServerParityServiceCatalogContractTest.mjs tests/Unit/ServerParitySystemMetricsContractTest.mjs tests/Unit/ServerParityOperatorDashboardTypesContractTest.mjs tests/Unit/ServerParityPrometheusMetricsContractTest.mjs tests/Unit/ServerParityPayloadCleanupContractTest.mjs tests/Unit/ServerParityRepairPassContractTest.mjs tests/Unit/ServerParityClusterInfoContractTest.mjs tests/Unit/ServerParityNamespaceDeleteContractTest.mjs tests/Unit/ServerParityMessageStreamsContractTest.mjs tests/Unit/ServerParityWorkflowPollRequestContractTest.mjs tests/Unit/ServerParityUpdateValidationContractTest.mjs tests/Unit/ServerParityFixtureSelectionTest.mjs tests/Unit/ServerParityUpdateValidationRetryTest.mjs tests/Unit/ServerParityWorkerSessionsContractTest.mjs tests/Unit/ServerParityWorkerSessionActivitiesContractTest.mjs > parity-evidence/comparator-tests.txt + "$image" --test tests/Unit/ServerParityRunnerTest.mjs tests/Unit/ServerParitySdkDirectionLanesTest.mjs tests/Unit/ServerParityCooperativeContractTest.mjs tests/Unit/ServerParityChildCancellationContractTest.mjs tests/Unit/ServerParityScheduleContractTest.mjs tests/Unit/ServerParityVisibilityContractTest.mjs tests/Unit/ServerParityAdmissionContractTest.mjs tests/Unit/ServerParityRoleTokenContractTest.mjs tests/Unit/ServerParityWorkerDeregistrationContractTest.mjs tests/Unit/ServerParitySearchAttributeContractTest.mjs tests/Unit/ServerParityWorkerListingContractTest.mjs tests/Unit/ServerParityWorkerReplacementContractTest.mjs tests/Unit/ServerParityPatchDeploymentContractTest.mjs tests/Unit/ServerParityWaitingHistoryContractTest.mjs tests/Unit/ServerParityTaskRetryContractTest.mjs tests/Unit/ServerParityPublishedPatchContractTest.mjs tests/Unit/ServerParityPublishedPatchTransportTest.mjs tests/Unit/ServerParityRecordFileTest.mjs tests/Unit/ServerParityFailureLocationTest.mjs tests/Unit/ServerParityFailureExcerptTest.mjs tests/Unit/ServerParityLateWorkflowCompletionContractTest.mjs tests/Unit/ServerParityStorageTestContractTest.mjs tests/Unit/ServerParitySchedulePoliciesContractTest.mjs tests/Unit/ServerParitySignalsBeforeWorkerContractTest.mjs tests/Unit/ServerParityScheduleCatalogContractTest.mjs tests/Unit/ServerParityOperatorMetricsContractTest.mjs tests/Unit/ServerParityActivityLeaseRenewalContractTest.mjs tests/Unit/ServerParityActivityCancellationAckContractTest.mjs tests/Unit/ServerParityServiceCatalogContractTest.mjs tests/Unit/ServerParitySystemMetricsContractTest.mjs tests/Unit/ServerParityOperatorDashboardTypesContractTest.mjs tests/Unit/ServerParityPrometheusMetricsContractTest.mjs tests/Unit/ServerParityPayloadCleanupContractTest.mjs tests/Unit/ServerParityRepairPassContractTest.mjs tests/Unit/ServerParityClusterInfoContractTest.mjs tests/Unit/ServerParityNamespaceDeleteContractTest.mjs tests/Unit/ServerParityMessageStreamsContractTest.mjs tests/Unit/ServerParityWorkflowPollRequestContractTest.mjs tests/Unit/ServerParityUpdateValidationContractTest.mjs tests/Unit/ServerParityFixtureSelectionTest.mjs tests/Unit/ServerParityUpdateValidationRetryTest.mjs tests/Unit/ServerParityImagePullTest.mjs tests/Unit/ServerParityWorkerSessionsContractTest.mjs tests/Unit/ServerParityWorkerSessionActivitiesContractTest.mjs > parity-evidence/comparator-tests.txt docker run --rm --user=1000:1000 --cpus=1 --memory=256m --memory-swap=256m \ --network=none --entrypoint node -v "$PWD:/source:ro" -w /source \ "$image" --test tests/Unit/ServerParityCodecDeadlineLoadTest.mjs > parity-evidence/codec-load-tests.txt diff --git a/scripts/ci/pull-images.sh b/scripts/ci/pull-images.sh new file mode 100755 index 000000000..86ea298cd --- /dev/null +++ b/scripts/ci/pull-images.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# Pulls each named image before a job's first container, retrying registry +# errors. A registry briefly answering "unknown blob" for a pinned digest once +# failed 24 fixture jobs at their first `docker run` (run 40164159608); a pull +# that retries with backoff absorbs that, and later runs find the image local. +# +# Usage: pull-images.sh IMAGE... (empty arguments are skipped) +# DW_PULL_ATTEMPTS (default 4) and DW_PULL_BACKOFF_SECONDS (default 15, doubled +# after each failed attempt) bound the wait; the last failure fails the step. +set -euo pipefail + +attempts="${DW_PULL_ATTEMPTS:-4}" +backoff="${DW_PULL_BACKOFF_SECONDS:-15}" +[[ "$attempts" =~ ^[1-9][0-9]*$ && "$backoff" =~ ^[0-9]+$ ]] || { + echo "pull-images.sh: DW_PULL_ATTEMPTS must be positive and DW_PULL_BACKOFF_SECONDS a whole number" >&2 + exit 2 +} + +for image in "$@"; do + [[ -n "$image" ]] || continue + wait="$backoff" + for ((attempt = 1; ; attempt++)); do + if docker pull --quiet "$image" >/dev/null; then + echo "pulled $image (attempt $attempt)" + break + fi + if ((attempt >= attempts)); then + echo "pull-images.sh: could not pull $image after $attempts attempts" >&2 + exit 1 + fi + echo "pull of $image failed (attempt $attempt of $attempts); retrying in ${wait}s" >&2 + sleep "$wait" + wait=$((wait * 2)) + done +done diff --git a/tests/Unit/ServerParityImagePullTest.mjs b/tests/Unit/ServerParityImagePullTest.mjs new file mode 100644 index 000000000..441dc79e1 --- /dev/null +++ b/tests/Unit/ServerParityImagePullTest.mjs @@ -0,0 +1,96 @@ +// The fixture jobs pull their common pinned images with bounded retries before +// their first container (run 40164159608 lost 24 jobs to one "unknown blob"). +import test from 'node:test'; +import assert from 'node:assert/strict'; +import {chmodSync, mkdtempSync, readFileSync, writeFileSync} from 'node:fs'; +import {tmpdir} from 'node:os'; +import {join, resolve} from 'node:path'; +import {spawnSync} from 'node:child_process'; + +const workflow = readFileSync(resolve('.github/workflows/server-parity.yml'), 'utf8'); +const script = resolve('scripts/ci/pull-images.sh'); + +function job(name) { + const start = workflow.indexOf(`\n ${name}:\n`); + assert.ok(start >= 0, name); + const next = workflow.slice(start + 1).search(/\n {2}[a-z0-9-]+:\n/); + return next < 0 ? workflow.slice(start) : workflow.slice(start, start + 1 + next); +} + +const pulls = { + 'published-sdk-adapters': [], + 'mysql-execution': ['"$MYSQL_IMAGE"', '"$(baseline php-2.5.14)"'], + 'postgres-storage': ['"$POSTGRES_IMAGE"', '"$(baseline php-2.5.14)"'], + 'execution-and-codec': ['"$(baseline php-2.5.14)"'], +}; + +test('each container-heavy fixture job pulls its pinned images first, with retries', () => { + for (const [name, extra] of Object.entries(pulls)) { + const block = job(name); + const steps = block.split(/\n {6}(?=- (?:name|uses):)/).slice(1); + assert.match(steps[0], /^- uses: actions\/checkout@/, name); + const pull = steps[1]; + assert.match(pull, /- name: Pull the pinned images, retrying registry errors\n/, name); + assert.match(pull, /bash scripts\/ci\/pull-images\.sh \\\n/, name); + assert.match(pull, /"\$\(baseline php-2\.11\.1\)"/, name); + for (const image of extra) assert.ok(pull.includes(image), `${name} pulls ${image}`); + // Every Rust image the job runs is in its pull list, so a digest bump + // cannot leave the pull behind. + const rust = new Set(block.match(/mirror\.gcr\.io\/library\/rust@sha256:[0-9a-f]{64}/g)); + assert.ok(rust.size > 0, name); + for (const image of rust) assert.ok(pull.includes(image), `${name} pulls ${image}`); + } +}); + +test('the workflow runs when the pull script changes', () => { + assert.match(workflow, /\n {6}- scripts\/ci\/pull-images\.sh\n/); +}); + +// Each `docker pull` fails while the counter file says so; every call is logged. +const fakeDocker = `#!/usr/bin/env bash +dir=$(dirname "$0") +echo "$*" >> "$dir/calls" +test "$1" = pull || exit 3 +remaining=$(cat "$dir/failures") +if ((remaining > 0)); then echo $((remaining - 1)) > "$dir/failures"; echo "unknown blob" >&2; exit 1; fi +`; + +function run(failures, args, env = {}) { + const dir = mkdtempSync(join(tmpdir(), 'pull-images-')); + writeFileSync(join(dir, 'docker'), fakeDocker); + chmodSync(join(dir, 'docker'), 0o755); + writeFileSync(join(dir, 'failures'), `${failures}\n`); + writeFileSync(join(dir, 'calls'), ''); + const result = spawnSync('bash', [script, ...args], { + encoding: 'utf8', + env: {...process.env, PATH: `${dir}:${process.env.PATH}`, DW_PULL_BACKOFF_SECONDS: '0', ...env}, + }); + return {...result, calls: readFileSync(join(dir, 'calls'), 'utf8').trim().split('\n').filter(Boolean)}; +} + +test('a pull that fails retries until it succeeds', () => { + const result = run(2, ['registry/a@sha256:1', 'registry/b@sha256:2']); + assert.equal(result.status, 0, result.stderr); + assert.deepEqual(result.calls, [ + 'pull --quiet registry/a@sha256:1', 'pull --quiet registry/a@sha256:1', 'pull --quiet registry/a@sha256:1', + 'pull --quiet registry/b@sha256:2', + ]); + assert.match(result.stderr, /pull of registry\/a@sha256:1 failed \(attempt 1 of 4\)/); + assert.match(result.stdout, /pulled registry\/a@sha256:1 \(attempt 3\)\npulled registry\/b@sha256:2 \(attempt 1\)/); +}); + +test('a pull that keeps failing stops at the attempt bound and fails the step', () => { + const result = run(99, ['registry/a@sha256:1', 'registry/b@sha256:2'], {DW_PULL_ATTEMPTS: '3'}); + assert.equal(result.status, 1); + assert.equal(result.calls.length, 3); + assert.match(result.stderr, /could not pull registry\/a@sha256:1 after 3 attempts/); +}); + +test('empty arguments are skipped and malformed bounds are refused', () => { + assert.deepEqual(run(0, ['', 'registry/a@sha256:1', '']).calls, ['pull --quiet registry/a@sha256:1']); + for (const env of [{DW_PULL_ATTEMPTS: '0'}, {DW_PULL_ATTEMPTS: 'x'}, {DW_PULL_BACKOFF_SECONDS: '-1'}]) { + const result = run(0, ['registry/a@sha256:1'], env); + assert.equal(result.status, 2, JSON.stringify(env)); + assert.deepEqual(result.calls, []); + } +}); diff --git a/tests/Unit/ServerParityPartsTest.mjs b/tests/Unit/ServerParityPartsTest.mjs index bc707fddf..309a3bc10 100644 --- a/tests/Unit/ServerParityPartsTest.mjs +++ b/tests/Unit/ServerParityPartsTest.mjs @@ -11,8 +11,8 @@ import {spawnSync} from 'node:child_process'; const workflow = readFileSync('.github/workflows/server-parity.yml', 'utf8'); const PARTS = ['native-tests', 'corpus', 'restarts', 'cohorts', 'sdk-directions', 'corrected-markers']; const SHARED_NODES = 'Start the published PHP and native nodes the corpus parts share'; -const EVERY_PART = [/^actions\/checkout@/, /^Fetch /, /^Create TLS fixture certificates$/, /^Bootstrap independent /, - /^Collect /, /^Summarize /, /^Retain /]; +const EVERY_PART = [/^actions\/checkout@/, /^Pull the pinned images, retrying registry errors$/, /^Fetch /, + /^Create TLS fixture certificates$/, /^Bootstrap independent /, /^Collect /, /^Summarize /, /^Retain /]; function job(name) { const start = workflow.indexOf(`\n ${name}:\n`);