CI: pull each fixture job's pinned images with bounded retries first

.github/workflows/server-parity.yml

@@ -10,6 +10,7 @@ on:
       - .github/workflows/server-parity.yml
       - scripts/conformance/server-parity*
       - scripts/conformance/server-parity/**
+      - scripts/ci/pull-images.sh
       - tests/Fixtures/ServerParity/**
       - tests/Fixtures/ServerParityBaselines/**
       - tests/Fixtures/ServerParityNormalization/**
@@ -55,6 +56,7 @@ on:
       - tests/Unit/ServerParityUpdateValidationContractTest.mjs
       - tests/Unit/ServerParityFixtureSelectionTest.mjs
       - tests/Unit/ServerParityUpdateValidationRetryTest.mjs
+      - tests/Unit/ServerParityImagePullTest.mjs
       - tests/Unit/ServerParityMessageStreamsContractTest.mjs
       - tests/Unit/ServerParityWaitingHistoryContractTest.mjs
       - tests/Unit/ServerParityTaskRetryContractTest.mjs
@@ -116,6 +118,19 @@ jobs:
       RESOURCE_SCOPE: dw-codec-build-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }}
     steps:
       - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
+      # A registry briefly answering "unknown blob" once failed every job at its
+      # first container; pull this job's common images with bounded retries.
+      - name: Pull the pinned images, retrying registry errors
+        run: |
+          set -euo pipefail
+          baseline() {
+            jq -r '.server.images["linux/amd64"] | sub("^durableworkflow/server@"; "ghcr.io/durable-workflow/server@")' \
+              "tests/Fixtures/ServerParityBaselines/$1.json"
+          }
+          bash scripts/ci/pull-images.sh \
+            mirror.gcr.io/library/rust@sha256:300ec56abce8cc9448ddea2172747d048ed902a3090e6b57babb2bf19f754081 \
+            mirror.gcr.io/library/python@sha256:bf44cdfcb76cd3b41e879bc058fc37ec5872002ccfde7fcb765e218cde0cd79c \
+            "$(baseline php-2.11.1)"
       - name: Build the locked published SDK adapter once
         run: bash scripts/conformance/server-parity/published-sdk-patch-adapters.sh
       - name: Build genuine legacy-marker adapters with the existing cache
@@ -349,6 +364,20 @@ jobs:
       DB_MEMORY: ${{ matrix.memory }}
     steps:
       - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
+      # A registry briefly answering "unknown blob" once failed every job at its
+      # first container; pull this job's common images with bounded retries.
+      - name: Pull the pinned images, retrying registry errors
+        run: |
+          set -euo pipefail
+          baseline() {
+            jq -r '.server.images["linux/amd64"] | sub("^durableworkflow/server@"; "ghcr.io/durable-workflow/server@")' \
+              "tests/Fixtures/ServerParityBaselines/$1.json"
+          }
+          bash scripts/ci/pull-images.sh \
+            mirror.gcr.io/library/rust@sha256:ba81bc3eaa4422af576c0262515d96b0111a628a6ccc2c86557cf55c9a4bbee0 \
+            "$MYSQL_IMAGE" \
+            "$(baseline php-2.11.1)" \
+            "$(baseline php-2.5.14)"
       - name: Fetch this run's published SDK adapters
         uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
         with:
@@ -988,6 +1017,20 @@ jobs:
       POSTGRES_IMAGE: ${{ matrix.image }}
     steps:
       - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
+      # A registry briefly answering "unknown blob" once failed every job at its
+      # first container; pull this job's common images with bounded retries.
+      - name: Pull the pinned images, retrying registry errors
+        run: |
+          set -euo pipefail
+          baseline() {
+            jq -r '.server.images["linux/amd64"] | sub("^durableworkflow/server@"; "ghcr.io/durable-workflow/server@")' \
+              "tests/Fixtures/ServerParityBaselines/$1.json"
+          }
+          bash scripts/ci/pull-images.sh \
+            mirror.gcr.io/library/rust@sha256:ba81bc3eaa4422af576c0262515d96b0111a628a6ccc2c86557cf55c9a4bbee0 \
+            "$POSTGRES_IMAGE" \
+            "$(baseline php-2.11.1)" \
+            "$(baseline php-2.5.14)"
       - name: Fetch this run's published SDK adapters
         uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
         with:
@@ -1642,6 +1685,19 @@ jobs:
       DW_PARITY_PART: ${{ matrix.part }}
     steps:
       - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
+      # A registry briefly answering "unknown blob" once failed every job at its
+      # first container; pull this job's common images with bounded retries.
+      - name: Pull the pinned images, retrying registry errors
+        run: |
+          set -euo pipefail
+          baseline() {
+            jq -r '.server.images["linux/amd64"] | sub("^durableworkflow/server@"; "ghcr.io/durable-workflow/server@")' \
+              "tests/Fixtures/ServerParityBaselines/$1.json"
+          }
+          bash scripts/ci/pull-images.sh \
+            mirror.gcr.io/library/rust@sha256:ba81bc3eaa4422af576c0262515d96b0111a628a6ccc2c86557cf55c9a4bbee0 \
+            "$(baseline php-2.11.1)" \
+            "$(baseline php-2.5.14)"
       - name: Fetch this run's published SDK adapters
         uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
         with:
@@ -1744,7 +1800,7 @@ jobs:
           bash scripts/ci/run-composer.sh --rm -v "$PWD:/source" -w /source -- \
             install --working-dir=scripts/conformance/server-parity --no-interaction --no-progress
           docker run --rm --user=1000:1000 --entrypoint node -v "$PWD:/source:ro" -w /source \
-            "$image" --test tests/Unit/ServerParityRunnerTest.mjs tests/Unit/ServerParitySdkDirectionLanesTest.mjs tests/Unit/ServerParityCooperativeContractTest.mjs tests/Unit/ServerParityChildCancellationContractTest.mjs tests/Unit/ServerParityScheduleContractTest.mjs tests/Unit/ServerParityVisibilityContractTest.mjs tests/Unit/ServerParityAdmissionContractTest.mjs tests/Unit/ServerParityRoleTokenContractTest.mjs tests/Unit/ServerParityWorkerDeregistrationContractTest.mjs tests/Unit/ServerParitySearchAttributeContractTest.mjs tests/Unit/ServerParityWorkerListingContractTest.mjs tests/Unit/ServerParityWorkerReplacementContractTest.mjs tests/Unit/ServerParityPatchDeploymentContractTest.mjs tests/Unit/ServerParityWaitingHistoryContractTest.mjs tests/Unit/ServerParityTaskRetryContractTest.mjs tests/Unit/ServerParityPublishedPatchContractTest.mjs tests/Unit/ServerParityPublishedPatchTransportTest.mjs tests/Unit/ServerParityRecordFileTest.mjs tests/Unit/ServerParityFailureLocationTest.mjs tests/Unit/ServerParityFailureExcerptTest.mjs tests/Unit/ServerParityLateWorkflowCompletionContractTest.mjs tests/Unit/ServerParityStorageTestContractTest.mjs tests/Unit/ServerParitySchedulePoliciesContractTest.mjs tests/Unit/ServerParitySignalsBeforeWorkerContractTest.mjs tests/Unit/ServerParityScheduleCatalogContractTest.mjs tests/Unit/ServerParityOperatorMetricsContractTest.mjs tests/Unit/ServerParityActivityLeaseRenewalContractTest.mjs tests/Unit/ServerParityActivityCancellationAckContractTest.mjs tests/Unit/ServerParityServiceCatalogContractTest.mjs tests/Unit/ServerParitySystemMetricsContractTest.mjs tests/Unit/ServerParityOperatorDashboardTypesContractTest.mjs tests/Unit/ServerParityPrometheusMetricsContractTest.mjs tests/Unit/ServerParityPayloadCleanupContractTest.mjs tests/Unit/ServerParityRepairPassContractTest.mjs tests/Unit/ServerParityClusterInfoContractTest.mjs tests/Unit/ServerParityNamespaceDeleteContractTest.mjs tests/Unit/ServerParityMessageStreamsContractTest.mjs tests/Unit/ServerParityWorkflowPollRequestContractTest.mjs tests/Unit/ServerParityUpdateValidationContractTest.mjs tests/Unit/ServerParityFixtureSelectionTest.mjs tests/Unit/ServerParityUpdateValidationRetryTest.mjs tests/Unit/ServerParityWorkerSessionsContractTest.mjs tests/Unit/ServerParityWorkerSessionActivitiesContractTest.mjs > parity-evidence/comparator-tests.txt
+            "$image" --test tests/Unit/ServerParityRunnerTest.mjs tests/Unit/ServerParitySdkDirectionLanesTest.mjs tests/Unit/ServerParityCooperativeContractTest.mjs tests/Unit/ServerParityChildCancellationContractTest.mjs tests/Unit/ServerParityScheduleContractTest.mjs tests/Unit/ServerParityVisibilityContractTest.mjs tests/Unit/ServerParityAdmissionContractTest.mjs tests/Unit/ServerParityRoleTokenContractTest.mjs tests/Unit/ServerParityWorkerDeregistrationContractTest.mjs tests/Unit/ServerParitySearchAttributeContractTest.mjs tests/Unit/ServerParityWorkerListingContractTest.mjs tests/Unit/ServerParityWorkerReplacementContractTest.mjs tests/Unit/ServerParityPatchDeploymentContractTest.mjs tests/Unit/ServerParityWaitingHistoryContractTest.mjs tests/Unit/ServerParityTaskRetryContractTest.mjs tests/Unit/ServerParityPublishedPatchContractTest.mjs tests/Unit/ServerParityPublishedPatchTransportTest.mjs tests/Unit/ServerParityRecordFileTest.mjs tests/Unit/ServerParityFailureLocationTest.mjs tests/Unit/ServerParityFailureExcerptTest.mjs tests/Unit/ServerParityLateWorkflowCompletionContractTest.mjs tests/Unit/ServerParityStorageTestContractTest.mjs tests/Unit/ServerParitySchedulePoliciesContractTest.mjs tests/Unit/ServerParitySignalsBeforeWorkerContractTest.mjs tests/Unit/ServerParityScheduleCatalogContractTest.mjs tests/Unit/ServerParityOperatorMetricsContractTest.mjs tests/Unit/ServerParityActivityLeaseRenewalContractTest.mjs tests/Unit/ServerParityActivityCancellationAckContractTest.mjs tests/Unit/ServerParityServiceCatalogContractTest.mjs tests/Unit/ServerParitySystemMetricsContractTest.mjs tests/Unit/ServerParityOperatorDashboardTypesContractTest.mjs tests/Unit/ServerParityPrometheusMetricsContractTest.mjs tests/Unit/ServerParityPayloadCleanupContractTest.mjs tests/Unit/ServerParityRepairPassContractTest.mjs tests/Unit/ServerParityClusterInfoContractTest.mjs tests/Unit/ServerParityNamespaceDeleteContractTest.mjs tests/Unit/ServerParityMessageStreamsContractTest.mjs tests/Unit/ServerParityWorkflowPollRequestContractTest.mjs tests/Unit/ServerParityUpdateValidationContractTest.mjs tests/Unit/ServerParityFixtureSelectionTest.mjs tests/Unit/ServerParityUpdateValidationRetryTest.mjs tests/Unit/ServerParityImagePullTest.mjs tests/Unit/ServerParityWorkerSessionsContractTest.mjs tests/Unit/ServerParityWorkerSessionActivitiesContractTest.mjs > parity-evidence/comparator-tests.txt
           docker run --rm --user=1000:1000 --cpus=1 --memory=256m --memory-swap=256m \
             --network=none --entrypoint node -v "$PWD:/source:ro" -w /source \
             "$image" --test tests/Unit/ServerParityCodecDeadlineLoadTest.mjs > parity-evidence/codec-load-tests.txt

scripts/ci/pull-images.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+# Pulls each named image before a job's first container, retrying registry
+# errors. A registry briefly answering "unknown blob" for a pinned digest once
+# failed 24 fixture jobs at their first `docker run` (run 40164159608); a pull
+# that retries with backoff absorbs that, and later runs find the image local.
+#
+# Usage: pull-images.sh IMAGE...   (empty arguments are skipped)
+# DW_PULL_ATTEMPTS (default 4) and DW_PULL_BACKOFF_SECONDS (default 15, doubled
+# after each failed attempt) bound the wait; the last failure fails the step.
+set -euo pipefail
+
+attempts="${DW_PULL_ATTEMPTS:-4}"
+backoff="${DW_PULL_BACKOFF_SECONDS:-15}"
+[[ "$attempts" =~ ^[1-9][0-9]*$ && "$backoff" =~ ^[0-9]+$ ]] || {
+    echo "pull-images.sh: DW_PULL_ATTEMPTS must be positive and DW_PULL_BACKOFF_SECONDS a whole number" >&2
+    exit 2
+}
+
+for image in "$@"; do
+    [[ -n "$image" ]] || continue
+    wait="$backoff"
+    for ((attempt = 1; ; attempt++)); do
+        if docker pull --quiet "$image" >/dev/null; then
+            echo "pulled $image (attempt $attempt)"
+            break
+        fi
+        if ((attempt >= attempts)); then
+            echo "pull-images.sh: could not pull $image after $attempts attempts" >&2
+            exit 1
+        fi
+        echo "pull of $image failed (attempt $attempt of $attempts); retrying in ${wait}s" >&2
+        sleep "$wait"
+        wait=$((wait * 2))
+    done
+done

tests/Unit/ServerParityImagePullTest.mjs

@@ -0,0 +1,96 @@
+// The fixture jobs pull their common pinned images with bounded retries before
+// their first container (run 40164159608 lost 24 jobs to one "unknown blob").
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import {chmodSync, mkdtempSync, readFileSync, writeFileSync} from 'node:fs';
+import {tmpdir} from 'node:os';
+import {join, resolve} from 'node:path';
+import {spawnSync} from 'node:child_process';
+
+const workflow = readFileSync(resolve('.github/workflows/server-parity.yml'), 'utf8');
+const script = resolve('scripts/ci/pull-images.sh');
+
+function job(name) {
+  const start = workflow.indexOf(`\n  ${name}:\n`);
+  assert.ok(start >= 0, name);
+  const next = workflow.slice(start + 1).search(/\n {2}[a-z0-9-]+:\n/);
+  return next < 0 ? workflow.slice(start) : workflow.slice(start, start + 1 + next);
+}
+
+const pulls = {
+  'published-sdk-adapters': [],
+  'mysql-execution': ['"$MYSQL_IMAGE"', '"$(baseline php-2.5.14)"'],
+  'postgres-storage': ['"$POSTGRES_IMAGE"', '"$(baseline php-2.5.14)"'],
+  'execution-and-codec': ['"$(baseline php-2.5.14)"'],
+};
+
+test('each container-heavy fixture job pulls its pinned images first, with retries', () => {
+  for (const [name, extra] of Object.entries(pulls)) {
+    const block = job(name);
+    const steps = block.split(/\n {6}(?=- (?:name|uses):)/).slice(1);
+    assert.match(steps[0], /^- uses: actions\/checkout@/, name);
+    const pull = steps[1];
+    assert.match(pull, /- name: Pull the pinned images, retrying registry errors\n/, name);
+    assert.match(pull, /bash scripts\/ci\/pull-images\.sh \\\n/, name);
+    assert.match(pull, /"\$\(baseline php-2\.11\.1\)"/, name);
+    for (const image of extra) assert.ok(pull.includes(image), `${name} pulls ${image}`);
+    // Every Rust image the job runs is in its pull list, so a digest bump
+    // cannot leave the pull behind.
+    const rust = new Set(block.match(/mirror\.gcr\.io\/library\/rust@sha256:[0-9a-f]{64}/g));
+    assert.ok(rust.size > 0, name);
+    for (const image of rust) assert.ok(pull.includes(image), `${name} pulls ${image}`);
+  }
+});
+
+test('the workflow runs when the pull script changes', () => {
+  assert.match(workflow, /\n {6}- scripts\/ci\/pull-images\.sh\n/);
+});
+
+// Each `docker pull` fails while the counter file says so; every call is logged.
+const fakeDocker = `#!/usr/bin/env bash
+dir=$(dirname "$0")
+echo "$*" >> "$dir/calls"
+test "$1" = pull || exit 3
+remaining=$(cat "$dir/failures")
+if ((remaining > 0)); then echo $((remaining - 1)) > "$dir/failures"; echo "unknown blob" >&2; exit 1; fi
+`;
+
+function run(failures, args, env = {}) {
+  const dir = mkdtempSync(join(tmpdir(), 'pull-images-'));
+  writeFileSync(join(dir, 'docker'), fakeDocker);
+  chmodSync(join(dir, 'docker'), 0o755);
+  writeFileSync(join(dir, 'failures'), `${failures}\n`);
+  writeFileSync(join(dir, 'calls'), '');
+  const result = spawnSync('bash', [script, ...args], {
+    encoding: 'utf8',
+    env: {...process.env, PATH: `${dir}:${process.env.PATH}`, DW_PULL_BACKOFF_SECONDS: '0', ...env},
+  });
+  return {...result, calls: readFileSync(join(dir, 'calls'), 'utf8').trim().split('\n').filter(Boolean)};
+}
+
+test('a pull that fails retries until it succeeds', () => {
+  const result = run(2, ['registry/a@sha256:1', 'registry/b@sha256:2']);
+  assert.equal(result.status, 0, result.stderr);
+  assert.deepEqual(result.calls, [
+    'pull --quiet registry/a@sha256:1', 'pull --quiet registry/a@sha256:1', 'pull --quiet registry/a@sha256:1',
+    'pull --quiet registry/b@sha256:2',
+  ]);
+  assert.match(result.stderr, /pull of registry\/a@sha256:1 failed \(attempt 1 of 4\)/);
+  assert.match(result.stdout, /pulled registry\/a@sha256:1 \(attempt 3\)\npulled registry\/b@sha256:2 \(attempt 1\)/);
+});
+
+test('a pull that keeps failing stops at the attempt bound and fails the step', () => {
+  const result = run(99, ['registry/a@sha256:1', 'registry/b@sha256:2'], {DW_PULL_ATTEMPTS: '3'});
+  assert.equal(result.status, 1);
+  assert.equal(result.calls.length, 3);
+  assert.match(result.stderr, /could not pull registry\/a@sha256:1 after 3 attempts/);
+});
+
+test('empty arguments are skipped and malformed bounds are refused', () => {
+  assert.deepEqual(run(0, ['', 'registry/a@sha256:1', '']).calls, ['pull --quiet registry/a@sha256:1']);
+  for (const env of [{DW_PULL_ATTEMPTS: '0'}, {DW_PULL_ATTEMPTS: 'x'}, {DW_PULL_BACKOFF_SECONDS: '-1'}]) {
+    const result = run(0, ['registry/a@sha256:1'], env);
+    assert.equal(result.status, 2, JSON.stringify(env));
+    assert.deepEqual(result.calls, []);
+  }
+});

tests/Unit/ServerParityPartsTest.mjs

@@ -11,8 +11,8 @@ import {spawnSync} from 'node:child_process';
 const workflow = readFileSync('.github/workflows/server-parity.yml', 'utf8');
 const PARTS = ['native-tests', 'corpus', 'restarts', 'cohorts', 'sdk-directions', 'corrected-markers'];
 const SHARED_NODES = 'Start the published PHP and native nodes the corpus parts share';
-const EVERY_PART = [/^actions\/checkout@/, /^Fetch /, /^Create TLS fixture certificates$/, /^Bootstrap independent /,
-  /^Collect /, /^Summarize /, /^Retain /];
+const EVERY_PART = [/^actions\/checkout@/, /^Pull the pinned images, retrying registry errors$/, /^Fetch /,
+  /^Create TLS fixture certificates$/, /^Bootstrap independent /, /^Collect /, /^Summarize /, /^Retain /];
 
 function job(name) {
   const start = workflow.indexOf(`\n  ${name}:\n`);