From 3ff3606a781b089d32db17e2b1e560f8c5cc622f Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Tue, 27 Oct 2026 01:42:55 +0000 Subject: [PATCH] Cache the fixture adapters' Cargo registry and build tree from main Every Shared Server fixtures matrix job waits for published-sdk-adapters, which spent about four of its 6.4 minutes building the same pinned, --locked adapter dependencies each run (PR run 40414928426). The job now restores the public Cargo registry data (registry, git) and the build tree, never the rest of CARGO_HOME, under one exact key: - there are no restore keys, so a run restores this key's tree or none; - only a successful push to main saves it (the top-level && guard the organization policy accepts since durable-workflow/.github#269), so no pull request or merge group writes what a later run reads; - every run still builds, stamps and hashes its own adapters, and the final adapter directories and revision checks are unchanged. scripts/ci/sdk-adapter-cache-key.sh computes the key from the runner OS and architecture, every pinned Rust image among the inputs (the build scripts' and the query-authority profile's) and a digest of every tracked adapter input: the five adapter crates, their build scripts, the pip wrapper and the SDK profiles. It fails unless all five crates and build scripts are inputs. The helper and the pip wrapper join the push paths so a change to either saves a fresh cache on main. A unit test pins the boundary. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/server-parity.yml | 32 ++++++++- scripts/ci/sdk-adapter-cache-key.sh | 47 +++++++++++++ tests/Unit/ServerParityAdapterCacheTest.mjs | 78 +++++++++++++++++++++ 3 files changed, 156 insertions(+), 1 deletion(-) create mode 100755 scripts/ci/sdk-adapter-cache-key.sh create mode 100644 tests/Unit/ServerParityAdapterCacheTest.mjs diff --git a/.github/workflows/server-parity.yml b/.github/workflows/server-parity.yml index 281b88f36..57db9b6a9 100644 --- a/.github/workflows/server-parity.yml +++ b/.github/workflows/server-parity.yml @@ -11,6 +11,8 @@ on: - scripts/conformance/server-parity* - scripts/conformance/server-parity/** - scripts/ci/pull-images.sh + - scripts/ci/patched-pip.sh + - scripts/ci/sdk-adapter-cache-key.sh - tests/Fixtures/ServerParity/** - tests/Fixtures/ServerParityBaselines/** - tests/Fixtures/ServerParityNormalization/** @@ -67,6 +69,7 @@ on: - tests/Unit/ServerParityUpdateValidationRetryTest.mjs - tests/Unit/ServerParityKilledAuthorPollTest.mjs - tests/Unit/ServerParityImagePullTest.mjs + - tests/Unit/ServerParityAdapterCacheTest.mjs - tests/Unit/ServerParityMessageStreamsContractTest.mjs - tests/Unit/ServerParityWorkflowStreamWritesContractTest.mjs - tests/Unit/ServerParityWaitingHistoryContractTest.mjs @@ -142,6 +145,24 @@ jobs: mirror.gcr.io/library/rust@sha256:300ec56abce8cc9448ddea2172747d048ed902a3090e6b57babb2bf19f754081 \ mirror.gcr.io/library/python@sha256:bf44cdfcb76cd3b41e879bc058fc37ec5872002ccfde7fcb765e218cde0cd79c \ "$(baseline php-2.11.1)" + # The adapters' pinned, --locked dependencies build the same way every + # run. Restore their public Cargo registry data and build tree, keyed + # by the toolchain image, the runner and every tracked adapter input. + # Only successful pushes to main save them, so no pull request or merge + # group writes what a later run reads. Every run still builds, stamps + # and hashes its own adapters, and the final adapters are never cached. + - name: Key the adapters' Cargo cache by every tracked adapter input + id: adapter_key + run: bash scripts/ci/sdk-adapter-cache-key.sh + - name: Restore the adapters' Cargo registry and build tree + id: adapter_cargo + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + sdk-cargo/registry + sdk-cargo/git + sdk-target + key: ${{ steps.adapter_key.outputs.key }} - name: Build the locked published SDK adapter once run: bash scripts/conformance/server-parity/published-sdk-patch-adapters.sh - name: Build genuine legacy-marker adapters with the existing cache @@ -156,6 +177,15 @@ jobs: run: bash scripts/conformance/server-parity/corrected-marker-adapters.sh - name: Build the separate current query consumers once run: bash scripts/conformance/server-parity/query-authority-adapters.sh + - name: Save the adapters' Cargo registry and build tree from main + if: ${{ success() && github.event_name == 'push' && github.ref == 'refs/heads/main' && steps.adapter_cargo.outputs.cache-hit != 'true' }} + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + sdk-cargo/registry + sdk-cargo/git + sdk-target + key: ${{ steps.adapter_key.outputs.key }} - name: Share current query consumers without compiler caches uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: @@ -1873,7 +1903,7 @@ jobs: bash scripts/ci/run-composer.sh --rm -v "$PWD:/source" -w /source -- \ install --working-dir=scripts/conformance/server-parity --no-interaction --no-progress docker run --rm --user=1000:1000 --entrypoint node -v "$PWD:/source:ro" -w /source \ - "$image" --test tests/Unit/ServerParityRunnerTest.mjs tests/Unit/ServerParitySdkDirectionLanesTest.mjs tests/Unit/ServerParityCodecDeadlineCellsTest.mjs tests/Unit/ServerParityCooperativeContractTest.mjs tests/Unit/ServerParityChildCancellationContractTest.mjs tests/Unit/ServerParityScheduleContractTest.mjs tests/Unit/ServerParityVisibilityContractTest.mjs tests/Unit/ServerParityAdmissionContractTest.mjs tests/Unit/ServerParityRoleTokenContractTest.mjs tests/Unit/ServerParityWorkerDeregistrationContractTest.mjs tests/Unit/ServerParitySearchAttributeContractTest.mjs tests/Unit/ServerParityWorkerListingContractTest.mjs tests/Unit/ServerParityWorkerReplacementContractTest.mjs tests/Unit/ServerParityPatchDeploymentContractTest.mjs tests/Unit/ServerParityWaitingHistoryContractTest.mjs tests/Unit/ServerParityTaskRetryContractTest.mjs tests/Unit/ServerParityPublishedPatchContractTest.mjs tests/Unit/ServerParityPublishedPatchTransportTest.mjs tests/Unit/ServerParityRecordFileTest.mjs tests/Unit/ServerParityFailureLocationTest.mjs tests/Unit/ServerParityFailureExcerptTest.mjs tests/Unit/ServerParityLateWorkflowCompletionContractTest.mjs tests/Unit/ServerParityStorageTestContractTest.mjs tests/Unit/ServerParitySchedulePoliciesContractTest.mjs tests/Unit/ServerParitySignalsBeforeWorkerContractTest.mjs tests/Unit/ServerParityScheduleCatalogContractTest.mjs tests/Unit/ServerParityOperatorMetricsContractTest.mjs tests/Unit/ServerParityActivityLeaseRenewalContractTest.mjs tests/Unit/ServerParityRecordLocalActivityContractTest.mjs tests/Unit/ServerParityActivityCancellationAckContractTest.mjs tests/Unit/ServerParityServiceCatalogContractTest.mjs tests/Unit/ServerParityServiceEndpointsContractTest.mjs tests/Unit/ServerParityServiceCallsContractTest.mjs tests/Unit/ServerParityWorkflowImportContractTest.mjs tests/Unit/ServerParityWaterlineImportContractTest.mjs tests/Unit/ServerParitySystemMetricsContractTest.mjs tests/Unit/ServerParityOperatorDashboardTypesContractTest.mjs tests/Unit/ServerParityPrometheusMetricsContractTest.mjs tests/Unit/ServerParityPayloadCleanupContractTest.mjs tests/Unit/ServerParityRepairPassContractTest.mjs tests/Unit/ServerParityClusterInfoContractTest.mjs tests/Unit/ServerParityNamespaceDeleteContractTest.mjs tests/Unit/ServerParityMessageStreamsContractTest.mjs tests/Unit/ServerParityCancellationScopesContractTest.mjs tests/Unit/ServerParityWorkflowPollRequestContractTest.mjs tests/Unit/ServerParityUpdateValidationContractTest.mjs tests/Unit/ServerParityFixtureSelectionTest.mjs tests/Unit/ServerParityUpdateValidationRetryTest.mjs tests/Unit/ServerParityKilledAuthorPollTest.mjs tests/Unit/ServerParityImagePullTest.mjs tests/Unit/ServerParityWorkerSessionsContractTest.mjs tests/Unit/ServerParityWorkerSessionActivitiesContractTest.mjs tests/Unit/ServerParityNexusOperationsContractTest.mjs tests/Unit/ServerParityBridgeAdaptersContractTest.mjs tests/Unit/ServerParityWorkflowStreamWritesContractTest.mjs > parity-evidence/comparator-tests.txt + "$image" --test tests/Unit/ServerParityRunnerTest.mjs tests/Unit/ServerParitySdkDirectionLanesTest.mjs tests/Unit/ServerParityCodecDeadlineCellsTest.mjs tests/Unit/ServerParityCooperativeContractTest.mjs tests/Unit/ServerParityChildCancellationContractTest.mjs tests/Unit/ServerParityScheduleContractTest.mjs tests/Unit/ServerParityVisibilityContractTest.mjs tests/Unit/ServerParityAdmissionContractTest.mjs tests/Unit/ServerParityRoleTokenContractTest.mjs tests/Unit/ServerParityWorkerDeregistrationContractTest.mjs tests/Unit/ServerParitySearchAttributeContractTest.mjs tests/Unit/ServerParityWorkerListingContractTest.mjs tests/Unit/ServerParityWorkerReplacementContractTest.mjs tests/Unit/ServerParityPatchDeploymentContractTest.mjs tests/Unit/ServerParityWaitingHistoryContractTest.mjs tests/Unit/ServerParityTaskRetryContractTest.mjs tests/Unit/ServerParityPublishedPatchContractTest.mjs tests/Unit/ServerParityPublishedPatchTransportTest.mjs tests/Unit/ServerParityRecordFileTest.mjs tests/Unit/ServerParityFailureLocationTest.mjs tests/Unit/ServerParityFailureExcerptTest.mjs tests/Unit/ServerParityLateWorkflowCompletionContractTest.mjs tests/Unit/ServerParityStorageTestContractTest.mjs tests/Unit/ServerParitySchedulePoliciesContractTest.mjs tests/Unit/ServerParitySignalsBeforeWorkerContractTest.mjs tests/Unit/ServerParityScheduleCatalogContractTest.mjs tests/Unit/ServerParityOperatorMetricsContractTest.mjs tests/Unit/ServerParityActivityLeaseRenewalContractTest.mjs tests/Unit/ServerParityRecordLocalActivityContractTest.mjs tests/Unit/ServerParityActivityCancellationAckContractTest.mjs tests/Unit/ServerParityServiceCatalogContractTest.mjs tests/Unit/ServerParityServiceEndpointsContractTest.mjs tests/Unit/ServerParityServiceCallsContractTest.mjs tests/Unit/ServerParityWorkflowImportContractTest.mjs tests/Unit/ServerParityWaterlineImportContractTest.mjs tests/Unit/ServerParitySystemMetricsContractTest.mjs tests/Unit/ServerParityOperatorDashboardTypesContractTest.mjs tests/Unit/ServerParityPrometheusMetricsContractTest.mjs tests/Unit/ServerParityPayloadCleanupContractTest.mjs tests/Unit/ServerParityRepairPassContractTest.mjs tests/Unit/ServerParityClusterInfoContractTest.mjs tests/Unit/ServerParityNamespaceDeleteContractTest.mjs tests/Unit/ServerParityMessageStreamsContractTest.mjs tests/Unit/ServerParityCancellationScopesContractTest.mjs tests/Unit/ServerParityWorkflowPollRequestContractTest.mjs tests/Unit/ServerParityUpdateValidationContractTest.mjs tests/Unit/ServerParityFixtureSelectionTest.mjs tests/Unit/ServerParityUpdateValidationRetryTest.mjs tests/Unit/ServerParityKilledAuthorPollTest.mjs tests/Unit/ServerParityImagePullTest.mjs tests/Unit/ServerParityAdapterCacheTest.mjs tests/Unit/ServerParityWorkerSessionsContractTest.mjs tests/Unit/ServerParityWorkerSessionActivitiesContractTest.mjs tests/Unit/ServerParityNexusOperationsContractTest.mjs tests/Unit/ServerParityBridgeAdaptersContractTest.mjs tests/Unit/ServerParityWorkflowStreamWritesContractTest.mjs > parity-evidence/comparator-tests.txt docker run --rm --user=1000:1000 --cpus=1 --memory=256m --memory-swap=256m \ --network=none --entrypoint node -v "$PWD:/source:ro" -w /source \ "$image" --test tests/Unit/ServerParityCodecDeadlineLoadTest.mjs > parity-evidence/codec-load-tests.txt diff --git a/scripts/ci/sdk-adapter-cache-key.sh b/scripts/ci/sdk-adapter-cache-key.sh new file mode 100755 index 000000000..ae9fe5b80 --- /dev/null +++ b/scripts/ci/sdk-adapter-cache-key.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# Print the exact cache key for the published SDK adapters' public Cargo +# registry data and build tree (Shared Server fixtures, Server #325), and +# write it to $GITHUB_OUTPUT as `key` when that is set. +# +# The key names the runner OS and architecture, every pinned Rust image the +# adapter builds use and a digest of every tracked adapter input: the five +# adapter crates, their build scripts, the pip wrapper and the SDK profiles. +# It is an exact key with no restore prefix. Only pushes to main save it. +set -euo pipefail +: "${RUNNER_OS:?RUNNER_OS must be set}" "${RUNNER_ARCH:?RUNNER_ARCH must be set}" + +parity=scripts/conformance/server-parity +profiles=tests/Fixtures/ServerParityProfiles +mapfile -t inputs < <(git ls-files -- \ + "$parity/published-sdk-patch" \ + "$parity/sdk-directions" \ + "$parity/legacy-marker-aliases" \ + "$parity/corrected-markers" \ + "$parity/query-authority" \ + ":(glob)$parity/*-adapters.sh" \ + scripts/ci/patched-pip.sh \ + "$profiles/published-sdk-patch-replay" \ + "$profiles/published-sdk-directions" \ + "$profiles/legacy-marker-aliases" \ + "$profiles/corrected-markers" \ + "$profiles/query-authority" \ + "$profiles/version-range-repair" | LC_ALL=C sort) + +# Never a partial key: all five adapter crates and build scripts must be +# inputs. A sixth adapter must join the list above. +test "$(printf '%s\n' "${inputs[@]}" | grep -c '/rust/Cargo\.lock$')" -eq 5 +test "$(printf '%s\n' "${inputs[@]}" | grep -c -- '-adapters\.sh$')" -eq 5 + +# The key names every pinned Rust image the inputs build with: the build +# scripts pin one and the query-authority profile another. The input digest +# below covers them too; naming them keeps a toolchain change readable. +mapfile -t toolchains < <(grep -ho 'library/rust@sha256:[0-9a-f]\{64\}' -- "${inputs[@]}" | sed 's/.*@sha256://' | LC_ALL=C sort -u) +test "${#toolchains[@]}" -ge 1 +rust=$(IFS=-; printf '%s' "${toolchains[*]}") + +digest=$(sha256sum -- "${inputs[@]}" | sha256sum | cut -c1-64) +key="sdk-adapter-cargo-v2-$RUNNER_OS-$RUNNER_ARCH-rust-$rust-inputs-$digest" +if test -n "${GITHUB_OUTPUT:-}"; then + printf 'key=%s\n' "$key" >> "$GITHUB_OUTPUT" +fi +printf '%s\n' "$key" diff --git a/tests/Unit/ServerParityAdapterCacheTest.mjs b/tests/Unit/ServerParityAdapterCacheTest.mjs new file mode 100644 index 000000000..59d152fe6 --- /dev/null +++ b/tests/Unit/ServerParityAdapterCacheTest.mjs @@ -0,0 +1,78 @@ +// The published SDK adapters' public Cargo registry data and build tree are +// cached between runs, but only successful pushes to main save them: a pull +// request or merge group can restore main's cache and never write one a +// later run reads. The key covers the toolchain, the runner and every +// tracked adapter input, and an empty input list fails the job. +import test from 'node:test'; +import assert from 'node:assert/strict'; +import {readFileSync} from 'node:fs'; +import {resolve} from 'node:path'; + +const workflow = readFileSync(resolve('.github/workflows/server-parity.yml'), 'utf8'); + +function job(name) { + const start = workflow.indexOf(`\n ${name}:\n`); + assert.ok(start >= 0, name); + const next = workflow.slice(start + 1).search(/\n {2}[a-z0-9-]+:\n/); + return next < 0 ? workflow.slice(start) : workflow.slice(start, start + 1 + next); +} + +const steps = block => block.split(/\n {6}(?=- (?:name|uses):)/).slice(1); + +test('only the adapter job caches, and only public registry data and the build tree', () => { + const uses = workflow.match(/uses: actions\/cache[^\s]*/g) ?? []; + assert.deepEqual(uses, [ + 'uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9', + 'uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9', + ]); + const adapters = steps(job('published-sdk-adapters')); + const restore = adapters.findIndex(step => step.includes('actions/cache/restore@')); + const save = adapters.findIndex(step => step.includes('actions/cache/save@')); + const builds = adapters.map((step, index) => [step, index]).filter(([step]) => /-adapters\.sh\n?$/m.test(step)).map(([, index]) => index); + assert.equal(builds.length, 5); + assert.ok(restore >= 0 && restore < Math.min(...builds), 'restored before the first build'); + assert.ok(save > Math.max(...builds), 'saved after the last build'); + for (const index of [restore, save]) { + // Never the whole CARGO_HOME, where credentials would live. + assert.match(adapters[index], /path: \|\n {12}sdk-cargo\/registry\n {12}sdk-cargo\/git\n {12}sdk-target\n/); + assert.match(adapters[index], /key: \$\{\{ steps\.adapter_key\.outputs\.key \}\}$/m); + } + // No restore-keys: a run gets this exact key's tree or builds from nothing. + assert.doesNotMatch(adapters[restore], /restore-keys/); + assert.match(adapters[save], /if: \$\{\{ success\(\) && github\.event_name == 'push' && github\.ref == 'refs\/heads\/main' && /); +}); + +test('the cache key follows the toolchains, the runner and every tracked adapter input', () => { + const block = job('published-sdk-adapters'); + const key = steps(block).find(step => step.includes('id: adapter_key')); + assert.ok(key, 'a key step'); + // The key comes from one repository helper, never inline shell. + assert.match(key, /\n {8}run: bash scripts\/ci\/sdk-adapter-cache-key\.sh\n?$/); + const helper = readFileSync(resolve('scripts/ci/sdk-adapter-cache-key.sh'), 'utf8'); + assert.match(helper, /^set -euo pipefail$/m); + assert.match(helper, /git ls-files --/); + for (const adapter of ['published-sdk-patch', 'sdk-directions', 'legacy-marker-aliases', 'corrected-markers', 'query-authority']) { + assert.ok(helper.includes(`"$parity/${adapter}" \\\n`), adapter); + } + assert.ok(helper.includes('":(glob)$parity/*-adapters.sh" \\\n')); + assert.ok(helper.includes('scripts/ci/patched-pip.sh \\\n')); + // An empty or partial input list fails instead of keying a cache. + assert.match(helper, /grep -c '\/rust\/Cargo\\\.lock\$'\)" -eq 5/); + assert.match(helper, /grep -c -- '-adapters\\\.sh\$'\)" -eq 5/); + // Every pinned Rust image among the inputs is named, as are the runner + // and a digest of every input's name and content. + assert.match(helper, /grep -ho 'library\/rust@sha256:\[0-9a-f\]\\\{64\\\}' -- "\$\{inputs\[@\]\}"/); + assert.match(helper, /test "\$\{#toolchains\[@\]\}" -ge 1/); + assert.match(helper, /digest=\$\(sha256sum -- "\$\{inputs\[@\]\}" \| sha256sum \| cut -c1-64\)/); + assert.ok(helper.includes('key="sdk-adapter-cargo-v2-$RUNNER_OS-$RUNNER_ARCH-rust-$rust-inputs-$digest"')); + // The five build scripts are the job's adapter build steps. + const builds = steps(block).filter(step => /-adapters\.sh\n?$/m.test(step)); + assert.equal(builds.length, 5); +}); + +test('changing the key helper or the pip wrapper runs the workflow on main', () => { + const push = workflow.slice(workflow.indexOf('\n push:\n'), workflow.indexOf('\njobs:\n')); + for (const path of ['scripts/ci/sdk-adapter-cache-key.sh', 'scripts/ci/patched-pip.sh', 'tests/Unit/ServerParityAdapterCacheTest.mjs']) { + assert.ok(push.includes(`\n - ${path}\n`), path); + } +});