From cbb948b9269f9320886c4640ead2fc0cc3025fac Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 7 Oct 2026 13:57:02 +0000 Subject: [PATCH] ci: read the turbo remote cache on the Type Check, Test Core, dogfood and Temporal build steps Extends Build Core's remote-cache env block, verbatim (same write rule: merge_group and push/workflow_dispatch on main write, everything else reads; same signing; no new secret), to every `turbo run build` step of lint.yml's three Type Check build lanes and of ci.yml's dogfood shards and Temporal Conformance. Test Core built its closure inside the test step's own turbo run, which must stay off the remote, so each shard now builds its `^build` closure in a new guarded step that carries the block, and the test step replays it. Build Core's comment gains the PINS every carrier points back to: build tasks only, hash-identical to Build Core's plan, and root inputs declared. Claude-Session: https://claude.ai/code/session_01VF48aw8RPG6wzDnMgp6rtw Co-authored-by: Claude --- .github/workflows/ci.yml | 126 +++++++++++++++++++++++++++++++++++-- .github/workflows/lint.yml | 45 +++++++++++++ 2 files changed, 167 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 79d4aa4e064..0d2576cbf26 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -659,6 +659,58 @@ jobs: echo 'Items on this shard (a package name, or a package plus a k/n file-level slice):' cat "$RUNNER_TEMP/shard-packages.txt" + # ⛔ THIS SHARD'S `^build` CLOSURE IS BUILT HERE, ON THE TURBO REMOTE + # CACHE (#22077), so `Run this shard's tests` REPLAYS it from the local + # cache instead of building it. Before this step the closure was built + # inside the test step's own turbo run (the slice step below runs zero + # iterations since #21487). Merge-queue run 37623284168, `Test Core + # (3/6)`: `Tasks: 72 successful, 72 total` / `Cached: 1 cached, 72 + # total` / `Time: 11m43.373s` for the tests AND their closure in one + # run. That run cannot read the remote, because `test` / `test:repo` + # stay off it (Build Core's PINS), so the closure moves here. + # + # `--filter=PKG^...` selects PKG's dependencies without PKG: the + # `^build` closure that `test` and `test:repo` schedule. Measured on + # 3d9188502e, turbo 2.11.5, all six shards of the local partition: this + # plan equals the test plan's build tasks, 32 to 63 per shard, with + # identical hashes, 0 differing and 0 extra, and every one of them has + # Build Core's hash. A `test` task that `dependsOn: ["build"]` (cli and + # metadata in turbo.json) also needs its OWN package's build. This filter + # leaves that out on purpose, so a shard never builds a package its tests + # do not need; the test step builds it as before. Today that is one task, + # `cli#build` on shard 1/6. + # + # Its own guarded step, for the reason the slice step below gives: a + # guarded SITE is (file, job, step). No `--summarize`: `.turbo/runs/` + # stays the test step's alone, and that is what the drift check and the + # timings capture read. The four TURBO_* lines are Build Core's, + # verbatim; its "Turbo remote cache (#21186)" comment holds the write + # rule, the signing and the PINS. ⛔ An empty FILTERS must exit here: + # `turbo run build` with no filter builds the whole workspace. + - name: Build this shard's dependency closure + env: + NODE_OPTIONS: --report-on-signal --report-signal=SIGUSR2 --report-directory=${{ runner.temp }}/stall-reports + TURBO_CACHE: ${{ (github.event_name == 'merge_group' || ((github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.ref == 'refs/heads/main')) && 'local:rw,remote:rw' || 'local:rw,remote:r' }} + TURBO_TOKEN: ${{ secrets.TURBO_REMOTE_CACHE_SIGNATURE_KEY != '' && secrets.TURBO_TOKEN || '' }} + TURBO_TEAM: ${{ vars.TURBO_TEAM }} + TURBO_REMOTE_CACHE_SIGNATURE_KEY: ${{ secrets.TURBO_REMOTE_CACHE_SIGNATURE_KEY }} + run: | + FILTERS="" + if [ -s "$RUNNER_TEMP/shard-packages.txt" ]; then + while read -r PKG _; do + [ -n "$PKG" ] || continue + FILTERS="$FILTERS --filter=$PKG^..." + done < "$RUNNER_TEMP/shard-packages.txt" + fi + if [ -z "$FILTERS" ]; then + echo "No packages on this shard — nothing to build." + exit 0 + fi + mkdir -p "$RUNNER_TEMP/stall-reports" + node scripts/run-with-stall-guard.mjs --log "$RUNNER_TEMP/test-core-closure-build.log" --stall-minutes 10 \ + --report-dir "$RUNNER_TEMP/stall-reports" -- \ + pnpm turbo run build $FILTERS --concurrency=4 --log-order=stream + # ⛔ A FILE-LEVEL SLICE BUILDS ITS DEPENDENCY CLOSURE HERE, IN ITS OWN # GUARDED STEP, so the slice leg in the next step REPLAYS it. Since #19278 # that leg carries its slice in `OS_TEST_SHARD`, which only the `test` @@ -1503,7 +1555,18 @@ jobs: - name: Install dependencies run: pnpm install --frozen-lockfile + # Turbo remote cache (#22077), on all four of this job's build steps: the + # four TURBO_* lines are Build Core's, verbatim, and Build Core's "Turbo + # remote cache (#21186)" comment holds the write rule, the signing and the + # PINS. Each of these plans is all `build` tasks with Build Core's hashes + # (8, 18, 14 and 31 nodes, measured). The suites between them run vitest + # through `pnpm --filter`, not turbo, and carry no credentials. - name: Build driver-sql and its dependencies + env: + TURBO_CACHE: ${{ (github.event_name == 'merge_group' || ((github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.ref == 'refs/heads/main')) && 'local:rw,remote:rw' || 'local:rw,remote:r' }} + TURBO_TOKEN: ${{ secrets.TURBO_REMOTE_CACHE_SIGNATURE_KEY != '' && secrets.TURBO_TOKEN || '' }} + TURBO_TEAM: ${{ vars.TURBO_TEAM }} + TURBO_REMOTE_CACHE_SIGNATURE_KEY: ${{ secrets.TURBO_REMOTE_CACHE_SIGNATURE_KEY }} run: pnpm exec turbo run build --filter=@objectstack/driver-sql... --concurrency=4 # The whole driver-sql suite runs under the skewed process zone — not @@ -1561,6 +1624,11 @@ jobs: # which is what a ratchet is for — it makes the property enforced rather # than incidental. - name: Build the non-SQL temporal backends + env: + TURBO_CACHE: ${{ (github.event_name == 'merge_group' || ((github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.ref == 'refs/heads/main')) && 'local:rw,remote:rw' || 'local:rw,remote:r' }} + TURBO_TOKEN: ${{ secrets.TURBO_REMOTE_CACHE_SIGNATURE_KEY != '' && secrets.TURBO_TOKEN || '' }} + TURBO_TEAM: ${{ vars.TURBO_TEAM }} + TURBO_REMOTE_CACHE_SIGNATURE_KEY: ${{ secrets.TURBO_REMOTE_CACHE_SIGNATURE_KEY }} run: >- pnpm exec turbo run build --filter=@objectstack/service-analytics... @@ -1640,6 +1708,11 @@ jobs: # single shorter substring (`live-`) would select the same set today and # silently widen with the next unrelated file that happens to spell it. - name: Build metadata-protocol and its dependencies + env: + TURBO_CACHE: ${{ (github.event_name == 'merge_group' || ((github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.ref == 'refs/heads/main')) && 'local:rw,remote:rw' || 'local:rw,remote:r' }} + TURBO_TOKEN: ${{ secrets.TURBO_REMOTE_CACHE_SIGNATURE_KEY != '' && secrets.TURBO_TOKEN || '' }} + TURBO_TEAM: ${{ vars.TURBO_TEAM }} + TURBO_REMOTE_CACHE_SIGNATURE_KEY: ${{ secrets.TURBO_REMOTE_CACHE_SIGNATURE_KEY }} run: pnpm exec turbo run build --filter=@objectstack/metadata-protocol... --concurrency=4 - name: Run the metadata-protocol migration statements against live MySQL and PostgreSQL @@ -1697,6 +1770,11 @@ jobs: # speaks up when a named path selects no tests — a second net, not a # licence to change the form.) - name: Build runtime and its dependencies + env: + TURBO_CACHE: ${{ (github.event_name == 'merge_group' || ((github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.ref == 'refs/heads/main')) && 'local:rw,remote:rw' || 'local:rw,remote:r' }} + TURBO_TOKEN: ${{ secrets.TURBO_REMOTE_CACHE_SIGNATURE_KEY != '' && secrets.TURBO_TOKEN || '' }} + TURBO_TEAM: ${{ vars.TURBO_TEAM }} + TURBO_REMOTE_CACHE_SIGNATURE_KEY: ${{ secrets.TURBO_REMOTE_CACHE_SIGNATURE_KEY }} run: pnpm exec turbo run build --filter=@objectstack/runtime... --concurrency=4 - name: Run the runtime cascade-delete matrix against live PostgreSQL @@ -1820,9 +1898,20 @@ jobs: # own `--stall-minutes` and its own headroom row. The step is NOT allowed # to buy itself room by raising this job's `timeout-minutes`: that budget # is the instrument that shows this fix working (#16886). + # + # Turbo remote cache (#22077): the four TURBO_* lines are Build Core's, + # verbatim, and Build Core's "Turbo remote cache (#21186)" comment holds + # the write rule, the signing and the PINS. This plan is 66 `build` nodes + # (63 with a command), each with Build Core's hash (measured). ⛔ They stay on THIS step: the + # test step below runs `dogfood#test`, which must never read the remote, + # and replays this closure from the local cache this step fills. - name: Build the dogfood package's dependency closure env: NODE_OPTIONS: --report-on-signal --report-signal=SIGUSR2 --report-directory=${{ runner.temp }}/stall-reports + TURBO_CACHE: ${{ (github.event_name == 'merge_group' || ((github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.ref == 'refs/heads/main')) && 'local:rw,remote:rw' || 'local:rw,remote:r' }} + TURBO_TOKEN: ${{ secrets.TURBO_REMOTE_CACHE_SIGNATURE_KEY != '' && secrets.TURBO_TOKEN || '' }} + TURBO_TEAM: ${{ vars.TURBO_TEAM }} + TURBO_REMOTE_CACHE_SIGNATURE_KEY: ${{ secrets.TURBO_REMOTE_CACHE_SIGNATURE_KEY }} run: | mkdir -p "$RUNNER_TEMP/stall-reports" node scripts/run-with-stall-guard.mjs --log "$RUNNER_TEMP/dogfood-build.log" --stall-minutes 10 \ @@ -2189,10 +2278,15 @@ jobs: # in both places (72 of 72 build tasks, read from both CI logs at one # SHA), so a pin bump there can replay what this step built on `main`. # - # Scope: THIS step only, i.e. `build` tasks. `test` / `test:repo` stay - # off the remote: their cross-package inputs are hand-declared, and a - # wrong hash replayed from a remote reaches every run and cloud too. - # Build Docs forces execution and carries no credentials. + # Scope: `turbo run build` steps only, i.e. `build` tasks. Since #22077 + # that is this step plus the build steps of the Test Core shards, the + # dogfood shards, Temporal Conformance and lint.yml's three Type Check + # build lanes, each carrying the four lines below verbatim; the PINS + # paragraph at the end says what keeps that set honest. `test` / + # `test:repo` stay off the remote: their cross-package inputs are + # hand-declared, and a wrong hash replayed from a remote reaches every + # run and cloud too. Build Docs forces execution and carries no + # credentials. # # TURBO_CACHE is the env form of `--cache`. Writes happen on # `merge_group` and on `push` / `workflow_dispatch` against main; every @@ -2211,6 +2305,30 @@ jobs: # upload and verify every download with that key. Any of the three # unset: remote caching is off and the build is unchanged; the summary # then reads `Remote caching disabled (remote cache requested — …)`. + # + # PINS, for every step that carries these four lines (#22077). Each + # carrier points back here. + # - BUILD TASKS ONLY. #19086's lesson: a step that writes artifacts from + # a cache-served dist must not itself be served. Each carrier runs one + # `turbo run build` whose `--dry=json` plan is all `#build` tasks. + # ⛔ Never put these lines on a step that runs `test`, `test:repo`, + # `typecheck` or a `gen:*` task (turbo.json caches `gen:schema` and + # `gen:skill-refs`), or on a step that writes tracked files. + # - SAME HASHES AS THIS STEP. Every carrier's build tasks are a subset + # of this step's plan with identical hashes (measured on 3d9188502e, + # turbo 2.11.5: 8 to 77 tasks per carrier, 0 differing), so a PR read + # hits what this step wrote on `main` or in the queue. Filters and + # `--concurrency` do not enter a task hash; an env var a carrier sets + # that turbo.json hashes would, so do not add one to a carrier. + # - A HIT IS ONLY AS CORRECT AS THE HASH (#21193). Every root file a + # build reads is a `$TURBO_ROOT$` input in turbo.json. Re-measured for + # #22077 on 3d9188502e: one appended line in each of the 17 declared + # root inputs moves the build hashes (72, 71 or 6 of 72, #21193's + # numbers); the root tsup.config.ts control moves 72/72; an unrelated + # script, ci.yml and lint.yml move 0. The static import closure of + # every root script a build command or tsup config names is inside + # those declarations. A build that starts reading a new root file + # must declare it in turbo.json before this cache can be trusted. - name: Build packages (excluding docs) env: TURBO_CACHE: ${{ (github.event_name == 'merge_group' || ((github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.ref == 'refs/heads/main')) && 'local:rw,remote:rw' || 'local:rw,remote:r' }} diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index f8601f7a90d..deb5d9ea8e8 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -6104,7 +6104,20 @@ jobs: # showcase imports were previously built only by accident — through # dogfood's dependency chain, which broke when dogfood moved to # packages/qa/ (#3037). + # + # Turbo remote cache (#22077): the four TURBO_* lines are ci.yml Build + # Core's, verbatim, and its "Turbo remote cache (#21186)" comment holds + # the write rule, the signing and the PINS. This plan is 70 `build` nodes + # with Build Core's hashes (measured). ⛔ Build steps only: the typecheck + # step below stays off the remote, and every build task its `^build` + # closure schedules is already in this plan (0 extra, measured), so it + # replays them from the local cache this step fills. - name: Build workspace packages + env: + TURBO_CACHE: ${{ (github.event_name == 'merge_group' || ((github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.ref == 'refs/heads/main')) && 'local:rw,remote:rw' || 'local:rw,remote:r' }} + TURBO_TOKEN: ${{ secrets.TURBO_REMOTE_CACHE_SIGNATURE_KEY != '' && secrets.TURBO_TOKEN || '' }} + TURBO_TEAM: ${{ vars.TURBO_TEAM }} + TURBO_REMOTE_CACHE_SIGNATURE_KEY: ${{ secrets.TURBO_REMOTE_CACHE_SIGNATURE_KEY }} run: pnpm exec turbo run build --filter='./packages/*' --filter='./examples/*^...' # CLI transcripts in content/docs are bound to the registries they quote @@ -6320,7 +6333,18 @@ jobs: # showcase imports were previously built only by accident — through # dogfood's dependency chain, which broke when dogfood moved to # packages/qa/ (#3037). + # + # Turbo remote cache (#22077), on both of this lane's build steps: the + # four TURBO_* lines are ci.yml Build Core's, verbatim, and its "Turbo + # remote cache (#21186)" comment holds the write rule, the signing and + # the PINS. The two plans are 70 and 77 `build` nodes with Build Core's + # hashes (measured). The gates after them carry no credentials. - name: Build workspace packages + env: + TURBO_CACHE: ${{ (github.event_name == 'merge_group' || ((github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.ref == 'refs/heads/main')) && 'local:rw,remote:rw' || 'local:rw,remote:r' }} + TURBO_TOKEN: ${{ secrets.TURBO_REMOTE_CACHE_SIGNATURE_KEY != '' && secrets.TURBO_TOKEN || '' }} + TURBO_TEAM: ${{ vars.TURBO_TEAM }} + TURBO_REMOTE_CACHE_SIGNATURE_KEY: ${{ secrets.TURBO_REMOTE_CACHE_SIGNATURE_KEY }} run: pnpm exec turbo run build --filter='./packages/*' --filter='./examples/*^...' # The MEASURED half of the coverage gate (#5278). The cheap structural @@ -6384,6 +6408,11 @@ jobs: # local run, which is what #8271 was. Cost of keeping it: 9.5s of turbo # cache hits (70/70) against this lane's ~249s. - name: Build the ledgered packages' dependencies + env: + TURBO_CACHE: ${{ (github.event_name == 'merge_group' || ((github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.ref == 'refs/heads/main')) && 'local:rw,remote:rw' || 'local:rw,remote:r' }} + TURBO_TOKEN: ${{ secrets.TURBO_REMOTE_CACHE_SIGNATURE_KEY != '' && secrets.TURBO_TOKEN || '' }} + TURBO_TEAM: ${{ vars.TURBO_TEAM }} + TURBO_REMOTE_CACHE_SIGNATURE_KEY: ${{ secrets.TURBO_REMOTE_CACHE_SIGNATURE_KEY }} run: pnpm exec turbo run build --filter='./packages/*' --filter='./packages/*/*' # [#15042] The closure-wide declaration re-check. `check-dts-emitted` runs @@ -6559,7 +6588,18 @@ jobs: # showcase imports were previously built only by accident — through # dogfood's dependency chain, which broke when dogfood moved to # packages/qa/ (#3037). + # + # Turbo remote cache (#22077), on both of this lane's build steps: the + # four TURBO_* lines are ci.yml Build Core's, verbatim, and its "Turbo + # remote cache (#21186)" comment holds the write rule, the signing and + # the PINS. The two plans are 70 and 77 `build` nodes with Build Core's + # hashes (measured). The gates after them carry no credentials. - name: Build workspace packages + env: + TURBO_CACHE: ${{ (github.event_name == 'merge_group' || ((github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.ref == 'refs/heads/main')) && 'local:rw,remote:rw' || 'local:rw,remote:r' }} + TURBO_TOKEN: ${{ secrets.TURBO_REMOTE_CACHE_SIGNATURE_KEY != '' && secrets.TURBO_TOKEN || '' }} + TURBO_TEAM: ${{ vars.TURBO_TEAM }} + TURBO_REMOTE_CACHE_SIGNATURE_KEY: ${{ secrets.TURBO_REMOTE_CACHE_SIGNATURE_KEY }} run: pnpm exec turbo run build --filter='./packages/*' --filter='./examples/*^...' # The nested-package build. ⚠️ NOT redundant with "Build workspace @@ -6581,6 +6621,11 @@ jobs: # reporting "measured and clean" (#4690). It is a superset of the step # above, so in practice it is cache hits plus the remainder (~13s in CI). - name: Build the nested packages the consumer gates resolve through + env: + TURBO_CACHE: ${{ (github.event_name == 'merge_group' || ((github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.ref == 'refs/heads/main')) && 'local:rw,remote:rw' || 'local:rw,remote:r' }} + TURBO_TOKEN: ${{ secrets.TURBO_REMOTE_CACHE_SIGNATURE_KEY != '' && secrets.TURBO_TOKEN || '' }} + TURBO_TEAM: ${{ vars.TURBO_TEAM }} + TURBO_REMOTE_CACHE_SIGNATURE_KEY: ${{ secrets.TURBO_REMOTE_CACHE_SIGNATURE_KEY }} run: pnpm exec turbo run build --filter='./packages/*' --filter='./packages/*/*' # [#15042] The closure-wide declaration re-check. `check-dts-emitted` runs