Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
157 changes: 153 additions & 4 deletions scripts/ci/select-gate-families.selftest.sh
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,16 @@
# edit, an added test, an added changeset) must skip every tooling self-test,
# while the tool's own inputs must still run them.
#
# Since #22076 the same section pins the pm_dispatch_gates read-set to the
# files the battery OPENS, as measured: a ratchet baseline, prose under
# scripts/ and agent prose outside the pm-dispatch rulebook skip it, while
# scripts/pm/** whole, the rulebook, a `.claude/**` shell script, a data file a
# relative specifier names and a scripts/ file of an unlisted kind still run
# it; a test under a package's scripts/ skips it unless a manifest or a
# workflow names it; a pull request touching only packages/*/src/** runs no PM
# self-test, and `schedule` and `push` still run it over the very changes the
# PR path skips.
#
# Since #19753 it covers migration_registry, the one family that ADDS a gate
# to the PR path: an entry, the generated registry, the generator and the two
# package files that decide how it runs must each select it; a spec source
Expand Down Expand Up @@ -77,13 +87,17 @@ mkdir -p "$UP/packages/a/scripts" "$UP/packages/a/src" "$UP/apps/site/src" "$UP/
"$UP/packages/spec/src/migrations/entries/semantic" "$UP/packages/spec/scripts"
printf '{"name":"fixture","private":true}\n' > "$UP/package.json"
printf 'packages:\n - packages/*\n' > "$UP/pnpm-workspace.yaml"
printf '{"name":"a"}\n' > "$UP/packages/a/package.json"
printf '{"name":"a","scripts":{"check:gate":"vitest run scripts/gate.test.ts"}}\n' > "$UP/packages/a/package.json"
printf 'export const a = 1;\n' > "$UP/packages/a/src/index.ts"
printf 'export const t = 1;\n' > "$UP/packages/a/src/index.test.ts"
printf '{"rows":[]}\n' > "$UP/packages/a/src/data.json"
printf '#!/usr/bin/env bash\necho foo\n' > "$UP/packages/a/foo.sh"
printf 'dist/\n' > "$UP/packages/a/.gitignore"
printf 'console.log(1);\n' > "$UP/packages/a/scripts/build.mjs"
# Two tests under a package's scripts/: one a check:* command in its manifest
# names (a gate source), one nothing names (#22076).
printf 'export const g = 1;\n' > "$UP/packages/a/scripts/gate.test.ts"
printf 'export const b = 1;\n' > "$UP/packages/a/scripts/build.test.ts"
printf 'export const site = 1;\n' > "$UP/apps/site/src/page.tsx"
printf '# guide\n' > "$UP/docs/guide.md"
printf '# page\n' > "$UP/content/docs/page.mdx"
Expand All @@ -96,7 +110,20 @@ printf '#!/usr/bin/env bash\necho ci\n' > "$UP/scripts/ci/tool.sh"
printf 'name: lint\n' > "$UP/.github/workflows/lint.yml"
printf '# agent\n' > "$UP/.claude/agents/os-dev.md"
printf '# skill\n' > "$UP/skills/x/SKILL.md"
# The pm_dispatch_gates read-set as measured (#22076): the rulebook its live
# cases open, a hook shell script the shell-mask census opens, and a data file
# under scripts/ that a module imports through a relative specifier. The
# specifier is assembled from an unquoted word on purpose, for the reason the
# two repo paths at the top are spelled as they are.
mkdir -p "$UP/.claude/skills/pm-dispatch/references" "$UP/.claude/hooks"
printf '# rules\n' > "$UP/.claude/skills/pm-dispatch/references/rules.md"
printf '#!/usr/bin/env bash\necho guard\n' > "$UP/.claude/hooks/guard.sh"
printf '{}\n' > "$UP/.claude/settings.json"
printf '{"rows":[]}\n' > "$UP/scripts/table.json"
TABLE_SPECIFIER=./table.json
printf "import table from '%s' with { type: 'json' };\nexport const rows = table.rows;\n" "$TABLE_SPECIFIER" > "$UP/scripts/reads-table.mjs"
printf '# rules\n' > "$UP/AGENTS.md"
printf '# claude\n' > "$UP/CLAUDE.md"
printf '# readme\n' > "$UP/README.md"
printf -- '---\n"a": patch\n---\nchange\n' > "$UP/.changeset/first.md"
# The migration_registry read-set (#19753), and its nearest neighbours outside it.
Expand All @@ -107,6 +134,7 @@ printf 'export const registry = [];\n' > "$UP/packages/spec/src/migrations/regis
printf 'export const entry = 1;\n' > "$UP/packages/spec/src/migrations/entries/semantic/17.x.ts"
printf 'export const gen = 1;\n' > "$UP/packages/spec/scripts/build-migration-registry.ts"
printf 'export const schemas = 1;\n' > "$UP/packages/spec/scripts/build-schemas.ts"
printf 'export const merge = 1;\n' > "$UP/packages/spec/scripts/conversions-merge.test.ts"
git_q -C "$UP" add -A
git_q -C "$UP" commit -q -m 'C0: root'
C0=$(git_q -C "$UP" rev-parse HEAD)
Expand Down Expand Up @@ -297,6 +325,27 @@ run_case 'push: the event decides, not the variables that happen to be set' "$RE
expect_rc 0
expect_all_run

# The backstop, held over the very change set the PR path now lets the PM
# self-test skip (#22076): an unscoped event never reads the diff.
S=$(scenario M:scripts/slot-lookup-baseline.json M:AGENTS.md M:skills/x/SKILL.md)
run_case 'schedule: a change the PR path skips the PM self-test for still runs it -- the hourly full run is the backstop (#22076 control)' "$REPO" schedule '' ''
expect_rc 0
expect_warnings '' ''
expect_all_run
expect_reason pm_dispatch_gates "event 'schedule' is not scoped"

run_case 'push: the same change on main still runs the PM self-test (#22076 control)' "$REPO" push main "$C0"
expect_rc 0
expect_warnings '' ''
expect_all_run
expect_reason pm_dispatch_gates "event 'push' is not scoped"

run_case 'merge_group: the same change, scoped, skips the PM self-test -- so the two controls above are not vacuous' "$REPO" merge_group '' "$C0"
expect_rc 0
expect_warnings '' ''
expect_verdicts slot_lookup query_options_erasure entry_guard declared_population_live bare_root_worklist self_test_workflow_commands
expect_reason pm_dispatch_gates 'no changed path is in its read-set'

# ── merge_group: the card's four cases ──────────────────────────────────────
S=$(scenario M:scripts/pm/tool.mjs)
run_case 'merge_group: a scripts/pm change runs every tooling self-test (and, for a .mjs, the corpus walk)' "$REPO" merge_group '' "$C0"
Expand Down Expand Up @@ -502,6 +551,20 @@ run_case 'merge_group: a package-local script is a gate source (the derivation f
expect_rc 0
expect_verdicts comment_mask_corpus pm_dispatch_gates declared_population_live bare_root_worklist self_test_workflow_commands

S=$(scenario M:packages/a/scripts/build.test.ts)
run_case 'merge_group: a test under a package scripts/ that no manifest or workflow names is product test code to the PM self-test, which skips (#22076)' "$REPO" merge_group '' "$C0"
expect_rc 0
expect_warnings '' ''
expect_verdicts slot_lookup query_options_erasure comment_mask_corpus declared_population_live bare_root_worklist self_test_workflow_commands
expect_reason pm_dispatch_gates 'no changed path is in its read-set'

S=$(scenario M:packages/a/scripts/gate.test.ts)
run_case 'merge_group: a test under a package scripts/ that a check script names is a gate source and still runs the PM self-test' "$REPO" merge_group '' "$C0"
expect_rc 0
expect_warnings '' ''
expect_verdicts slot_lookup query_options_erasure comment_mask_corpus pm_dispatch_gates declared_population_live bare_root_worklist self_test_workflow_commands
expect_reason pm_dispatch_gates 'packages/a/scripts/gate.test.ts (M, workspace)'

S=$(scenario M:scripts/pm/os-verify-lock.sh)
run_case 'merge_group: the lock script runs its own self-test and every family that reads scripts/' "$REPO" merge_group '' "$C0"
expect_rc 0
Expand All @@ -516,19 +579,68 @@ expect_reason verify_lock scripts/helper.mjs
expect_reason migration_registry 'no changed path is in its read-set'

S=$(scenario M:scripts/slot-lookup-baseline.json)
run_case 'merge_group: a ratchet baseline runs the ratchets and every family that reads scripts/' "$REPO" merge_group '' "$C0"
run_case 'merge_group: a ratchet baseline runs the ratchets and the families that walk scripts/, and not the PM self-test -- the battery opens no data file (#22076)' "$REPO" merge_group '' "$C0"
expect_rc 0
expect_warnings '' ''
expect_verdicts slot_lookup query_options_erasure entry_guard declared_population_live bare_root_worklist self_test_workflow_commands
expect_reason pm_dispatch_gates 'no changed path is in its read-set'
expect_reason entry_guard '(M, scripts)'

S=$(scenario M:scripts/table.json)
run_case 'merge_group: a data file a relative import specifier names still runs the PM self-test -- the one edge by which the discovery opens a data file' "$REPO" merge_group '' "$C0"
expect_rc 0
expect_warnings '' ''
expect_verdicts slot_lookup query_options_erasure entry_guard pm_dispatch_gates declared_population_live bare_root_worklist self_test_workflow_commands
expect_reason pm_dispatch_gates 'scripts/table.json (M, scripts)'

S=$(scenario A:scripts/notes.md A:scripts/ci/fixtures/job-log.txt)
run_case 'merge_group: prose and a fixture log under scripts/ run the families that walk scripts/, and not the PM self-test (#22076)' "$REPO" merge_group '' "$C0"
expect_rc 0
expect_warnings '' ''
expect_verdicts entry_guard declared_population_live bare_root_worklist self_test_workflow_commands
expect_reason pm_dispatch_gates 'no changed path is in its read-set'

S=$(scenario A:scripts/ci/matrix.yaml)
run_case 'merge_group: a scripts/ file of a kind the arm does not list still runs the PM self-test -- fail-open on the extension' "$REPO" merge_group '' "$C0"
expect_rc 0
expect_verdicts entry_guard pm_dispatch_gates declared_population_live bare_root_worklist self_test_workflow_commands
expect_reason pm_dispatch_gates 'scripts/ci/matrix.yaml (A, scripts)'

S=$(scenario A:scripts/pm/ledger.json)
run_case 'merge_group: a data file under scripts/pm still runs the PM self-test -- scripts/pm is in its read-set whole' "$REPO" merge_group '' "$C0"
expect_rc 0
expect_verdicts entry_guard pm_dispatch_gates declared_population_live bare_root_worklist self_test_workflow_commands
expect_reason pm_dispatch_gates 'scripts/pm/ledger.json (A, scripts)'

S=$(scenario M:scripts/ci/tool.sh)
run_case 'merge_group: a scripts/ subdirectory script is a gate source only' "$REPO" merge_group '' "$C0"
expect_rc 0
expect_verdicts entry_guard pm_dispatch_gates declared_population_live bare_root_worklist self_test_workflow_commands

S=$(scenario M:.claude/agents/os-dev.md M:skills/x/SKILL.md M:AGENTS.md)
run_case 'merge_group: agent configuration runs the PM self-test alone -- it is the only battery here that reads it' "$REPO" merge_group '' "$C0"
run_case 'merge_group: agent prose the battery never opens (an agent definition, a published skill, AGENTS.md) runs no family (#22076)' "$REPO" merge_group '' "$C0"
expect_rc 0
expect_warnings '' ''
expect_verdicts
expect_reason pm_dispatch_gates 'no changed path is in its read-set'

S=$(scenario M:CLAUDE.md M:.claude/settings.json)
run_case 'merge_group: CLAUDE.md and the .claude settings JSON run no family (#22076)' "$REPO" merge_group '' "$C0"
expect_rc 0
expect_verdicts

S=$(scenario M:.claude/skills/pm-dispatch/references/rules.md)
run_case 'merge_group: the pm-dispatch rulebook runs the PM self-test alone -- its live cases open it' "$REPO" merge_group '' "$C0"
expect_rc 0
expect_warnings '' ''
expect_verdicts pm_dispatch_gates
expect_reason pm_dispatch_gates '(M, agent-config)'

S=$(scenario M:.claude/hooks/guard.sh)
run_case 'merge_group: a .claude hook shell script still runs the PM self-test alone -- only prose and JSON left the read-set' "$REPO" merge_group '' "$C0"
expect_rc 0
expect_verdicts pm_dispatch_gates
expect_reason pm_dispatch_gates '.claude/hooks/guard.sh (M, agent-config)'

S=$(scenario M:package.json)
run_case 'merge_group: root configuration runs every family' "$REPO" merge_group '' "$C0"
Expand Down Expand Up @@ -649,6 +761,43 @@ expect_verdicts slot_lookup query_options_erasure comment_mask_corpus
expect_reason pm_dispatch_gates 'no changed path is in its read-set'
expect_line 'Gate families: 3 run, 7 skipped'

# The card's pin and triage's (#22076): product code alone runs no PM
# self-test; scripts/pm/** runs it, prose included.
git_q -C "$REPO" checkout -q -B feature-22076-src "$C0"
printf 'export const a = 3;\n' > "$REPO/packages/a/src/index.ts"
printf 'export const index = 2;\n' > "$REPO/packages/spec/src/index.ts"
git_q -C "$REPO" commit -q -am 'F4: product source in two packages, nothing else'
run_case 'pull_request: a PR touching only packages/*/src/** runs no PM self-test (#22076 pin)' "$REPO" pull_request main ''
expect_rc 0
expect_warnings '' ''
expect_verdicts slot_lookup query_options_erasure comment_mask_corpus
expect_reason pm_dispatch_gates 'no changed path is in its read-set'
expect_changed 'two package sources' "M packages/a/src/index.ts
M packages/spec/src/index.ts"

git_q -C "$REPO" checkout -q -B feature-22076-pm "$C0"
printf '# pm, revised on a feature branch\n' > "$REPO/scripts/pm/README.md"
git_q -C "$REPO" commit -q -am 'F5: prose under scripts/pm'
run_case 'pull_request: a PR touching scripts/pm/** runs the PM self-test, prose included (#22076 pin)' "$REPO" pull_request main ''
expect_rc 0
expect_warnings '' ''
expect_verdicts entry_guard pm_dispatch_gates declared_population_live bare_root_worklist self_test_workflow_commands
expect_reason pm_dispatch_gates 'scripts/pm/README.md (M, scripts)'

# The shape the card measured paying the step: a spec source, tests under
# packages/spec/scripts/ that nothing wires as a gate, and a changeset.
git_q -C "$REPO" checkout -q -B feature-22076-spec-tests "$C0"
printf 'export const merge = 2;\n' > "$REPO/packages/spec/scripts/conversions-merge.test.ts"
printf 'export const index = 3;\n' > "$REPO/packages/spec/src/index.ts"
printf -- '---\n"spec": patch\n---\nthe shared entry\n' > "$REPO/.changeset/shared-entry.md"
git_q -C "$REPO" add -A
git_q -C "$REPO" commit -q -m 'F6: a spec source, an unwired test under packages/spec/scripts, a changeset'
run_case 'pull_request: a spec source, an unwired test under packages/spec/scripts and a changeset run no PM self-test (#22076)' "$REPO" pull_request main ''
expect_rc 0
expect_warnings '' ''
expect_verdicts slot_lookup query_options_erasure comment_mask_corpus declared_population_live bare_root_worklist self_test_workflow_commands
expect_reason pm_dispatch_gates 'no changed path is in its read-set'

git_q -C "$REPO" checkout -q -B feature-19753 "$C0"
printf 'export const entry = 2;\n' > "$REPO/packages/spec/src/migrations/entries/semantic/17.x.ts"
git_q -C "$REPO" commit -q -am 'F3: an entry edited on a feature branch, registry not regenerated'
Expand Down Expand Up @@ -753,7 +902,7 @@ pin_step migration_registry 'pnpm --filter @objectstack/spec check:migration-reg

# ── Verdict ─────────────────────────────────────────────────────────────────
# #4690: a battery that ran nothing is a failure, never a pass.
if [ "$cases" -lt 56 ] || [ "$checks" -lt 293 ]; then
if [ "$cases" -lt 71 ] || [ "$checks" -lt 369 ]; then
echo "SELFTEST FAILED: only $cases case(s) / $checks check(s) ran -- the battery is short"
exit 1
fi
Expand Down
Loading
Loading