|
33 | 33 | # quoted verbatim into the PR body and a human decides. See "THE PINS" below. |
34 | 34 | # ⛔ It never opens a PR when the regenerated file is byte-identical. A weekly |
35 | 35 | # no-op PR trains everyone to ignore this PR. |
| 36 | +# ⛔ It never moves a suite-duration ceiling (#16468). The dataset also |
| 37 | +# carries `ceilings`, which `Test Core` grades every executed package |
| 38 | +# against (scripts/check-test-suite-ceilings.mjs). The generator writes |
| 39 | +# them in the same pass and HOLDS every ceiling the committed dataset |
| 40 | +# already has; a package gets a new one only when it had none (its slowest |
| 41 | +# executed run in this window x 1.25). A ceiling rises only by a ruling, |
| 42 | +# recorded in that script, so a refresh whose window ran slower REPORTS |
| 43 | +# the gap in the PR body instead of absorbing it. The first refresh after |
| 44 | +# the check landed writes the first table; until it merges the check reads |
| 45 | +# NOT MEASURED. |
36 | 46 | # |
37 | 47 | # HOW IT GETS ITS INPUTS, AND WHY IT RUNS HERE RATHER THAN IN AN AGENT WORKTREE |
38 | 48 | # ---------------------------------------------------------------------------- |
@@ -634,6 +644,53 @@ jobs: |
634 | 644 | } |
635 | 645 | ') |
636 | 646 |
|
| 647 | + # The suite-duration ceilings (#16468), read out of the dataset like the |
| 648 | + # two lines above. The prior table is the COMMITTED dataset, read from |
| 649 | + # git because the bins step has already copied the refreshed file over |
| 650 | + # it. A refresh never moves a held ceiling, so the line counts what is |
| 651 | + # new and what is uncapped, and the spread table below marks every |
| 652 | + # held ceiling this window would have set differently -- and every one |
| 653 | + # a run of this window already went over. |
| 654 | + CEILING_LINE=$(node -e ' |
| 655 | + const fs = require("fs"); |
| 656 | + const { execFileSync } = require("child_process"); |
| 657 | + const d = JSON.parse(fs.readFileSync(process.env.RUNNER_TEMP + "/refresh/refreshed.json", "utf8")); |
| 658 | + if (!d.ceilings || typeof d.ceilings !== "object" || !d.uncapped || typeof d.uncapped !== "object") { |
| 659 | + throw new Error("the refreshed dataset carries no `ceilings` / `uncapped`"); |
| 660 | + } |
| 661 | + const prior = JSON.parse(execFileSync("git", ["show", "HEAD:scripts/test-shard-timings.json"], { encoding: "utf8" })); |
| 662 | + const held = prior.ceilings && typeof prior.ceilings === "object" ? prior.ceilings : null; |
| 663 | + const names = Object.keys(d.ceilings); |
| 664 | + const fresh = held === null ? names : names.filter((n) => !Object.hasOwn(held, n)); |
| 665 | + const uncapped = Object.entries(d.uncapped).map(([n, why]) => n + " (" + why + ")"); |
| 666 | + const h = d.provenance.ceilingHeadroom; |
| 667 | + const over = (fs.readFileSync(process.env.RUNNER_TEMP + "/refresh/spread.md", "utf8").match(/OVER its held ceiling/g) || []).length; |
| 668 | + const lines = []; |
| 669 | + if (held === null) { |
| 670 | + lines.push( |
| 671 | + "⚠️ This refresh writes the FIRST suite-duration ceiling table: " + names.length + " package(s) get a " + |
| 672 | + "ceiling, each its slowest executed run in this window x " + h + ". From the merge of this PR the `Test Core` " + |
| 673 | + "check (scripts/check-test-suite-ceilings.mjs) grades every executed package against it -- until then it " + |
| 674 | + "reads NOT MEASURED. Every later refresh HOLDS these numbers and a ceiling rises only by a ruling, so " + |
| 675 | + "review the ceiling column below before merging." |
| 676 | + ); |
| 677 | + } else { |
| 678 | + lines.push( |
| 679 | + "Suite-duration ceilings: " + (names.length - fresh.length) + " held unchanged (a refresh never moves one; " + |
| 680 | + "a raise is a ruling), " + fresh.length + " new" + (fresh.length ? " (" + fresh.join(", ") + ")" : "") + |
| 681 | + ", each new one its slowest executed run in this window x " + h + "." |
| 682 | + ); |
| 683 | + } |
| 684 | + lines.push(uncapped.length ? "No ceiling: " + uncapped.join(", ") + "." : "Every package carries a ceiling."); |
| 685 | + if (over > 0) { |
| 686 | + lines.push( |
| 687 | + "⚠️ " + over + " package(s) ran OVER their held ceiling in this window (marked in the table): the brake " + |
| 688 | + "fired on main. This refresh does not raise them; that takes a ruling." |
| 689 | + ); |
| 690 | + } |
| 691 | + console.log(lines.join("\n\n")); |
| 692 | + ') |
| 693 | +
|
637 | 694 | SHARDS=$(node -e ' |
638 | 695 | const rs = JSON.parse(require("fs").readFileSync(process.env.RUNNER_TEMP + "/candidates.json", "utf8")); |
639 | 696 | const r = rs.find((x) => String(x.run_id) === process.env.RUN_ID); |
@@ -675,10 +732,12 @@ jobs: |
675 | 732 | echo "because a weekly lane cannot know which cards a given run ought to retire. If this is" |
676 | 733 | echo "the first refresh to land, retire those two by hand as part of merging it." |
677 | 734 | echo |
678 | | - echo "## Sample depth and run-to-run spread" |
| 735 | + echo "## Sample depth, run-to-run spread and suite-duration ceilings" |
679 | 736 | echo |
680 | 737 | echo "$DEPTH_LINE" |
681 | 738 | echo |
| 739 | + echo "$CEILING_LINE" |
| 740 | + echo |
682 | 741 | cat "$RUNNER_TEMP/refresh/spread.md" |
683 | 742 | echo |
684 | 743 | echo "## Measured per-shard suite time on the newest run in the set" |
|
0 commit comments