From 3226716aebe8f7101633a9848123d675ece8b284 Mon Sep 17 00:00:00 2001 From: glennfaison Date: Mon, 16 Nov 2026 22:24:38 +0000 Subject: [PATCH] feat(validate): check the contract-first, Process and common-input rules (#106) --- .changeset/validate-contract-shape.md | 4 ++ scripts/contracts.mjs | 32 +++++++-- scripts/validate-skills.mjs | 5 +- skills/meta/agent-workflow-protocol/SKILL.md | 5 +- tests/contracts.test.mjs | 73 +++++++++++++++----- 5 files changed, 92 insertions(+), 27 deletions(-) create mode 100644 .changeset/validate-contract-shape.md diff --git a/.changeset/validate-contract-shape.md b/.changeset/validate-contract-shape.md new file mode 100644 index 0000000..ab53025 --- /dev/null +++ b/.changeset/validate-contract-shape.md @@ -0,0 +1,4 @@ +--- +"github-agent-workflow": patch +--- +`npm run validate` now checks four rules it let through: the contract is the first section of every skill, reference skills included; a reference skill may not keep a `## Never`, `## Result` or other folded section; every skill that runs has a `## Process` section; and every pipeline skill lists `runner` and `unattended`. The protocol's **Contracts** section now defines `types` and states these rules. A heading inside a code fence no longer counts as a section. diff --git a/scripts/contracts.mjs b/scripts/contracts.mjs index a42847a..ef2aa54 100644 --- a/scripts/contracts.mjs +++ b/scripts/contracts.mjs @@ -80,6 +80,9 @@ const REQUIRED = { const ALLOWED = new Set([...COMMON, "target", "types", "dispatches", "used_by"]); // Prose sections that the contract replaces. A skill that still has one has two sources of truth. +// Inputs every pipeline skill (item, group or sweep) lists. Defaults and meanings are in the protocol. +export const COMMON_INPUTS = ["runner", "unattended"]; + export const FOLDED_SECTIONS = ["Eligibility", "Context read", "Outputs", "Result", "Never", "Success criteria"]; const PROTOCOL = "agent-workflow-protocol"; @@ -128,7 +131,14 @@ export function purity(c) { /** Headings of the sections in a skill body. */ export function headings(body) { - return [...body.matchAll(/^## (.+?)\s*$/gm)].map((m) => m[1].replace(/[`*]/g, "").trim()); + let fenced = false; + const found = []; + for (const line of body.split("\n")) { + if (/^\s*```/.test(line)) { fenced = !fenced; continue; } + const m = !fenced && line.match(/^## (.+?)\s*$/); + if (m) found.push(m[1].replace(/[`*]/g, "").trim()); + } + return found; } /** @@ -158,6 +168,13 @@ export function checkContracts(skills, byName, external = new Set(), { pending = for (const k of Object.keys(c)) if (!ALLOWED.has(k)) err(`unknown key \`${k}\``); if (!KINDS.includes(c.kind)) { err(`kind must be one of ${KINDS.join(", ")}`); continue; } + // The shape every skill shares, reference skills included: the contract comes first, and the prose + // keeps no second copy of what the contract says. + const heads = headings(s.body); + if (heads[0] !== "Contract") err(`\`## Contract\` must be the first section, but \`## ${heads[0]}\` comes first`); + for (const h of heads) { + if (FOLDED_SECTIONS.includes(h)) err(`has a \`## ${h}\` section. That belongs in the contract now`); + } if (c.kind === "reference") { // Reference material is never run: no effects, no returns, no scenarios. Nothing else is allowed, so // a skill cannot claim the exemption and also carry half a contract. @@ -178,6 +195,14 @@ export function checkContracts(skills, byName, external = new Set(), { pending = if (extra.length) err(`\`params\` has ${extra.map((k) => `\`${k}\``).join(", ")}, which the Inputs table does not list`); } + // **Process** says how: every skill that runs names its how-section that + if (!heads.includes("Process")) err("has no `## Process` section. The how-section of every skill that runs is named Process"); + + // every pipeline skill lists the common inputs (see **Common inputs** in the protocol) + if (PIPELINE_KINDS.includes(c.kind) && Array.isArray(c.params)) { + for (const k of COMMON_INPUTS) if (!c.params.includes(k)) err(`a pipeline skill lists \`${k}\` in \`params\` and in its Inputs table, and this one does not`); + } + // target if (c.target !== undefined) { const t = c.target; @@ -230,11 +255,6 @@ export function checkContracts(skills, byName, external = new Set(), { pending = } } - // the prose must not keep a second copy of what the contract says - for (const h of headings(s.body)) { - if (FOLDED_SECTIONS.includes(h)) err(`has a \`## ${h}\` section. That belongs in the contract now`); - } - // A utility skill returns a document, so `returns` describes its shape: sections, or the verdicts it may give. if (c.kind === "utility" && isStrings(c.returns)) { for (const r of c.returns) if (/^#N\b/.test(r)) err(`result \`${r}\` is an item result. A utility skill returns a document, so \`returns\` describes its sections or verdicts`); diff --git a/scripts/validate-skills.mjs b/scripts/validate-skills.mjs index 3e19618..ab20be2 100755 --- a/scripts/validate-skills.mjs +++ b/scripts/validate-skills.mjs @@ -18,7 +18,10 @@ // - every promoted skill has a valid `## Contract` (scripts/contracts.mjs), except those in // CONTRACT_PENDING: its params are its Inputs table, its results are a closed set, its effects are // listed by scope, and the prose no longer carries Eligibility, Outputs, Result or Never sections of -// its own. Reference skills say `kind: reference` and are exempt from the rest. A skill that another +// its own. Reference skills say `kind: reference` and are exempt from the rest, except that they too +// open with `## Contract` and keep no folded section. The contract is the first section of every +// skill, every skill that runs has a `## Process` section, and every pipeline skill lists `runner` and +// `unattended`. A skill that another // skill calls must have one, and a utility skill's `used_by` must list its callers // - each skill's purity class (pure, claimed, writes elsewhere) is printed // Package-wide: diff --git a/skills/meta/agent-workflow-protocol/SKILL.md b/skills/meta/agent-workflow-protocol/SKILL.md index 9a4c942..d76cad2 100644 --- a/skills/meta/agent-workflow-protocol/SKILL.md +++ b/skills/meta/agent-workflow-protocol/SKILL.md @@ -17,11 +17,12 @@ kind: reference ## Contracts -Every skill opens with a `## Contract`: one YAML block that says what the skill is as a function. The utility skills that have none yet are listed in `CONTRACT_PENDING` in the validator, and that list only shrinks. **Process** says *how*. Where the two disagree, the contract wins, and `npm run validate` checks the parts it can. A skill is evaluated against its contract: give it a state of the world and inputs, then check what it returned and what it wrote. +Every skill opens with a `## Contract`: one YAML block that says what the skill is as a function. The utility skills that have none yet are listed in `CONTRACT_PENDING` in the validator, and that list only shrinks. **Process** says *how*. Where the two disagree, the contract wins, and `npm run validate` checks the parts it can. Every skill, reference skills included, opens with its `## Contract` and keeps none of the prose sections the contract replaced. Every skill that runs has a `## Process` section, and every pipeline skill (`item`, `group` or `sweep`) lists `runner` and `unattended`. A skill is evaluated against its contract: give it a state of the world and inputs, then check what it returned and what it wrote. -- `kind`: `item` (works on one issue or PR, or one batch of findings), `group` (selects work and dispatches an item skill) `sweep` (repairs state across many items without claiming them: only `reconcile-github-issues`), `utility` (a building block that other skills call through the Skill tool and that returns a document: a report, an outcome comment, a round of questions) or `reference` (material that is read and never run: this skill, `pipeline-guide`, `runner-prompts`) +- `kind`: `item` (works on one issue or PR, or one batch of findings), `group` (selects work and dispatches an item skill), `sweep` (repairs state across many items without claiming them: only `reconcile-github-issues`), `utility` (a building block that other skills call through the Skill tool and that returns a document: a report, an outcome comment, a round of questions) or `reference` (material that is read and never run: this skill, `pipeline-guide`, `runner-prompts`) - `step`: Exploring, Triaging, Implementing, Shepherding or Reviewing. Required for `item`, `group` and `sweep`. A `utility` skill may leave it out, because it belongs to no step - `target`: what the skill is given: `{param: issue|pr}` for an item skill, or `{from: free-text, type: …}` +- `types`: named shapes the contract refers to: a mapping from a type name to its fields, each field described in one line. `target.type` may name one, as in `candidate[]`. Only `file-github-issues` uses it today - `params`: every input the skill accepts, exactly the rows of its `## Inputs` table. Defaults and meanings stay in that table - `dispatches`: group skills: the item skill each selected item goes to - `used_by`: utility skills: the skills that call it, which must each list it in `calls`. A skill that has `calls: [x]` must be in `x`'s `used_by`. A user-invoked utility skill is called by no one, so its `used_by` is `[]` diff --git a/tests/contracts.test.mjs b/tests/contracts.test.mjs index cac03ea..b0e9a46 100644 --- a/tests/contracts.test.mjs +++ b/tests/contracts.test.mjs @@ -5,7 +5,7 @@ import { checkContracts, extractContract, resultHead, headings, purity, entryPar const ITEM = `kind: item step: Triaging target: { param: issue, type: issue } -params: [issue, runner] +params: [issue, runner, unattended] requires: ["The issue is open."] returns: ["#N done", "#N escalated: ", "#N skipped: ", "#N failed: "] ensures: ["The lease is released."] @@ -20,7 +20,7 @@ terminates: "One pass."`; const GROUP = `kind: group step: Triaging -params: [runner] +params: [runner, unattended] dispatches: do-it requires: ["None."] returns: ["nothing eligible"] @@ -34,10 +34,11 @@ calls: [] idempotent: "Yes." terminates: "One pass."`; -const INPUTS = "## Inputs\n\n| Input | Default | What |\n|---|---|---|\n| `issue` | *(required)* | x |\n| `runner` | `unknown` | y |\n"; -const skill = (name, yaml, rest = "", inputKeys = ["issue", "runner"]) => ({ +const INPUTS = "## Inputs\n\n| Input | Default | What |\n|---|---|---|\n| `issue` | *(required)* | x |\n| `runner` | `unknown` | y |\n| `unattended` | `true` | z |\n"; +const PROCESS = "## Process\n\n1. Do it.\n"; +const skill = (name, yaml, rest = "", inputKeys = ["issue", "runner", "unattended"]) => ({ name, rel: `skills/pipeline/${name}`, bucket: "pipeline", userInvoked: false, inputKeys, - body: `# ${name}\n\n## Contract\n\n\`\`\`yaml\n${yaml}\n\`\`\`\n\n${INPUTS}\n${rest}`, + body: `# ${name}\n\n## Contract\n\n\`\`\`yaml\n${yaml}\n\`\`\`\n\n${INPUTS}\n${PROCESS}\n${rest}`, }); const run = (skills, extra = [], pending = new Set()) => { const all = [...skills, ...extra]; @@ -50,15 +51,15 @@ test("a well-formed item contract has no problems", () => { }); test("a group dispatching an item of the same step is fine", () => { - assert.deepEqual(run([skill("do-them", GROUP, "", ["runner"]), item()]), []); + assert.deepEqual(run([skill("do-them", GROUP, "", ["runner", "unattended"]), item()]), []); }); test("a dispatched item that writes the run log must accept write_log", () => { const logging = ITEM.replace('claimed: ["A label."]', 'claimed: ["A label."]\n shared: ["One run-log entry."]'); - const errs = run([skill("do-them", GROUP, "", ["runner"]), item(logging)]); + const errs = run([skill("do-them", GROUP, "", ["runner", "unattended"]), item(logging)]); assert.ok(errs.some((e) => /no `write_log` param/.test(e))); - const ok = logging.replace("[issue, runner]", "[issue, runner, write_log]"); - assert.deepEqual(run([skill("do-them", GROUP, "", ["runner"]), item(ok, "", ["issue", "runner", "write_log"])]), []); + const ok = logging.replace("[issue, runner, unattended]", "[issue, runner, unattended, write_log]"); + assert.deepEqual(run([skill("do-them", GROUP, "", ["runner", "unattended"]), item(ok, "", ["issue", "runner", "unattended", "write_log"])]), []); }); test("a precondition may not call a helper that writes", () => { @@ -82,7 +83,7 @@ test("invalid YAML is reported, not thrown", () => { }); test("params must equal the Inputs table", () => { - const errs = run([item(ITEM.replace("[issue, runner]", "[issue, bogus]"))]); + const errs = run([item(ITEM.replace("[issue, runner, unattended]", "[issue, bogus, unattended]"))]); assert.ok(errs.some((e) => /missing `runner`/.test(e))); assert.ok(errs.some((e) => /has `bogus`/.test(e))); }); @@ -121,17 +122,17 @@ test("a prose section the contract replaces is reported", () => { test("a group skill cannot claim, and an item skill with a target must", () => { const claiming = GROUP.replace(" shared:", " claimed: [\"x\"]\n shared:"); - assert.ok(run([skill("do-them", claiming, "", ["runner"]), item()]).some((e) => /only an item skill claims/.test(e))); + assert.ok(run([skill("do-them", claiming, "", ["runner", "unattended"]), item()]).some((e) => /only an item skill claims/.test(e))); const noClaim = ITEM.replace(/ claimed: \["A label."\]/, " shared: [\"A log.\"]"); assert.ok(run([item(noClaim)]).some((e) => /needs `effects.writes.claimed`/.test(e))); }); test("a sweep skill needs no target or dispatch, but cannot claim", () => { const SWEEP = GROUP.replace("kind: group", "kind: sweep").replace("dispatches: do-it\n", ""); - assert.deepEqual(run([skill("tidy", SWEEP, "", ["runner"])]), []); - assert.ok(run([skill("tidy", SWEEP + "\ndispatches: do-it", "", ["runner"])]).some((e) => /only a group skill dispatches/.test(e))); + assert.deepEqual(run([skill("tidy", SWEEP, "", ["runner", "unattended"])]), []); + assert.ok(run([skill("tidy", SWEEP + "\ndispatches: do-it", "", ["runner", "unattended"])]).some((e) => /only a group skill dispatches/.test(e))); const claiming = SWEEP.replace(" shared:", " claimed: [\"x\"]\n shared:"); - assert.ok(run([skill("tidy", claiming, "", ["runner"])]).some((e) => /only an item skill claims/.test(e))); + assert.ok(run([skill("tidy", claiming, "", ["runner", "unattended"])]).some((e) => /only an item skill claims/.test(e))); }); test("an unknown write scope is reported", () => { @@ -149,8 +150,8 @@ test("calls: an unknown or user-invoked skill is reported, and a skill the text test("a group must dispatch an item skill of its own step", () => { const other = skill("do-it", ITEM.replace("step: Triaging", "step: Reviewing")); - assert.ok(run([skill("do-them", GROUP, "", ["runner"]), other]).some((e) => /belongs to Reviewing/.test(e))); - assert.ok(run([skill("do-them", GROUP.replace("do-it", "nope"), "", ["runner"])]).some((e) => /not a skill in this package/.test(e))); + assert.ok(run([skill("do-them", GROUP, "", ["runner", "unattended"]), other]).some((e) => /belongs to Reviewing/.test(e))); + assert.ok(run([skill("do-them", GROUP.replace("do-it", "nope"), "", ["runner", "unattended"])]).some((e) => /not a skill in this package/.test(e))); }); test("resultHead and headings", () => { @@ -246,8 +247,44 @@ test("a reference skill needs no params, results or scenarios, whatever its Inpu assert.deepEqual(run([skill("the-guide", REFERENCE, "## Process\n\nRead it.\n", ["issue", "runner"])]), []); }); -test("a reference skill may carry the prose sections a contract replaces", () => { - assert.deepEqual(run([skill("the-guide", REFERENCE, "## Result\n\ntext\n", [])]), []); +test("a reference skill may not carry the prose sections a contract replaces either", () => { + const errs = run([skill("the-guide", REFERENCE, "## Never\n\ntext\n", [])]); + assert.ok(errs.some((e) => /has a `## Never` section/.test(e))); +}); + +test("a heading inside a code fence is not a section", () => { + assert.deepEqual(headings("## A\n```\n## Never\n```\n## B\n"), ["A", "B"]); + assert.deepEqual(run([skill("the-guide", REFERENCE, "```md\n## Never\n```\n", [])]), []); +}); + +test("the contract is the first section, in every kind of skill", () => { + const late = (name, yaml, kind) => ({ + name, rel: `skills/pipeline/${name}`, bucket: "pipeline", userInvoked: false, inputKeys: kind === "reference" ? [] : ["issue", "runner", "unattended"], + body: `# ${name}\n\n## Overview\n\nText.\n\n## Contract\n\n\`\`\`yaml\n${yaml}\n\`\`\`\n\n${INPUTS}\n${PROCESS}`, + }); + assert.ok(run([late("do-it", ITEM, "item")]).some((e) => /must be the first section, but `## Overview` comes first/.test(e))); + assert.ok(run([late("the-guide", REFERENCE, "reference")]).some((e) => /must be the first section/.test(e))); +}); + +test("a skill that runs has a `## Process` section, named exactly that", () => { + const renamed = (s) => ({ ...s, body: s.body.replace("## Process", "## Workflow") }); + const errs = run([renamed(item())]); + assert.ok(errs.some((e) => /has no `## Process` section/.test(e))); + assert.deepEqual(run([skill("the-guide", REFERENCE, "", [])]), [], "a reference skill is never run, so it has no Process"); +}); + +test("every pipeline skill lists runner and unattended, and a utility skill need not", () => { + const without = (k) => ITEM.replace(`params: [issue, runner, unattended]`, `params: [${["issue", "runner", "unattended"].filter((x) => x !== k).join(", ")}]`); + for (const k of ["runner", "unattended"]) { + const keys = ["issue", "runner", "unattended"].filter((x) => x !== k); + const errs = run([item(without(k), "", keys)]); + assert.ok(errs.some((e) => e.includes("lists `" + k + "`")), `missing ${k} is reported`); + } + const groupWithout = GROUP.replace("params: [runner, unattended]", "params: [runner]"); + assert.ok(run([skill("do-them", groupWithout, "", ["runner"]), item()]).some((e) => /lists `unattended`/.test(e))); + // a utility skill is not a pipeline skill: it takes neither input and is not reported + assert.deepEqual(run([item(CALLER), util()]), []); + assert.ok(!run([item(CALLER), util()]).some((e) => /lists `(runner|unattended)`/.test(e))); }); test("a skill without a contract is reported, unless it is pending", () => {