Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions .changeset/validate-contract-shape.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
---
"github-agent-workflow": patch
---
`npm run validate` now checks four rules it let through: the contract is the first section of every skill, reference skills included; a reference skill may not keep a `## Never`, `## Result` or other folded section; every skill that runs has a `## Process` section; and every pipeline skill lists `runner` and `unattended`. The protocol's **Contracts** section now defines `types` and states these rules. A heading inside a code fence no longer counts as a section.
32 changes: 26 additions & 6 deletions scripts/contracts.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,9 @@ const REQUIRED = {
const ALLOWED = new Set([...COMMON, "target", "types", "dispatches", "used_by"]);

// Prose sections that the contract replaces. A skill that still has one has two sources of truth.
// Inputs every pipeline skill (item, group or sweep) lists. Defaults and meanings are in the protocol.
export const COMMON_INPUTS = ["runner", "unattended"];

export const FOLDED_SECTIONS = ["Eligibility", "Context read", "Outputs", "Result", "Never", "Success criteria"];

const PROTOCOL = "agent-workflow-protocol";
Expand Down Expand Up @@ -128,7 +131,14 @@ export function purity(c) {

/** Headings of the sections in a skill body. */
export function headings(body) {
return [...body.matchAll(/^## (.+?)\s*$/gm)].map((m) => m[1].replace(/[`*]/g, "").trim());
let fenced = false;
const found = [];
for (const line of body.split("\n")) {
if (/^\s*```/.test(line)) { fenced = !fenced; continue; }
const m = !fenced && line.match(/^## (.+?)\s*$/);
if (m) found.push(m[1].replace(/[`*]/g, "").trim());
}
return found;
}

/**
Expand Down Expand Up @@ -158,6 +168,13 @@ export function checkContracts(skills, byName, external = new Set(), { pending =

for (const k of Object.keys(c)) if (!ALLOWED.has(k)) err(`unknown key \`${k}\``);
if (!KINDS.includes(c.kind)) { err(`kind must be one of ${KINDS.join(", ")}`); continue; }
// The shape every skill shares, reference skills included: the contract comes first, and the prose
// keeps no second copy of what the contract says.
const heads = headings(s.body);
if (heads[0] !== "Contract") err(`\`## Contract\` must be the first section, but \`## ${heads[0]}\` comes first`);
for (const h of heads) {
if (FOLDED_SECTIONS.includes(h)) err(`has a \`## ${h}\` section. That belongs in the contract now`);
}
if (c.kind === "reference") {
// Reference material is never run: no effects, no returns, no scenarios. Nothing else is allowed, so
// a skill cannot claim the exemption and also carry half a contract.
Expand All @@ -178,6 +195,14 @@ export function checkContracts(skills, byName, external = new Set(), { pending =
if (extra.length) err(`\`params\` has ${extra.map((k) => `\`${k}\``).join(", ")}, which the Inputs table does not list`);
}

// **Process** says how: every skill that runs names its how-section that
if (!heads.includes("Process")) err("has no `## Process` section. The how-section of every skill that runs is named Process");

// every pipeline skill lists the common inputs (see **Common inputs** in the protocol)
if (PIPELINE_KINDS.includes(c.kind) && Array.isArray(c.params)) {
for (const k of COMMON_INPUTS) if (!c.params.includes(k)) err(`a pipeline skill lists \`${k}\` in \`params\` and in its Inputs table, and this one does not`);
}

// target
if (c.target !== undefined) {
const t = c.target;
Expand Down Expand Up @@ -230,11 +255,6 @@ export function checkContracts(skills, byName, external = new Set(), { pending =
}
}

// the prose must not keep a second copy of what the contract says
for (const h of headings(s.body)) {
if (FOLDED_SECTIONS.includes(h)) err(`has a \`## ${h}\` section. That belongs in the contract now`);
}

// A utility skill returns a document, so `returns` describes its shape: sections, or the verdicts it may give.
if (c.kind === "utility" && isStrings(c.returns)) {
for (const r of c.returns) if (/^#N\b/.test(r)) err(`result \`${r}\` is an item result. A utility skill returns a document, so \`returns\` describes its sections or verdicts`);
Expand Down
5 changes: 4 additions & 1 deletion scripts/validate-skills.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,10 @@
// - every promoted skill has a valid `## Contract` (scripts/contracts.mjs), except those in
// CONTRACT_PENDING: its params are its Inputs table, its results are a closed set, its effects are
// listed by scope, and the prose no longer carries Eligibility, Outputs, Result or Never sections of
// its own. Reference skills say `kind: reference` and are exempt from the rest. A skill that another
// its own. Reference skills say `kind: reference` and are exempt from the rest, except that they too
// open with `## Contract` and keep no folded section. The contract is the first section of every
// skill, every skill that runs has a `## Process` section, and every pipeline skill lists `runner` and
// `unattended`. A skill that another
// skill calls must have one, and a utility skill's `used_by` must list its callers
// - each skill's purity class (pure, claimed, writes elsewhere) is printed
// Package-wide:
Expand Down
5 changes: 3 additions & 2 deletions skills/meta/agent-workflow-protocol/SKILL.md
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,12 @@ kind: reference

## Contracts

Every skill opens with a `## Contract`: one YAML block that says what the skill is as a function. The utility skills that have none yet are listed in `CONTRACT_PENDING` in the validator, and that list only shrinks. **Process** says *how*. Where the two disagree, the contract wins, and `npm run validate` checks the parts it can. A skill is evaluated against its contract: give it a state of the world and inputs, then check what it returned and what it wrote.
Every skill opens with a `## Contract`: one YAML block that says what the skill is as a function. The utility skills that have none yet are listed in `CONTRACT_PENDING` in the validator, and that list only shrinks. **Process** says *how*. Where the two disagree, the contract wins, and `npm run validate` checks the parts it can. Every skill, reference skills included, opens with its `## Contract` and keeps none of the prose sections the contract replaced. Every skill that runs has a `## Process` section, and every pipeline skill (`item`, `group` or `sweep`) lists `runner` and `unattended`. A skill is evaluated against its contract: give it a state of the world and inputs, then check what it returned and what it wrote.

- `kind`: `item` (works on one issue or PR, or one batch of findings), `group` (selects work and dispatches an item skill) `sweep` (repairs state across many items without claiming them: only `reconcile-github-issues`), `utility` (a building block that other skills call through the Skill tool and that returns a document: a report, an outcome comment, a round of questions) or `reference` (material that is read and never run: this skill, `pipeline-guide`, `runner-prompts`)
- `kind`: `item` (works on one issue or PR, or one batch of findings), `group` (selects work and dispatches an item skill), `sweep` (repairs state across many items without claiming them: only `reconcile-github-issues`), `utility` (a building block that other skills call through the Skill tool and that returns a document: a report, an outcome comment, a round of questions) or `reference` (material that is read and never run: this skill, `pipeline-guide`, `runner-prompts`)
- `step`: Exploring, Triaging, Implementing, Shepherding or Reviewing. Required for `item`, `group` and `sweep`. A `utility` skill may leave it out, because it belongs to no step
- `target`: what the skill is given: `{param: issue|pr}` for an item skill, or `{from: free-text, type: …}`
- `types`: named shapes the contract refers to: a mapping from a type name to its fields, each field described in one line. `target.type` may name one, as in `candidate[]`. Only `file-github-issues` uses it today
- `params`: every input the skill accepts, exactly the rows of its `## Inputs` table. Defaults and meanings stay in that table
- `dispatches`: group skills: the item skill each selected item goes to
- `used_by`: utility skills: the skills that call it, which must each list it in `calls`. A skill that has `calls: [x]` must be in `x`'s `used_by`. A user-invoked utility skill is called by no one, so its `used_by` is `[]`
Expand Down
73 changes: 55 additions & 18 deletions tests/contracts.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import { checkContracts, extractContract, resultHead, headings, purity, entryPar
const ITEM = `kind: item
step: Triaging
target: { param: issue, type: issue }
params: [issue, runner]
params: [issue, runner, unattended]
requires: ["The issue is open."]
returns: ["#N done", "#N escalated: <reason>", "#N skipped: <reason>", "#N failed: <reason>"]
ensures: ["The lease is released."]
Expand All @@ -20,7 +20,7 @@ terminates: "One pass."`;

const GROUP = `kind: group
step: Triaging
params: [runner]
params: [runner, unattended]
dispatches: do-it
requires: ["None."]
returns: ["nothing eligible"]
Expand All @@ -34,10 +34,11 @@ calls: []
idempotent: "Yes."
terminates: "One pass."`;

const INPUTS = "## Inputs\n\n| Input | Default | What |\n|---|---|---|\n| `issue` | *(required)* | x |\n| `runner` | `unknown` | y |\n";
const skill = (name, yaml, rest = "", inputKeys = ["issue", "runner"]) => ({
const INPUTS = "## Inputs\n\n| Input | Default | What |\n|---|---|---|\n| `issue` | *(required)* | x |\n| `runner` | `unknown` | y |\n| `unattended` | `true` | z |\n";
const PROCESS = "## Process\n\n1. Do it.\n";
const skill = (name, yaml, rest = "", inputKeys = ["issue", "runner", "unattended"]) => ({
name, rel: `skills/pipeline/${name}`, bucket: "pipeline", userInvoked: false, inputKeys,
body: `# ${name}\n\n## Contract\n\n\`\`\`yaml\n${yaml}\n\`\`\`\n\n${INPUTS}\n${rest}`,
body: `# ${name}\n\n## Contract\n\n\`\`\`yaml\n${yaml}\n\`\`\`\n\n${INPUTS}\n${PROCESS}\n${rest}`,
});
const run = (skills, extra = [], pending = new Set()) => {
const all = [...skills, ...extra];
Expand All @@ -50,15 +51,15 @@ test("a well-formed item contract has no problems", () => {
});

test("a group dispatching an item of the same step is fine", () => {
assert.deepEqual(run([skill("do-them", GROUP, "", ["runner"]), item()]), []);
assert.deepEqual(run([skill("do-them", GROUP, "", ["runner", "unattended"]), item()]), []);
});

test("a dispatched item that writes the run log must accept write_log", () => {
const logging = ITEM.replace('claimed: ["A label."]', 'claimed: ["A label."]\n shared: ["One run-log entry."]');
const errs = run([skill("do-them", GROUP, "", ["runner"]), item(logging)]);
const errs = run([skill("do-them", GROUP, "", ["runner", "unattended"]), item(logging)]);
assert.ok(errs.some((e) => /no `write_log` param/.test(e)));
const ok = logging.replace("[issue, runner]", "[issue, runner, write_log]");
assert.deepEqual(run([skill("do-them", GROUP, "", ["runner"]), item(ok, "", ["issue", "runner", "write_log"])]), []);
const ok = logging.replace("[issue, runner, unattended]", "[issue, runner, unattended, write_log]");
assert.deepEqual(run([skill("do-them", GROUP, "", ["runner", "unattended"]), item(ok, "", ["issue", "runner", "unattended", "write_log"])]), []);
});

test("a precondition may not call a helper that writes", () => {
Expand All @@ -82,7 +83,7 @@ test("invalid YAML is reported, not thrown", () => {
});

test("params must equal the Inputs table", () => {
const errs = run([item(ITEM.replace("[issue, runner]", "[issue, bogus]"))]);
const errs = run([item(ITEM.replace("[issue, runner, unattended]", "[issue, bogus, unattended]"))]);
assert.ok(errs.some((e) => /missing `runner`/.test(e)));
assert.ok(errs.some((e) => /has `bogus`/.test(e)));
});
Expand Down Expand Up @@ -121,17 +122,17 @@ test("a prose section the contract replaces is reported", () => {

test("a group skill cannot claim, and an item skill with a target must", () => {
const claiming = GROUP.replace(" shared:", " claimed: [\"x\"]\n shared:");
assert.ok(run([skill("do-them", claiming, "", ["runner"]), item()]).some((e) => /only an item skill claims/.test(e)));
assert.ok(run([skill("do-them", claiming, "", ["runner", "unattended"]), item()]).some((e) => /only an item skill claims/.test(e)));
const noClaim = ITEM.replace(/ claimed: \["A label."\]/, " shared: [\"A log.\"]");
assert.ok(run([item(noClaim)]).some((e) => /needs `effects.writes.claimed`/.test(e)));
});

test("a sweep skill needs no target or dispatch, but cannot claim", () => {
const SWEEP = GROUP.replace("kind: group", "kind: sweep").replace("dispatches: do-it\n", "");
assert.deepEqual(run([skill("tidy", SWEEP, "", ["runner"])]), []);
assert.ok(run([skill("tidy", SWEEP + "\ndispatches: do-it", "", ["runner"])]).some((e) => /only a group skill dispatches/.test(e)));
assert.deepEqual(run([skill("tidy", SWEEP, "", ["runner", "unattended"])]), []);
assert.ok(run([skill("tidy", SWEEP + "\ndispatches: do-it", "", ["runner", "unattended"])]).some((e) => /only a group skill dispatches/.test(e)));
const claiming = SWEEP.replace(" shared:", " claimed: [\"x\"]\n shared:");
assert.ok(run([skill("tidy", claiming, "", ["runner"])]).some((e) => /only an item skill claims/.test(e)));
assert.ok(run([skill("tidy", claiming, "", ["runner", "unattended"])]).some((e) => /only an item skill claims/.test(e)));
});

test("an unknown write scope is reported", () => {
Expand All @@ -149,8 +150,8 @@ test("calls: an unknown or user-invoked skill is reported, and a skill the text

test("a group must dispatch an item skill of its own step", () => {
const other = skill("do-it", ITEM.replace("step: Triaging", "step: Reviewing"));
assert.ok(run([skill("do-them", GROUP, "", ["runner"]), other]).some((e) => /belongs to Reviewing/.test(e)));
assert.ok(run([skill("do-them", GROUP.replace("do-it", "nope"), "", ["runner"])]).some((e) => /not a skill in this package/.test(e)));
assert.ok(run([skill("do-them", GROUP, "", ["runner", "unattended"]), other]).some((e) => /belongs to Reviewing/.test(e)));
assert.ok(run([skill("do-them", GROUP.replace("do-it", "nope"), "", ["runner", "unattended"])]).some((e) => /not a skill in this package/.test(e)));
});

test("resultHead and headings", () => {
Expand Down Expand Up @@ -246,8 +247,44 @@ test("a reference skill needs no params, results or scenarios, whatever its Inpu
assert.deepEqual(run([skill("the-guide", REFERENCE, "## Process\n\nRead it.\n", ["issue", "runner"])]), []);
});

test("a reference skill may carry the prose sections a contract replaces", () => {
assert.deepEqual(run([skill("the-guide", REFERENCE, "## Result\n\ntext\n", [])]), []);
test("a reference skill may not carry the prose sections a contract replaces either", () => {
const errs = run([skill("the-guide", REFERENCE, "## Never\n\ntext\n", [])]);
assert.ok(errs.some((e) => /has a `## Never` section/.test(e)));
});

test("a heading inside a code fence is not a section", () => {
assert.deepEqual(headings("## A\n```\n## Never\n```\n## B\n"), ["A", "B"]);
assert.deepEqual(run([skill("the-guide", REFERENCE, "```md\n## Never\n```\n", [])]), []);
});

test("the contract is the first section, in every kind of skill", () => {
const late = (name, yaml, kind) => ({
name, rel: `skills/pipeline/${name}`, bucket: "pipeline", userInvoked: false, inputKeys: kind === "reference" ? [] : ["issue", "runner", "unattended"],
body: `# ${name}\n\n## Overview\n\nText.\n\n## Contract\n\n\`\`\`yaml\n${yaml}\n\`\`\`\n\n${INPUTS}\n${PROCESS}`,
});
assert.ok(run([late("do-it", ITEM, "item")]).some((e) => /must be the first section, but `## Overview` comes first/.test(e)));
assert.ok(run([late("the-guide", REFERENCE, "reference")]).some((e) => /must be the first section/.test(e)));
});

test("a skill that runs has a `## Process` section, named exactly that", () => {
const renamed = (s) => ({ ...s, body: s.body.replace("## Process", "## Workflow") });
const errs = run([renamed(item())]);
assert.ok(errs.some((e) => /has no `## Process` section/.test(e)));
assert.deepEqual(run([skill("the-guide", REFERENCE, "", [])]), [], "a reference skill is never run, so it has no Process");
});

test("every pipeline skill lists runner and unattended, and a utility skill need not", () => {
const without = (k) => ITEM.replace(`params: [issue, runner, unattended]`, `params: [${["issue", "runner", "unattended"].filter((x) => x !== k).join(", ")}]`);
for (const k of ["runner", "unattended"]) {
const keys = ["issue", "runner", "unattended"].filter((x) => x !== k);
const errs = run([item(without(k), "", keys)]);
assert.ok(errs.some((e) => e.includes("lists `" + k + "`")), `missing ${k} is reported`);
}
const groupWithout = GROUP.replace("params: [runner, unattended]", "params: [runner]");
assert.ok(run([skill("do-them", groupWithout, "", ["runner"]), item()]).some((e) => /lists `unattended`/.test(e)));
// a utility skill is not a pipeline skill: it takes neither input and is not reported
assert.deepEqual(run([item(CALLER), util()]), []);
assert.ok(!run([item(CALLER), util()]).some((e) => /lists `(runner|unattended)`/.test(e)));
});

test("a skill without a contract is reported, unless it is pending", () => {
Expand Down
Loading