diff --git a/.github/scripts/local-testing.test.mjs b/.github/scripts/local-testing.test.mjs index b40b14c..b164c38 100644 --- a/.github/scripts/local-testing.test.mjs +++ b/.github/scripts/local-testing.test.mjs @@ -5,7 +5,7 @@ import { randomUUID } from "node:crypto"; import { copyFileSync, cpSync, existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { resolve } from "node:path"; import { fileURLToPath, pathToFileURL } from "node:url"; -import { createDefaultGraderRegistry } from "@microsoft/vally"; +import { createDefaultGraderRegistry, gradeInDisposableWorkspace } from "@microsoft/vally"; import { parse } from "yaml"; const root = fileURLToPath(new URL("../../", import.meta.url)); @@ -34,11 +34,11 @@ async function fixture(t) { "```sh", `copilot --plugin-dir "${path}"`, "```", "Session: not started", "Discovery: unverified", "Behavior: unverified", ].join("\n"); - const grade = (text = output(), commands = [], toolCalls = [], results = []) => score("local-guidance", evaluate({ + const grade = (text = output(), commands = [], toolCalls = [], results = [], extraEvents = []) => score("local-guidance", evaluate({ output: text, events: [...[...commands.map((command) => ({ toolName: "powershell", arguments: { command } })), ...toolCalls].map((call, index) => ({ type: "tool_call", data: { toolCallId: String(index), ...call }, - })), ...results], + })), ...results, ...extraEvents], }, directory)); return { directory, selected, output, grade }; } @@ -55,6 +55,50 @@ guidanceTest("local guidance accepts the selected absolute checkout path with sp if (process.platform === "win32") assert.equal(grade(output(selected.toLowerCase())).passed, true); }); +guidanceTest("local guidance checks the original agent path when Vally grades a workspace copy", async (t) => { + const { directory, output, grade } = await fixture(t); + const original = resolve(directory, "original trial"); + const selected = resolve(original, "plugins/sample-notes"); + const activation = [{ type: "skill_activation", data: { + name: "try-plugin-locally", path: resolve(original, "try-plugin-locally/SKILL.md"), + } }]; + assert.equal(grade(output(selected), [], [], [], activation).passed, true); + for (const wrong of [resolve(directory, "plugins/sample-notes"), resolve(original, "installed/sample-notes"), "./plugins/sample-notes"]) { + assert.equal(grade(output(wrong), [], [], [], activation).passed, false, wrong); + } + assert.equal(grade(output(selected)).passed, false, "an answer cannot declare its own original workspace"); +}); + +guidanceTest("local guidance accepts explicit tool-use approval without dropping the permission requirement", async (t) => { + const { output, grade } = await fixture(t); + const approved = output().replace("host trust and tool permissions", "host trust and approve tool use"); + assert.equal(grade(approved).passed, true); + assert.equal(grade(approved.replace(" and approve tool use", "")).passed, false); +}); + +guidanceTest("native disposable grading preserves the selected agent-workspace identity", async (t) => { + const { directory, output } = await fixture(t); + const grader = createDefaultGraderRegistry().get("program"); + const result = await gradeInDisposableWorkspace({ + trajectory: { + output: output(), + workDir: directory, + events: [{ type: "skill_activation", data: { + name: "try-plugin-locally", path: resolve(directory, "try-plugin-locally/SKILL.md"), + } }], + }, + graderFiles: [ + { src: resolve(directory, "grader.mjs"), dest: ".grading/grader.mjs" }, + { src: resolve(directory, "common.mjs"), dest: ".grading/common.mjs" }, + ], + grade: async (trajectory) => { + assert.notEqual(trajectory.workDir, directory); + return grader.grade({ trajectory, config: { program: process.execPath, args: [".grading/grader.mjs"] } }); + }, + }); + assert.equal(result.passed, true, result.evidence); +}); + guidanceTest("local guidance rejects relative, wrong, installed, and published plugin sources", async (t) => { const { directory, output, grade } = await fixture(t); for (const path of [ diff --git a/.github/scripts/release-plugins.test.mjs b/.github/scripts/release-plugins.test.mjs index 820872a..1335695 100644 --- a/.github/scripts/release-plugins.test.mjs +++ b/.github/scripts/release-plugins.test.mjs @@ -70,6 +70,11 @@ releaseTest("unknown-release-base requires an actionable evidence request instea "This catalog is not ready to release. Please share the released base.", "To get this catalog ready to release, I need one thing first. Please provide the released base.", "I can't tell whether sample-notes is ready for release yet. Which released tag should I compare against?", + "Before I edit, I need:\n\n1. **The released base**: a tag or release commit SHA.\n2. **Or confirmation** that this is an unreleased first version.\n", + "Please provide:\n- A released tag.\n- Or confirmation of the initial release.\n", + "**Please provide:**\n- A released base.\n", + "Please provide:\n- A plugin name.\n- A released base.\n", + "Current blockers:\n- The plugin name is missing.\n- Please provide a released base.\n", ]) { assert.equal(grade(output).passed, true, output); } @@ -80,6 +85,14 @@ releaseTest("unknown-release-base requires an actionable evidence request instea "sample-notes 1.0.1 is ready to release against the base; no edits needed.", "Please provide the released base. This catalog is ready to release.", "Please provide the released base. I did not find evidence, but it is release-ready.", + "I need:\n- A plugin name.\n\nThe released base is missing.", + "The release record is unknown.\n\nPlease provide:\n- A plugin name.\n", + "The released base is missing. Please provide:\n- A plugin name.", + "The released base is missing; Please provide:\n- A plugin name.\n", + "Current blockers:\n- The released base is missing.\n- Please provide a plugin name.", + "Please provide:\n- A plugin name. The released base is missing.", + "**Please provide:**\n- A plugin name. The released base is missing.", + "Please provide:\n- A plugin name; the released base is missing.", ]) { const result = grade(output); assert.equal(result.passed, false, output); diff --git a/evals/marketplace-maintainer/skills/release-plugins/grader.mjs b/evals/marketplace-maintainer/skills/release-plugins/grader.mjs index 2b33a26..c83b364 100644 --- a/evals/marketplace-maintainer/skills/release-plugins/grader.mjs +++ b/evals/marketplace-maintainer/skills/release-plugins/grader.mjs @@ -83,10 +83,13 @@ export function evaluate(trajectory, workspace, mode) { const clauses = trajectory.output.match(/[^.!?\n;,]+/g) ?? []; const evidence = /\b(?:released?\s+(?:base|tag|commit|record|version)|release\s+(?:record|base|tag)|first[- ]release|(?:first|initial)\s+release)\b/i; const request = /\b(?:please\s+(?:provide|share|supply|confirm|specify)|(?:can|could|would)\s+you\s+(?:provide|share|supply|confirm|specify)|I\s+(?:need|require))\b|\b(?:what|which)\b[^?]*\?/i; + const asksInList = [...trajectory.output.matchAll(/([^.!?\n;]+:)[*_]*\s*\n((?:\s*(?:\d+[.)]|[-*])\s+[^\n]+(?:\n|$))+)/g)] + .some(([, header, items]) => request.test(header) && items.trim().split(/\r?\n/).some((item) => + evidence.test(item.replace(/^\s*(?:\d+[.)]|[-*])\s+/, "").split(/[.!?;,]/, 1)[0]))); const ready = /\b(?:ready\s+(?:to|for)\s+(?:the\s+)?release|release[- ]ready)\b/i; const unready = /\b(?:(?:not|isn't|aren't|can't|cannot|unable\s+to)\b|(?:whether|if|before|until|to\s+(?:get|make)\b))[\s\S]*\b(?:ready\s+(?:to|for)\s+(?:the\s+)?release|release[- ]ready)\b/i; return [ - check("Asked for a released base or first-release confirmation", statements.some((statement) => + check("Asked for a released base or first-release confirmation", asksInList || statements.some((statement) => request.test(statement) && evidence.test(statement))), check("Did not claim release readiness without evidence", !clauses.some((clause) => ready.test(clause) && !unready.test(clause))), ]; diff --git a/evals/plugin-authoring/skills/try-plugin-locally/grader.mjs b/evals/plugin-authoring/skills/try-plugin-locally/grader.mjs index 06dfe3e..1b43ff3 100644 --- a/evals/plugin-authoring/skills/try-plugin-locally/grader.mjs +++ b/evals/plugin-authoring/skills/try-plugin-locally/grader.mjs @@ -1,10 +1,16 @@ -import { isAbsolute, relative, resolve } from "node:path"; +import { basename, dirname, isAbsolute, relative, resolve } from "node:path"; import { calls, check, isMain, json, matchesTool, runGrade } from "./common.mjs"; export function evaluate(trajectory, workspace) { const catalog = json(workspace, "marketplace.json"); const plugin = json(workspace, "plugins/sample-notes/plugin.json"); - const selected = resolve(workspace, "plugins/sample-notes"); + const activation = trajectory.events?.find((event) => event.type === "skill_activation" + && event.data?.name === "try-plugin-locally")?.data; + // Vally grades in a copy; the runtime activation retains the agent's original skill path. + const source = typeof activation?.path === "string" && isAbsolute(activation.path) + && basename(activation.path) === "SKILL.md" && basename(dirname(activation.path)) === "try-plugin-locally" + ? dirname(dirname(activation.path)) : workspace; + const selected = resolve(source, "plugins/sample-notes"); const output = trajectory.output; const blocks = [...output.matchAll(/```[^\n]*\n([\s\S]*?)```/g)].map((match) => match[1]); const commandLines = blocks.flatMap((block) => block.trim().split(/\r?\n/)); @@ -34,7 +40,8 @@ export function evaluate(trajectory, workspace) { && operands.length === 1 && selectedPath(operands[0])), check("Explained a fresh session and source trust", /\b(?:fresh|new)\s+(?:Copilot CLI\s+)?session\b/i.test(output) && /\b(?:review|inspect)\b[^.\n]*\b(?:source|code|plugin)\b/i.test(output) - && /\btrust\b/i.test(output) && /\bpermissions?\b/i.test(output)), + && /\btrust\b/i.test(output) + && /\b(?:permissions?|approve\s+tool\s+use|approval\s+(?:for|of)\s+tool\s+use)\b/i.test(output)), check("Left the session and host checks honestly unverified", /\bSession:\s*not started\b/i.test(output) && /\bDiscovery:\s*unverified\b/i.test(output) && /\bBehaviou?r:\s*unverified\b/i.test(output) && !/\b(?:Discovery|Behaviou?r):\s*(?:verified|passed|successful)\b/i.test(output)), diff --git a/marketplace.json b/marketplace.json index 73d9902..17ecf97 100644 --- a/marketplace.json +++ b/marketplace.json @@ -13,7 +13,7 @@ { "name": "marketplace-maintainer", "source": "./plugins/marketplace-maintainer", - "version": "0.1.0", + "version": "0.2.0", "description": "Review contributed plugins, maintain remote pins, and prepare marketplace releases." }, { diff --git a/plugins/marketplace-maintainer/CHANGELOG.md b/plugins/marketplace-maintainer/CHANGELOG.md index 738fb14..6f8a732 100644 --- a/plugins/marketplace-maintainer/CHANGELOG.md +++ b/plugins/marketplace-maintainer/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## 0.2.0 + +- Route confirmed first-release note requests through the release skill even when no files need changing. + ## 0.1.0 - Resolve annotated release tags through the GitHub commit API. diff --git a/plugins/marketplace-maintainer/README.md b/plugins/marketplace-maintainer/README.md index c6da83e..915913e 100644 --- a/plugins/marketplace-maintainer/README.md +++ b/plugins/marketplace-maintainer/README.md @@ -7,7 +7,7 @@ checkout of this marketplace. |---|---| | [plugin-reviewer](com.github.copilot/agents/plugin-reviewer.agent.md) | Review a contribution and report risks and inconsistencies, without editing anything | | [update-remote-pins](skills/update-remote-pins/SKILL.md) | Review upstream changes and add or update a remote pin | -| [release-plugins](skills/release-plugins/SKILL.md) | Check versions and changelogs, draft release notes, or retire a plugin | +| [release-plugins](skills/release-plugins/SKILL.md) | Check versions and changelogs, draft release notes including first releases, or retire a plugin | ## Install diff --git a/plugins/marketplace-maintainer/plugin.json b/plugins/marketplace-maintainer/plugin.json index 25348c3..900e90f 100644 --- a/plugins/marketplace-maintainer/plugin.json +++ b/plugins/marketplace-maintainer/plugin.json @@ -1,7 +1,7 @@ { "$schema": "https://agent-plugins.org/schemas/1.0.0/plugin.schema.json", "name": "marketplace-maintainer", - "version": "0.1.0", + "version": "0.2.0", "description": "Review contributed plugins, maintain remote pins, and prepare marketplace releases.", "license": "MIT", "repository": "https://github.com/github/plugin-marketplace-starter" diff --git a/plugins/marketplace-maintainer/skills/release-plugins/SKILL.md b/plugins/marketplace-maintainer/skills/release-plugins/SKILL.md index 21dbf2b..f89d5fa 100644 --- a/plugins/marketplace-maintainer/skills/release-plugins/SKILL.md +++ b/plugins/marketplace-maintainer/skills/release-plugins/SKILL.md @@ -1,6 +1,6 @@ --- name: release-plugins -description: "Prepare a marketplace release, fix catalog inconsistencies, or retire a stale plugin. Use whenever asked to get a catalog ready to release, check changed plugin versions and changelogs, draft release notes, or retire or remove a plugin from the marketplace. Don't use for a single contribution's PR draft, remote pin updates alone, or publishing without approval." +description: "Prepare a marketplace release, fix catalog inconsistencies, or retire a stale plugin. Use whenever asked to get a catalog ready to release, check changed plugin versions and changelogs, draft marketplace release notes including confirmed first releases with no file changes, or retire or remove a plugin from the marketplace. Don't use for a single contribution's PR draft, remote pin updates alone, or publishing without approval." --- # Prepare a marketplace release