Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
42 commits
Select commit Hold shift + click to select a range
9a5cf0c
Replace the sample plugins with two marketplace maintenance plugins
ronniegeraghty Nov 2, 2026
f67f03d
Simplify the starter to a plugin marketplace and its two plugins
ronniegeraghty Nov 2, 2026
d838c30
Maintain the marketplace by hand: remove scripts, tests, and templates
ronniegeraghty Nov 2, 2026
ee7ca30
Remove the repository settings overlay so nothing installs on checkout
ronniegeraghty Nov 3, 2026
5cc87d2
Simplify the validate and evals workflows
ronniegeraghty Nov 3, 2026
827da79
Validate plugin.json files and remote pins in CI
ronniegeraghty Nov 3, 2026
46e8e74
Trim CODEOWNERS to the OSPO template
ronniegeraghty Nov 3, 2026
464aecb
Move shared eval files to evals/_shared
ronniegeraghty Nov 3, 2026
15245f0
Replace the custom eval graders with Vally's built-in graders
ronniegeraghty Nov 3, 2026
4cf610d
Turn on the two maintenance plugins for people working in the repository
ronniegeraghty Nov 3, 2026
deb49e5
Rename shared eval fixtures to evals/_shared/fixtures
ronniegeraghty Nov 3, 2026
f5caa39
Restore the custom eval graders
ronniegeraghty Nov 3, 2026
7f17da6
Split each plugin's eval grader into one grader per skill
ronniegeraghty Nov 3, 2026
9b1ba52
Simplify plugin READMEs, drop hard-coded license, add repository, neu…
ronniegeraghty Nov 3, 2026
64a3f37
Fix eval findings: tighten skill triggers and reviewer output, fix br…
ronniegeraghty Nov 3, 2026
f559bc6
Extract CI validation scripts and address review findings
ronniegeraghty Nov 3, 2026
d244e21
Merge main into maintenance-plugins
ronniegeraghty Nov 3, 2026
fe8dd8f
Guard local plugin testing against checkout instructions
ronniegeraghty Nov 3, 2026
35e5ac1
Protect pin updates and submissions from checkout instructions
ronniegeraghty Nov 3, 2026
b5157e6
Validate local install inputs before fixed CLI commands
ronniegeraghty Nov 3, 2026
127b249
Keep starter documentation README-first
ronniegeraghty Nov 4, 2026
2f1de7e
Add optional public Copilot Evals plugin pin
ronniegeraghty Nov 5, 2026
a22aa9c
Address marketplace adoption and maintenance feedback
ronniegeraghty Nov 9, 2026
d003cdd
Make local checkout testing the default plugin workflow
ronniegeraghty Nov 9, 2026
c8ea66e
Address Copilot review findings
ronniegeraghty Nov 9, 2026
3a6a89d
Correct tool-call and skill-usage grading
ronniegeraghty Nov 9, 2026
64591cf
Validate upstream refs and require native skill activation
ronniegeraghty Nov 10, 2026
f7beb1a
Validate catalog sources and retired-plugin settings
ronniegeraghty Nov 10, 2026
2c710dd
Fix remote-only starter checks and eval grading
ronniegeraghty Nov 10, 2026
bdc8756
Avoid backtracking in smoke-grader Git matching
ronniegeraghty Nov 10, 2026
0f8c197
Ignore non-directory entries when enumerating plugins/
jongio Nov 10, 2026
156416a
feat: add pr-description golden-path plugin, multi-skill eval, and ev…
jongio Nov 10, 2026
0eb1ff0
Address PR #7 security and eval reporting review findings
jongio Nov 10, 2026
e347340
Address remaining Copilot review feedback on pr-description grader an…
jongio Nov 10, 2026
45e6c8b
fix: strip terminal controls in eval-report and handle negated/zero-c…
jongio Nov 10, 2026
c8e5667
fix(pr-description): treat supplied diffs and notes as untrusted data
jongio Nov 10, 2026
f7baf29
Refactor: remove custom eval-report script in favor of core Evals too…
jongio Nov 10, 2026
dafa67c
Keep starter-repo demo fixtures and eval-report CLI for live template…
jongio Nov 10, 2026
90dbb2d
Remove temporary install-test directory
jongio Nov 10, 2026
d1fb340
Merge pull request #7 from github/ignite-skill-eval-golden-paths
jongio Nov 10, 2026
e55dcce
Address Copilot review findings across workflow and eval graders
jongio Nov 10, 2026
6f246d7
Tokenize shell invocations in release-plugins publication check
jongio Nov 10, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 2 additions & 12 deletions .github/CODEOWNERS
Original file line number Diff line number Diff line change
@@ -1,15 +1,5 @@
# Code owners review changes to the paths below. Replace these examples with
# real GitHub users or teams before you ship, then require code owner review in
# a branch protection rule or ruleset. This file alone doesn't enforce review.
# https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners
#
# * @your-org/ai-customizations
# /.github/workflows/ @your-org/ai-customizations-admins
# /.github/copilot/settings.json @your-org/ai-customizations-admins
# /scripts/ @your-org/ai-customizations-admins
# /evals/ @your-org/ai-customizations
# /marketplace.json @your-org/ai-customizations-admins
# /plugins/<plugin-with-hooks>/ @your-org/ai-customizations-admins
# For more information, see [docs](https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners#codeowners-syntax)

# This repository is maintained by:

* @github/code-ai-evals
80 changes: 0 additions & 80 deletions .github/ISSUE_TEMPLATE/new-plugin.yml

This file was deleted.

35 changes: 0 additions & 35 deletions .github/PULL_REQUEST_TEMPLATE.md

This file was deleted.

17 changes: 0 additions & 17 deletions .github/copilot-instructions.md

This file was deleted.

8 changes: 2 additions & 6 deletions .github/copilot/settings.json
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,7 @@
}
},
"enabledPlugins": {
"bug-report@plugin-marketplace-starter": true,
"team-brainstorm@plugin-marketplace-starter": true,
"format-json@plugin-marketplace-starter": true,
"customization-authoring@plugin-marketplace-starter": true,
"code-review@plugin-marketplace-starter": true,
"json-configuration@plugin-marketplace-starter": true
"marketplace-maintainer@plugin-marketplace-starter": true,
"plugin-authoring@plugin-marketplace-starter": true
}
}
33 changes: 33 additions & 0 deletions .github/scripts/check-local-catalog.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
import { existsSync, readFileSync, readdirSync, realpathSync } from "node:fs";
import { resolve } from "node:path";
import { fileURLToPath } from "node:url";

export function checkLocalCatalog(root = process.cwd()) {
const read = (path) => JSON.parse(readFileSync(resolve(root, path), "utf8"));
const folders = existsSync(resolve(root, "plugins"))
? readdirSync(resolve(root, "plugins"), { withFileTypes: true }).filter((entry) => entry.isDirectory()).map((entry) => entry.name)
: [];
for (const folder of folders) {
read(`plugins/${folder}/plugin.json`);
}
for (const plugin of read("marketplace.json").plugins) {
const source = plugin?.source;
const local = typeof source === "string" && source.trim().length > 0;
const remote = source !== null && typeof source === "object" && !Array.isArray(source);
if (!local && !remote) {
throw new Error(`${plugin?.name ?? "(unnamed plugin)"}: source must be a non-empty local folder path or a remote source object.`);
}
if (!local) continue;
const manifest = read(`${plugin.source}/plugin.json`);
for (const field of ["name", "version", "description"]) {
if (manifest[field] !== plugin[field]) {
throw new Error(`${plugin.name}: marketplace.json and ${plugin.source}/plugin.json disagree on ${field}.`);
}
}
}
}

if (process.argv[1] && realpathSync(fileURLToPath(import.meta.url)) === realpathSync(resolve(process.argv[1]))) {
checkLocalCatalog();
console.log("Local catalog entries match their plugin.json files.");
}
109 changes: 109 additions & 0 deletions .github/scripts/check-remote-pins.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
import { spawnSync } from "node:child_process";
import { readFileSync, realpathSync } from "node:fs";
import { resolve } from "node:path";
import { fileURLToPath } from "node:url";

const agentPluginsSchema = /^https:\/\/agent-plugins\.org\/schemas\/\d+\.\d+\.\d+\/plugin\.schema\.json$/;
const supportedSchemas = new Set(["1.0.0", "1.1.0"].map((version) => `https://agent-plugins.org/schemas/${version}/plugin.schema.json`));
// Copilot CLI's plugin reference (github/docs@8e7396e); legacy order also tested in d838c30^.
const legacyManifestPaths = [".plugin/plugin.json", "plugin.json", ".github/plugin/plugin.json", ".claude-plugin/plugin.json"];

export function githubApi(endpoint, run = spawnSync) {
const result = run("gh", ["api", endpoint, "--include", "-H", "Accept: application/vnd.github.raw+json"], {
encoding: "utf8",
stdio: ["ignore", "pipe", "pipe"],
});
if (result.error) throw new Error(`Couldn't run gh api (${result.error.code ?? "process error"}).`);
const response = /^HTTP\/[\d.]+\s+(\d{3})[^\r\n]*\r?\n[\s\S]*?\r?\n\r?\n([\s\S]*)$/.exec(result.stdout ?? "");
if (!response) {
throw new Error(`GitHub API request failed for ${endpoint} (exit ${result.status}; no HTTP response).`);
}
const status = Number(response[1]);
if (result.status !== 0 && status === 200) {
throw new Error(`gh api failed for ${endpoint} (exit ${result.status}) despite HTTP 200.`);
}
return { status, body: response[2] };
}

function parseJson(body, label) {
try {
return JSON.parse(body);
} catch {
throw new Error(`${label} isn't valid JSON.`);
}
}

function requireSuccess(response, label) {
if (response.status !== 200) {
throw new Error(`${label}: GitHub API returned HTTP ${response.status}${response.status === 404 ? " (not found, or your token can't read it)" : ""}.`);
}
}

export function checkRemotePin(entry, api = githubApi) {
const fail = (message) => { throw new Error(`${entry.name}: ${message}`); };
const { source, repo, path, sha, ref } = entry.source ?? {};
if (source !== "github") fail("only github remote sources are supported.");
if (!/^[0-9a-f]{40}$/.test(sha ?? "")) fail("source.sha must be a full 40-character commit.");
if (ref !== undefined && (typeof ref !== "string" || !ref.trim())) {
fail("source.ref must be a non-empty string when supplied.");
}

const commit = api(`repos/${repo}/commits/${sha}`);
requireSuccess(commit, `${entry.name}: commit ${sha} in ${repo}`);
if (parseJson(commit.body, `${entry.name}: commit response`).sha !== sha) {
fail(`GitHub didn't return the pinned commit ${sha}.`);
}
if (ref !== undefined) {
const refCommit = api(`repos/${repo}/commits/${encodeURIComponent(ref)}`);
requireSuccess(refCommit, `${entry.name}: ref ${ref} in ${repo}`);
if (parseJson(refCommit.body, `${entry.name}: ref response`).sha !== sha) {
fail(`source.ref ${ref} doesn't resolve to the pinned commit ${sha}.`);
}
}

const fetched = new Map();
const load = (candidate) => {
if (fetched.has(candidate)) return fetched.get(candidate);
const file = path ? `${path}/${candidate}` : candidate;
const encoded = file.split("/").map(encodeURIComponent).join("/");
const response = api(`repos/${repo}/contents/${encoded}?ref=${sha}`);
if (response.status === 404) {
fetched.set(candidate, undefined);
return undefined;
}
requireSuccess(response, `${entry.name}: ${repo}/${file}`);
const manifest = { file, data: parseJson(response.body, `${entry.name}: ${file} at ${sha}`) };
fetched.set(candidate, manifest);
return manifest;
};

const root = load("plugin.json");
let manifest;
if (typeof root?.data?.$schema === "string" && agentPluginsSchema.test(root.data.$schema)) {
if (!supportedSchemas.has(root.data.$schema)) fail(`root plugin.json declares an unsupported Agent Plugins schema: ${root.data.$schema}.`);
manifest = root;
} else {
for (const candidate of legacyManifestPaths) {
manifest = load(candidate);
if (manifest) break;
}
}
if (!manifest) fail(`no plugin.json under ${path || "the repository root"} at ${sha}.`);
if (manifest.data?.name !== entry.name) {
fail(`upstream ${manifest.file} is named ${manifest.data?.name}, expected ${entry.name}.`);
}
if (entry.version !== undefined && manifest.data.version !== undefined && manifest.data.version !== entry.version) {
fail(`upstream ${manifest.file} has version ${manifest.data.version}, expected ${entry.version}.`);
}
return manifest.file;
}

if (process.argv[1] && realpathSync(fileURLToPath(import.meta.url)) === realpathSync(resolve(process.argv[1]))) {
const entries = JSON.parse(readFileSync("marketplace.json", "utf8")).plugins
.filter((entry) => typeof entry.source === "object");
for (const entry of entries) {
const file = checkRemotePin(entry);
console.log(`${entry.name}: ${entry.source.repo}@${entry.source.sha} (${file}) ok`);
}
if (!entries.length) console.log("No remote pins to check.");
}
Loading
Loading