diff --git a/.github/scripts/eval-report.test.mjs b/.github/scripts/eval-report.test.mjs index 3c7c691..6286a10 100644 --- a/.github/scripts/eval-report.test.mjs +++ b/.github/scripts/eval-report.test.mjs @@ -393,7 +393,9 @@ prDescriptionTest("pr-description eval stages the multi-skill marketplace cohort const evalDir = resolve(root, "evals/pr-description/skills/pr-description"); const spec = parse(readFileSync(resolve(evalDir, "eval.yaml"), "utf8")); assert.equal(spec.scoring.threshold, 1); - const skills = spec.agent_environment.skills; + const project = parse(readFileSync(resolve(root, ".vally.yaml"), "utf8")); + assert.equal(spec.agent_environment, "marketplace"); + const skills = project.environments.marketplace.skills; for (const configuredSkill of skills) { assert.ok(existsSync(resolve(evalDir, configuredSkill)), `configured cohort skill path is missing: ${configuredSkill}`); } diff --git a/.vally.yaml b/.vally.yaml index 051c0bd..02efee6 100644 --- a/.vally.yaml +++ b/.vally.yaml @@ -8,6 +8,23 @@ paths: - eval.yaml - "*.eval.yaml" results: vally-results +environments: + marketplace: + # Vally resolves these paths relative to each eval, not this file. + skills: + - ../../../../plugins/plugin-authoring/skills/add-plugin + - ../../../../plugins/plugin-authoring/skills/try-plugin-locally + - ../../../../plugins/plugin-authoring/skills/submit-plugin + - ../../../../plugins/marketplace-maintainer/skills/update-remote-pins + - ../../../../plugins/marketplace-maintainer/skills/release-plugins + - ../../../../plugins/pr-description/skills/pr-description + # Vally runs setup in each trial workspace; Copilot discovers these skills natively. + commands: + - git clone --filter=blob:none --no-checkout --depth 1 --branch copilot-evals-v0.24.0 --sparse https://github.com/github/evals.git .vally-eval-source + - git -C .vally-eval-source sparse-checkout set packages/plugin/skills + - git -C .vally-eval-source checkout --detach 34ddb5803aeaee8330ab70c19ad1a3f37fb77ded + - >- + node -e "const fs = require('node:fs'); fs.cpSync('.vally-eval-source/packages/plugin/skills', '.github/skills', { recursive: true }); fs.rmSync('.vally-eval-source', { recursive: true });" suites: smoke: description: One representative case for each skill and agent diff --git a/evals/marketplace-maintainer/skills/release-plugins/eval.yaml b/evals/marketplace-maintainer/skills/release-plugins/eval.yaml index 36742fb..442dffd 100644 --- a/evals/marketplace-maintainer/skills/release-plugins/eval.yaml +++ b/evals/marketplace-maintainer/skills/release-plugins/eval.yaml @@ -6,12 +6,7 @@ defaults: model: claude-haiku-5.5 timeout: 5m runs: 1 -# Both of the plugin's skills load, as they do when it's installed, so these -# cases also check that Copilot picks the right one. -agent_environment: - skills: - - ../../../../plugins/marketplace-maintainer/skills/update-remote-pins - - ../../../../plugins/marketplace-maintainer/skills/release-plugins +agent_environment: marketplace grading_environment: files: - src: grader.mjs diff --git a/evals/marketplace-maintainer/skills/update-remote-pins/eval.yaml b/evals/marketplace-maintainer/skills/update-remote-pins/eval.yaml index c6f73a2..90386d5 100644 --- a/evals/marketplace-maintainer/skills/update-remote-pins/eval.yaml +++ b/evals/marketplace-maintainer/skills/update-remote-pins/eval.yaml @@ -6,12 +6,7 @@ defaults: model: claude-haiku-5.5 timeout: 5m runs: 1 -# Both of the plugin's skills load, as they do when it's installed, so these -# cases also check that Copilot picks the right one. -agent_environment: - skills: - - ../../../../plugins/marketplace-maintainer/skills/update-remote-pins - - ../../../../plugins/marketplace-maintainer/skills/release-plugins +agent_environment: marketplace grading_environment: files: - src: grader.mjs diff --git a/evals/plugin-authoring/skills/add-plugin/eval.yaml b/evals/plugin-authoring/skills/add-plugin/eval.yaml index 5986458..59ff2d6 100644 --- a/evals/plugin-authoring/skills/add-plugin/eval.yaml +++ b/evals/plugin-authoring/skills/add-plugin/eval.yaml @@ -6,13 +6,7 @@ defaults: model: claude-haiku-5.5 timeout: 5m runs: 1 -# All of the plugin's skills load, as they do when it's installed, so these -# cases also check that Copilot picks the right one. -agent_environment: - skills: - - ../../../../plugins/plugin-authoring/skills/add-plugin - - ../../../../plugins/plugin-authoring/skills/try-plugin-locally - - ../../../../plugins/plugin-authoring/skills/submit-plugin +agent_environment: marketplace grading_environment: files: - src: grader.mjs diff --git a/evals/plugin-authoring/skills/submit-plugin/eval.yaml b/evals/plugin-authoring/skills/submit-plugin/eval.yaml index a828115..a86f54d 100644 --- a/evals/plugin-authoring/skills/submit-plugin/eval.yaml +++ b/evals/plugin-authoring/skills/submit-plugin/eval.yaml @@ -6,13 +6,7 @@ defaults: model: claude-haiku-5.5 timeout: 5m runs: 1 -# All of the plugin's skills load, as they do when it's installed, so these -# cases also check that Copilot picks the right one. -agent_environment: - skills: - - ../../../../plugins/plugin-authoring/skills/add-plugin - - ../../../../plugins/plugin-authoring/skills/try-plugin-locally - - ../../../../plugins/plugin-authoring/skills/submit-plugin +agent_environment: marketplace grading_environment: files: - src: grader.mjs diff --git a/evals/plugin-authoring/skills/try-plugin-locally/eval.yaml b/evals/plugin-authoring/skills/try-plugin-locally/eval.yaml index 1259348..04263ca 100644 --- a/evals/plugin-authoring/skills/try-plugin-locally/eval.yaml +++ b/evals/plugin-authoring/skills/try-plugin-locally/eval.yaml @@ -6,13 +6,7 @@ defaults: model: claude-haiku-5.5 timeout: 5m runs: 1 -# All of the plugin's skills load, as they do when it's installed, so these -# cases also check that Copilot picks the right one. -agent_environment: - skills: - - ../../../../plugins/plugin-authoring/skills/add-plugin - - ../../../../plugins/plugin-authoring/skills/try-plugin-locally - - ../../../../plugins/plugin-authoring/skills/submit-plugin +agent_environment: marketplace grading_environment: files: - src: grader.mjs diff --git a/evals/pr-description/skills/pr-description/eval.yaml b/evals/pr-description/skills/pr-description/eval.yaml index f696de0..dee5500 100644 --- a/evals/pr-description/skills/pr-description/eval.yaml +++ b/evals/pr-description/skills/pr-description/eval.yaml @@ -6,16 +6,7 @@ defaults: model: claude-haiku-5.5 timeout: 5m runs: 1 -# Stage pr-description alongside the other marketplace skills so cases verify -# realistic multi-skill routing and non-collision. -agent_environment: - skills: - - ../../../../plugins/pr-description/skills/pr-description - - ../../../../plugins/plugin-authoring/skills/add-plugin - - ../../../../plugins/plugin-authoring/skills/try-plugin-locally - - ../../../../plugins/plugin-authoring/skills/submit-plugin - - ../../../../plugins/marketplace-maintainer/skills/update-remote-pins - - ../../../../plugins/marketplace-maintainer/skills/release-plugins +agent_environment: marketplace grading_environment: files: - src: grader.mjs