From da37fd44a39bf525bfc218c145c0d85b4b05b7ee Mon Sep 17 00:00:00 2001
From: Ronnie Geraghty <rgeraghty@microsoft.com>
Date: Tue, 10 Nov 2026 16:51:12 -0800
Subject: [PATCH 1/3] Run skill evals with a shared marketplace environment

Share the native Vally skill environment across the marketplace evals, prepare immutable remote skill sources, and check runtime loaded-skill evidence without changing plugin activation or CI.

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
---
 .github/scripts/check-remote-pins.mjs         |   4 +-
 .github/scripts/eval-report.test.mjs          |   6 +-
 .github/scripts/lint.mjs                      |   2 +
 .github/scripts/marketplace-context.test.mjs  | 206 ++++++++++++++++++
 .github/scripts/prepare-eval-context.mjs      | 120 ++++++++++
 .github/scripts/validation.test.mjs           |   2 +
 .vally.yaml                                   |  17 ++
 README.md                                     |  13 +-
 docs/checks-and-evals.md                      |  53 ++++-
 evals/_shared/graders/marketplace-context.mjs |  14 ++
 .../skills/release-plugins/eval.yaml          |   7 +-
 .../skills/update-remote-pins/eval.yaml       |   7 +-
 .../skills/add-plugin/eval.yaml               |   8 +-
 .../skills/submit-plugin/eval.yaml            |   8 +-
 .../skills/try-plugin-locally/eval.yaml       |   8 +-
 .../skills/pr-description/eval.yaml           |  31 ++-
 package.json                                  |   2 +
 17 files changed, 456 insertions(+), 52 deletions(-)
 create mode 100644 .github/scripts/marketplace-context.test.mjs
 create mode 100644 .github/scripts/prepare-eval-context.mjs
 create mode 100644 evals/_shared/graders/marketplace-context.mjs

diff --git a/.github/scripts/check-remote-pins.mjs b/.github/scripts/check-remote-pins.mjs
index 7fb520e..1d011cf 100644
--- a/.github/scripts/check-remote-pins.mjs
+++ b/.github/scripts/check-remote-pins.mjs
@@ -8,8 +8,8 @@ const supportedSchemas = new Set(["1.0.0", "1.1.0"].map((version) => `https://ag
 // Copilot CLI's plugin reference (github/docs@8e7396e); legacy order also tested in d838c30^.
 const legacyManifestPaths = [".plugin/plugin.json", "plugin.json", ".github/plugin/plugin.json", ".claude-plugin/plugin.json"];
 
-export function githubApi(endpoint, run = spawnSync) {
-  const result = run("gh", ["api", endpoint, "--include", "-H", "Accept: application/vnd.github.raw+json"], {
+export function githubApi(endpoint, run = spawnSync, accept = "application/vnd.github.raw+json") {
+  const result = run("gh", ["api", endpoint, "--include", "-H", `Accept: ${accept}`], {
     encoding: "utf8",
     stdio: ["ignore", "pipe", "pipe"],
   });
diff --git a/.github/scripts/eval-report.test.mjs b/.github/scripts/eval-report.test.mjs
index 3c7c691..4ef3318 100644
--- a/.github/scripts/eval-report.test.mjs
+++ b/.github/scripts/eval-report.test.mjs
@@ -393,8 +393,10 @@ prDescriptionTest("pr-description eval stages the multi-skill marketplace cohort
   const evalDir = resolve(root, "evals/pr-description/skills/pr-description");
   const spec = parse(readFileSync(resolve(evalDir, "eval.yaml"), "utf8"));
   assert.equal(spec.scoring.threshold, 1);
-  const skills = spec.agent_environment.skills;
-  for (const configuredSkill of skills) {
+  const project = parse(readFileSync(resolve(root, ".vally.yaml"), "utf8"));
+  assert.equal(spec.agent_environment, "marketplace");
+  const skills = project.environments.marketplace.skills;
+  for (const configuredSkill of skills.filter((path) => path.includes("/plugins/"))) {
     assert.ok(existsSync(resolve(evalDir, configuredSkill)), `configured cohort skill path is missing: ${configuredSkill}`);
   }
   for (const expectedSkill of [
diff --git a/.github/scripts/lint.mjs b/.github/scripts/lint.mjs
index 70056b7..295029e 100644
--- a/.github/scripts/lint.mjs
+++ b/.github/scripts/lint.mjs
@@ -2,8 +2,10 @@ import { spawnSync } from "node:child_process";
 import { existsSync } from "node:fs";
 import { createRequire } from "node:module";
 import { checkLocalCatalog } from "./check-local-catalog.mjs";
+import { checkMarketplaceEnvironment } from "./prepare-eval-context.mjs";
 
 checkLocalCatalog();
+if (existsSync(".vally.yaml")) checkMarketplaceEnvironment(process.cwd());
 const args = [];
 if (existsSync("plugins")) args.push("plugins");
 if (existsSync("evals")) args.push("--eval-spec", "evals");
diff --git a/.github/scripts/marketplace-context.test.mjs b/.github/scripts/marketplace-context.test.mjs
new file mode 100644
index 0000000..2fdde4c
--- /dev/null
+++ b/.github/scripts/marketplace-context.test.mjs
@@ -0,0 +1,206 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import { createHash, randomUUID } from "node:crypto";
+import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { basename, dirname, resolve } from "node:path";
+import { fileURLToPath, pathToFileURL } from "node:url";
+import { createDefaultGraderRegistry, loadEvalSpec, ProjectContext, resolveStimulus, runEval } from "@microsoft/vally";
+import { parse, stringify } from "yaml";
+import { checkMarketplaceEnvironment, prepareEvalContext } from "./prepare-eval-context.mjs";
+
+const root = fileURLToPath(new URL("../../", import.meta.url));
+const sha = "a".repeat(40);
+const schema = "https://agent-plugins.org/schemas/1.0.0/plugin.schema.json";
+const hash = (bytes) => createHash("sha1").update(`blob ${bytes.length}\0`).update(bytes).digest("hex");
+
+function fixture(t) {
+  const directory = resolve(root, ".github/scripts", `.context-test-${randomUUID()} space`);
+  mkdirSync(directory);
+  t.after(() => rmSync(directory, { recursive: true, force: true }));
+  const write = (path, content) => {
+    const file = resolve(directory, path);
+    mkdirSync(dirname(file), { recursive: true });
+    writeFileSync(file, content);
+  };
+  write("plugins/fixture-local/skills/local-notes/SKILL.md", "---\nname: local-notes\ndescription: Summarize local notes.\n---\nLocal notes instructions.\n");
+  write("marketplace.json", JSON.stringify({ plugins: [
+    { name: "fixture-local", source: "./plugins/fixture-local" },
+    { name: "fixture-remote", version: "1.0.0", source: {
+      source: "github", repo: "example/fixture", path: "packages/plugin", ref: "v1.0.0", sha,
+    } },
+  ] }));
+  const environment = { skills: [
+    "../../../../plugins/fixture-local/skills/local-notes",
+    `../../../../.vally/marketplace/fixture-remote/${sha}/skills/remote-notes`,
+  ] };
+  const saveConfig = () => write(".vally.yaml", stringify({ environments: { marketplace: environment } }));
+  saveConfig();
+  const contents = new Map([
+    ["remote-notes/SKILL.md", Buffer.from("---\nname: remote-notes\ndescription: Summarize remote notes.\n---\nRemote notes instructions.\n")],
+    ["remote-notes/references/example.bin", Buffer.from([0, 128, 255])],
+  ]);
+  const tree = { truncated: false, tree: [...contents].map(([path, bytes]) => ({
+    path: `packages/plugin/skills/${path}`, type: "blob", mode: "100644", sha: hash(bytes),
+  })).concat([{ path: "packages/plugin/mcp.json", type: "blob", mode: "100644", sha: "b".repeat(40) }]) };
+  const calls = [];
+  const api = (endpoint, _run, accept) => {
+    calls.push(endpoint);
+    const response = (data) => ({ status: 200, body: JSON.stringify(data) });
+    if (endpoint.startsWith("repos/example/fixture/commits/")) return response({ sha });
+    if (endpoint.includes("/contents/")) return response({
+      $schema: schema, name: "fixture-remote", version: "1.0.0",
+    });
+    if (endpoint.includes("/git/trees/")) return response(tree);
+    for (const bytes of contents.values()) {
+      if (endpoint.endsWith(`/git/blobs/${hash(bytes)}`)) {
+        assert.equal(accept, "application/vnd.github+json", "binary sources need JSON/base64, not raw UTF-8");
+        return response({ encoding: "base64", content: bytes.toString("base64") });
+      }
+    }
+    assert.fail(`Unexpected API request: ${endpoint}`);
+  };
+  return { directory, write, environment, saveConfig, tree, api, calls, contents };
+}
+
+test("preparation stages pinned remote skill files, verifies cached bytes and excludes plugin runtimes", (t) => {
+  const { directory, api, calls, contents } = fixture(t);
+  const skills = prepareEvalContext(directory, api);
+  assert.deepEqual(skills.map((path) => basename(path)), ["local-notes", "remote-notes"]);
+  const remote = skills[1];
+  for (const [path, bytes] of contents) {
+    assert.deepEqual(readFileSync(resolve(dirname(remote), path)), bytes);
+  }
+  assert.equal(existsSync(resolve(dirname(remote), "mcp.json")), false);
+  const firstBlobCalls = calls.filter((path) => path.includes("/git/blobs/")).length;
+  assert.equal(firstBlobCalls, contents.size);
+  prepareEvalContext(directory, api);
+  assert.equal(calls.filter((path) => path.includes("/git/blobs/")).length, firstBlobCalls);
+  writeFileSync(resolve(remote, "SKILL.md"), "changed");
+  assert.throws(() => prepareEvalContext(directory, api), /Modified cached skill source/);
+});
+
+test("preparation rejects incomplete cohorts, stale pins, truncated trees and unavailable sources", (t) => {
+  const { directory, environment, saveConfig, tree, api } = fixture(t);
+  environment.skills.pop();
+  saveConfig();
+  assert.throws(() => prepareEvalContext(directory, api), /must match the catalog/);
+  environment.skills.push(`../../../../.vally/marketplace/fixture-remote/${"b".repeat(40)}/skills/remote-notes`);
+  saveConfig();
+  assert.throws(() => checkMarketplaceEnvironment(directory), /must match the catalog/);
+  environment.skills[1] = environment.skills[1].replace("b".repeat(40), sha);
+  saveConfig();
+  tree.truncated = true;
+  assert.throws(() => prepareEvalContext(directory, api), /truncated/);
+  tree.truncated = false;
+  assert.throws(() => prepareEvalContext(directory, () => ({ status: 403, body: "Forbidden" })), /HTTP 403/);
+  tree.tree[0].mode = "120000";
+  assert.throws(() => prepareEvalContext(directory, api), /unsupported skill file mode/);
+});
+
+test("completeness checks detect new local skills and unexpected cached remote files", (t) => {
+  const { directory, write, api } = fixture(t);
+  const remote = prepareEvalContext(directory, api)[1];
+  writeFileSync(resolve(remote, "extra.md"), "unexpected");
+  assert.throws(() => prepareEvalContext(directory, api), /Unexpected cached skill file/);
+  rmSync(resolve(remote, "extra.md"));
+  write("plugins/fixture-local/skills/another/SKILL.md", "---\nname: another\ndescription: Another skill.\n---\nInstructions.\n");
+  assert.throws(() => checkMarketplaceEnvironment(directory), /must match the catalog/);
+});
+
+test("native Vally resolves the named environment, merges a case fixture and supplies both skills to an executor without a model", async (t) => {
+  const { directory, write, api } = fixture(t);
+  prepareEvalContext(directory, api);
+  const evalDirectory = resolve(directory, "evals/fixture-local/skills/local-notes");
+  write("evals/fixture-local/skills/local-notes/fixture.txt", "case fixture");
+  write("evals/fixture-local/skills/local-notes/eval.yaml", stringify({
+    name: "fixture-context", agent_environment: "marketplace",
+    stimuli: [{ name: "fixture-case", prompt: "Summarize notes.",
+      agent_environment: { files: [{ src: "fixture.txt", dest: "input.txt" }] } }],
+  }));
+  const project = await ProjectContext.load(directory);
+  const spec = await loadEvalSpec(resolve(evalDirectory, "eval.yaml"));
+  const stimulus = resolveStimulus(spec.stimuli[0], project.config.environments[spec.environment],
+    project.config.environments, spec.tags, evalDirectory, spec.grading_environment);
+  let observed;
+  const result = await runEval({
+    prompt: stimulus.prompt, stimulus, environment: stimulus.environment, baseDir: evalDirectory,
+    workDir: directory, skills: [],
+    executor: { name: "test-only-no-model", execute: async (_stimulus, options) => {
+      observed = options.skills.map((skill) => skill.name);
+      assert.equal(readFileSync(resolve(options.workDir, "input.txt"), "utf8"), "case fixture");
+      for (const name of observed) {
+        assert.ok(readFileSync(resolve(options.workDir, name, "SKILL.md"), "utf8").includes(`name: ${name}`));
+      }
+      assert.deepEqual(readFileSync(resolve(options.workDir, "remote-notes/references/example.bin")), Buffer.from([0, 128, 255]));
+      return { output: "No model was called.", workDir: options.workDir, events: [],
+        metadata: { executor: "test-only-no-model" }, metrics: {}, endReason: "completed" };
+    } },
+  });
+  t.after(() => result.cleanup());
+  assert.deepEqual(observed, ["local-notes", "remote-notes"]);
+  assert.equal(result.trajectory.metadata.skillsLoaded, undefined,
+    "native staging is not a live Copilot skills-loaded observation");
+});
+
+test("loaded-context grader requires observed SDK inventory, not answers, paths, activation alone or mock metadata", {
+  skip: !existsSync(resolve(root, "evals/_shared/graders/marketplace-context.mjs")),
+}, async () => {
+  const { evaluate } = await import(pathToFileURL(resolve(root, "evals/_shared/graders/marketplace-context.mjs")));
+  const { score } = await import(pathToFileURL(resolve(root, "evals/_shared/graders/common.mjs")));
+  const expected = ["local-notes", "remote-notes"];
+  const trajectory = { output: "Both skills are available.", events: [] };
+  for (const candidate of [
+    trajectory,
+    { ...trajectory, metadata: { executor: "copilot-sdk", skillsLoaded: ["local-notes"] } },
+    { ...trajectory, metadata: { executor: "mock", skillsLoaded: expected } },
+    { ...trajectory, events: [{ type: "skill_activation", data: { name: "remote-notes" } }] },
+    { ...trajectory, events: [{ type: "tool_call", data: { toolName: "view", arguments: { path: "remote-notes/SKILL.md" } } }] },
+  ]) {
+    assert.equal(score("context", evaluate(candidate, expected)).passed, false);
+  }
+  assert.equal(score("context", evaluate({
+    ...trajectory, metadata: { executor: "copilot-sdk", skillsLoaded: expected },
+  }, expected)).passed, true);
+});
+
+test("all local skill evals use the shared environment and the runtime inventory check covers that environment", async () => {
+  const skills = checkMarketplaceEnvironment();
+  const project = parse(readFileSync(resolve(root, ".vally.yaml"), "utf8"));
+  for (const path of skills.filter((path) => !path.includes(`${resolve(root, ".vally")}`))) {
+    const plugin = basename(resolve(path, "../../"));
+    const spec = await loadEvalSpec(resolve(root, "evals", plugin, "skills", basename(path), "eval.yaml"));
+    assert.equal(spec.environment, "marketplace", spec.name);
+    for (const raw of spec.stimuli) {
+      const stimulus = resolveStimulus(raw, project.environments.marketplace, project.environments);
+      assert.deepEqual(stimulus.environment.skills, project.environments.marketplace.skills);
+    }
+  }
+  const prDescription = resolve(root, "evals/pr-description/skills/pr-description/eval.yaml");
+  if (existsSync(resolve(root, "plugins/pr-description"))) {
+    const spec = await loadEvalSpec(prDescription);
+    const grader = spec.stimuli.flatMap((item) => item.graders).find((item) => item.name === "marketplace-skills-loaded");
+    assert.deepEqual(grader.config.args.slice(1).sort(), skills.map((path) => basename(path)).sort());
+  }
+});
+
+test("native program grading preserves the SDK inventory for the context check without calling a model", {
+  skip: !existsSync(resolve(root, "evals/_shared/graders/marketplace-context.mjs")),
+}, async (t) => {
+  const { directory } = fixture(t);
+  const grader = createDefaultGraderRegistry().get("program");
+  const config = {
+    program: process.execPath,
+    args: [resolve(root, "evals/_shared/graders/marketplace-context.mjs"), "local-notes", "remote-notes"],
+  };
+  for (const [skillsLoaded, passed] of [
+    [["local-notes", "remote-notes"], true],
+    [["local-notes"], false],
+  ]) {
+    const result = await grader.grade({
+      config, workDir: directory, stimulus: { name: "synthetic-inventory", prompt: "Synthetic evidence only." },
+      trajectory: { output: "Synthetic evidence only.", events: [], workDir: directory,
+        metadata: { executor: "copilot-sdk", skillsLoaded } },
+    });
+    assert.equal(result.passed, passed, result.evidence);
+  }
+});
diff --git a/.github/scripts/prepare-eval-context.mjs b/.github/scripts/prepare-eval-context.mjs
new file mode 100644
index 0000000..1062964
--- /dev/null
+++ b/.github/scripts/prepare-eval-context.mjs
@@ -0,0 +1,120 @@
+import { createHash, randomUUID } from "node:crypto";
+import { existsSync, mkdirSync, readFileSync, readdirSync, realpathSync, renameSync, rmSync, writeFileSync } from "node:fs";
+import { basename, dirname, isAbsolute, relative, resolve, sep } from "node:path";
+import { fileURLToPath } from "node:url";
+import { parse } from "yaml";
+import { checkRemotePin, githubApi } from "./check-remote-pins.mjs";
+
+const root = fileURLToPath(new URL("../../", import.meta.url));
+const readJson = (file) => JSON.parse(readFileSync(file, "utf8"));
+const gitHash = (bytes) => createHash("sha1").update(`blob ${bytes.length}\0`).update(bytes).digest("hex");
+
+function inside(directory, path) {
+  const result = resolve(directory, path);
+  const rel = relative(directory, result);
+  if (!rel || isAbsolute(rel) || rel === ".." || rel.startsWith(`..${sep}`)) {
+    throw new Error(`Expected a path inside ${directory}: ${path}`);
+  }
+  return result;
+}
+
+function localSkills(directory) {
+  const skills = resolve(directory, "skills");
+  if (!existsSync(skills)) return [];
+  return readdirSync(skills, { withFileTypes: true })
+    .filter((entry) => entry.isDirectory())
+    .map((entry) => {
+      const path = resolve(skills, entry.name);
+      if (!existsSync(resolve(path, "SKILL.md"))) throw new Error(`Missing SKILL.md: ${path}`);
+      return path;
+    });
+}
+
+export function checkMarketplaceEnvironment(directory = root, remoteSkills = undefined) {
+  const catalog = readJson(resolve(directory, "marketplace.json"));
+  const config = parse(readFileSync(resolve(directory, ".vally.yaml"), "utf8"));
+  const configured = config.environments?.marketplace?.skills;
+  if (!Array.isArray(configured)) throw new Error("Missing environments.marketplace.skills in .vally.yaml.");
+  const evalDirectory = resolve(directory, "evals/_plugin/skills/_skill");
+  const paths = configured.map((path) => resolve(evalDirectory, path));
+  const expected = [];
+  for (const entry of catalog.plugins) {
+    if (typeof entry.source === "string") {
+      expected.push(...localSkills(inside(directory, entry.source)));
+    } else {
+      const { source, repo, path = "", sha } = entry.source ?? {};
+      if (source !== "github" || !/^[\w.-]+\/[\w.-]+$/.test(repo ?? "") || !/^[a-f0-9]{40}$/.test(sha ?? "")) {
+        throw new Error(`${entry.name}: expected a GitHub source pinned to a full commit SHA.`);
+      }
+      if (!/^[a-z0-9][a-z0-9-]*$/.test(entry.name)
+        || (path && (!/^[\w.-]+(?:\/[\w.-]+)*$/.test(path) || path.split("/").some((part) => [".", ".."].includes(part))))) {
+        throw new Error(`${entry.name}: invalid remote plugin name or source path.`);
+      }
+      const skillsRoot = resolve(directory, ".vally/marketplace", entry.name, sha, "skills");
+      expected.push(...(remoteSkills
+        ? remoteSkills.get(entry.name)
+        : paths.filter((candidate) => dirname(candidate) === skillsRoot)));
+    }
+  }
+  if (new Set(paths.map((path) => basename(path))).size !== paths.length) throw new Error("Duplicate marketplace skill names.");
+  if (paths.length !== expected.length || paths.some((path) => !expected.includes(path))) {
+    throw new Error("marketplace.skills must match the catalog's skill directories and remote pins. Update .vally.yaml.");
+  }
+  return paths;
+}
+
+export function prepareEvalContext(directory = root, api = githubApi) {
+  checkMarketplaceEnvironment(directory);
+  const remoteSkills = new Map();
+  for (const entry of readJson(resolve(directory, "marketplace.json")).plugins) {
+    if (typeof entry.source === "string") continue;
+    checkRemotePin(entry, api);
+    const { repo, path = "", sha } = entry.source;
+    const response = api(`repos/${repo}/git/trees/${sha}?recursive=1`);
+    if (response.status !== 200) throw new Error(`${entry.name}: couldn't read the pinned skill tree (HTTP ${response.status}).`);
+    const tree = JSON.parse(response.body);
+    if (tree.truncated) throw new Error(`${entry.name}: GitHub truncated the tree; skill completeness is unknown.`);
+    const prefix = `${path ? `${path}/` : ""}skills/`;
+    const files = tree.tree.filter((item) => item.type === "blob" && item.path.startsWith(prefix));
+    const destination = resolve(directory, ".vally/marketplace", entry.name, sha, "skills");
+    const expectedFiles = files.map((item) => inside(destination, item.path.slice(prefix.length)));
+    for (const [index, item] of files.entries()) {
+      if (!["100644", "100755"].includes(item.mode)) throw new Error(`${entry.name}: unsupported skill file mode at ${item.path}.`);
+      const file = expectedFiles[index];
+      if (existsSync(file)) {
+        if (gitHash(readFileSync(file)) !== item.sha) throw new Error(`Modified cached skill source: ${file}. Remove that cache and prepare again.`);
+        continue;
+      }
+      const blob = api(`repos/${repo}/git/blobs/${item.sha}`, undefined, "application/vnd.github+json");
+      if (blob.status !== 200) throw new Error(`${entry.name}: couldn't fetch ${item.path} (HTTP ${blob.status}).`);
+      const content = JSON.parse(blob.body);
+      if (content.encoding !== "base64") throw new Error(`${entry.name}: unexpected blob encoding at ${item.path}.`);
+      const bytes = Buffer.from(content.content, "base64");
+      if (gitHash(bytes) !== item.sha) throw new Error(`${entry.name}: content hash mismatch at ${item.path}.`);
+      mkdirSync(dirname(file), { recursive: true });
+      const temporary = `${file}.${randomUUID()}.tmp`;
+      try {
+        writeFileSync(temporary, bytes, { mode: item.mode === "100755" ? 0o755 : 0o644 });
+        renameSync(temporary, file);
+      } finally {
+        rmSync(temporary, { force: true });
+      }
+    }
+    if (existsSync(destination)) {
+      for (const file of readdirSync(destination, { recursive: true, withFileTypes: true })) {
+        if (file.isDirectory()) continue;
+        const absolute = resolve(file.parentPath, file.name);
+        if (!expectedFiles.includes(absolute)) throw new Error(`Unexpected cached skill file: ${absolute}. Remove that cache and prepare again.`);
+      }
+    }
+    remoteSkills.set(entry.name, files
+      .filter((item) => /^[^/]+\/SKILL\.md$/.test(item.path.slice(prefix.length)))
+      .map((item) => dirname(inside(destination, item.path.slice(prefix.length)))));
+  }
+  return checkMarketplaceEnvironment(directory, remoteSkills);
+}
+
+if (process.argv[1] && realpathSync(fileURLToPath(import.meta.url)) === realpathSync(resolve(process.argv[1]))) {
+  const skills = prepareEvalContext(process.cwd());
+  console.log(`Prepared ${skills.length} marketplace skills. No plugins installed or evals run.`);
+}
diff --git a/.github/scripts/validation.test.mjs b/.github/scripts/validation.test.mjs
index ba64bdf..d967efb 100644
--- a/.github/scripts/validation.test.mjs
+++ b/.github/scripts/validation.test.mjs
@@ -385,6 +385,8 @@ test("validation calls CI scripts, and paid eval activation is manual-only", ()
   assert.equal(job.steps.find((step) => step.uses?.startsWith("actions/checkout@")).with["persist-credentials"], false);
   assert.deepEqual(JSON.parse(readFileSync(resolve(root, "package.json"), "utf8")).scripts, {
     lint: "node .github/scripts/lint.mjs",
+    "eval:prepare": "node .github/scripts/prepare-eval-context.mjs",
+    preeval: "npm run eval:prepare",
     eval: "vally eval --workers 1 --max-retries 1",
     "eval:report": "node .github/scripts/eval-report.mjs",
     "eval:compare": "node .github/scripts/eval-report.mjs",
diff --git a/.vally.yaml b/.vally.yaml
index 051c0bd..0e11e0a 100644
--- a/.vally.yaml
+++ b/.vally.yaml
@@ -8,6 +8,23 @@ paths:
     - eval.yaml
     - "*.eval.yaml"
   results: vally-results
+environments:
+  marketplace:
+    # Vally resolves these paths relative to each eval, not this file.
+    # npm run eval:prepare fetches the remote skills without installing plugins.
+    skills:
+      - ../../../../plugins/plugin-authoring/skills/add-plugin
+      - ../../../../plugins/plugin-authoring/skills/try-plugin-locally
+      - ../../../../plugins/plugin-authoring/skills/submit-plugin
+      - ../../../../plugins/marketplace-maintainer/skills/update-remote-pins
+      - ../../../../plugins/marketplace-maintainer/skills/release-plugins
+      - ../../../../plugins/pr-description/skills/pr-description
+      - ../../../../.vally/marketplace/eval/34ddb5803aeaee8330ab70c19ad1a3f37fb77ded/skills/compare
+      - ../../../../.vally/marketplace/eval/34ddb5803aeaee8330ab70c19ad1a3f37fb77ded/skills/create
+      - ../../../../.vally/marketplace/eval/34ddb5803aeaee8330ab70c19ad1a3f37fb77ded/skills/improve
+      - ../../../../.vally/marketplace/eval/34ddb5803aeaee8330ab70c19ad1a3f37fb77ded/skills/review
+      - ../../../../.vally/marketplace/eval/34ddb5803aeaee8330ab70c19ad1a3f37fb77ded/skills/run
+      - ../../../../.vally/marketplace/eval/34ddb5803aeaee8330ab70c19ad1a3f37fb77ded/skills/setup
 suites:
   smoke:
     description: One representative case for each skill and agent
diff --git a/README.md b/README.md
index 3ca8f26..7d39831 100644
--- a/README.md
+++ b/README.md
@@ -177,8 +177,9 @@ The `release-plugins` skill can retire a plugin for you. By hand, delete
 `plugins/<name>/` and `evals/<name>/`, then remove the plugin's entry from
 `marketplace.json` and its row and install command from this README. If it's
 one of the two plugins in `.github/copilot/settings.json`, remove its key there
-too. If surviving cohort evals under `evals/` list the retired plugin's skills
-in `agent_environment.skills` or `disallowed`, remove those entries as well.
+too. Remove its skills from the shared `marketplace` environment in
+[.vally.yaml](.vally.yaml), and from surviving evals' `disallowed` lists and
+loaded-skill checks.
 
 You can keep only remote entries by removing both bundled local plugins and
 their eval folders. No empty `plugins/` or `evals/` directory is needed.
@@ -196,11 +197,15 @@ validation, catalog consistency checks, and remote-pin checks on every pull
 request.
 
 **Paid evals:** `npm run eval` runs real Copilot sessions and costs AI credits.
+The six skill evals share the `marketplace` environment in
+[.vally.yaml](.vally.yaml), with all local and pinned remote marketplace skills
+available. Before Vally starts, npm fetches the remote skill sources into ignored
+storage; it does not install plugins or enable their MCP servers or extensions.
 [Evals](.github/workflows/evals.yml) runs only when manually started; pull
 requests never trigger paid evals.
 
-See [Checks and evals](docs/checks-and-evals.md) for CI details, local test
-commands, and Bash and PowerShell eval setup.
+See [Checks and evals](docs/checks-and-evals.md) for the shared environment,
+remote-source preparation, CI details, and Bash and PowerShell eval setup.
 
 ## License
 
diff --git a/docs/checks-and-evals.md b/docs/checks-and-evals.md
index a4d6988..2e218ac 100644
--- a/docs/checks-and-evals.md
+++ b/docs/checks-and-evals.md
@@ -12,7 +12,10 @@ npm run lint
 
 `npm run lint` checks local catalog consistency, then checks every skill under
 `plugins/` and every eval under `evals/` with the pinned
-[Vally](https://microsoft.github.io/vally/) CLI. A remote-only marketplace can
+[Vally](https://microsoft.github.io/vally/) CLI. It also checks the shared
+environment's local skill coverage and remote pin paths without accessing the
+network. Remote skill completeness is checked during preparation below.
+A remote-only marketplace can
 omit both directories; schema validation also skips when no local manifests
 remain. Regression tests for the bundled maintenance plugins run only while
 those local plugins remain. Missing files for a retained plugin still fail.
@@ -69,6 +72,54 @@ call a model or prove how a plugin behaves in a model session.
 
 **These runs start real Copilot sessions and spend AI credits.**
 
+### Shared marketplace environment
+
+The six skill evals reference `agent_environment: marketplace`. Its definition in
+[.vally.yaml](../.vally.yaml) lists all six local skills and the six skills from
+the catalog-pinned remote Evals plugin. Vally merges that skill set with each
+case's existing fixtures; grading files stay separate. The independent
+`plugin-reviewer` delegation eval is unchanged.
+
+`npm run eval` first runs `npm run eval:prepare`. Preparation makes read-only
+GitHub API requests through `gh`, checks remote pins and fetches only skill
+sources and their supporting files into ignored `.vally/marketplace/` storage.
+It calls no model and does not install plugins, change profiles, run fetched
+code, or start plugin MCP servers, hooks or extensions. Use a signed-in `gh`
+account that can read the remote repositories. Review the pinned skill sources
+before approving an eval; an eval agent can follow their instructions and use
+tools.
+
+Preparation fails rather than omitting a missing skill, using a moving ref,
+accepting a truncated tree, or reusing modified cached sources. Verified files
+are reused at their exact commit. If the cache was changed, remove only the
+reported `.vally/marketplace/<plugin>/<sha>/` cache and prepare it again.
+Preparation alone is model-free:
+
+```sh
+npm run eval:prepare
+```
+
+Run this before invoking Vally directly instead of through `npm run eval`.
+When adding or removing a skill, update the shared environment. When changing a
+remote pin, update its paths to the new SHA and list all skills at that commit.
+Update the `marketplace-skills-loaded` grader's expected names in the
+`pr-description` eval too. Source paths are relative to the eval directory,
+not `.vally.yaml`; the six skill evals currently share the same directory depth.
+
+That positive case checks the native Copilot SDK trajectory's
+`metadata.skillsLoaded` inventory for every marketplace skill, alongside its
+existing exact target-invocation check. Listed paths and model answers cannot
+satisfy the loaded-skill check. Model-free tests exercise native Vally staging
+with a test executor and synthetic grader inputs, not a live Copilot session.
+An approved paid run is still needed to demonstrate actual runtime loading.
+The check's result is stored with the existing grader results.
+
+This is a shared **skills** environment, not full plugin-runtime activation.
+It does not establish availability of plugin agents, MCP tools, hooks, extensions
+or interactive canvases.
+
+### Running an eval
+
 Skill evals use Vally's built-in `skill-invocation` grader: `required` for every
 positive and boundary case, and `disallowed` for negative cases. Invocation
 credit requires a matching `skill_activation` event. Successful file reads,
diff --git a/evals/_shared/graders/marketplace-context.mjs b/evals/_shared/graders/marketplace-context.mjs
new file mode 100644
index 0000000..cedc1e1
--- /dev/null
+++ b/evals/_shared/graders/marketplace-context.mjs
@@ -0,0 +1,14 @@
+import { check, isMain, runGrade } from "./common.mjs";
+
+export function evaluate(trajectory, expected) {
+  const loaded = trajectory.metadata?.skillsLoaded;
+  return [
+    check("Context evidence comes from the native Copilot SDK executor", trajectory.metadata?.executor === "copilot-sdk"),
+    check("Every marketplace skill was reported loaded by the session",
+      expected.length > 0 && Array.isArray(loaded) && expected.every((name) => loaded.includes(name))),
+  ];
+}
+
+if (isMain(import.meta.url)) {
+  await runGrade("marketplace-context", (trajectory) => evaluate(trajectory, process.argv.slice(2)));
+}
diff --git a/evals/marketplace-maintainer/skills/release-plugins/eval.yaml b/evals/marketplace-maintainer/skills/release-plugins/eval.yaml
index 36742fb..442dffd 100644
--- a/evals/marketplace-maintainer/skills/release-plugins/eval.yaml
+++ b/evals/marketplace-maintainer/skills/release-plugins/eval.yaml
@@ -6,12 +6,7 @@ defaults:
   model: claude-haiku-5.5
   timeout: 5m
   runs: 1
-# Both of the plugin's skills load, as they do when it's installed, so these
-# cases also check that Copilot picks the right one.
-agent_environment:
-  skills:
-    - ../../../../plugins/marketplace-maintainer/skills/update-remote-pins
-    - ../../../../plugins/marketplace-maintainer/skills/release-plugins
+agent_environment: marketplace
 grading_environment:
   files:
     - src: grader.mjs
diff --git a/evals/marketplace-maintainer/skills/update-remote-pins/eval.yaml b/evals/marketplace-maintainer/skills/update-remote-pins/eval.yaml
index c6f73a2..90386d5 100644
--- a/evals/marketplace-maintainer/skills/update-remote-pins/eval.yaml
+++ b/evals/marketplace-maintainer/skills/update-remote-pins/eval.yaml
@@ -6,12 +6,7 @@ defaults:
   model: claude-haiku-5.5
   timeout: 5m
   runs: 1
-# Both of the plugin's skills load, as they do when it's installed, so these
-# cases also check that Copilot picks the right one.
-agent_environment:
-  skills:
-    - ../../../../plugins/marketplace-maintainer/skills/update-remote-pins
-    - ../../../../plugins/marketplace-maintainer/skills/release-plugins
+agent_environment: marketplace
 grading_environment:
   files:
     - src: grader.mjs
diff --git a/evals/plugin-authoring/skills/add-plugin/eval.yaml b/evals/plugin-authoring/skills/add-plugin/eval.yaml
index 5986458..59ff2d6 100644
--- a/evals/plugin-authoring/skills/add-plugin/eval.yaml
+++ b/evals/plugin-authoring/skills/add-plugin/eval.yaml
@@ -6,13 +6,7 @@ defaults:
   model: claude-haiku-5.5
   timeout: 5m
   runs: 1
-# All of the plugin's skills load, as they do when it's installed, so these
-# cases also check that Copilot picks the right one.
-agent_environment:
-  skills:
-    - ../../../../plugins/plugin-authoring/skills/add-plugin
-    - ../../../../plugins/plugin-authoring/skills/try-plugin-locally
-    - ../../../../plugins/plugin-authoring/skills/submit-plugin
+agent_environment: marketplace
 grading_environment:
   files:
     - src: grader.mjs
diff --git a/evals/plugin-authoring/skills/submit-plugin/eval.yaml b/evals/plugin-authoring/skills/submit-plugin/eval.yaml
index a828115..a86f54d 100644
--- a/evals/plugin-authoring/skills/submit-plugin/eval.yaml
+++ b/evals/plugin-authoring/skills/submit-plugin/eval.yaml
@@ -6,13 +6,7 @@ defaults:
   model: claude-haiku-5.5
   timeout: 5m
   runs: 1
-# All of the plugin's skills load, as they do when it's installed, so these
-# cases also check that Copilot picks the right one.
-agent_environment:
-  skills:
-    - ../../../../plugins/plugin-authoring/skills/add-plugin
-    - ../../../../plugins/plugin-authoring/skills/try-plugin-locally
-    - ../../../../plugins/plugin-authoring/skills/submit-plugin
+agent_environment: marketplace
 grading_environment:
   files:
     - src: grader.mjs
diff --git a/evals/plugin-authoring/skills/try-plugin-locally/eval.yaml b/evals/plugin-authoring/skills/try-plugin-locally/eval.yaml
index 1259348..04263ca 100644
--- a/evals/plugin-authoring/skills/try-plugin-locally/eval.yaml
+++ b/evals/plugin-authoring/skills/try-plugin-locally/eval.yaml
@@ -6,13 +6,7 @@ defaults:
   model: claude-haiku-5.5
   timeout: 5m
   runs: 1
-# All of the plugin's skills load, as they do when it's installed, so these
-# cases also check that Copilot picks the right one.
-agent_environment:
-  skills:
-    - ../../../../plugins/plugin-authoring/skills/add-plugin
-    - ../../../../plugins/plugin-authoring/skills/try-plugin-locally
-    - ../../../../plugins/plugin-authoring/skills/submit-plugin
+agent_environment: marketplace
 grading_environment:
   files:
     - src: grader.mjs
diff --git a/evals/pr-description/skills/pr-description/eval.yaml b/evals/pr-description/skills/pr-description/eval.yaml
index f696de0..74d7baf 100644
--- a/evals/pr-description/skills/pr-description/eval.yaml
+++ b/evals/pr-description/skills/pr-description/eval.yaml
@@ -6,22 +6,15 @@ defaults:
   model: claude-haiku-5.5
   timeout: 5m
   runs: 1
-# Stage pr-description alongside the other marketplace skills so cases verify
-# realistic multi-skill routing and non-collision.
-agent_environment:
-  skills:
-    - ../../../../plugins/pr-description/skills/pr-description
-    - ../../../../plugins/plugin-authoring/skills/add-plugin
-    - ../../../../plugins/plugin-authoring/skills/try-plugin-locally
-    - ../../../../plugins/plugin-authoring/skills/submit-plugin
-    - ../../../../plugins/marketplace-maintainer/skills/update-remote-pins
-    - ../../../../plugins/marketplace-maintainer/skills/release-plugins
+agent_environment: marketplace
 grading_environment:
   files:
     - src: grader.mjs
       dest: .grading/grader.mjs
     - src: ../../../_shared/graders/common.mjs
       dest: .grading/common.mjs
+    - src: ../../../_shared/graders/marketplace-context.mjs
+      dest: .grading/marketplace-context.mjs
 stimuli:
   - name: pr-description-drafts-structured-pr-body
     prompt: >-
@@ -45,6 +38,24 @@ stimuli:
         config:
           program: node
           args: [.grading/grader.mjs]
+      - type: program
+        name: marketplace-skills-loaded
+        config:
+          program: node
+          args:
+            - .grading/marketplace-context.mjs
+            - add-plugin
+            - try-plugin-locally
+            - submit-plugin
+            - update-remote-pins
+            - release-plugins
+            - pr-description
+            - compare
+            - create
+            - improve
+            - review
+            - run
+            - setup
       - type: diff-empty
   - name: pr-description-ignores-bug-and-review-question
     prompt: >-
diff --git a/package.json b/package.json
index 57c490c..bdc3818 100644
--- a/package.json
+++ b/package.json
@@ -6,6 +6,8 @@
   "engines": { "node": ">=22.14.0" },
   "scripts": {
     "lint": "node .github/scripts/lint.mjs",
+    "eval:prepare": "node .github/scripts/prepare-eval-context.mjs",
+    "preeval": "npm run eval:prepare",
     "eval": "vally eval --workers 1 --max-retries 1",
     "eval:report": "node .github/scripts/eval-report.mjs",
     "eval:compare": "node .github/scripts/eval-report.mjs"

From 8f02682429ec1b344be3aa42fbb3e57bf56a1d45 Mon Sep 17 00:00:00 2001
From: Ronnie Geraghty <rgeraghty@microsoft.com>
Date: Tue, 10 Nov 2026 17:11:45 -0800
Subject: [PATCH 2/3] Reduce marketplace context change to the shared
 environment

Remove the extra source-preparation machinery, custom inventory grader, new tests, package/helper changes and expanded docs. Retain all marketplace skill paths and the minimal existing test adaptation. Remote source materialization and live runtime confirmation remain unresolved, not implicitly supplied by configuration.

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
---
 .github/scripts/check-remote-pins.mjs         |   4 +-
 .github/scripts/lint.mjs                      |   2 -
 .github/scripts/marketplace-context.test.mjs  | 206 ------------------
 .github/scripts/prepare-eval-context.mjs      | 120 ----------
 .github/scripts/validation.test.mjs           |   2 -
 .vally.yaml                                   |   2 +-
 README.md                                     |  13 +-
 docs/checks-and-evals.md                      |  53 +----
 evals/_shared/graders/marketplace-context.mjs |  14 --
 .../skills/pr-description/eval.yaml           |  20 --
 package.json                                  |   2 -
 11 files changed, 8 insertions(+), 430 deletions(-)
 delete mode 100644 .github/scripts/marketplace-context.test.mjs
 delete mode 100644 .github/scripts/prepare-eval-context.mjs
 delete mode 100644 evals/_shared/graders/marketplace-context.mjs

diff --git a/.github/scripts/check-remote-pins.mjs b/.github/scripts/check-remote-pins.mjs
index 1d011cf..7fb520e 100644
--- a/.github/scripts/check-remote-pins.mjs
+++ b/.github/scripts/check-remote-pins.mjs
@@ -8,8 +8,8 @@ const supportedSchemas = new Set(["1.0.0", "1.1.0"].map((version) => `https://ag
 // Copilot CLI's plugin reference (github/docs@8e7396e); legacy order also tested in d838c30^.
 const legacyManifestPaths = [".plugin/plugin.json", "plugin.json", ".github/plugin/plugin.json", ".claude-plugin/plugin.json"];
 
-export function githubApi(endpoint, run = spawnSync, accept = "application/vnd.github.raw+json") {
-  const result = run("gh", ["api", endpoint, "--include", "-H", `Accept: ${accept}`], {
+export function githubApi(endpoint, run = spawnSync) {
+  const result = run("gh", ["api", endpoint, "--include", "-H", "Accept: application/vnd.github.raw+json"], {
     encoding: "utf8",
     stdio: ["ignore", "pipe", "pipe"],
   });
diff --git a/.github/scripts/lint.mjs b/.github/scripts/lint.mjs
index 295029e..70056b7 100644
--- a/.github/scripts/lint.mjs
+++ b/.github/scripts/lint.mjs
@@ -2,10 +2,8 @@ import { spawnSync } from "node:child_process";
 import { existsSync } from "node:fs";
 import { createRequire } from "node:module";
 import { checkLocalCatalog } from "./check-local-catalog.mjs";
-import { checkMarketplaceEnvironment } from "./prepare-eval-context.mjs";
 
 checkLocalCatalog();
-if (existsSync(".vally.yaml")) checkMarketplaceEnvironment(process.cwd());
 const args = [];
 if (existsSync("plugins")) args.push("plugins");
 if (existsSync("evals")) args.push("--eval-spec", "evals");
diff --git a/.github/scripts/marketplace-context.test.mjs b/.github/scripts/marketplace-context.test.mjs
deleted file mode 100644
index 2fdde4c..0000000
--- a/.github/scripts/marketplace-context.test.mjs
+++ /dev/null
@@ -1,206 +0,0 @@
-import test from "node:test";
-import assert from "node:assert/strict";
-import { createHash, randomUUID } from "node:crypto";
-import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
-import { basename, dirname, resolve } from "node:path";
-import { fileURLToPath, pathToFileURL } from "node:url";
-import { createDefaultGraderRegistry, loadEvalSpec, ProjectContext, resolveStimulus, runEval } from "@microsoft/vally";
-import { parse, stringify } from "yaml";
-import { checkMarketplaceEnvironment, prepareEvalContext } from "./prepare-eval-context.mjs";
-
-const root = fileURLToPath(new URL("../../", import.meta.url));
-const sha = "a".repeat(40);
-const schema = "https://agent-plugins.org/schemas/1.0.0/plugin.schema.json";
-const hash = (bytes) => createHash("sha1").update(`blob ${bytes.length}\0`).update(bytes).digest("hex");
-
-function fixture(t) {
-  const directory = resolve(root, ".github/scripts", `.context-test-${randomUUID()} space`);
-  mkdirSync(directory);
-  t.after(() => rmSync(directory, { recursive: true, force: true }));
-  const write = (path, content) => {
-    const file = resolve(directory, path);
-    mkdirSync(dirname(file), { recursive: true });
-    writeFileSync(file, content);
-  };
-  write("plugins/fixture-local/skills/local-notes/SKILL.md", "---\nname: local-notes\ndescription: Summarize local notes.\n---\nLocal notes instructions.\n");
-  write("marketplace.json", JSON.stringify({ plugins: [
-    { name: "fixture-local", source: "./plugins/fixture-local" },
-    { name: "fixture-remote", version: "1.0.0", source: {
-      source: "github", repo: "example/fixture", path: "packages/plugin", ref: "v1.0.0", sha,
-    } },
-  ] }));
-  const environment = { skills: [
-    "../../../../plugins/fixture-local/skills/local-notes",
-    `../../../../.vally/marketplace/fixture-remote/${sha}/skills/remote-notes`,
-  ] };
-  const saveConfig = () => write(".vally.yaml", stringify({ environments: { marketplace: environment } }));
-  saveConfig();
-  const contents = new Map([
-    ["remote-notes/SKILL.md", Buffer.from("---\nname: remote-notes\ndescription: Summarize remote notes.\n---\nRemote notes instructions.\n")],
-    ["remote-notes/references/example.bin", Buffer.from([0, 128, 255])],
-  ]);
-  const tree = { truncated: false, tree: [...contents].map(([path, bytes]) => ({
-    path: `packages/plugin/skills/${path}`, type: "blob", mode: "100644", sha: hash(bytes),
-  })).concat([{ path: "packages/plugin/mcp.json", type: "blob", mode: "100644", sha: "b".repeat(40) }]) };
-  const calls = [];
-  const api = (endpoint, _run, accept) => {
-    calls.push(endpoint);
-    const response = (data) => ({ status: 200, body: JSON.stringify(data) });
-    if (endpoint.startsWith("repos/example/fixture/commits/")) return response({ sha });
-    if (endpoint.includes("/contents/")) return response({
-      $schema: schema, name: "fixture-remote", version: "1.0.0",
-    });
-    if (endpoint.includes("/git/trees/")) return response(tree);
-    for (const bytes of contents.values()) {
-      if (endpoint.endsWith(`/git/blobs/${hash(bytes)}`)) {
-        assert.equal(accept, "application/vnd.github+json", "binary sources need JSON/base64, not raw UTF-8");
-        return response({ encoding: "base64", content: bytes.toString("base64") });
-      }
-    }
-    assert.fail(`Unexpected API request: ${endpoint}`);
-  };
-  return { directory, write, environment, saveConfig, tree, api, calls, contents };
-}
-
-test("preparation stages pinned remote skill files, verifies cached bytes and excludes plugin runtimes", (t) => {
-  const { directory, api, calls, contents } = fixture(t);
-  const skills = prepareEvalContext(directory, api);
-  assert.deepEqual(skills.map((path) => basename(path)), ["local-notes", "remote-notes"]);
-  const remote = skills[1];
-  for (const [path, bytes] of contents) {
-    assert.deepEqual(readFileSync(resolve(dirname(remote), path)), bytes);
-  }
-  assert.equal(existsSync(resolve(dirname(remote), "mcp.json")), false);
-  const firstBlobCalls = calls.filter((path) => path.includes("/git/blobs/")).length;
-  assert.equal(firstBlobCalls, contents.size);
-  prepareEvalContext(directory, api);
-  assert.equal(calls.filter((path) => path.includes("/git/blobs/")).length, firstBlobCalls);
-  writeFileSync(resolve(remote, "SKILL.md"), "changed");
-  assert.throws(() => prepareEvalContext(directory, api), /Modified cached skill source/);
-});
-
-test("preparation rejects incomplete cohorts, stale pins, truncated trees and unavailable sources", (t) => {
-  const { directory, environment, saveConfig, tree, api } = fixture(t);
-  environment.skills.pop();
-  saveConfig();
-  assert.throws(() => prepareEvalContext(directory, api), /must match the catalog/);
-  environment.skills.push(`../../../../.vally/marketplace/fixture-remote/${"b".repeat(40)}/skills/remote-notes`);
-  saveConfig();
-  assert.throws(() => checkMarketplaceEnvironment(directory), /must match the catalog/);
-  environment.skills[1] = environment.skills[1].replace("b".repeat(40), sha);
-  saveConfig();
-  tree.truncated = true;
-  assert.throws(() => prepareEvalContext(directory, api), /truncated/);
-  tree.truncated = false;
-  assert.throws(() => prepareEvalContext(directory, () => ({ status: 403, body: "Forbidden" })), /HTTP 403/);
-  tree.tree[0].mode = "120000";
-  assert.throws(() => prepareEvalContext(directory, api), /unsupported skill file mode/);
-});
-
-test("completeness checks detect new local skills and unexpected cached remote files", (t) => {
-  const { directory, write, api } = fixture(t);
-  const remote = prepareEvalContext(directory, api)[1];
-  writeFileSync(resolve(remote, "extra.md"), "unexpected");
-  assert.throws(() => prepareEvalContext(directory, api), /Unexpected cached skill file/);
-  rmSync(resolve(remote, "extra.md"));
-  write("plugins/fixture-local/skills/another/SKILL.md", "---\nname: another\ndescription: Another skill.\n---\nInstructions.\n");
-  assert.throws(() => checkMarketplaceEnvironment(directory), /must match the catalog/);
-});
-
-test("native Vally resolves the named environment, merges a case fixture and supplies both skills to an executor without a model", async (t) => {
-  const { directory, write, api } = fixture(t);
-  prepareEvalContext(directory, api);
-  const evalDirectory = resolve(directory, "evals/fixture-local/skills/local-notes");
-  write("evals/fixture-local/skills/local-notes/fixture.txt", "case fixture");
-  write("evals/fixture-local/skills/local-notes/eval.yaml", stringify({
-    name: "fixture-context", agent_environment: "marketplace",
-    stimuli: [{ name: "fixture-case", prompt: "Summarize notes.",
-      agent_environment: { files: [{ src: "fixture.txt", dest: "input.txt" }] } }],
-  }));
-  const project = await ProjectContext.load(directory);
-  const spec = await loadEvalSpec(resolve(evalDirectory, "eval.yaml"));
-  const stimulus = resolveStimulus(spec.stimuli[0], project.config.environments[spec.environment],
-    project.config.environments, spec.tags, evalDirectory, spec.grading_environment);
-  let observed;
-  const result = await runEval({
-    prompt: stimulus.prompt, stimulus, environment: stimulus.environment, baseDir: evalDirectory,
-    workDir: directory, skills: [],
-    executor: { name: "test-only-no-model", execute: async (_stimulus, options) => {
-      observed = options.skills.map((skill) => skill.name);
-      assert.equal(readFileSync(resolve(options.workDir, "input.txt"), "utf8"), "case fixture");
-      for (const name of observed) {
-        assert.ok(readFileSync(resolve(options.workDir, name, "SKILL.md"), "utf8").includes(`name: ${name}`));
-      }
-      assert.deepEqual(readFileSync(resolve(options.workDir, "remote-notes/references/example.bin")), Buffer.from([0, 128, 255]));
-      return { output: "No model was called.", workDir: options.workDir, events: [],
-        metadata: { executor: "test-only-no-model" }, metrics: {}, endReason: "completed" };
-    } },
-  });
-  t.after(() => result.cleanup());
-  assert.deepEqual(observed, ["local-notes", "remote-notes"]);
-  assert.equal(result.trajectory.metadata.skillsLoaded, undefined,
-    "native staging is not a live Copilot skills-loaded observation");
-});
-
-test("loaded-context grader requires observed SDK inventory, not answers, paths, activation alone or mock metadata", {
-  skip: !existsSync(resolve(root, "evals/_shared/graders/marketplace-context.mjs")),
-}, async () => {
-  const { evaluate } = await import(pathToFileURL(resolve(root, "evals/_shared/graders/marketplace-context.mjs")));
-  const { score } = await import(pathToFileURL(resolve(root, "evals/_shared/graders/common.mjs")));
-  const expected = ["local-notes", "remote-notes"];
-  const trajectory = { output: "Both skills are available.", events: [] };
-  for (const candidate of [
-    trajectory,
-    { ...trajectory, metadata: { executor: "copilot-sdk", skillsLoaded: ["local-notes"] } },
-    { ...trajectory, metadata: { executor: "mock", skillsLoaded: expected } },
-    { ...trajectory, events: [{ type: "skill_activation", data: { name: "remote-notes" } }] },
-    { ...trajectory, events: [{ type: "tool_call", data: { toolName: "view", arguments: { path: "remote-notes/SKILL.md" } } }] },
-  ]) {
-    assert.equal(score("context", evaluate(candidate, expected)).passed, false);
-  }
-  assert.equal(score("context", evaluate({
-    ...trajectory, metadata: { executor: "copilot-sdk", skillsLoaded: expected },
-  }, expected)).passed, true);
-});
-
-test("all local skill evals use the shared environment and the runtime inventory check covers that environment", async () => {
-  const skills = checkMarketplaceEnvironment();
-  const project = parse(readFileSync(resolve(root, ".vally.yaml"), "utf8"));
-  for (const path of skills.filter((path) => !path.includes(`${resolve(root, ".vally")}`))) {
-    const plugin = basename(resolve(path, "../../"));
-    const spec = await loadEvalSpec(resolve(root, "evals", plugin, "skills", basename(path), "eval.yaml"));
-    assert.equal(spec.environment, "marketplace", spec.name);
-    for (const raw of spec.stimuli) {
-      const stimulus = resolveStimulus(raw, project.environments.marketplace, project.environments);
-      assert.deepEqual(stimulus.environment.skills, project.environments.marketplace.skills);
-    }
-  }
-  const prDescription = resolve(root, "evals/pr-description/skills/pr-description/eval.yaml");
-  if (existsSync(resolve(root, "plugins/pr-description"))) {
-    const spec = await loadEvalSpec(prDescription);
-    const grader = spec.stimuli.flatMap((item) => item.graders).find((item) => item.name === "marketplace-skills-loaded");
-    assert.deepEqual(grader.config.args.slice(1).sort(), skills.map((path) => basename(path)).sort());
-  }
-});
-
-test("native program grading preserves the SDK inventory for the context check without calling a model", {
-  skip: !existsSync(resolve(root, "evals/_shared/graders/marketplace-context.mjs")),
-}, async (t) => {
-  const { directory } = fixture(t);
-  const grader = createDefaultGraderRegistry().get("program");
-  const config = {
-    program: process.execPath,
-    args: [resolve(root, "evals/_shared/graders/marketplace-context.mjs"), "local-notes", "remote-notes"],
-  };
-  for (const [skillsLoaded, passed] of [
-    [["local-notes", "remote-notes"], true],
-    [["local-notes"], false],
-  ]) {
-    const result = await grader.grade({
-      config, workDir: directory, stimulus: { name: "synthetic-inventory", prompt: "Synthetic evidence only." },
-      trajectory: { output: "Synthetic evidence only.", events: [], workDir: directory,
-        metadata: { executor: "copilot-sdk", skillsLoaded } },
-    });
-    assert.equal(result.passed, passed, result.evidence);
-  }
-});
diff --git a/.github/scripts/prepare-eval-context.mjs b/.github/scripts/prepare-eval-context.mjs
deleted file mode 100644
index 1062964..0000000
--- a/.github/scripts/prepare-eval-context.mjs
+++ /dev/null
@@ -1,120 +0,0 @@
-import { createHash, randomUUID } from "node:crypto";
-import { existsSync, mkdirSync, readFileSync, readdirSync, realpathSync, renameSync, rmSync, writeFileSync } from "node:fs";
-import { basename, dirname, isAbsolute, relative, resolve, sep } from "node:path";
-import { fileURLToPath } from "node:url";
-import { parse } from "yaml";
-import { checkRemotePin, githubApi } from "./check-remote-pins.mjs";
-
-const root = fileURLToPath(new URL("../../", import.meta.url));
-const readJson = (file) => JSON.parse(readFileSync(file, "utf8"));
-const gitHash = (bytes) => createHash("sha1").update(`blob ${bytes.length}\0`).update(bytes).digest("hex");
-
-function inside(directory, path) {
-  const result = resolve(directory, path);
-  const rel = relative(directory, result);
-  if (!rel || isAbsolute(rel) || rel === ".." || rel.startsWith(`..${sep}`)) {
-    throw new Error(`Expected a path inside ${directory}: ${path}`);
-  }
-  return result;
-}
-
-function localSkills(directory) {
-  const skills = resolve(directory, "skills");
-  if (!existsSync(skills)) return [];
-  return readdirSync(skills, { withFileTypes: true })
-    .filter((entry) => entry.isDirectory())
-    .map((entry) => {
-      const path = resolve(skills, entry.name);
-      if (!existsSync(resolve(path, "SKILL.md"))) throw new Error(`Missing SKILL.md: ${path}`);
-      return path;
-    });
-}
-
-export function checkMarketplaceEnvironment(directory = root, remoteSkills = undefined) {
-  const catalog = readJson(resolve(directory, "marketplace.json"));
-  const config = parse(readFileSync(resolve(directory, ".vally.yaml"), "utf8"));
-  const configured = config.environments?.marketplace?.skills;
-  if (!Array.isArray(configured)) throw new Error("Missing environments.marketplace.skills in .vally.yaml.");
-  const evalDirectory = resolve(directory, "evals/_plugin/skills/_skill");
-  const paths = configured.map((path) => resolve(evalDirectory, path));
-  const expected = [];
-  for (const entry of catalog.plugins) {
-    if (typeof entry.source === "string") {
-      expected.push(...localSkills(inside(directory, entry.source)));
-    } else {
-      const { source, repo, path = "", sha } = entry.source ?? {};
-      if (source !== "github" || !/^[\w.-]+\/[\w.-]+$/.test(repo ?? "") || !/^[a-f0-9]{40}$/.test(sha ?? "")) {
-        throw new Error(`${entry.name}: expected a GitHub source pinned to a full commit SHA.`);
-      }
-      if (!/^[a-z0-9][a-z0-9-]*$/.test(entry.name)
-        || (path && (!/^[\w.-]+(?:\/[\w.-]+)*$/.test(path) || path.split("/").some((part) => [".", ".."].includes(part))))) {
-        throw new Error(`${entry.name}: invalid remote plugin name or source path.`);
-      }
-      const skillsRoot = resolve(directory, ".vally/marketplace", entry.name, sha, "skills");
-      expected.push(...(remoteSkills
-        ? remoteSkills.get(entry.name)
-        : paths.filter((candidate) => dirname(candidate) === skillsRoot)));
-    }
-  }
-  if (new Set(paths.map((path) => basename(path))).size !== paths.length) throw new Error("Duplicate marketplace skill names.");
-  if (paths.length !== expected.length || paths.some((path) => !expected.includes(path))) {
-    throw new Error("marketplace.skills must match the catalog's skill directories and remote pins. Update .vally.yaml.");
-  }
-  return paths;
-}
-
-export function prepareEvalContext(directory = root, api = githubApi) {
-  checkMarketplaceEnvironment(directory);
-  const remoteSkills = new Map();
-  for (const entry of readJson(resolve(directory, "marketplace.json")).plugins) {
-    if (typeof entry.source === "string") continue;
-    checkRemotePin(entry, api);
-    const { repo, path = "", sha } = entry.source;
-    const response = api(`repos/${repo}/git/trees/${sha}?recursive=1`);
-    if (response.status !== 200) throw new Error(`${entry.name}: couldn't read the pinned skill tree (HTTP ${response.status}).`);
-    const tree = JSON.parse(response.body);
-    if (tree.truncated) throw new Error(`${entry.name}: GitHub truncated the tree; skill completeness is unknown.`);
-    const prefix = `${path ? `${path}/` : ""}skills/`;
-    const files = tree.tree.filter((item) => item.type === "blob" && item.path.startsWith(prefix));
-    const destination = resolve(directory, ".vally/marketplace", entry.name, sha, "skills");
-    const expectedFiles = files.map((item) => inside(destination, item.path.slice(prefix.length)));
-    for (const [index, item] of files.entries()) {
-      if (!["100644", "100755"].includes(item.mode)) throw new Error(`${entry.name}: unsupported skill file mode at ${item.path}.`);
-      const file = expectedFiles[index];
-      if (existsSync(file)) {
-        if (gitHash(readFileSync(file)) !== item.sha) throw new Error(`Modified cached skill source: ${file}. Remove that cache and prepare again.`);
-        continue;
-      }
-      const blob = api(`repos/${repo}/git/blobs/${item.sha}`, undefined, "application/vnd.github+json");
-      if (blob.status !== 200) throw new Error(`${entry.name}: couldn't fetch ${item.path} (HTTP ${blob.status}).`);
-      const content = JSON.parse(blob.body);
-      if (content.encoding !== "base64") throw new Error(`${entry.name}: unexpected blob encoding at ${item.path}.`);
-      const bytes = Buffer.from(content.content, "base64");
-      if (gitHash(bytes) !== item.sha) throw new Error(`${entry.name}: content hash mismatch at ${item.path}.`);
-      mkdirSync(dirname(file), { recursive: true });
-      const temporary = `${file}.${randomUUID()}.tmp`;
-      try {
-        writeFileSync(temporary, bytes, { mode: item.mode === "100755" ? 0o755 : 0o644 });
-        renameSync(temporary, file);
-      } finally {
-        rmSync(temporary, { force: true });
-      }
-    }
-    if (existsSync(destination)) {
-      for (const file of readdirSync(destination, { recursive: true, withFileTypes: true })) {
-        if (file.isDirectory()) continue;
-        const absolute = resolve(file.parentPath, file.name);
-        if (!expectedFiles.includes(absolute)) throw new Error(`Unexpected cached skill file: ${absolute}. Remove that cache and prepare again.`);
-      }
-    }
-    remoteSkills.set(entry.name, files
-      .filter((item) => /^[^/]+\/SKILL\.md$/.test(item.path.slice(prefix.length)))
-      .map((item) => dirname(inside(destination, item.path.slice(prefix.length)))));
-  }
-  return checkMarketplaceEnvironment(directory, remoteSkills);
-}
-
-if (process.argv[1] && realpathSync(fileURLToPath(import.meta.url)) === realpathSync(resolve(process.argv[1]))) {
-  const skills = prepareEvalContext(process.cwd());
-  console.log(`Prepared ${skills.length} marketplace skills. No plugins installed or evals run.`);
-}
diff --git a/.github/scripts/validation.test.mjs b/.github/scripts/validation.test.mjs
index d967efb..ba64bdf 100644
--- a/.github/scripts/validation.test.mjs
+++ b/.github/scripts/validation.test.mjs
@@ -385,8 +385,6 @@ test("validation calls CI scripts, and paid eval activation is manual-only", ()
   assert.equal(job.steps.find((step) => step.uses?.startsWith("actions/checkout@")).with["persist-credentials"], false);
   assert.deepEqual(JSON.parse(readFileSync(resolve(root, "package.json"), "utf8")).scripts, {
     lint: "node .github/scripts/lint.mjs",
-    "eval:prepare": "node .github/scripts/prepare-eval-context.mjs",
-    preeval: "npm run eval:prepare",
     eval: "vally eval --workers 1 --max-retries 1",
     "eval:report": "node .github/scripts/eval-report.mjs",
     "eval:compare": "node .github/scripts/eval-report.mjs",
diff --git a/.vally.yaml b/.vally.yaml
index 0e11e0a..948ef94 100644
--- a/.vally.yaml
+++ b/.vally.yaml
@@ -11,7 +11,7 @@ paths:
 environments:
   marketplace:
     # Vally resolves these paths relative to each eval, not this file.
-    # npm run eval:prepare fetches the remote skills without installing plugins.
+    # Remote skill directories must already exist at the catalog-pinned commit.
     skills:
       - ../../../../plugins/plugin-authoring/skills/add-plugin
       - ../../../../plugins/plugin-authoring/skills/try-plugin-locally
diff --git a/README.md b/README.md
index 7d39831..3ca8f26 100644
--- a/README.md
+++ b/README.md
@@ -177,9 +177,8 @@ The `release-plugins` skill can retire a plugin for you. By hand, delete
 `plugins/<name>/` and `evals/<name>/`, then remove the plugin's entry from
 `marketplace.json` and its row and install command from this README. If it's
 one of the two plugins in `.github/copilot/settings.json`, remove its key there
-too. Remove its skills from the shared `marketplace` environment in
-[.vally.yaml](.vally.yaml), and from surviving evals' `disallowed` lists and
-loaded-skill checks.
+too. If surviving cohort evals under `evals/` list the retired plugin's skills
+in `agent_environment.skills` or `disallowed`, remove those entries as well.
 
 You can keep only remote entries by removing both bundled local plugins and
 their eval folders. No empty `plugins/` or `evals/` directory is needed.
@@ -197,15 +196,11 @@ validation, catalog consistency checks, and remote-pin checks on every pull
 request.
 
 **Paid evals:** `npm run eval` runs real Copilot sessions and costs AI credits.
-The six skill evals share the `marketplace` environment in
-[.vally.yaml](.vally.yaml), with all local and pinned remote marketplace skills
-available. Before Vally starts, npm fetches the remote skill sources into ignored
-storage; it does not install plugins or enable their MCP servers or extensions.
 [Evals](.github/workflows/evals.yml) runs only when manually started; pull
 requests never trigger paid evals.
 
-See [Checks and evals](docs/checks-and-evals.md) for the shared environment,
-remote-source preparation, CI details, and Bash and PowerShell eval setup.
+See [Checks and evals](docs/checks-and-evals.md) for CI details, local test
+commands, and Bash and PowerShell eval setup.
 
 ## License
 
diff --git a/docs/checks-and-evals.md b/docs/checks-and-evals.md
index 2e218ac..a4d6988 100644
--- a/docs/checks-and-evals.md
+++ b/docs/checks-and-evals.md
@@ -12,10 +12,7 @@ npm run lint
 
 `npm run lint` checks local catalog consistency, then checks every skill under
 `plugins/` and every eval under `evals/` with the pinned
-[Vally](https://microsoft.github.io/vally/) CLI. It also checks the shared
-environment's local skill coverage and remote pin paths without accessing the
-network. Remote skill completeness is checked during preparation below.
-A remote-only marketplace can
+[Vally](https://microsoft.github.io/vally/) CLI. A remote-only marketplace can
 omit both directories; schema validation also skips when no local manifests
 remain. Regression tests for the bundled maintenance plugins run only while
 those local plugins remain. Missing files for a retained plugin still fail.
@@ -72,54 +69,6 @@ call a model or prove how a plugin behaves in a model session.
 
 **These runs start real Copilot sessions and spend AI credits.**
 
-### Shared marketplace environment
-
-The six skill evals reference `agent_environment: marketplace`. Its definition in
-[.vally.yaml](../.vally.yaml) lists all six local skills and the six skills from
-the catalog-pinned remote Evals plugin. Vally merges that skill set with each
-case's existing fixtures; grading files stay separate. The independent
-`plugin-reviewer` delegation eval is unchanged.
-
-`npm run eval` first runs `npm run eval:prepare`. Preparation makes read-only
-GitHub API requests through `gh`, checks remote pins and fetches only skill
-sources and their supporting files into ignored `.vally/marketplace/` storage.
-It calls no model and does not install plugins, change profiles, run fetched
-code, or start plugin MCP servers, hooks or extensions. Use a signed-in `gh`
-account that can read the remote repositories. Review the pinned skill sources
-before approving an eval; an eval agent can follow their instructions and use
-tools.
-
-Preparation fails rather than omitting a missing skill, using a moving ref,
-accepting a truncated tree, or reusing modified cached sources. Verified files
-are reused at their exact commit. If the cache was changed, remove only the
-reported `.vally/marketplace/<plugin>/<sha>/` cache and prepare it again.
-Preparation alone is model-free:
-
-```sh
-npm run eval:prepare
-```
-
-Run this before invoking Vally directly instead of through `npm run eval`.
-When adding or removing a skill, update the shared environment. When changing a
-remote pin, update its paths to the new SHA and list all skills at that commit.
-Update the `marketplace-skills-loaded` grader's expected names in the
-`pr-description` eval too. Source paths are relative to the eval directory,
-not `.vally.yaml`; the six skill evals currently share the same directory depth.
-
-That positive case checks the native Copilot SDK trajectory's
-`metadata.skillsLoaded` inventory for every marketplace skill, alongside its
-existing exact target-invocation check. Listed paths and model answers cannot
-satisfy the loaded-skill check. Model-free tests exercise native Vally staging
-with a test executor and synthetic grader inputs, not a live Copilot session.
-An approved paid run is still needed to demonstrate actual runtime loading.
-The check's result is stored with the existing grader results.
-
-This is a shared **skills** environment, not full plugin-runtime activation.
-It does not establish availability of plugin agents, MCP tools, hooks, extensions
-or interactive canvases.
-
-### Running an eval
-
 Skill evals use Vally's built-in `skill-invocation` grader: `required` for every
 positive and boundary case, and `disallowed` for negative cases. Invocation
 credit requires a matching `skill_activation` event. Successful file reads,
diff --git a/evals/_shared/graders/marketplace-context.mjs b/evals/_shared/graders/marketplace-context.mjs
deleted file mode 100644
index cedc1e1..0000000
--- a/evals/_shared/graders/marketplace-context.mjs
+++ /dev/null
@@ -1,14 +0,0 @@
-import { check, isMain, runGrade } from "./common.mjs";
-
-export function evaluate(trajectory, expected) {
-  const loaded = trajectory.metadata?.skillsLoaded;
-  return [
-    check("Context evidence comes from the native Copilot SDK executor", trajectory.metadata?.executor === "copilot-sdk"),
-    check("Every marketplace skill was reported loaded by the session",
-      expected.length > 0 && Array.isArray(loaded) && expected.every((name) => loaded.includes(name))),
-  ];
-}
-
-if (isMain(import.meta.url)) {
-  await runGrade("marketplace-context", (trajectory) => evaluate(trajectory, process.argv.slice(2)));
-}
diff --git a/evals/pr-description/skills/pr-description/eval.yaml b/evals/pr-description/skills/pr-description/eval.yaml
index 74d7baf..dee5500 100644
--- a/evals/pr-description/skills/pr-description/eval.yaml
+++ b/evals/pr-description/skills/pr-description/eval.yaml
@@ -13,8 +13,6 @@ grading_environment:
       dest: .grading/grader.mjs
     - src: ../../../_shared/graders/common.mjs
       dest: .grading/common.mjs
-    - src: ../../../_shared/graders/marketplace-context.mjs
-      dest: .grading/marketplace-context.mjs
 stimuli:
   - name: pr-description-drafts-structured-pr-body
     prompt: >-
@@ -38,24 +36,6 @@ stimuli:
         config:
           program: node
           args: [.grading/grader.mjs]
-      - type: program
-        name: marketplace-skills-loaded
-        config:
-          program: node
-          args:
-            - .grading/marketplace-context.mjs
-            - add-plugin
-            - try-plugin-locally
-            - submit-plugin
-            - update-remote-pins
-            - release-plugins
-            - pr-description
-            - compare
-            - create
-            - improve
-            - review
-            - run
-            - setup
       - type: diff-empty
   - name: pr-description-ignores-bug-and-review-question
     prompt: >-
diff --git a/package.json b/package.json
index bdc3818..57c490c 100644
--- a/package.json
+++ b/package.json
@@ -6,8 +6,6 @@
   "engines": { "node": ">=22.14.0" },
   "scripts": {
     "lint": "node .github/scripts/lint.mjs",
-    "eval:prepare": "node .github/scripts/prepare-eval-context.mjs",
-    "preeval": "npm run eval:prepare",
     "eval": "vally eval --workers 1 --max-retries 1",
     "eval:report": "node .github/scripts/eval-report.mjs",
     "eval:compare": "node .github/scripts/eval-report.mjs"

From 698e510ad0b4c6f19b06b25617f012f9bc0567ce Mon Sep 17 00:00:00 2001
From: Ronnie Geraghty <rgeraghty@microsoft.com>
Date: Tue, 10 Nov 2026 17:17:33 -0800
Subject: [PATCH 3/3] Use native Vally setup for pinned remote skills

Stage remote Evals skills through shared environment commands and native workspace discovery, without separate preparation infrastructure.

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
---
 .github/scripts/eval-report.test.mjs |  2 +-
 .vally.yaml                          | 14 +++++++-------
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/.github/scripts/eval-report.test.mjs b/.github/scripts/eval-report.test.mjs
index 4ef3318..6286a10 100644
--- a/.github/scripts/eval-report.test.mjs
+++ b/.github/scripts/eval-report.test.mjs
@@ -396,7 +396,7 @@ prDescriptionTest("pr-description eval stages the multi-skill marketplace cohort
   const project = parse(readFileSync(resolve(root, ".vally.yaml"), "utf8"));
   assert.equal(spec.agent_environment, "marketplace");
   const skills = project.environments.marketplace.skills;
-  for (const configuredSkill of skills.filter((path) => path.includes("/plugins/"))) {
+  for (const configuredSkill of skills) {
     assert.ok(existsSync(resolve(evalDir, configuredSkill)), `configured cohort skill path is missing: ${configuredSkill}`);
   }
   for (const expectedSkill of [
diff --git a/.vally.yaml b/.vally.yaml
index 948ef94..02efee6 100644
--- a/.vally.yaml
+++ b/.vally.yaml
@@ -11,7 +11,6 @@ paths:
 environments:
   marketplace:
     # Vally resolves these paths relative to each eval, not this file.
-    # Remote skill directories must already exist at the catalog-pinned commit.
     skills:
       - ../../../../plugins/plugin-authoring/skills/add-plugin
       - ../../../../plugins/plugin-authoring/skills/try-plugin-locally
@@ -19,12 +18,13 @@ environments:
       - ../../../../plugins/marketplace-maintainer/skills/update-remote-pins
       - ../../../../plugins/marketplace-maintainer/skills/release-plugins
       - ../../../../plugins/pr-description/skills/pr-description
-      - ../../../../.vally/marketplace/eval/34ddb5803aeaee8330ab70c19ad1a3f37fb77ded/skills/compare
-      - ../../../../.vally/marketplace/eval/34ddb5803aeaee8330ab70c19ad1a3f37fb77ded/skills/create
-      - ../../../../.vally/marketplace/eval/34ddb5803aeaee8330ab70c19ad1a3f37fb77ded/skills/improve
-      - ../../../../.vally/marketplace/eval/34ddb5803aeaee8330ab70c19ad1a3f37fb77ded/skills/review
-      - ../../../../.vally/marketplace/eval/34ddb5803aeaee8330ab70c19ad1a3f37fb77ded/skills/run
-      - ../../../../.vally/marketplace/eval/34ddb5803aeaee8330ab70c19ad1a3f37fb77ded/skills/setup
+    # Vally runs setup in each trial workspace; Copilot discovers these skills natively.
+    commands:
+      - git clone --filter=blob:none --no-checkout --depth 1 --branch copilot-evals-v0.24.0 --sparse https://github.com/github/evals.git .vally-eval-source
+      - git -C .vally-eval-source sparse-checkout set packages/plugin/skills
+      - git -C .vally-eval-source checkout --detach 34ddb5803aeaee8330ab70c19ad1a3f37fb77ded
+      - >-
+        node -e "const fs = require('node:fs'); fs.cpSync('.vally-eval-source/packages/plugin/skills', '.github/skills', { recursive: true }); fs.rmSync('.vally-eval-source', { recursive: true });"
 suites:
   smoke:
     description: One representative case for each skill and agent