diff --git a/.github/scripts/validation.test.mjs b/.github/scripts/validation.test.mjs index ba64bdf..6063235 100644 --- a/.github/scripts/validation.test.mjs +++ b/.github/scripts/validation.test.mjs @@ -386,6 +386,7 @@ test("validation calls CI scripts, and paid eval activation is manual-only", () assert.deepEqual(JSON.parse(readFileSync(resolve(root, "package.json"), "utf8")).scripts, { lint: "node .github/scripts/lint.mjs", eval: "vally eval --workers 1 --max-retries 1", + "eval:models": "vally experiment run evals/standard-models.experiment.yaml --output-dir evals/results/standard-models --require-pass", "eval:report": "node .github/scripts/eval-report.mjs", "eval:compare": "node .github/scripts/eval-report.mjs", }); diff --git a/README.md b/README.md index 3ca8f26..ff6cdf4 100644 --- a/README.md +++ b/README.md @@ -199,6 +199,14 @@ request. [Evals](.github/workflows/evals.yml) runs only when manually started; pull requests never trigger paid evals. +**Standard models:** `npm run eval:models` uses the native +[model experiment](evals/standard-models.experiment.yaml) to test every retained +skill and agent eval with Claude Sonnet 5.5 and GPT-6.1 Terra. These are starter +defaults: during setup, customize `matrix.model.values` and its `baseline.model` +to the models you want to test and can access. Preview without model calls with +`npm run eval:models -- --dry-run`. Individual evals and Actions keep their +existing single-model defaults; the experiment does not override them globally. + See [Checks and evals](docs/checks-and-evals.md) for CI details, local test commands, and Bash and PowerShell eval setup. diff --git a/docs/checks-and-evals.md b/docs/checks-and-evals.md index 3604cec..84b6b7a 100644 --- a/docs/checks-and-evals.md +++ b/docs/checks-and-evals.md @@ -79,6 +79,34 @@ side effects. The `plugin-reviewer` eval needs the full Copilot CLI installed by `npm ci`. Point `COPILOT_CLI_PATH` at that pinned CLI before running the smoke suite. +### Standard model roster + +[The native experiment](../evals/standard-models.experiment.yaml) is the single +source for the standard model roster: `claude-sonnet-5.5` and `gpt-6.1-terra`. +When adopting the template, replace `matrix.model.values` with your own model +IDs and set `baseline.model` to one of them. Vally requires at least two variants; +for a single model, use `npm run eval -- --model ` instead. + +After setting `COPILOT_CLI_PATH` as shown below, preview the selected evals, +variants and configuration hashes without running setup, Docker or a model: + +```sh +npm run eval:models -- --dry-run +``` + +`npm run eval:models` executes the planned variants and spends AI credits for +each case under each model. The two-model roster is not a guarantee of account +access or a quality claim. The baseline identifies the experiment's control, +not a preferred winner; the command does not invoke an extra comparison judge. +Only the model varies, preserving the evals, graders, fixtures and shared +marketplace environment. The existing single-model commands and Actions do +not consume this experiment. + +Vally 0.19 experiments have a separate output flag rather than reading +`paths.results`. The npm command explicitly writes timestamped per-variant +results under `evals/results/standard-models/`. If you customize the output +location, update the command's `--output-dir` too. + The `try-plugin-locally` smoke case prepares a fresh-session `--plugin-dir` command for the selected plugin in a developer's existing checkout. Vally already stages the candidate skills. The case checks the selected absolute diff --git a/evals/standard-models.experiment.yaml b/evals/standard-models.experiment.yaml new file mode 100644 index 0000000..0858834 --- /dev/null +++ b/evals/standard-models.experiment.yaml @@ -0,0 +1,14 @@ +name: standard-models +evals: + - "*/skills/*/eval.yaml" + - "*/agents/*/eval.yaml" +matrix: + model: + path: /defaults/model + values: + - claude-sonnet-5.5 + - gpt-6.1-terra +baseline: + model: claude-sonnet-5.5 +execution: + workers: 1 diff --git a/package.json b/package.json index 57c490c..d206d88 100644 --- a/package.json +++ b/package.json @@ -7,6 +7,7 @@ "scripts": { "lint": "node .github/scripts/lint.mjs", "eval": "vally eval --workers 1 --max-retries 1", + "eval:models": "vally experiment run evals/standard-models.experiment.yaml --output-dir evals/results/standard-models --require-pass", "eval:report": "node .github/scripts/eval-report.mjs", "eval:compare": "node .github/scripts/eval-report.mjs" },