From c9652371396d12b7aea61cd2c707991035bb6b4f Mon Sep 17 00:00:00 2001 From: Andriy Svyryd Date: Tue, 15 Sep 2026 18:29:11 -0700 Subject: [PATCH] Add a Vally-based evaluation harness for repository-owned Copilot customizations. Discovers and validates instructions, skills, agents, prompts, and agentic workflows by convention. Runs paired treatment and unskilled-control evaluations with token grading and Vally comparison. Selectively evaluates changed components and publishes progressive PR results. Restricts secret-backed runs to trusted PR authors; later commits require manual reruns. Replace repository MCP configuration and duplicated skills with plugins. Update skill guidance and add matching repository-grounded evaluations. Add Dependabot coverage for Vally dependencies. --- .agents/skills/change-tracking/SKILL.md | 13 +- .agents/skills/make-custom-agent/SKILL.md | 3 +- .agents/skills/make-instructions/SKILL.md | 32 +- .agents/skills/make-skill/SKILL.md | 58 +- .../make-skill/references/testing-patterns.md | 15 +- .agents/skills/migrations/SKILL.md | 5 +- .agents/skills/run-apichief/SKILL.md | 2 +- .agents/skills/servicing-pr/SKILL.md | 3 +- .agents/skills/sqlite-adonet/SKILL.md | 3 + .agents/skills/testing/SKILL.md | 94 - .claude/settings.json | 15 +- .github/copilot-instructions.md | 8 + .github/copilot/settings.json | 21 +- .github/dependabot.yml | 12 + .github/workflows/agent-harness-coverage.yml | 56 + .../workflows/agent-harness-evaluation.yml | 543 +++++ .vscode/mcp.json | 48 - eng/harness-evaluation/README.md | 98 + .../copilot-instructions/eval.yaml | 83 + eng/harness-evaluation/package-lock.json | 2090 +++++++++++++++++ eng/harness-evaluation/package.json | 22 + .../skills/change-tracking/eval.yaml | 59 + .../skills/cosmos-provider/eval.yaml | 65 + .../skills/make-custom-agent/eval.yaml | 59 + .../make-github-actions-workflow/eval.yaml | 64 + .../skills/make-instructions/eval.yaml | 66 + .../skills/make-skill/eval.yaml | 63 + .../skills/migrations/eval.yaml | 66 + .../skills/model-building/eval.yaml | 52 + .../skills/query-pipeline/eval.yaml | 52 + .../skills/run-apichief/eval.yaml | 51 + .../skills/scaffolding/eval.yaml | 52 + .../skills/servicing-pr/eval.yaml | 56 + .../skills/sqlite-adonet/eval.yaml | 55 + .../skills/tooling/eval.yaml | 52 + .../skills/triage/eval.yaml | 79 + .../skills/update-pipeline/eval.yaml | 54 + eng/harness-evaluation/src/cli.mjs | 225 ++ eng/harness-evaluation/src/harness.mjs | 532 +++++ eng/harness-evaluation/test/harness.test.mjs | 476 ++++ 40 files changed, 5218 insertions(+), 184 deletions(-) delete mode 100644 .agents/skills/testing/SKILL.md create mode 100644 .github/workflows/agent-harness-coverage.yml create mode 100644 .github/workflows/agent-harness-evaluation.yml delete mode 100644 .vscode/mcp.json create mode 100644 eng/harness-evaluation/README.md create mode 100644 eng/harness-evaluation/instructions/copilot-instructions/eval.yaml create mode 100644 eng/harness-evaluation/package-lock.json create mode 100644 eng/harness-evaluation/package.json create mode 100644 eng/harness-evaluation/skills/change-tracking/eval.yaml create mode 100644 eng/harness-evaluation/skills/cosmos-provider/eval.yaml create mode 100644 eng/harness-evaluation/skills/make-custom-agent/eval.yaml create mode 100644 eng/harness-evaluation/skills/make-github-actions-workflow/eval.yaml create mode 100644 eng/harness-evaluation/skills/make-instructions/eval.yaml create mode 100644 eng/harness-evaluation/skills/make-skill/eval.yaml create mode 100644 eng/harness-evaluation/skills/migrations/eval.yaml create mode 100644 eng/harness-evaluation/skills/model-building/eval.yaml create mode 100644 eng/harness-evaluation/skills/query-pipeline/eval.yaml create mode 100644 eng/harness-evaluation/skills/run-apichief/eval.yaml create mode 100644 eng/harness-evaluation/skills/scaffolding/eval.yaml create mode 100644 eng/harness-evaluation/skills/servicing-pr/eval.yaml create mode 100644 eng/harness-evaluation/skills/sqlite-adonet/eval.yaml create mode 100644 eng/harness-evaluation/skills/tooling/eval.yaml create mode 100644 eng/harness-evaluation/skills/triage/eval.yaml create mode 100644 eng/harness-evaluation/skills/update-pipeline/eval.yaml create mode 100644 eng/harness-evaluation/src/cli.mjs create mode 100644 eng/harness-evaluation/src/harness.mjs create mode 100644 eng/harness-evaluation/test/harness.test.mjs diff --git a/.agents/skills/change-tracking/SKILL.md b/.agents/skills/change-tracking/SKILL.md index c7ef97dbe3a..5ce64693ddb 100644 --- a/.agents/skills/change-tracking/SKILL.md +++ b/.agents/skills/change-tracking/SKILL.md @@ -10,12 +10,19 @@ Manages entity states and detects changes for `SaveChanges()`. ## Core Components -- `StateManager` — central engine, identity maps, tracks all entities -- `InternalEntityEntry` — per-entity state, property flags, snapshots -- `SnapshotFactoryFactory` subclasses build snapshot factories for change detection +- `StateManager` — owns tracked entries, identity/reference maps, fixup, cascades, notifications, and changed counts +- `InternalEntryBase` and derived classes — own per-entry state, flags, values, snapshots, and ordered state transitions +- `ChangeDetector` — compares current values with baselines and reports changes through entry mutation APIs +- `SnapshotFactoryFactory` subclasses — create passive original- and relationship-value baselines +- `IdentityMap` — permits one active entry per key; shared identity pairs a replacement with the prior `Deleted` entry - `PropertyAccessorsFactory`, `ClrPropertyGetterFactory` and `ClrPropertySetterFactory` compile property accessors for efficient snapshotting and change detection - Ordinals in `indices` parameter specify element at each complex collection depth +## Change Detection + +- Snapshot and notification strategies both call `SetPropertyModified()` to keep property flags and entity state consistent. +- `SetEntityState()` validates values, updates flags and complex entries, changes state, then runs manager bookkeeping hooks. + ## Testing Unit tests: `test/EFCore.Tests/ChangeTracking/`. Functional tests: `test/EFCore.Specification.Tests/GraphUpdates/`. diff --git a/.agents/skills/make-custom-agent/SKILL.md b/.agents/skills/make-custom-agent/SKILL.md index e9c1f81a7e6..8c77ab0fa00 100644 --- a/.agents/skills/make-custom-agent/SKILL.md +++ b/.agents/skills/make-custom-agent/SKILL.md @@ -242,5 +242,4 @@ After creating or modifying an agent, verify: - [GitHub Copilot Extensions documentation](https://docs.github.com/en/copilot/building-copilot-extensions/about-building-copilot-extensions) - [GitHub Copilot Custom agents configuration](https://docs.github.com/en/copilot/reference/custom-agents-configuration) - [Agent Skills Specification](https://agentskills.io/specification) -- [make-skill](../make-skill/SKILL.md) -- [make-instructions](../make-instructions/SKILL.md) +- Related repository skills: `make-skill` and `make-instructions` diff --git a/.agents/skills/make-instructions/SKILL.md b/.agents/skills/make-instructions/SKILL.md index 551dc3eddd6..655ae6d517b 100644 --- a/.agents/skills/make-instructions/SKILL.md +++ b/.agents/skills/make-instructions/SKILL.md @@ -1,6 +1,6 @@ --- name: make-instructions -description: 'Create VS Code file-based instructions (.instructions.md files). Use when asked to create, scaffold, or add file-based instructions for Copilot. Generates .instructions.md with YAML frontmatter and background knowledge content.' +description: 'Create and evaluate VS Code file-based instructions (.instructions.md files). Use when asked to create, scaffold, or add file-based instructions for Copilot. Generates scoped instructions and a paired Vally harness eval.' --- # Create File-Based Instructions @@ -24,6 +24,10 @@ Build understanding of the area the instructions should cover. Identify: - [ ] Common pitfalls that Copilot should avoid - [ ] Non-obvious domain knowledge that isn't discoverable from code alone +Read the repository-wide instruction files that apply to the same paths and make an explicit exclusion list. Do not +repeat those rules in the new file, even when they are relevant examples for the scoped area; include only guidance that +becomes more specific or materially different at the narrower scope. + If the scope is unclear or overlaps with existing instructions, ask the user for clarification. ### Step 2: Choose the file location @@ -73,7 +77,30 @@ Recommended sections (adapt as needed): 5. **Key Files** — table of important files for orientation (optional) 6. **Common Pitfalls** — traps to avoid (optional) -### Step 5: Validate +### Step 5: Author the harness evaluation + +Create `eng/harness-evaluation/instructions//eval.yaml`, where `` is the instruction path relative to +`.github/instructions/` with its suffix removed and nested path separators replaced by `--`. + +- Use the same ID for the eval `name` and set `defaults.runs: 5` with the repository's configured executor and judge. +- Add bounded turn, token, and duration constraints. +- Ground each stimulus in at least one real repository path outside the customization and evaluation directories; name that path in the prompt and ask the agent to inspect, explain, modify, or validate it. +- Exercise guidance distinctive to the instruction file. Prefer deterministic output graders, with a narrow semantic rubric for behavior that cannot be checked mechanically. +- Commit a meaningful `scoring.threshold`. The harness runs treatment and omitted-instruction control arms, so the task should discriminate between them. + +### Step 6: Validate with the harness + +Run: + +```powershell +npm --prefix eng/harness-evaluation run validate +npm --prefix eng/harness-evaluation run lint +npm --prefix eng/harness-evaluation run eval -- --require-pass +``` + +The first command must discover the instruction and same-ID eval, the second must pass strict Vally lint, and the final +command must pass the committed threshold and produce a treatment-versus-control comparison. Review its quality verdict +and token delta. After creating the file, verify: @@ -84,6 +111,7 @@ After creating the file, verify: - [ ] Content is concise (aim for under 500 lines or 5000 tokens) — long instructions dilute effectiveness - [ ] No secrets, tokens, or internal URLs included - [ ] Instructions don't duplicate what's already in `.github/copilot-instructions.md` or under `.agents/skills/` +- [ ] The paired eval demonstrates behavior that the omitted-instruction control does not provide reliably ## Common Pitfalls diff --git a/.agents/skills/make-skill/SKILL.md b/.agents/skills/make-skill/SKILL.md index 9b8ec3dc1ba..2f5649e3008 100644 --- a/.agents/skills/make-skill/SKILL.md +++ b/.agents/skills/make-skill/SKILL.md @@ -1,6 +1,6 @@ --- name: make-skill -description: 'Create new Agent Skills for GitHub Copilot. Use when asked to create, scaffold, or add a skill. Generates SKILL.md with frontmatter, directory structure, and optional resources.' +description: 'Create and evaluate new Agent Skills for GitHub Copilot. Use when asked to create, scaffold, or add a skill. Generates SKILL.md, optional resources, and a paired Vally harness eval.' --- # Create Skill @@ -92,28 +92,41 @@ Include these recommended sections, following this file's structure: > ❌ **NEVER** count API failures as success. Return "Unknown" and exclude from positive counts. -### Step 7: Validate the skill - -Ensure the name: -- Does not start or end with a hyphen -- Does not contain consecutive hyphens -- Is between 1-64 characters -- YAML frontmatter name matches directory name exactly - -After creating a skill, verify: -- [ ] frontmatter fields are valid -- [ ] SKILL.md is under 500 lines and 5000 tokens, split into references if needed -- [ ] File references use relative paths -- [ ] Instructions are actionable and specific -- [ ] Instructions don't duplicate what's already in `.github/copilot-instructions.md` or under `.github/instructions/` -- [ ] Workflow has numbered steps with clear checkpoints -- [ ] Validation section exists with observable success criteria -- [ ] No secrets, tokens, or internal URLs included -- [ ] Common pitfalls are relevant and have solutions +### Step 7: Author the harness evaluation + +Create `eng/harness-evaluation/skills//eval.yaml` with: + +- A matching eval `name`, `defaults.runs: 5`, bounded turns/tokens/duration, and the repository's configured executor and judge. +- At least one real repository anchor in `tags.repo-path` outside the skill and evaluation directories. Name an anchor in the prompt and ask the agent to inspect, explain, modify, or validate it. +- A task that needs the skill's distinctive guidance, an exact `skill-invocation` requirement for ``, deterministic output graders where possible, and a narrow semantic rubric for the remainder. +- A committed `scoring.threshold` that reflects the intended quality gate. The harness runs both skilled and unskilled arms with Vally compare, so do not weaken the stimulus merely to make the control pass. + +### Step 8: Validate with the harness + +Use the repository harness scripts as the authoritative validation: + +```powershell +npm --prefix eng/harness-evaluation run validate +npm --prefix eng/harness-evaluation run lint +npm --prefix eng/harness-evaluation run eval -- --require-pass +``` + +The first command must discover the skill and its same-name eval without missing or orphaned coverage. The second must pass strict Vally lint. The final command must pass the eval's committed threshold and produce a treatment-versus-unskilled-control comparison; review its quality verdict and token delta. + +Also verify: + +- [ ] The skill name does not start or end with a hyphen, contain consecutive hyphens, or exceed 64 characters +- [ ] YAML frontmatter name matches the directory name exactly and all frontmatter fields are valid +- [ ] SKILL.md is under 500 lines and 5000 tokens, splitting stable detail into references when needed +- [ ] File references are relative and instructions are actionable and specific +- [ ] Instructions do not duplicate `.github/copilot-instructions.md` or `.github/instructions/` +- [ ] The workflow has numbered steps and observable success criteria +- [ ] No secrets, tokens, or internal URLs are included - [ ] Optional directories are used appropriately -- [ ] Scripts handle edge cases gracefully and return structured outputs and helpful error messages when applicable +- [ ] Scripts handle edge cases, fail closed, and return structured, helpful errors +- [ ] The paired Vally comparison demonstrates distinctive value over the unskilled control -### Step 8: Test with Multi-Model Subagents +### Step 9: Test with Multi-Model Subagents Follow [references/testing-patterns.md](references/testing-patterns.md): @@ -141,5 +154,4 @@ Follow [references/testing-patterns.md](references/testing-patterns.md): ## References - [Agent Skills Specification](https://agentskills.io/specification) -- [Copilot Instructions](../../../.github/copilot-instructions.md) -- [Contributing Guidelines](../../../.github/CONTRIBUTING.md) +- Repository guidance: `.github/copilot-instructions.md` and `.github/CONTRIBUTING.md` diff --git a/.agents/skills/make-skill/references/testing-patterns.md b/.agents/skills/make-skill/references/testing-patterns.md index 44799155b23..5d5ec5f9625 100644 --- a/.agents/skills/make-skill/references/testing-patterns.md +++ b/.agents/skills/make-skill/references/testing-patterns.md @@ -184,16 +184,16 @@ task agent_type="general-purpose" model="{different-model}" prompt="Review the s The two approaches complement each other: writer-critic for creation/iteration, multi-model for validation. -## Waza Eval Testing +## Vally Evaluation -For repeatable, quantitative skill testing, use the **waza-eval** skill. It provides: +For repeatable, quantitative skill testing, author the repository's paired Vally eval. It provides: -- **Structured eval suites** — define tasks with prompts, expected outputs, and graders -- **Progression testing** — compare tool efficiency across skill versions from git history -- **Session capture** — commit result transcripts as golden sessions for regression detection -- **CI integration** — gate PRs on eval pass rates +- **Structured eval suites** — define repository-grounded stimuli, expected outputs, and graders +- **Unskilled controls** — compare identical tasks with and without the target skill +- **Quality and efficiency comparison** — report judge preference and token, turn, tool-call, time, and error deltas +- **CI integration** — require the treatment threshold and fail statistically significant regressions -Use waza evals when you need to *measure* whether a skill change improved behavior. Use multi-model review (above) when you need *qualitative* structural feedback. +Use `node eng/harness-evaluation/src/cli.mjs eval --runs 5 --workers 1 --require-pass` to measure whether a skill improves behavior. Use multi-model review (above) for qualitative structural feedback. ### Regression Heuristics @@ -220,6 +220,7 @@ Evals should include trigger tests (does the skill activate correctly?): Before shipping a skill change: - [ ] Description matches trigger tests (USE FOR phrases appear in should-trigger prompts) +- [ ] `eng/harness-evaluation/skills//eval.yaml` passes treatment/control comparison - [ ] Stop signals are explicit with numeric bounds - [ ] Domain examples present (not just tool schemas) - [ ] Token budget met (SKILL.md under 4K orchestrating / 15K knowledge) diff --git a/.agents/skills/migrations/SKILL.md b/.agents/skills/migrations/SKILL.md index 96e5223a200..5e40a36ad91 100644 --- a/.agents/skills/migrations/SKILL.md +++ b/.agents/skills/migrations/SKILL.md @@ -16,7 +16,10 @@ user-invocable: false - Model snapshots use `typeof(Dictionary)` (property bag format), not the actual CLR type. When examining the `ClrType` in a snapshot, don't assume it matches the real entity type. - `SnapshotModelProcessor.Process()` is used at design-time to fixup older model snapshots for backward compatibility. +- `MigrationsModelDiffer` uses provider-agnostic structural comparison between relational models to determine what migration operations are necessary. ## Testing -Migration operation tests: `test/EFCore.Relational.Tests/Migrations/`. Functional tests: `test/EFCore.{Provider}.FunctionalTests/Migrations/`. Model differ tests: `test/EFCore.Relational.Tests/Migrations/Internal/MigrationsModelDifferTest*.cs`. \ No newline at end of file +Migration operation tests: `test/EFCore.Relational.Tests/Migrations/`. Functional tests: `test/EFCore.{Provider}.FunctionalTests/Migrations/`. Model differ tests: `test/EFCore.Relational.Tests/Migrations/Internal/MigrationsModelDifferTest*.cs`. + +To simulate a snapshot model use `ModelBuilder` calls without conventions. \ No newline at end of file diff --git a/.agents/skills/run-apichief/SKILL.md b/.agents/skills/run-apichief/SKILL.md index e78bbca1c44..b50d7084507 100644 --- a/.agents/skills/run-apichief/SKILL.md +++ b/.agents/skills/run-apichief/SKILL.md @@ -6,7 +6,7 @@ user-invocable: false # Run ApiChief -Use the [ApiChief tool](../../../eng/Tools/ApiChief/README.md) to inspect or refresh EF Core public API baselines for projects under `src/`. +Use the ApiChief tool documented at `eng/Tools/ApiChief/README.md` to inspect or refresh EF Core public API baselines for projects under `src/`. ApiChief can run against either a compiled assembly or a previously emitted baseline JSON file. Prefer the repo-local `.dotnet` SDK and the checked-in build scripts in this repo. diff --git a/.agents/skills/servicing-pr/SKILL.md b/.agents/skills/servicing-pr/SKILL.md index e9dabb573ed..1dbe0d53904 100644 --- a/.agents/skills/servicing-pr/SKILL.md +++ b/.agents/skills/servicing-pr/SKILL.md @@ -44,7 +44,7 @@ Brief risk assessment ranked from "extremely low" to "high". Note amount of code ## Quirk (AppContext Switch) -A quirk lets users opt out of the fix at runtime, reducing patch risk. Add for all cases where it makes sense. Skip when the fix is 100% obvious/risk-free, or when the quirk couldn't be used, like in tools or analyzers. +A quirk lets users opt out of the fix at runtime, reducing patch risk. Skip when the fix is 100% obvious/risk-free, or when the quirk couldn't be used, like in tools or analyzers. ### Adding a Quirk @@ -58,6 +58,7 @@ private static readonly bool UseOldBehavior37585 = - Change `37585` to the relevant issue number - Wrap changes with a condition on `!UseOldBehavior37585` so activating the switch bypasses the fix, prefer to minimize the number of times the switch is checked - If the PR closes multiple issues, pick the most appropriate one for the switch name +- Distinguish a recommendation from the staged implementation. Do not say "Quirk added" unless the source change actually includes the switch. ## Validation diff --git a/.agents/skills/sqlite-adonet/SKILL.md b/.agents/skills/sqlite-adonet/SKILL.md index 6f07513f044..156b27488ef 100644 --- a/.agents/skills/sqlite-adonet/SKILL.md +++ b/.agents/skills/sqlite-adonet/SKILL.md @@ -12,3 +12,6 @@ Standalone ADO.NET provider in `src/Microsoft.Data.Sqlite.Core/`, independent of - Static constructor calls `SQLitePCL.Batteries_V2.Init()` reflectively - `CreateFunction()`/`CreateAggregate()` overloads generated from T4 templates (`.tt` files) +- `DbConnection.Close()` closes the connection but does not dispose commands associated with it; a reusable + `DbCommand` may remain associated and execute after that same connection is reopened. Keep connection-driven statement + cleanup separate from `DbCommand.Dispose()`, which is the terminal command-lifetime operation. diff --git a/.agents/skills/testing/SKILL.md b/.agents/skills/testing/SKILL.md deleted file mode 100644 index ea148cdd365..00000000000 --- a/.agents/skills/testing/SKILL.md +++ /dev/null @@ -1,94 +0,0 @@ ---- -name: testing -description: 'Implementation details for EF Core test infrastructure. Use when changing test fixtures, SQL baseline assertions, test helpers, the test class hierarchy, or when adding new tests.' -user-invocable: false ---- - -# Testing - -## Test Categories - -### Unit Tests (`test/EFCore.Tests/`, `test/EFCore.Relational.Tests/`, `test/EFCore.{Provider}.Tests/`) -Isolated logic tests. Build models via `*TestHelpers.Instance.CreateConventionBuilder()`, resolve services from `CreateContextServices()`. No database needed. - -### Specification Tests (provider-agnostic abstract bases) -Define WHAT to test (LINQ queries, expected results). Can't be run directly — provider tests override to verify HOW (generated SQL). - -- Core → `test/EFCore.Specification.Tests/` -- Relational → `test/EFCore.Relational.Specification.Tests/` - -### Functional Tests (`test/EFCore.{Provider}.FunctionalTests/`) -Concrete provider tests inheriting specification tests. Most include SQL baseline assertions. - -## Test Class Hierarchy (Query Example) - -``` -QueryTestBase # Core - └─ NorthwindWhereQueryTestBase # Specification - └─ NorthwindWhereQueryRelationalTestBase # Relational specification - └─ NorthwindWhereQuerySqlServerTest # Provider (asserts SQL) -``` - -Provider override pattern: -```csharp -public override async Task Where_simple(bool async) -{ - await base.Where_simple(async); // runs LINQ + asserts results - AssertSql("""..."""); // asserts provider-specific SQL -} -``` - -## TestHelpers Hierarchy - -``` -TestHelpers (abstract) # EFCore.Specification.Tests - ├─ InMemoryTestHelpers # non-relational - └─ RelationalTestHelpers (abstract) # EFCore.Relational.Specification.Tests - ├─ SqlServerTestHelpers - └─ SqliteTestHelpers -``` - -Key methods: `CreateConventionBuilder()`, `CreateContextServices(model)`, `CreateOptions()` - -## Fixtures - -### SharedStoreFixtureBase -Many tests share one database. Creates `TestStore` + pooled `DbContextFactory` in `InitializeAsync()`. Seeds data once. Use for read-heavy tests (e.g., Northwind query tests). - -### NonSharedModelTestBase -Each test gets a fresh model/store. Call `InitializeAsync(onModelCreating, seed, ...)` per test. Use for tests needing unique schemas. - -## SQL Baseline Assertions - -`TestSqlLoggerFactory` captures SQL. `AssertSql("""...""")` compares against expected. Set `EF_TEST_REWRITE_BASELINES=1` to auto-rewrite baselines via Roslyn. - -## Workflow: Adding New Tests - -1. **Specification test**: Add to `EFCore.Specification.Tests` (core) or `EFCore.Relational.Specification.Tests` (relational) -2. **Provider overrides**: Override in **every** provider functional test class (`EFCore.{Provider}.FunctionalTests`) that inherits the base with provider-appropriate assertions. -3. **Unit test**: Add to `EFCore.{Provider}.Tests` -4. Run with `EF_TEST_REWRITE_BASELINES=1` to capture initial baselines -5. Run tests with project rebuilding enabled (don't use `--no-build`) to ensure code changes are picked up -6. When testing cross-platform code (e.g., file paths, path separators), verify the fix works on both Windows and Linux/macOS - -## Common Pitfalls - -| Pitfall | Solution | -|---------|----------| -| Baseline mismatch (SQL or compiled model) | Re-run with `EF_TEST_REWRITE_BASELINES=1` | -| `Check_all_tests_overridden` fails | Override the new test in every inheriting provider class | -| SQL Server feature missing at lower compat level | Gate with `[SqlServerCondition(...)]`| - -## Running Tests Directly Against the Built Assembly - -When invoking the test DLL via `dotnet exec` (e.g. to isolate a single test method without rebuilding through `dotnet test`), always append `--filter-not-trait category=failing --ignore-exit-code 8`: - -```pwsh -dotnet exec ./Microsoft.EntityFrameworkCore.SqlServer.FunctionalTests.dll ` - --filter-method '*MigrationsSqlServerTest.Create_json_index_over_whole_complex_collection' ` - --filter-not-trait category=failing --ignore-exit-code 8 -``` - -These flags mirror what `test/Directory.Build.props` passes via `TestingPlatformCommandLineArguments`. - -Don't add `--no-build` unless the test assembly was built in the immediate previous step. diff --git a/.claude/settings.json b/.claude/settings.json index 35358294c16..ae67a740cb2 100644 --- a/.claude/settings.json +++ b/.claude/settings.json @@ -5,9 +5,22 @@ "source": "github", "repo": "dotnet/arcade-skills" } + }, + "dotnet-skills": { + "source": { + "source": "github", + "repo": "dotnet/skills" + } } }, "enabledPlugins": { - "dotnet-dnceng@dotnet-arcade-skills": true + "dotnet-dnceng@dotnet-arcade-skills": true, + "dotnet-helix@dotnet-arcade-skills": true, + "dotnet-codeflow@dotnet-arcade-skills": true, + "dotnet-test@dotnet-skills": true, + "dotnet-diag@dotnet-skills": true, + "dotnet-msbuild@dotnet-skills": true, + "dotnet-nuget@dotnet-skills": true, + "microsoft-docs@claude-plugins-official": true } } diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 035f331c7f2..c3ff4ddc85c 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -23,6 +23,14 @@ Don't rely just on names to guess its function, evaluate the code based on the i - **NEVER** hardcode package versions in `.csproj` files - Use `eng/Versions.props` and `Directory.Packages.props` for NuGet package version management +## Testing + +- Put provider-independent behavior in specification tests. Override the test in every inheriting provider functional test class, call the base test, and use `AssertSql` only for providers that produce SQL. +- `NonSharedModelTestBase` supports both the tests that share a model as well as those that do not. +- Set `EF_TEST_REWRITE_BASELINES=1` to rewrite SQL and compiled-model baselines. +- Tests execute using MTP, call `dotnet exec --filter-method '' --filter-not-trait category=failing --ignore-exit-code 8` for focused runs. +- Do not add `--no-build` unless the test assembly was built in the immediately preceding step. + ## Implementation Guidelines - Write code that is secure by default. Avoid exposing potentially private or sensitive data diff --git a/.github/copilot/settings.json b/.github/copilot/settings.json index 35358294c16..d71884fbdca 100644 --- a/.github/copilot/settings.json +++ b/.github/copilot/settings.json @@ -5,9 +5,28 @@ "source": "github", "repo": "dotnet/arcade-skills" } + }, + "dotnet-skills": { + "source": { + "source": "github", + "repo": "dotnet/skills" + } + }, + "microsoft-docs-marketplace": { + "source": { + "source": "github", + "repo": "microsoftdocs/mcp" + } } }, "enabledPlugins": { - "dotnet-dnceng@dotnet-arcade-skills": true + "dotnet-dnceng@dotnet-arcade-skills": true, + "dotnet-helix@dotnet-arcade-skills": true, + "dotnet-codeflow@dotnet-arcade-skills": true, + "dotnet-test@dotnet-skills": true, + "dotnet-diag@dotnet-skills": true, + "dotnet-msbuild@dotnet-skills": true, + "dotnet-nuget@dotnet-skills": true, + "microsoft-docs@microsoft-docs-marketplace": true } } diff --git a/.github/dependabot.yml b/.github/dependabot.yml index b899b4dcda1..374f5c8788f 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -287,3 +287,15 @@ updates: interval: "weekly" labels: - "area-infrastructure" + + - package-ecosystem: "npm" + directory: "/eng/harness-evaluation" + open-pull-requests-limit: 5 + schedule: + interval: "weekly" + day: "monday" + labels: + - "area-infrastructure" + groups: + npm: + patterns: ["*"] diff --git a/.github/workflows/agent-harness-coverage.yml b/.github/workflows/agent-harness-coverage.yml new file mode 100644 index 00000000000..030763b140d --- /dev/null +++ b/.github/workflows/agent-harness-coverage.yml @@ -0,0 +1,56 @@ +# Validates that every repository-owned instruction, skill, agent, agentic +# workflow, and prompt has a well-formed EF Core-grounded Vally evaluation. +# MCPs and plugins are excluded. + +name: Agent Harness Coverage + +on: + pull_request: + +permissions: + contents: read + +jobs: + coverage: + runs-on: ubuntu-24.04 + steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Restore repository SDK + shell: bash + run: | + ./restore.sh + . ./activate.sh + echo "DOTNET_ROOT=$PWD/.dotnet" >> "$GITHUB_ENV" + echo "$PWD/.dotnet" >> "$GITHUB_PATH" + + - name: Set up Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 + with: + node-version: 22 + cache: npm + cache-dependency-path: eng/harness-evaluation/package-lock.json + + - name: Install evaluation tools + working-directory: eng/harness-evaluation + run: npm ci --ignore-scripts + + - name: Audit evaluation dependencies + working-directory: eng/harness-evaluation + run: npm audit --audit-level=moderate + + - name: Test evaluation harness + working-directory: eng/harness-evaluation + run: npm test + + - name: Validate customization coverage + working-directory: eng/harness-evaluation + run: npm run validate + + - name: Lint skills and evaluations + working-directory: eng/harness-evaluation + run: npm run lint diff --git a/.github/workflows/agent-harness-evaluation.yml b/.github/workflows/agent-harness-evaluation.yml new file mode 100644 index 00000000000..8b4fb45e69e --- /dev/null +++ b/.github/workflows/agent-harness-evaluation.yml @@ -0,0 +1,543 @@ +# Discovers affected harness components for every PR. Automatic evaluation is +# limited to trusted authors; contributors can explicitly evaluate any PR. + +name: Agent Harness Evaluation + +on: + pull_request_target: + types: [opened, reopened, ready_for_review] + workflow_dispatch: + inputs: + pull_request: + description: Pull request number to evaluate + required: true + type: number + component: + description: Affected component to evaluate; leave empty to evaluate all affected components + required: false + type: string + +permissions: + contents: read + +concurrency: + group: agent-harness-evaluation-${{ github.event.pull_request.number || inputs.pull_request }} + cancel-in-progress: true + +jobs: + discover: + runs-on: ubuntu-24.04 + permissions: + contents: read + issues: write + pull-requests: read + outputs: + base_sha: ${{ steps.pr.outputs.base_sha }} + head_sha: ${{ steps.pr.outputs.head_sha }} + pull_request: ${{ steps.pr.outputs.pull_request }} + can_use_secrets: ${{ steps.pr.outputs.can_use_secrets }} + component_filter: ${{ steps.pr.outputs.component_filter }} + author: ${{ steps.pr.outputs.author }} + matrix: ${{ steps.components.outputs.matrix }} + has_entries: ${{ steps.components.outputs.has_entries }} + components: ${{ steps.components.outputs.components }} + steps: + - name: Resolve pull request and authorization + id: pr + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 + env: + MANUAL_COMPONENT: ${{ inputs.component }} + MANUAL_PULL_REQUEST: ${{ inputs.pull_request }} + with: + script: | + const manual = context.eventName === 'workflow_dispatch'; + if (manual && context.ref !== `refs/heads/${context.payload.repository.default_branch}`) { + throw new Error('Run manual harness evaluations from the repository default branch.'); + } + const pullNumber = manual + ? Number(process.env.MANUAL_PULL_REQUEST) + : context.payload.pull_request.number; + if (!Number.isInteger(pullNumber) || pullNumber <= 0) { + throw new Error(`Invalid pull request number: ${process.env.MANUAL_PULL_REQUEST}`); + } + const { data: pull } = await github.rest.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: pullNumber + }); + const author = pull.user.login; + let permission = 'none'; + try { + const response = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: context.repo.owner, + repo: context.repo.repo, + username: author + }); + permission = response.data.permission; + } catch (error) { + if (error.status !== 404) throw error; + } + + // Helper function to check if a user is a bot + const isBot = (username) => { + const lower = username.toLowerCase(); + return lower === 'copilot' || + lower === 'dotnet-bot' || + lower.startsWith('app/') || + lower.includes('[bot]'); + }; + const fromBranch = pull.head.repo?.id === pull.base.repo.id; + const canUseSecrets = isBot(author) + ? fromBranch + : manual || ['admin', 'maintain', 'write'].includes(permission); + + core.setOutput('base_sha', pull.base.sha); + core.setOutput('head_sha', pull.head.sha); + core.setOutput('pull_request', String(pullNumber)); + core.setOutput('can_use_secrets', String(canUseSecrets)); + core.setOutput('component_filter', manual ? process.env.MANUAL_COMPONENT.trim() : ''); + core.setOutput('author', author); + await core.summary + .addHeading('Agent Harness Evaluation') + .addRaw(`PR #${pullNumber} by ${author}: repository permission ${permission}; ` + + `evaluation ${canUseSecrets ? 'authorized' : 'requires manual authorization'}.`) + .write(); + + # Execute only the base branch's trusted harness code during discovery. + - name: Check out trusted harness + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + ref: ${{ steps.pr.outputs.base_sha }} + fetch-depth: 0 + persist-credentials: false + + # The PR checkout is data inspected by the trusted harness; no PR scripts run. + - name: Check out pull request data + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + ref: ${{ steps.pr.outputs.head_sha }} + path: _pr + fetch-depth: 0 + persist-credentials: false + + - name: Set up Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 + with: + node-version: 22 + cache: npm + cache-dependency-path: eng/harness-evaluation/package-lock.json + + - name: Install trusted evaluation tools + working-directory: eng/harness-evaluation + run: npm ci --ignore-scripts + + - name: Discover affected components + id: components + env: + BASE_SHA: ${{ steps.pr.outputs.base_sha }} + COMPONENT_FILTER: ${{ steps.pr.outputs.component_filter }} + HEAD_SHA: ${{ steps.pr.outputs.head_sha }} + run: | + arguments=( + --repo-root "$GITHUB_WORKSPACE/_pr" + --base "$BASE_SHA" + --head "$HEAD_SHA" + --github-output "$GITHUB_OUTPUT" + ) + if [[ -n "$COMPONENT_FILTER" ]]; then + arguments+=(--component "$COMPONENT_FILTER") + fi + node eng/harness-evaluation/src/cli.mjs discover "${arguments[@]}" + + - name: Update pull request metadata + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 + env: + CAN_USE_SECRETS: ${{ steps.pr.outputs.can_use_secrets }} + COMPONENT_FILTER: ${{ steps.pr.outputs.component_filter }} + HEAD_SHA: ${{ steps.pr.outputs.head_sha }} + PULL_REQUEST: ${{ steps.pr.outputs.pull_request }} + SELECTED_COMPONENTS: ${{ steps.components.outputs.components }} + with: + script: | + const issueNumber = Number(process.env.PULL_REQUEST); + const selected = new Set(JSON.parse(process.env.SELECTED_COMPONENTS)); + if (selected.size === 0) return; + + const label = 'area-harness'; + try { + await github.rest.issues.getLabel({ + owner: context.repo.owner, + repo: context.repo.repo, + name: label + }); + } catch (error) { + if (error.status === 404) { + throw new Error(`Required label '${label}' does not exist.`); + } + throw error; + } + await github.rest.issues.addLabels({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issueNumber, + labels: [label] + }); + + const comments = await github.paginate(github.rest.issues.listComments, { + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issueNumber, + per_page: 100 + }); + if (!process.env.COMPONENT_FILTER) { + for (const comment of comments) { + const match = comment.body?.match(//); + if (comment.user?.login !== 'github-actions[bot]' || !match || selected.has(match[1])) continue; + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: comment.id, + body: `${match[0]}\n### Agent evaluation: \`${match[1]}\`\n\n` + + `Status: **superseded** by \`${process.env.HEAD_SHA}\`; this component is no longer affected.` + }); + } + } + + const promptMarker = ''; + const prompt = comments.find(comment => + comment.user?.login === 'github-actions[bot]' && comment.body?.includes(promptMarker)); + if (process.env.CAN_USE_SECRETS === 'true') { + if (prompt && !process.env.COMPONENT_FILTER) { + await github.rest.issues.deleteComment({ + owner: context.repo.owner, repo: context.repo.repo, comment_id: prompt.id + }); + } + return; + } + + const manualUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}` + + '/actions/workflows/agent-harness-evaluation.yml'; + const body = `${promptMarker}\n### Agent harness evaluation required\n\n` + + `This PR affects: ${[...selected].map(id => `\`${id}\``).join(', ')}. ` + + `Its author cannot use repository secrets in an automatic run. A contributor with write access should ` + + `[run the workflow manually](${manualUrl}) and enter pull request **#${issueNumber}**.`; + if (prompt) { + await github.rest.issues.updateComment({ + owner: context.repo.owner, repo: context.repo.repo, comment_id: prompt.id, body + }); + } else { + await github.rest.issues.createComment({ + owner: context.repo.owner, repo: context.repo.repo, issue_number: issueNumber, body + }); + } + + evaluate: + name: evaluate (${{ matrix.id }}) + needs: discover + if: needs.discover.outputs.has_entries == 'true' && needs.discover.outputs.can_use_secrets == 'true' + strategy: + fail-fast: false + max-parallel: 3 + matrix: ${{ fromJSON(needs.discover.outputs.matrix) }} + runs-on: ubuntu-24.04 + timeout-minutes: ${{ matrix.timeout_minutes }} + permissions: + contents: read + copilot-requests: write + issues: write + pull-requests: read + env: + COMPONENT: ${{ matrix.id }} + RESULT_ROOT: artifacts/TestResults/harness-evaluation/${{ matrix.id }} + steps: + - name: Check out pull request + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + ref: ${{ needs.discover.outputs.head_sha }} + fetch-depth: 0 + persist-credentials: false + + - name: Report evaluation started + continue-on-error: true + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 + env: + HEAD_SHA: ${{ needs.discover.outputs.head_sha }} + PULL_REQUEST: ${{ needs.discover.outputs.pull_request }} + COMPONENT: ${{ matrix.id }} + with: + script: | + const marker = ``; + const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; + const manualUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}` + + '/actions/workflows/agent-harness-evaluation.yml'; + const body = `${marker}\n### Agent evaluation: \`${process.env.COMPONENT}\`\n\n` + + `Status: **running** for \`${process.env.HEAD_SHA}\`\n\n` + + `PR updates do not rerun evaluations automatically. [Run again manually](${manualUrl}) after later commits.\n\n` + + `[View workflow run](${runUrl})`; + const issueNumber = Number(process.env.PULL_REQUEST); + const comments = await github.paginate(github.rest.issues.listComments, { + owner: context.repo.owner, repo: context.repo.repo, issue_number: issueNumber, per_page: 100 + }); + const existing = comments.find(comment => + comment.user?.login === 'github-actions[bot]' && comment.body?.includes(marker)); + if (existing) { + await github.rest.issues.updateComment({ + owner: context.repo.owner, repo: context.repo.repo, comment_id: existing.id, body + }); + } else { + await github.rest.issues.createComment({ + owner: context.repo.owner, repo: context.repo.repo, issue_number: issueNumber, body + }); + } + + - name: Restore repository SDK + shell: bash + run: | + ./restore.sh + . ./activate.sh + echo "DOTNET_ROOT=$PWD/.dotnet" >> "$GITHUB_ENV" + echo "$PWD/.dotnet" >> "$GITHUB_PATH" + + - name: Set up Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 + with: + node-version: 22 + cache: npm + cache-dependency-path: eng/harness-evaluation/package-lock.json + + - name: Install evaluation tools + working-directory: eng/harness-evaluation + run: npm ci --ignore-scripts + + - name: Run treatment, unskilled control, and comparison + id: run + shell: bash + env: + GITHUB_TOKEN: ${{ github.token }} + EFCORE_SOURCE_ROOT: ${{ github.workspace }} + run: | + timeout --signal=TERM --kill-after=30s "${{ matrix.watchdog_minutes }}m" \ + node eng/harness-evaluation/src/cli.mjs eval "$COMPONENT" \ + --runs 5 \ + --workers 1 \ + --require-pass \ + --output "$RESULT_ROOT" + + - name: Upload component results + id: upload + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + with: + name: harness-eval-result-${{ matrix.id }} + path: ${{ env.RESULT_ROOT }}/ + include-hidden-files: true + if-no-files-found: warn + retention-days: 14 + + - name: Verify Vally reports + id: verify + if: steps.run.outcome == 'success' + shell: bash + run: | + node --input-type=module <<'EOF' + import fs from 'node:fs'; + import path from 'node:path'; + + const resultFiles = []; + const visit = directory => { + for (const entry of fs.readdirSync(directory, { withFileTypes: true })) { + const fullPath = path.join(directory, entry.name); + if (entry.isDirectory()) visit(fullPath); + else if (entry.name === 'results.jsonl') resultFiles.push(fullPath); + } + }; + visit(process.env.RESULT_ROOT); + const controlFile = resultFiles.find(file => file.split(path.sep).includes('control')); + const treatmentFile = resultFiles.find(file => file.split(path.sep).includes('treatment')); + if (!controlFile || !treatmentFile || resultFiles.length !== 2) { + throw new Error(`Expected control and treatment results.jsonl files, found ${resultFiles.length}.`); + } + + const readRecords = file => fs.readFileSync(file, 'utf8').trim().split(/\r?\n/).map(JSON.parse); + const control = readRecords(controlFile); + const treatment = readRecords(treatmentFile); + const controlSummary = control.findLast(record => record.type === 'run-summary'); + const treatmentSummary = treatment.findLast(record => record.type === 'run-summary'); + const controlTrials = control.filter(record => record.type === 'trial-result'); + const treatmentTrials = treatment.filter(record => record.type === 'trial-result'); + if (!controlSummary || controlSummary.hadExecutionErrors || + !treatmentSummary || !treatmentSummary.passed || treatmentSummary.hadExecutionErrors || + controlTrials.length !== treatmentTrials.length || + treatmentTrials.some(trial => trial.totalTrials !== 5)) { + throw new Error('Vally did not produce complete control and passing treatment results.'); + } + + const comparisons = readRecords(path.join(process.env.RESULT_ROOT, 'comparison.jsonl')); + if (comparisons.length !== 1 || comparisons[0].type !== 'comparison' || + comparisons[0].summary.erroredCount !== 0 || + comparisons[0].summary.trialCount !== treatmentTrials.length || + !comparisons[0].summary.metricDeltas.some(metric => + metric.metric === 'totalTokens' && Number.isFinite(metric.mean))) { + throw new Error('Vally did not produce a complete comparison with token metrics.'); + } + EOF + + - name: Report component result + if: always() + continue-on-error: true + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 + env: + ARTIFACT_URL: ${{ steps.upload.outputs.artifact-url }} + BASE_SHA: ${{ needs.discover.outputs.base_sha }} + COMPONENT: ${{ matrix.id }} + EVAL_PATH: ${{ matrix.eval_path }} + EVAL_OUTCOME: ${{ steps.verify.outcome }} + HEAD_SHA: ${{ needs.discover.outputs.head_sha }} + PULL_REQUEST: ${{ needs.discover.outputs.pull_request }} + with: + script: | + const fs = require('fs'); + const path = require('path'); + const issueNumber = Number(process.env.PULL_REQUEST); + const marker = ``; + const passed = process.env.EVAL_OUTCOME === 'success'; + let details = '_No complete Vally comparison was produced._'; + + try { + const comparisonPath = path.join(process.env.RESULT_ROOT, 'comparison.jsonl'); + if (fs.existsSync(comparisonPath)) { + const lines = fs.readFileSync(comparisonPath, 'utf8').split(/\r?\n/).filter(Boolean); + if (lines.length !== 1) throw new Error(`expected one comparison record, found ${lines.length}`); + const comparison = JSON.parse(lines[0]); + const summary = comparison?.summary; + const tokens = summary?.metricDeltas?.find(metric => metric.metric === 'totalTokens'); + if (comparison?.type !== 'comparison' || + !Number.isFinite(summary?.meanScore) || + !Number.isFinite(summary?.wins) || + !Number.isFinite(summary?.ties) || + !Number.isFinite(summary?.losses) || + !Number.isFinite(tokens?.mean)) { + throw new Error('comparison summary or token metrics are incomplete'); + } + const tokenDelta = `${tokens.mean >= 0 ? '+' : ''}${Math.round(tokens.mean).toLocaleString()}`; + details = `Quality score (treatment relative to control): **${summary.meanScore.toFixed(2)}** ` + + `(${summary.wins} wins, ${summary.ties} ties, ${summary.losses} losses)\n\n` + + `Mean token delta: **${tokenDelta} tokens** per paired trial.`; + } + } catch (error) { + core.warning(`Could not parse comparison report: ${error.message}`); + } + + let baseline = ''; + try { + let existedAtBase = true; + try { + await github.rest.repos.getContent({ + owner: context.repo.owner, + repo: context.repo.repo, + path: process.env.EVAL_PATH, + ref: process.env.BASE_SHA + }); + } catch (error) { + if (error.status !== 404) throw error; + existedAtBase = false; + } + + if (existedAtBase) { + const query = `repo:${context.repo.owner}/${context.repo.repo} is:pr label:area-harness ` + + `in:comments "agent-harness-eval:${process.env.COMPONENT}"`; + const { data } = await github.rest.search.issuesAndPullRequests({ + q: query, sort: 'updated', order: 'desc', per_page: 100 + }); + let previous; + for (const pull of data.items) { + if (pull.number === issueNumber) continue; + const comments = await github.paginate(github.rest.issues.listComments, { + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: pull.number, + per_page: 100 + }); + for (const comment of comments) { + if (comment.user?.login !== 'github-actions[bot]' || + !comment.body?.includes(marker) || + !/Status: \*\*(passed|failed)\*\*/.test(comment.body)) continue; + if (!previous || Date.parse(comment.updated_at) > Date.parse(previous.comment.updated_at)) { + previous = { comment, pullNumber: pull.number }; + } + } + } + if (previous) { + baseline = `\n\n[Previous result from PR #${previous.pullNumber}]` + + `(${previous.comment.html_url}) provides the baseline.`; + } + } + } catch (error) { + core.warning(`Could not locate a previous baseline: ${error.message}`); + } + + const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; + const manualUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}` + + '/actions/workflows/agent-harness-evaluation.yml'; + const artifactLink = process.env.ARTIFACT_URL + ? `[Download Vally reports](${process.env.ARTIFACT_URL}) · ` + : ''; + const body = `${marker}\n### Agent evaluation: \`${process.env.COMPONENT}\`\n\n` + + `Status: **${passed ? 'passed' : 'failed'}** for \`${process.env.HEAD_SHA}\`\n\n` + + `${details}${baseline}\n\nPR updates do not rerun evaluations automatically. ` + + `[Run again manually](${manualUrl}) after later commits.\n\n` + + `${artifactLink}[View workflow run](${runUrl})`; + const comments = await github.paginate(github.rest.issues.listComments, { + owner: context.repo.owner, repo: context.repo.repo, issue_number: issueNumber, per_page: 100 + }); + const existing = comments.find(comment => + comment.user?.login === 'github-actions[bot]' && comment.body?.includes(marker)); + if (existing) { + await github.rest.issues.updateComment({ + owner: context.repo.owner, repo: context.repo.repo, comment_id: existing.id, body + }); + } else { + await github.rest.issues.createComment({ + owner: context.repo.owner, repo: context.repo.repo, issue_number: issueNumber, body + }); + } + + evaluation: + needs: [discover, evaluate] + if: always() + runs-on: ubuntu-24.04 + steps: + - name: Fail when discovery did not complete + if: needs.discover.result != 'success' + run: | + echo "::error::Agent harness component discovery failed." + exit 1 + + - name: No affected components + if: needs.discover.result == 'success' && needs.discover.outputs.has_entries != 'true' + run: echo "No repository-owned harness evaluations are affected." + + - name: Report deferred evaluation + if: >- + needs.discover.outputs.has_entries == 'true' && + needs.discover.outputs.can_use_secrets != 'true' + run: echo "Evaluation requires a contributor-triggered manual run." + + - name: Fail when a component comparison failed + if: >- + needs.discover.outputs.has_entries == 'true' && + needs.discover.outputs.can_use_secrets == 'true' && + needs.evaluate.result != 'success' + run: | + echo "::error::At least one agent harness component comparison failed." + exit 1 + + - name: Report successful evaluation + if: >- + needs.discover.outputs.has_entries == 'true' && + needs.discover.outputs.can_use_secrets == 'true' && + needs.evaluate.result == 'success' + env: + COMPONENTS: ${{ needs.discover.outputs.components }} + run: | + echo "All selected components passed treatment and control comparison: $COMPONENTS." \ No newline at end of file diff --git a/.vscode/mcp.json b/.vscode/mcp.json deleted file mode 100644 index 9d5b8f44c12..00000000000 --- a/.vscode/mcp.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "servers": { - "microsoft-docs": { - "type": "http", - "url": "https://learn.microsoft.com/api/mcp" - }, - "nuget": { - "type": "stdio", - "command": "dnx", - "args": [ - "NuGet.Mcp.Server", - "--prerelease", - "--yes" - ] - }, - "Community-Mcp-DotNet": { - "type": "stdio", - "command": "dnx", - "args": [ - "Community.Mcp.DotNet", - "--prerelease", - "--yes" - ], - "env": { - "DOTNET_SKIP_FIRST_TIME_EXPERIENCE": "1", - "DOTNET_NOLOGO": "1" - } - }, - "helix-azdo": { - "type": "stdio", - "command": "dnx", - "args": [ - "lewing.helix.mcp", - "--prerelease", - "--yes" - ] - }, - "msbuild-binlog": { - "type": "stdio", - "command": "dnx", - "args": [ - "baronfel.binlog.mcp", - "--prerelease", - "--yes" - ] - } - } -} diff --git a/eng/harness-evaluation/README.md b/eng/harness-evaluation/README.md new file mode 100644 index 00000000000..322aad5e4a2 --- /dev/null +++ b/eng/harness-evaluation/README.md @@ -0,0 +1,98 @@ +# Agent harness evaluations + +This directory contains Vally evaluations for repository-owned Copilot instructions, skills, agents, agentic workflows, +and prompts. + +## Layout + +- `instructions//eval.yaml` evaluates a repository instruction file. +- `skills//eval.yaml` evaluates `.agents/skills//SKILL.md`. +- `agents//eval.yaml` evaluates `.github/agents/.agent.md` or `.md`. +- `workflows//eval.yaml` evaluates a `.github/workflows/.md` agentic workflow source, not its compiled `.lock.yml`. +- `prompts//eval.yaml` evaluates `.github/prompts/.prompt.md`. + +Component IDs and eval paths are derived from these conventions. Nested instruction, agent, agentic-workflow, or prompt +paths use `--` between path segments. + +MCP servers and plugins are explicitly outside behavioral-eval coverage. Prefer shared plugins over repository-owned MCP +servers or skills when one owns the capability. This repository enables `dotnet`, `dotnet-test`, `dotnet-diag`, +`dotnet-msbuild`, and `dotnet-nuget` from `dotnet/skills`, plus `dotnet-helix`, `dotnet-codeflow`, and `dotnet-dnceng` from +`dotnet/arcade-skills`. `dotnet-msbuild` supplies binlog tooling; enabling Arcade's `dotnet-binlog` too would register a +second MCP named `binlog`. The Microsoft Docs plugin supplies the Microsoft Learn MCP endpoint and its usage skills. +`eng/common/AGENTS.md` is generated Arcade guidance and is also outside the inventory. + +## Authoring rules + +Every stimulus must: + +1. Set `defaults.runs: 5` in its eval. +2. Declare one or more current checkout paths in `tags.repo-path`. +3. Include at least one path outside `.agents/`, `.github/copilot-instructions.md`, and `eng/harness-evaluation/`. +4. Explicitly ask the agent to inspect, explain, modify, or validate a declared path. +5. Use deterministic graders where possible and a narrow `prompt` rubric only for semantic quality. + +The runner stages the declared paths from the evaluated commit into an isolated Vally workspace. Fixtures may supplement repository anchors, but a generic or fixture-only task is invalid. + +Skill evals load only their target skill and require a `skill-invocation` grader. The runner adds a `token-budget` grader +to every prepared stimulus and gives it 10% of the aggregate score while preserving the relative quality weights. + +## Acceptance + +Pull request and manual evaluation run five unskilled-control trials and five treatment trials, then use `vally compare` +to judge paired trajectories. The treatment must meet the eval's committed `scoring.threshold`, comparison judging must +complete, and `--fail-on-regression` rejects a statistically significant treatment regression. Reports include the +treatment-relative quality verdict and mean token delta. + +The workflow discovers and labels affected PRs when they are opened, reopened, or marked ready for review. Evaluation +starts automatically for authorized contributors and the Copilot coding-agent bot. Other authors receive a comment asking a +contributor to run the workflow from the repository default branch with its `pull_request` input. Manual dispatch +is also required to evaluate later commits. + +## Local validation + +From the repository root in PowerShell: + +```powershell +# Restore the repository-managed .NET SDK and build dependencies. +.\restore.cmd + +# Use the repository-managed .NET SDK. +. .\activate.ps1 + +Set-Location eng\harness-evaluation + +# Install the Vally toolchain. +npm ci --ignore-scripts + +# Exercise component discovery, selection, staging, and timeout logic. +npm test + +# Check that every repository-owned instruction, skill, agent, agentic workflow, and prompt has a valid matching eval. +npm run validate + +# Strict-lint all skills and prepared eval specifications. +npm run lint + +# Run five treatment trials, five unskilled controls, and compare quality and token use. +node src/cli.mjs eval --runs 5 --workers 1 --require-pass +``` + +Set `defaults.timeout` to five times the slowest observed trial, rounded up to the next five-minute boundary. + +Behavioral runs use the Copilot SDK and require a valid local Copilot login or `COPILOT_GITHUB_TOKEN`. Results are written +below `artifacts/TestResults/harness-evaluation//` as separate `control` and `treatment` runs plus +`comparison.jsonl`. They may contain prompts, model output, and tool payloads; do not commit them. + +The runner defaults to one active trial per component. GitHub Actions parallelizes separate component jobs, so increasing Vally workers would multiply concurrent Copilot sessions and can cause session destruction or rate limiting. + +Before accepting a new or materially changed eval, inspect both arms and `comparison.jsonl`. A control that consistently +matches or beats the treatment means the eval is not discriminating enough. + +## Adding components + +- **Skill:** add `.agents/skills//SKILL.md` and `eng/harness-evaluation/skills//eval.yaml`. +- **Instruction, agent, agentic workflow, or prompt:** add the customization and a same-ID eval under the corresponding + `eng/harness-evaluation/` directory. + +The required coverage workflow rejects missing or orphaned repository-customization evals. MCP and plugin changes do not +require eval coverage. diff --git a/eng/harness-evaluation/instructions/copilot-instructions/eval.yaml b/eng/harness-evaluation/instructions/copilot-instructions/eval.yaml new file mode 100644 index 00000000000..5516f5ba2c6 --- /dev/null +++ b/eng/harness-evaluation/instructions/copilot-instructions/eval.yaml @@ -0,0 +1,83 @@ +name: copilot-instructions +description: Evaluates repository-wide Copilot guidance for EF Core internal APIs and provider test hierarchies. +type: capability +defaults: + runs: 5 + timeout: 10m + model: claude-sonnet-5 + judge_model: gpt-5.6-terra + executor: copilot-sdk +stimuli: + - name: review-an-internal-public-api-change + prompt: | + Inspect src/EFCore/Metadata/Internal/EntityType.cs. A proposed change adds a new public virtual member in this + internal namespace; the member activates a CLR type through reflection and awaits a callback. Write api-review.md + listing every repository-wide implementation, documentation, compatibility, and validation requirement that must + be satisfied before accepting the change. Ground the review by naming at least two relevant existing members or + annotations found in the staged file. Do not edit source. + constraints: + max_turns: 50 + max_tokens: 1000000 + max_duration: 10m + tags: + repo-path: + - src/EFCore/Metadata/Internal/EntityType.cs + rubric: + - The review requires the standard EF Core internal-API warning XML documentation on the public member. + - It addresses NativeAOT compatibility by avoiding reflection or requiring the appropriate annotations or exception. + - It requires ConfigureAwait(false) on the awaited callback and EFCore.ApiBaseline.Tests for the public API change. + - The advice names at least two relevant members or annotations from the staged EntityType implementation rather + than giving generic library guidance. + graders: + - type: file-matches + config: + path: api-review.md + pattern: (?=[\s\S]*internal API)(?=[\s\S]*(NativeAOT|RequiresDynamicCode|DynamicallyAccessedMembers))(?=[\s\S]*ConfigureAwait\(false\))(?=[\s\S]*EFCore\.ApiBaseline\.Tests)[\s\S]+ + - type: prompt + config: + scoring: binary + threshold: 1.0 + evidence: + - trajectory + - repo + output_delivery: workspace + - name: add-a-query-test-through-provider-hierarchies + prompt: | + Inspect test/EFCore.Specification.Tests/Query/QueryTestBase.cs, + test/EFCore.Specification.Tests/Query/NorthwindWhereQueryTestBase.cs, + test/EFCore.SqlServer.FunctionalTests/Query/NorthwindWhereQuerySqlServerTest.cs, and + test/EFCore.Sqlite.FunctionalTests/Query/NorthwindWhereQuerySqliteTest.cs. Write test-plan.md for adding a new + Where_with_parameterized_limit case through this hierarchy, including result assertions, provider overrides, + SQL baselines, and focused commands to validate it. Include `restore.cmd` and dot-sourced `activate.ps1` setup, + then use `dotnet exec` on each built provider test DLL with `--filter-method`, `--filter-not-trait category=failing`, + and `--ignore-exit-code 8`. Do not put `--no-build` on test commands. Do not edit source. + constraints: + max_turns: 40 + max_tokens: 750000 + max_duration: 5m + tags: + repo-path: + - test/EFCore.Specification.Tests/Query/QueryTestBase.cs + - test/EFCore.Specification.Tests/Query/NorthwindWhereQueryTestBase.cs + - test/EFCore.SqlServer.FunctionalTests/Query/NorthwindWhereQuerySqlServerTest.cs + - test/EFCore.Sqlite.FunctionalTests/Query/NorthwindWhereQuerySqliteTest.cs + rubric: + - The plan puts provider-independent behavior in the specification test and identifies every relevant staged provider override. + - Provider overrides call the base test and use AssertSql only where SQL is produced. + - The commands follow this repository's restore, activation, and focused Microsoft Testing Platform conventions without stale --no-build usage. + graders: + - type: file-matches + config: + path: test-plan.md + pattern: (?=[\s\S]*Where_with_parameterized_limit)(?=[\s\S]*restore\.cmd)(?=[\s\S]*activate\.ps1)(?=[\s\S]*base)(?=[\s\S]*AssertSql)(?=[\s\S]*dotnet exec)(?=[\s\S]*--filter-method)(?=[\s\S]*--filter-not-trait category=failing)(?=[\s\S]*--ignore-exit-code 8)[\s\S]+ + - type: prompt + config: + scoring: binary + threshold: 1.0 + evidence: [trajectory, repo] + output_delivery: workspace +scoring: + weights: + file-matches: 0.4 + prompt: 0.6 + threshold: 0.8 diff --git a/eng/harness-evaluation/package-lock.json b/eng/harness-evaluation/package-lock.json new file mode 100644 index 00000000000..986082430ad --- /dev/null +++ b/eng/harness-evaluation/package-lock.json @@ -0,0 +1,2090 @@ +{ + "name": "efcore-harness-evaluation", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "efcore-harness-evaluation", + "version": "0.1.0", + "devDependencies": { + "@microsoft/vally": "0.16.0", + "@microsoft/vally-cli": "0.16.0", + "yaml": "2.9.0" + }, + "engines": { + "node": ">=22.12.0" + } + }, + "node_modules/@azure-rest/core-client": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/@azure-rest/core-client/-/core-client-2.9.0.tgz", + "integrity": "sha512-6933vNLqh06RR7rumnrq3UZIeBtRLeBHDPMrieDPIljiCaMMwt0nRw++nd3TOxiohtasNtm29rC+b/9ZVKCOqg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.1.2", + "@azure/core-auth": "^1.10.0", + "@azure/core-rest-pipeline": "^1.24.0", + "@azure/core-tracing": "^1.3.0", + "@typespec/ts-http-runtime": "^0.3.8", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/abort-controller": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-2.2.0.tgz", + "integrity": "sha512-fNAjWnA/nZ2jz31kxR/AqRaUT8ewHBw/WuBIosK0moMy1C9e5ValbDfFdIxJzVOOYaYkV/b2F1S4H/aHiqfVQg==", + "dev": true, + "license": "MIT", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/core-auth": { + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/@azure/core-auth/-/core-auth-1.11.0.tgz", + "integrity": "sha512-IUZydyTUkDnYdstOW9pFOOUQlBjAepK5teihDE3x6yxsPJs/hsAaaYpeGxdxrgtOiJbBKSjKW7MDk7AEhb4LRg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.1.2", + "@azure/core-util": "^1.13.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/core-client": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@azure/core-client/-/core-client-1.11.1.tgz", + "integrity": "sha512-2QygG2F76ZpMP2eMztiJvAiFMu71M9rDeU7vO/QKg5Css7MgM4frUOslFjhVjRhbGaCNPtz/S8M6y46/fFKVuQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.1.2", + "@azure/core-auth": "^1.10.0", + "@azure/core-rest-pipeline": "^1.22.0", + "@azure/core-tracing": "^1.3.0", + "@azure/core-util": "^1.13.0", + "@azure/logger": "^1.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/core-process": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@azure/core-process/-/core-process-1.0.0.tgz", + "integrity": "sha512-/shnJ+ooO8WPxDhPEeI/2oRQuubn16gZ6CvlbpWbEswZfzwI9tI/sMAHmF3x1LuQ9yZYXfLW3TjzGMLEC5blKg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/core-rest-pipeline": { + "version": "1.25.0", + "resolved": "https://registry.npmjs.org/@azure/core-rest-pipeline/-/core-rest-pipeline-1.25.0.tgz", + "integrity": "sha512-bMs8ekJLjX8wPV+9IPBges1SLPyuDtE9g5gLDWOpxzKcoOFQnpLGkbcT1tdw3FaAmDS1gnPmMmJ6y/T5B96kIA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.1.2", + "@azure/core-auth": "^1.10.0", + "@azure/core-tracing": "^1.3.0", + "@azure/core-util": "^1.13.0", + "@azure/logger": "^1.3.0", + "@typespec/ts-http-runtime": "^0.3.4", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/core-tracing": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@azure/core-tracing/-/core-tracing-1.4.0.tgz", + "integrity": "sha512-eGwxD0AtncrxeBM4tG8R55Pc3rdX1hNW2WibJAgYpCVA6E93mvvVH+LcssoVjOBrSKWS55yEIHsk0X8ctHmfOQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/core-util": { + "version": "1.14.0", + "resolved": "https://registry.npmjs.org/@azure/core-util/-/core-util-1.14.0.tgz", + "integrity": "sha512-9n2pWK61veAuN0V20t9lOuoV4CFMdyAZ1ygZzvBGk/pBBJRib/PjL9PLXa/aI2CcPpyHfqVsxxqLCYl6uZlfDw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.1.2", + "@typespec/ts-http-runtime": "^0.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/logger": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@azure/logger/-/logger-1.4.0.tgz", + "integrity": "sha512-rbAE25KUfjU/s3XHUdJgceoCP5dEOpMx85J04kF+QMdta73XkuG9JGHHinch+XIoKpBdqljin+KqURpJriSzLA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@typespec/ts-http-runtime": "^0.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/monitor-opentelemetry-exporter": { + "version": "1.0.0-beta.45", + "resolved": "https://registry.npmjs.org/@azure/monitor-opentelemetry-exporter/-/monitor-opentelemetry-exporter-1.0.0-beta.45.tgz", + "integrity": "sha512-e23akpV6sm5YGlZEAD2TDidPZ5LdeNH99bGnx/OkrvWCOEYO8Js+HaNqjO6ctq252v1BtnTwJ53bXsc3/SGFBQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@azure-rest/core-client": "^2.5.2", + "@azure/core-auth": "^1.9.0", + "@azure/core-client": "^1.9.2", + "@azure/core-process": "^1.0.0", + "@azure/core-rest-pipeline": "^1.19.0", + "@azure/core-util": "^1.11.0", + "@opentelemetry/api": "^1.9.1", + "@opentelemetry/api-logs": "^0.221.0", + "@opentelemetry/core": "^2.10.0", + "@opentelemetry/resources": "^2.10.0", + "@opentelemetry/sdk-logs": "^0.221.0", + "@opentelemetry/sdk-metrics": "^2.10.0", + "@opentelemetry/sdk-trace-base": "^2.10.0", + "@opentelemetry/semantic-conventions": "^1.43.0", + "tslib": "^2.8.1" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@github/copilot": { + "version": "1.0.80", + "resolved": "https://registry.npmjs.org/@github/copilot/-/copilot-1.0.80.tgz", + "integrity": "sha512-6tf93ZF56KOiTTAjK/UhLZkl1W543IzaTQly288kockJZFswpRTnQEI00Yvacpb39DTvTYu3/ha9SeKpo/pgZQ==", + "dev": true, + "license": "SEE LICENSE IN LICENSE.md", + "dependencies": { + "detect-libc": "^2.1.2" + }, + "bin": { + "copilot": "npm-loader.js" + }, + "optionalDependencies": { + "@github/copilot-darwin-arm64": "1.0.80", + "@github/copilot-darwin-x64": "1.0.80", + "@github/copilot-linux-arm64": "1.0.80", + "@github/copilot-linux-x64": "1.0.80", + "@github/copilot-linuxmusl-arm64": "1.0.80", + "@github/copilot-linuxmusl-x64": "1.0.80", + "@github/copilot-win32-arm64": "1.0.80", + "@github/copilot-win32-x64": "1.0.80" + } + }, + "node_modules/@github/copilot-darwin-arm64": { + "version": "1.0.80", + "resolved": "https://registry.npmjs.org/@github/copilot-darwin-arm64/-/copilot-darwin-arm64-1.0.80.tgz", + "integrity": "sha512-fzn4PnSx3+O/a3ip72KVsjnzORsEygK+0i21bFAnFBYS+0Wi1Pk+o/CmNsJ7aRbf1enSJrcH8UDVkyc9pMGEBg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "SEE LICENSE IN LICENSE.md", + "optional": true, + "os": [ + "darwin" + ], + "bin": { + "copilot-darwin-arm64": "copilot" + } + }, + "node_modules/@github/copilot-darwin-x64": { + "version": "1.0.80", + "resolved": "https://registry.npmjs.org/@github/copilot-darwin-x64/-/copilot-darwin-x64-1.0.80.tgz", + "integrity": "sha512-PKsyGk5DccNzR3bYXcYTGB9N6sHzhzGqEwq/2t1qBwqPbrC98Zo2dOT2G40/QYpJ4XdrGmTmdmfPJQ9PJknlIQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "SEE LICENSE IN LICENSE.md", + "optional": true, + "os": [ + "darwin" + ], + "bin": { + "copilot-darwin-x64": "copilot" + } + }, + "node_modules/@github/copilot-linux-arm64": { + "version": "1.0.80", + "resolved": "https://registry.npmjs.org/@github/copilot-linux-arm64/-/copilot-linux-arm64-1.0.80.tgz", + "integrity": "sha512-8oXwN2luyHEjIoSk8AkATBjXDhRoQtuiUvC93GpfQKFHI+I1eoOVwIsAq5fKP8jNCF2rOrYFIcTjwmRt38kCcQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "SEE LICENSE IN LICENSE.md", + "optional": true, + "os": [ + "linux" + ], + "bin": { + "copilot-linux-arm64": "copilot" + } + }, + "node_modules/@github/copilot-linux-x64": { + "version": "1.0.80", + "resolved": "https://registry.npmjs.org/@github/copilot-linux-x64/-/copilot-linux-x64-1.0.80.tgz", + "integrity": "sha512-qv1ytVNwA3IDK7kcQow+fAikD67t42+AQ8X42bK/7oudNiv4frVZMO0yh1DYIebVRcmEhmPvbVPY/ptVUK3cbA==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "SEE LICENSE IN LICENSE.md", + "optional": true, + "os": [ + "linux" + ], + "bin": { + "copilot-linux-x64": "copilot" + } + }, + "node_modules/@github/copilot-linuxmusl-arm64": { + "version": "1.0.80", + "resolved": "https://registry.npmjs.org/@github/copilot-linuxmusl-arm64/-/copilot-linuxmusl-arm64-1.0.80.tgz", + "integrity": "sha512-Qjyi+OlVnPC4Lkuy7blDMMwMUQI/yELl7gDnqQlaN8TEbhZqZueuf3p0a+kEjXcNsw4XtNYQc0eMJqSIYy/Pjg==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "SEE LICENSE IN LICENSE.md", + "optional": true, + "os": [ + "linux" + ], + "bin": { + "copilot-linuxmusl-arm64": "copilot" + } + }, + "node_modules/@github/copilot-linuxmusl-x64": { + "version": "1.0.80", + "resolved": "https://registry.npmjs.org/@github/copilot-linuxmusl-x64/-/copilot-linuxmusl-x64-1.0.80.tgz", + "integrity": "sha512-rBg8pugf+5FhiZxi2zkOr+rlcOVF6Xg63j1FvryfwPT4DJ2w5Na7O3lpS4sgu8QmsP5H+dAqjlXYLYsvSoVQ0g==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "SEE LICENSE IN LICENSE.md", + "optional": true, + "os": [ + "linux" + ], + "bin": { + "copilot-linuxmusl-x64": "copilot" + } + }, + "node_modules/@github/copilot-sdk": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/@github/copilot-sdk/-/copilot-sdk-1.0.11.tgz", + "integrity": "sha512-ngrnfa9052fLTOMoY0iiQS2B6pFDYJpWNj3syCdjzdje0R5mWoij9b8exJZciLvX7BbJjKz2/lIdwo24av3e3A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@github/copilot": "^1.0.79", + "koffi": "^3.1.0", + "vscode-jsonrpc": "^8.2.1", + "zod": "^4.3.6" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@github/copilot-win32-arm64": { + "version": "1.0.80", + "resolved": "https://registry.npmjs.org/@github/copilot-win32-arm64/-/copilot-win32-arm64-1.0.80.tgz", + "integrity": "sha512-+f7Vkd3vt2DYOxRnS8dStvYu3DY638N/AuLuIjxZp1F9GgwCUZK69wspqIxg2L59PmRRQcH4AGTrRDR60ENIZA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "SEE LICENSE IN LICENSE.md", + "optional": true, + "os": [ + "win32" + ], + "bin": { + "copilot-win32-arm64": "copilot.exe" + } + }, + "node_modules/@github/copilot-win32-x64": { + "version": "1.0.80", + "resolved": "https://registry.npmjs.org/@github/copilot-win32-x64/-/copilot-win32-x64-1.0.80.tgz", + "integrity": "sha512-PO0kPqhRTWQfsqGaj4UN3cj8ttkcJYy4wmXiArtFm+03AIFu8xTvuhQDPn2xEOsUome7m7t2XomKoavcrCcRsw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "SEE LICENSE IN LICENSE.md", + "optional": true, + "os": [ + "win32" + ], + "bin": { + "copilot-win32-x64": "copilot.exe" + } + }, + "node_modules/@hono/node-server": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/@hono/node-server/-/node-server-2.1.1.tgz", + "integrity": "sha512-ELuehkj5VCBdgEw9zs+ivkKwyzzUCSQuE96YmiPvn1ECBoZCczbFXJLeEGMTYjphP6gydh4pHMqEYPVMYUVgQg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=20" + }, + "peerDependencies": { + "hono": "^4" + } + }, + "node_modules/@koromix/koffi-android-arm64": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/@koromix/koffi-android-arm64/-/koffi-android-arm64-3.3.0.tgz", + "integrity": "sha512-lHXE2mq+uG6CVeX1iepn2ATcAWMZPagr+J0ICYge7wTMDYc5pPRQMq0Rr/16775i6QlUqQqpk8c5Gu66+u3gGA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-android-x64": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/@koromix/koffi-android-x64/-/koffi-android-x64-3.3.0.tgz", + "integrity": "sha512-7EeVK/KS9XNbOsUfy0B3uuSjPQZenjJ35y9QSi6R6eK4Fv0oGvREQaBzIJRJwonxp7sJfGObnwnQTnKTVK2DWA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-darwin-arm64": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/@koromix/koffi-darwin-arm64/-/koffi-darwin-arm64-3.3.0.tgz", + "integrity": "sha512-2R9bihYmYFyRP98wRG2m5WQVKMONf7DoPJI8RkGCOUB2afOXJEVXFepiSKxBZGPml8i1USwAxMUwkzbJ5uMG1A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-darwin-x64": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/@koromix/koffi-darwin-x64/-/koffi-darwin-x64-3.3.0.tgz", + "integrity": "sha512-OecYCQfzhUx+XyAtatIHlk0jIz87wKNjtYvwlwgMwgaGQwO/TxFeReNo+s3167Rh1a9c8rUIOjq1jfzAPBxbKA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-freebsd-arm64": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/@koromix/koffi-freebsd-arm64/-/koffi-freebsd-arm64-3.3.0.tgz", + "integrity": "sha512-6ytHzV1DuyxTP5LtFmQ3imBk71j/F/Nb0Hcwhjcn35esnknm5rQbgRl6tNRldrkGnM4GIAJpsagooUBh1xAzYg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-freebsd-ia32": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/@koromix/koffi-freebsd-ia32/-/koffi-freebsd-ia32-3.3.0.tgz", + "integrity": "sha512-hINo5z1Iz2u3Rx3oHfzcmi5A03gYAoYf4nCJJYaY0bb5IuGdIC7noQTyyI9YxfyTJgoqw3wKFWWYtvsAmEo9Hw==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-freebsd-x64": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/@koromix/koffi-freebsd-x64/-/koffi-freebsd-x64-3.3.0.tgz", + "integrity": "sha512-7XQYAZh828nW6+y3IdyUNdo2VbVHL5ZC8Qb+yqHrlDjTjVXSyGYZJHAyJZ5c7eSTTKBhtIMTX238rIsdexxrog==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-linux-arm": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/@koromix/koffi-linux-arm/-/koffi-linux-arm-3.3.0.tgz", + "integrity": "sha512-NbahRKKKFkHoFN/ZpLY+w4F6eYSqV4KMN40PmjZ1xQjG/XjYb6PIYgE41c7/acAPxOYqujrcaI3B4BAgWYm5/A==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-linux-arm64": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/@koromix/koffi-linux-arm64/-/koffi-linux-arm64-3.3.0.tgz", + "integrity": "sha512-Eo/+wGeawGICzJkUlkzi/KerAJ6U1L/Hf7cnJKH7aGgCOY5RRnjUqXs/eL+UDpRcFHzGeUn+Fq+8q2VxYxvAfw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-linux-ia32": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/@koromix/koffi-linux-ia32/-/koffi-linux-ia32-3.3.0.tgz", + "integrity": "sha512-u9tKuj42RFWJU8ZgZqcBhkFCkIZE/SE6oR1ljpukttJUhOVuaf/lmsWxyOxiVu3U8T1gtD658FlLgrbMVIn6aQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-linux-loong64": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/@koromix/koffi-linux-loong64/-/koffi-linux-loong64-3.3.0.tgz", + "integrity": "sha512-2hKlNZ6OXkTUp6fcueZcP0sQ6yhpdfANg9/mWxDKzT1o8qIj4qJD7UcZLCcRIC8JXiP8JQwZBtlPuyXIqofDQg==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-linux-ppc64": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/@koromix/koffi-linux-ppc64/-/koffi-linux-ppc64-3.3.0.tgz", + "integrity": "sha512-bIVAoHgzyRN7xs529BK7rF3BbZUTJgIr0t/wvBvAgaIpreKsdZ/QWxhedKRbTN7ljlDHkPZFmu5XcCjb/vqUag==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-linux-riscv64": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/@koromix/koffi-linux-riscv64/-/koffi-linux-riscv64-3.3.0.tgz", + "integrity": "sha512-2niYAe+TBXbvf7yog/aiHiDUrKQn+RkMSH5x3ZYo65OQyII1CY3Jk2kP3Pe735gq0LzIYJsDuGMW+02ighSadQ==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-linux-x64": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/@koromix/koffi-linux-x64/-/koffi-linux-x64-3.3.0.tgz", + "integrity": "sha512-apU8ZMOrOh1eqMP/Pp0sDZ/WY6A4r6JWph26h63VM9nZ+5KCTYuPgojfoLPNvrfI/bn1PY3+bpXKRhZHI3gxbQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-openbsd-arm64": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/@koromix/koffi-openbsd-arm64/-/koffi-openbsd-arm64-3.3.0.tgz", + "integrity": "sha512-iTyHMCSSs+X579sWJv7XVFMp9gOfROI8IUEwp09YxJpGvmiSoipI0J7kytFuFq6dyuwO0Ymy4JdlmKnCM0qkxg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-openbsd-ia32": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/@koromix/koffi-openbsd-ia32/-/koffi-openbsd-ia32-3.3.0.tgz", + "integrity": "sha512-xCvMV+4VNHjTxgVPEsBzU0k8dlA7aoFO5QjRF9lVU/NzGhUedlYvFgaF6IWsGOoWEaNt9hgP4KK/wxMbm9v8EQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-openbsd-x64": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/@koromix/koffi-openbsd-x64/-/koffi-openbsd-x64-3.3.0.tgz", + "integrity": "sha512-xItIwx1elM8k1pTudEfuV/1U2gQ06+rFfxCVXrZkyWYYGTzmA5To00f8garJFR+E9gbjQJbn8QDFvWMI+i2sMA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-win32-arm64": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/@koromix/koffi-win32-arm64/-/koffi-win32-arm64-3.3.0.tgz", + "integrity": "sha512-fD41LgKBx5i1YlJDuds4tRHAaG5vV55tvagzJwKFDR/5BDeaFvH9QFMUMDoNl2LkmVyeug0dIElHr/oxejh3IA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-win32-ia32": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/@koromix/koffi-win32-ia32/-/koffi-win32-ia32-3.3.0.tgz", + "integrity": "sha512-BfDbDVEuQwPZOvWXawc3JT/7i9cN7QHYotgo014jjXb9pNv46Ykhb0RRglkcHg+SdytW/7pxYn18YwcQP6LLvQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-win32-x64": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/@koromix/koffi-win32-x64/-/koffi-win32-x64-3.3.0.tgz", + "integrity": "sha512-D+ORp73rTmvGVfq01+4uBmgVZqrPqU9N9xCZCfexaCDlVk0WQsaqid1dGsi6eScJuHt4Cp8s1i5/4CsecxFTKQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@microsoft/vally": { + "version": "0.16.0", + "resolved": "https://registry.npmjs.org/@microsoft/vally/-/vally-0.16.0.tgz", + "integrity": "sha512-SxpK/kfWaD+CZvPVRxmR8Y/I2MBwqBcamSvTXpU6sZCK352QdSOYiEPjvXFqx9vyt47GdhB3hOe3Nag7dM5/3w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@github/copilot": "1.0.80", + "@github/copilot-sdk": "1.0.11", + "@opentelemetry/api": "^1.9.1", + "js-tiktoken": "^1.0.21", + "mdast-util-from-markdown": "^2.0.3", + "picomatch": "^4.0.7", + "yaml": "^2.9.0", + "zod": "^4.4.3" + }, + "engines": { + "node": ">=22.12.0" + } + }, + "node_modules/@microsoft/vally-cli": { + "version": "0.16.0", + "resolved": "https://registry.npmjs.org/@microsoft/vally-cli/-/vally-cli-0.16.0.tgz", + "integrity": "sha512-mEX7xd4gNIOmM79GQJOWo1ewss6kc8Lg3AOAP95Cz+jWlQ+zK3MphRuA0skfr4x6Zddm7DfVy9lHOgqyNNBdQg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@azure/monitor-opentelemetry-exporter": "^1.0.0-beta.32", + "@microsoft/vally": "^0.16.0", + "@microsoft/vally-server": "^0.16.0", + "@opentelemetry/api": "^1.9.1", + "@opentelemetry/exporter-trace-otlp-http": "^0.221.0", + "@opentelemetry/resources": "^2.10.0", + "@opentelemetry/sdk-trace-base": "^2.10.0", + "@opentelemetry/sdk-trace-node": "^2.10.0", + "commander": "^15.0.0" + }, + "bin": { + "vally": "dist/index.js" + }, + "engines": { + "node": ">=22.12.0" + }, + "optionalDependencies": { + "@vscode/deviceid": "~0.1.5" + } + }, + "node_modules/@microsoft/vally-server": { + "version": "0.16.0", + "resolved": "https://registry.npmjs.org/@microsoft/vally-server/-/vally-server-0.16.0.tgz", + "integrity": "sha512-G11QONnAe+c+WzcRIjvz1xn9Ly2FctT3CZ3vK2Gcm74Vc4P+Yf9XK5jVzKM1ixB0rR1F2p4kv6ZfpQQjmTjMYw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@hono/node-server": "^2.1.1", + "@microsoft/vally": "^0.16.0", + "better-sqlite3": "^13.0.3", + "hono": "^4.13.4" + }, + "engines": { + "node": ">=22.12.0" + } + }, + "node_modules/@opentelemetry/api": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.1.tgz", + "integrity": "sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/@opentelemetry/api-logs": { + "version": "0.221.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/api-logs/-/api-logs-0.221.0.tgz", + "integrity": "sha512-OlanaW1vv7ufTqQ3/fPLI4arGt5ZoM+P8abOMki6uEYnpRazepSWDwDnnw+la7kE26SHVC18//SMccrDvLKOXQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api": "^1.3.0" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/@opentelemetry/context-async-hooks": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/context-async-hooks/-/context-async-hooks-2.11.0.tgz", + "integrity": "sha512-Tr79DyWI8itsBdg+jH+opjfrwLzX+erk1/ExkIwhWoAVjVrJIn2y5+cGjTC0Vy8fyNIA/y8wuJPZwr1T3xCZeQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/core": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.11.0.tgz", + "integrity": "sha512-7YP44XH0tV6+Mb54x2YGf84i7yi+31MBZlE8JwvozkxyTvXbSp10X7cI7YE49ChJ3shMJoBmCJF3+1QFBJctGA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/exporter-trace-otlp-http": { + "version": "0.221.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-trace-otlp-http/-/exporter-trace-otlp-http-0.221.0.tgz", + "integrity": "sha512-AySXiKoC+meiWm6zdVj5T2LnPDZuatveBby1cMOeQteIWsYXAUxs8Sru13G2pVSPrUXz6vF+og7QVBX6GdC/oQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/otlp-exporter-base": "0.221.0", + "@opentelemetry/otlp-transformer": "0.221.0", + "@opentelemetry/sdk-trace": "2.10.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/otlp-exporter-base": { + "version": "0.221.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/otlp-exporter-base/-/otlp-exporter-base-0.221.0.tgz", + "integrity": "sha512-UFPIq80OH3Ns/oPFHRj14d4DTOxUo+MUFU8hUiCq5jTqFhdeJnfVSANHT+xp92409cA+oxzvlZCe6NM1wvCuBA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.10.0", + "@opentelemetry/otlp-transformer": "0.221.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/otlp-exporter-base/node_modules/@opentelemetry/core": { + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.10.0.tgz", + "integrity": "sha512-/wNZ8twnEQQA4HoHu22+vcsdru6pWPWxW+7w+FlxT6Id7PE/WIbZmVKkte+PF72e0F2dnImFeHD2syyE1Mw6MQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/otlp-transformer": { + "version": "0.221.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/otlp-transformer/-/otlp-transformer-0.221.0.tgz", + "integrity": "sha512-lg6lkOU08Az23jVcn/0Els9HP+V8PnR4Km6p0KgpTggS0n/WuhnmY64rSh83Of9iR9nD+dpWr6adlcX8KzAwjg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api-logs": "0.221.0", + "@opentelemetry/core": "2.10.0", + "@opentelemetry/resources": "2.10.0", + "@opentelemetry/sdk-logs": "0.221.0", + "@opentelemetry/sdk-metrics": "2.10.0", + "@opentelemetry/sdk-trace": "2.10.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/otlp-transformer/node_modules/@opentelemetry/core": { + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.10.0.tgz", + "integrity": "sha512-/wNZ8twnEQQA4HoHu22+vcsdru6pWPWxW+7w+FlxT6Id7PE/WIbZmVKkte+PF72e0F2dnImFeHD2syyE1Mw6MQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/otlp-transformer/node_modules/@opentelemetry/resources": { + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-2.10.0.tgz", + "integrity": "sha512-q6MMm2zhggzsHVNbabYwut+a6nbuQQe3URUoxaojM/8K1IBfwwPzvxIjNi2/lI1TFe+fMHMW9MWhrtDLEXEnkA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.10.0", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/otlp-transformer/node_modules/@opentelemetry/sdk-metrics": { + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-metrics/-/sdk-metrics-2.10.0.tgz", + "integrity": "sha512-t6r1VSvXNtSDnPXU1FbZeetJb7yyovHmgu0wRSoftxtE0g2rSNhQZQUy69sRUCL+iioJpX8SN/S6wq6ZtvLySQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.10.0", + "@opentelemetry/resources": "2.10.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.9.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/resources": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-2.11.0.tgz", + "integrity": "sha512-Ie7+8q8MDF4FAEQCKVMTx3ReUvxiIAgIiiW3c9JdmP8+HMcDy20puT+AHjexnExgnbvBxjQ9fjkFDWrikJ2jQA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.11.0", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-logs": { + "version": "0.221.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-logs/-/sdk-logs-0.221.0.tgz", + "integrity": "sha512-FaDcazjyMp7TZZZAsqbo4IkovP0UegoCu0EBkiNt+qCqvUf7FPAsfcrZ3+ZEkKgXZ/jHafop+JoGPDk3A0SmLg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api-logs": "0.221.0", + "@opentelemetry/core": "2.10.0", + "@opentelemetry/resources": "2.10.0", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.4.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-logs/node_modules/@opentelemetry/core": { + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.10.0.tgz", + "integrity": "sha512-/wNZ8twnEQQA4HoHu22+vcsdru6pWPWxW+7w+FlxT6Id7PE/WIbZmVKkte+PF72e0F2dnImFeHD2syyE1Mw6MQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-logs/node_modules/@opentelemetry/resources": { + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-2.10.0.tgz", + "integrity": "sha512-q6MMm2zhggzsHVNbabYwut+a6nbuQQe3URUoxaojM/8K1IBfwwPzvxIjNi2/lI1TFe+fMHMW9MWhrtDLEXEnkA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.10.0", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-metrics": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-metrics/-/sdk-metrics-2.11.0.tgz", + "integrity": "sha512-7GXXcObyHyDUUSG+L+kJoquty01bzm7ivE7+SSgXXJcHuPzGviptxwARmI2c+bnnxjexGQbJnyNlN8HxBP/Y7A==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.11.0", + "@opentelemetry/resources": "2.11.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.9.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-trace": { + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace/-/sdk-trace-2.10.0.tgz", + "integrity": "sha512-MfQGq3GRmTh5fM/y+OjaO0vj6+luCB1XO2gfXCalKCfgKw0eHL++sm75DNweC6ohlp+aFvACqeE0fYayqdRaoQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.10.0", + "@opentelemetry/resources": "2.10.0", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-trace-base": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-base/-/sdk-trace-base-2.11.0.tgz", + "integrity": "sha512-H19x/TX/LZdqiYOjM7fqtSxwlplC5pgelavqbQdHbhdq0q/AI/TGkM2dfGuuynTXmJPeF2HoZVoPDu+TGoW78A==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.11.0", + "@opentelemetry/resources": "2.11.0", + "@opentelemetry/sdk-trace": "2.11.0", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-trace-base/node_modules/@opentelemetry/sdk-trace": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace/-/sdk-trace-2.11.0.tgz", + "integrity": "sha512-fFnTqGm8/G73GQVnxYi7LXa1ZVYEUvgL6XI1LpvV0bPC7WQ/ZGgKxCSl8FnlZBKto9JHHEFTO6s6CUpvvtwFrA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.11.0", + "@opentelemetry/resources": "2.11.0", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-trace-node": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-node/-/sdk-trace-node-2.11.0.tgz", + "integrity": "sha512-CuvCMJmZxswhNLlM2LfuLOW3h3fZujA4hsG4B+Sz4dX2zvaXO8Ng74cnDHWD64gLszTlhiG3c0iNUjj4g+0/sA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/context-async-hooks": "2.11.0", + "@opentelemetry/core": "2.11.0", + "@opentelemetry/sdk-trace-base": "2.11.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-trace/node_modules/@opentelemetry/core": { + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.10.0.tgz", + "integrity": "sha512-/wNZ8twnEQQA4HoHu22+vcsdru6pWPWxW+7w+FlxT6Id7PE/WIbZmVKkte+PF72e0F2dnImFeHD2syyE1Mw6MQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-trace/node_modules/@opentelemetry/resources": { + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-2.10.0.tgz", + "integrity": "sha512-q6MMm2zhggzsHVNbabYwut+a6nbuQQe3URUoxaojM/8K1IBfwwPzvxIjNi2/lI1TFe+fMHMW9MWhrtDLEXEnkA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.10.0", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/semantic-conventions": { + "version": "1.43.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.43.0.tgz", + "integrity": "sha512-eSYWTm620tTk45EKSedaUL8MFYI8hW164hIXsgIHyxu3VobUB3fFCu5t0hQby6OoWRPsG1KkKUG2M5UadiLiVg==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=14" + } + }, + "node_modules/@types/debug": { + "version": "4.1.13", + "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.13.tgz", + "integrity": "sha512-KSVgmQmzMwPlmtljOomayoR89W4FynCAi3E8PPs7vmDVPe84hT+vGPKkJfThkmXs0x0jAaa9U8uW8bbfyS2fWw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/ms": "*" + } + }, + "node_modules/@types/mdast": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/@types/mdast/-/mdast-4.0.4.tgz", + "integrity": "sha512-kGaNbPh1k7AFzgpud/gMdvIm5xuECykRR+JnWKQno9TAXVa6WIVCGTPvYGekIDL4uwCZQSYbUxNBSb1aUo79oA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/unist": "*" + } + }, + "node_modules/@types/ms": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@types/ms/-/ms-2.1.0.tgz", + "integrity": "sha512-GsCCIZDE/p3i96vtEqx+7dBUGXrc7zeSK3wwPHIaRThS+9OhWIXRqzs4d6k1SVU8g91DrNRWxWUGhp5KXQb2VA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/unist": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/unist/-/unist-3.0.3.tgz", + "integrity": "sha512-ko/gIFJRv177XgZsZcBwnqJN5x/Gien8qNOn0D5bQU/zAzVf9Zt3BlcUiLqhV9y4ARk0GbT3tnUiPNgnTXzc/Q==", + "dev": true, + "license": "MIT" + }, + "node_modules/@typespec/ts-http-runtime": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@typespec/ts-http-runtime/-/ts-http-runtime-0.3.9.tgz", + "integrity": "sha512-edSdeAqkdxBVzA1yL1LrLCml1YjyCVvPMtMqJpbF+6K609tHe8V6sQUzFQSGcYNhcuhOceZtjvN32+mpIth30A==", + "dev": true, + "license": "MIT", + "dependencies": { + "http-proxy-agent": "^7.0.0", + "https-proxy-agent": "^7.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@vscode/deviceid": { + "version": "0.1.5", + "resolved": "https://registry.npmjs.org/@vscode/deviceid/-/deviceid-0.1.5.tgz", + "integrity": "sha512-D0be67wWo7WyyBqHnRkL2bK7lp7CDH/EMN4kMV6INoKc7kxRL3nsTtngt9JZrOcZdnW59gquGRk+6KFIDyD3QA==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "dependencies": { + "fs-extra": "^11.2.0", + "uuid": "^14.0.0" + } + }, + "node_modules/agent-base": { + "version": "7.1.4", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", + "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/better-sqlite3": { + "version": "13.0.3", + "resolved": "https://registry.npmjs.org/better-sqlite3/-/better-sqlite3-13.0.3.tgz", + "integrity": "sha512-RbOBxmLBG8uvFUc15X9+9SFemKcQ0WBuISBVkpuiaUB2qblC8UWlHEjdWVoZ8AdhSwmoEgsiXKfopX0CQxaACQ==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "dependencies": { + "node-addon-api": "^8.0.0" + }, + "engines": { + "node": ">=22" + } + }, + "node_modules/character-entities": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/character-entities/-/character-entities-2.0.2.tgz", + "integrity": "sha512-shx7oQ0Awen/BRIdkjkvz54PnEEI/EjwXDSIZp86/KKdbafHh1Df/RYGBhn4hbe2+uKC9FnT5UCEdyPz3ai9hQ==", + "dev": true, + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/commander": { + "version": "15.0.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-15.0.0.tgz", + "integrity": "sha512-z67u4ZhzCL/Tydu1lJARtEZYWbWaN7oYLHbsuzocr6y4N6WZAagG3RQ4FW61V1/0+jImpj293XfrcYnd1qxtPg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=22.12.0" + } + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/decode-named-character-reference": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/decode-named-character-reference/-/decode-named-character-reference-1.3.0.tgz", + "integrity": "sha512-GtpQYB283KrPp6nRw50q3U9/VfOutZOe103qlN7BPP6Ad27xYnOIWv4lPzo8HCAL+mMZofJ9KEy30fq6MfaK6Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "character-entities": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/dequal": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz", + "integrity": "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=8" + } + }, + "node_modules/devlop": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/devlop/-/devlop-1.1.0.tgz", + "integrity": "sha512-RWmIqhcFf1lRYBvNmr7qTNuyCt/7/ns2jbpp1+PalgE/rDQcBT0fioSMUpJ93irlUhC5hrg4cYqe6U+0ImW0rA==", + "dev": true, + "license": "MIT", + "dependencies": { + "dequal": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/fs-extra": { + "version": "11.4.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.4.0.tgz", + "integrity": "sha512-EQsFzMUJkCKGr1ePqlYADkIUmHW1s3ZXr5Yqy6wbGrfUCphpl2maM/kyOIRA2HpP3AaFQTZXD4ldjek+nccddA==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + }, + "engines": { + "node": ">=14.14" + } + }, + "node_modules/graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "dev": true, + "license": "ISC", + "optional": true + }, + "node_modules/hono": { + "version": "4.13.7", + "resolved": "https://registry.npmjs.org/hono/-/hono-4.13.7.tgz", + "integrity": "sha512-c8/gF9ac8Y78/agExVocyLevgR+JlpNB444Py0FSX8pJoPdYUfUzRcXtYEYGwt6l19qIlVZPN5Mfsw9jFShmQQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=16.9.0" + } + }, + "node_modules/http-proxy-agent": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", + "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", + "dev": true, + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.0", + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/https-proxy-agent": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", + "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", + "dev": true, + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/js-tiktoken": { + "version": "1.0.21", + "resolved": "https://registry.npmjs.org/js-tiktoken/-/js-tiktoken-1.0.21.tgz", + "integrity": "sha512-biOj/6M5qdgx5TKjDnFT1ymSpM5tbd3ylwDtrQvFQSu0Z7bBYko2dF+W/aUkXUPuk6IVpRxk/3Q2sHOzGlS36g==", + "dev": true, + "license": "MIT", + "dependencies": { + "base64-js": "^1.5.1" + } + }, + "node_modules/jsonfile": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.2.1.tgz", + "integrity": "sha512-zwOTdL3rFQ/lRdBnntKVOX6k5cKJwEc1HdilT71BWEu7J41gXIB2MRp+vxduPSwZJPWBxEzv4yH1wYLJGUHX4Q==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "universalify": "^2.0.0" + }, + "optionalDependencies": { + "graceful-fs": "^4.1.6" + } + }, + "node_modules/koffi": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/koffi/-/koffi-3.3.0.tgz", + "integrity": "sha512-UeyQtptuPeCP0BqKD+Q8ftTQhIUBHO7bXyK9DVIaW9hJSg0BrICm4K5N8Qtn7zKk20sb5vpnES9KBqVksM2kpQ==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "funding": { + "url": "https://liberapay.com/Koromix" + }, + "optionalDependencies": { + "@koromix/koffi-android-arm64": "3.3.0", + "@koromix/koffi-android-x64": "3.3.0", + "@koromix/koffi-darwin-arm64": "3.3.0", + "@koromix/koffi-darwin-x64": "3.3.0", + "@koromix/koffi-freebsd-arm64": "3.3.0", + "@koromix/koffi-freebsd-ia32": "3.3.0", + "@koromix/koffi-freebsd-x64": "3.3.0", + "@koromix/koffi-linux-arm": "3.3.0", + "@koromix/koffi-linux-arm64": "3.3.0", + "@koromix/koffi-linux-ia32": "3.3.0", + "@koromix/koffi-linux-loong64": "3.3.0", + "@koromix/koffi-linux-ppc64": "3.3.0", + "@koromix/koffi-linux-riscv64": "3.3.0", + "@koromix/koffi-linux-x64": "3.3.0", + "@koromix/koffi-openbsd-arm64": "3.3.0", + "@koromix/koffi-openbsd-ia32": "3.3.0", + "@koromix/koffi-openbsd-x64": "3.3.0", + "@koromix/koffi-win32-arm64": "3.3.0", + "@koromix/koffi-win32-ia32": "3.3.0", + "@koromix/koffi-win32-x64": "3.3.0" + } + }, + "node_modules/mdast-util-from-markdown": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/mdast-util-from-markdown/-/mdast-util-from-markdown-2.0.3.tgz", + "integrity": "sha512-W4mAWTvSlKvf8L6J+VN9yLSqQ9AOAAvHuoDAmPkz4dHf553m5gVj2ejadHJhoJmcmxEnOv6Pa8XJhpxE93kb8Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "@types/unist": "^3.0.0", + "decode-named-character-reference": "^1.0.0", + "devlop": "^1.0.0", + "mdast-util-to-string": "^4.0.0", + "micromark": "^4.0.0", + "micromark-util-decode-numeric-character-reference": "^2.0.0", + "micromark-util-decode-string": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0", + "unist-util-stringify-position": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-to-string": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/mdast-util-to-string/-/mdast-util-to-string-4.0.0.tgz", + "integrity": "sha512-0H44vDimn51F0YwvxSJSm0eCDOJTRlmN0R1yBh4HLj9wiV1Dn0QoXGbvFAWj2hSItVTlCmBF1hqKlIyUBVFLPg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/micromark/-/micromark-4.0.2.tgz", + "integrity": "sha512-zpe98Q6kvavpCr1NPVSCMebCKfD7CA2NqZ+rykeNhONIJBpc1tFKt9hucLGwha3jNTNI8lHpctWJWoimVF4PfA==", + "dev": true, + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "@types/debug": "^4.0.0", + "debug": "^4.0.0", + "decode-named-character-reference": "^1.0.0", + "devlop": "^1.0.0", + "micromark-core-commonmark": "^2.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-chunked": "^2.0.0", + "micromark-util-combine-extensions": "^2.0.0", + "micromark-util-decode-numeric-character-reference": "^2.0.0", + "micromark-util-encode": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0", + "micromark-util-resolve-all": "^2.0.0", + "micromark-util-sanitize-uri": "^2.0.0", + "micromark-util-subtokenize": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-core-commonmark": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/micromark-core-commonmark/-/micromark-core-commonmark-2.0.3.tgz", + "integrity": "sha512-RDBrHEMSxVFLg6xvnXmb1Ayr2WzLAWjeSATAoxwKYJV94TeNavgoIdA0a9ytzDSVzBy2YKFK+emCPOEibLeCrg==", + "dev": true, + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "decode-named-character-reference": "^1.0.0", + "devlop": "^1.0.0", + "micromark-factory-destination": "^2.0.0", + "micromark-factory-label": "^2.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-factory-title": "^2.0.0", + "micromark-factory-whitespace": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-chunked": "^2.0.0", + "micromark-util-classify-character": "^2.0.0", + "micromark-util-html-tag-name": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0", + "micromark-util-resolve-all": "^2.0.0", + "micromark-util-subtokenize": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-destination": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-destination/-/micromark-factory-destination-2.0.1.tgz", + "integrity": "sha512-Xe6rDdJlkmbFRExpTOmRj9N3MaWmbAgdpSrBQvCFqhezUn4AHqJHbaEnfbVYYiexVSs//tqOdY/DxhjdCiJnIA==", + "dev": true, + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-label": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-label/-/micromark-factory-label-2.0.1.tgz", + "integrity": "sha512-VFMekyQExqIW7xIChcXn4ok29YE3rnuyveW3wZQWWqF4Nv9Wk5rgJ99KzPvHjkmPXF93FXIbBp6YdW3t71/7Vg==", + "dev": true, + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-space": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-space/-/micromark-factory-space-2.0.1.tgz", + "integrity": "sha512-zRkxjtBxxLd2Sc0d+fbnEunsTj46SWXgXciZmHq0kDYGnck/ZSGj9/wULTV95uoeYiK5hRXP2mJ98Uo4cq/LQg==", + "dev": true, + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-title": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-title/-/micromark-factory-title-2.0.1.tgz", + "integrity": "sha512-5bZ+3CjhAd9eChYTHsjy6TGxpOFSKgKKJPJxr293jTbfry2KDoWkhBb6TcPVB4NmzaPhMs1Frm9AZH7OD4Cjzw==", + "dev": true, + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-whitespace": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-whitespace/-/micromark-factory-whitespace-2.0.1.tgz", + "integrity": "sha512-Ob0nuZ3PKt/n0hORHyvoD9uZhr+Za8sFoP+OnMcnWK5lngSzALgQYKMr9RJVOWLqQYuyn6ulqGWSXdwf6F80lQ==", + "dev": true, + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-character": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/micromark-util-character/-/micromark-util-character-2.1.1.tgz", + "integrity": "sha512-wv8tdUTJ3thSFFFJKtpYKOYiGP2+v96Hvk4Tu8KpCAsTMs6yi+nVmGh1syvSCsaxz45J6Jbw+9DD6g97+NV67Q==", + "dev": true, + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-chunked": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-chunked/-/micromark-util-chunked-2.0.1.tgz", + "integrity": "sha512-QUNFEOPELfmvv+4xiNg2sRYeS/P84pTW0TCgP5zc9FpXetHY0ab7SxKyAQCNCc1eK0459uoLI1y5oO5Vc1dbhA==", + "dev": true, + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-classify-character": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-classify-character/-/micromark-util-classify-character-2.0.1.tgz", + "integrity": "sha512-K0kHzM6afW/MbeWYWLjoHQv1sgg2Q9EccHEDzSkxiP/EaagNzCm7T/WMKZ3rjMbvIpvBiZgwR3dKMygtA4mG1Q==", + "dev": true, + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-combine-extensions": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-combine-extensions/-/micromark-util-combine-extensions-2.0.1.tgz", + "integrity": "sha512-OnAnH8Ujmy59JcyZw8JSbK9cGpdVY44NKgSM7E9Eh7DiLS2E9RNQf0dONaGDzEG9yjEl5hcqeIsj4hfRkLH/Bg==", + "dev": true, + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-chunked": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-decode-numeric-character-reference": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/micromark-util-decode-numeric-character-reference/-/micromark-util-decode-numeric-character-reference-2.0.2.tgz", + "integrity": "sha512-ccUbYk6CwVdkmCQMyr64dXz42EfHGkPQlBj5p7YVGzq8I7CtjXZJrubAYezf7Rp+bjPseiROqe7G6foFd+lEuw==", + "dev": true, + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-decode-string": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-decode-string/-/micromark-util-decode-string-2.0.1.tgz", + "integrity": "sha512-nDV/77Fj6eH1ynwscYTOsbK7rR//Uj0bZXBwJZRfaLEJ1iGBR6kIfNmlNqaqJf649EP0F3NWNdeJi03elllNUQ==", + "dev": true, + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "decode-named-character-reference": "^1.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-decode-numeric-character-reference": "^2.0.0", + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-encode": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-encode/-/micromark-util-encode-2.0.1.tgz", + "integrity": "sha512-c3cVx2y4KqUnwopcO9b/SCdo2O67LwJJ/UyqGfbigahfegL9myoEFoDYZgkT7f36T0bLrM9hZTAaAyH+PCAXjw==", + "dev": true, + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-util-html-tag-name": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-html-tag-name/-/micromark-util-html-tag-name-2.0.1.tgz", + "integrity": "sha512-2cNEiYDhCWKI+Gs9T0Tiysk136SnR13hhO8yW6BGNyhOC4qYFnwF1nKfD3HFAIXA5c45RrIG1ub11GiXeYd1xA==", + "dev": true, + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-util-normalize-identifier": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-normalize-identifier/-/micromark-util-normalize-identifier-2.0.1.tgz", + "integrity": "sha512-sxPqmo70LyARJs0w2UclACPUUEqltCkJ6PhKdMIDuJ3gSf/Q+/GIe3WKl0Ijb/GyH9lOpUkRAO2wp0GVkLvS9Q==", + "dev": true, + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-resolve-all": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-resolve-all/-/micromark-util-resolve-all-2.0.1.tgz", + "integrity": "sha512-VdQyxFWFT2/FGJgwQnJYbe1jjQoNTS4RjglmSjTUlpUMa95Htx9NHeYW4rGDJzbjvCsl9eLjMQwGeElsqmzcHg==", + "dev": true, + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-sanitize-uri": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-sanitize-uri/-/micromark-util-sanitize-uri-2.0.1.tgz", + "integrity": "sha512-9N9IomZ/YuGGZZmQec1MbgxtlgougxTodVwDzzEouPKo3qFWvymFHWcnDi2vzV1ff6kas9ucW+o3yzJK9YB1AQ==", + "dev": true, + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-encode": "^2.0.0", + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-subtokenize": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/micromark-util-subtokenize/-/micromark-util-subtokenize-2.1.0.tgz", + "integrity": "sha512-XQLu552iSctvnEcgXw6+Sx75GflAPNED1qx7eBJ+wydBb2KCbRZe+NwvIEEMM83uml1+2WSXpBAcp9IUCgCYWA==", + "dev": true, + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-util-chunked": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-symbol": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-symbol/-/micromark-util-symbol-2.0.1.tgz", + "integrity": "sha512-vs5t8Apaud9N28kgCrRUdEed4UJ+wWNvicHLPxCa9ENlYuAY31M0ETy5y1vA33YoNPDFTghEbnh6efaE8h4x0Q==", + "dev": true, + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-util-types": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/micromark-util-types/-/micromark-util-types-2.0.2.tgz", + "integrity": "sha512-Yw0ECSpJoViF1qTU4DC6NwtC4aWGt1EkzaQB8KPPyCRR8z9TWeV0HbEFGTO+ZY1wB22zmxnJqhPyTpOVCpeHTA==", + "dev": true, + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true, + "license": "MIT" + }, + "node_modules/node-addon-api": { + "version": "8.9.2", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.9.2.tgz", + "integrity": "sha512-VijLXbi3UACN69I0JVXJsX4tjACjNoQDgv2gTF6sx2wWEi8tkSg2eX8p5gSIFi8z2+DL3oHmY6OyKce38SDolg==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^18 || ^20 || >= 21" + } + }, + "node_modules/picomatch": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.7.tgz", + "integrity": "sha512-qcJu88Q2IWqJsDD529JKMdwGm/dvInW4HvQnRwiH9JtihJvzGOscDtHE3x1pBKeUOTysQ8kVmLnJ2kJu7yhcGA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "dev": true, + "license": "0BSD" + }, + "node_modules/unist-util-stringify-position": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/unist-util-stringify-position/-/unist-util-stringify-position-4.0.0.tgz", + "integrity": "sha512-0ASV06AAoKCDkS2+xw5RXJywruurpbC4JZSm7nr7MOt1ojAzvyyaO+UxZf18j8FCF6kmzCZKcAgN/yu2gm2XgQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/universalify": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.1.tgz", + "integrity": "sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==", + "dev": true, + "license": "MIT", + "optional": true, + "engines": { + "node": ">= 10.0.0" + } + }, + "node_modules/uuid": { + "version": "14.0.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-14.0.2.tgz", + "integrity": "sha512-xZe/16rV4aa+HGSOCiY2YeLT1OybRLrrkL/Rqaq7p7GMVXjFh+6wN4oMYgjFmnSnhY8t6Xpdl2l9qmnHYuMHwQ==", + "dev": true, + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "license": "MIT", + "optional": true, + "bin": { + "uuid": "dist-node/bin/uuid" + } + }, + "node_modules/vscode-jsonrpc": { + "version": "8.2.1", + "resolved": "https://registry.npmjs.org/vscode-jsonrpc/-/vscode-jsonrpc-8.2.1.tgz", + "integrity": "sha512-kdjOSJ2lLIn7r1rtrMbbNCHjyMPfRnowdKjBQ+mGq6NAW5QY2bEZC/khaC5OR8svbbjvLEaIXkOq45e2X9BIbQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/yaml": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.9.0.tgz", + "integrity": "sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA==", + "dev": true, + "license": "ISC", + "bin": { + "yaml": "bin.mjs" + }, + "engines": { + "node": ">= 14.6" + }, + "funding": { + "url": "https://github.com/sponsors/eemeli" + } + }, + "node_modules/zod": { + "version": "4.6.5", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.6.5.tgz", + "integrity": "sha512-v5l/aFXZQeai4awLbOpSoHecE9UiMrnfx75tEXLjNonXVARxQ5mOeipTjROUchszUNCqnE+hqAMujRsRHsut2Q==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + } + } +} diff --git a/eng/harness-evaluation/package.json b/eng/harness-evaluation/package.json new file mode 100644 index 00000000000..fe1051a5b5c --- /dev/null +++ b/eng/harness-evaluation/package.json @@ -0,0 +1,22 @@ +{ + "name": "efcore-harness-evaluation", + "version": "0.1.0", + "private": true, + "type": "module", + "engines": { + "node": ">=22.12.0" + }, + "scripts": { + "test": "node --test test/*.test.mjs", + "validate": "node src/cli.mjs validate", + "discover": "node src/cli.mjs discover", + "prepare": "node src/cli.mjs prepare", + "lint": "node src/cli.mjs lint", + "eval": "node src/cli.mjs eval" + }, + "devDependencies": { + "@microsoft/vally": "0.16.0", + "@microsoft/vally-cli": "0.16.0", + "yaml": "2.9.0" + } +} diff --git a/eng/harness-evaluation/skills/change-tracking/eval.yaml b/eng/harness-evaluation/skills/change-tracking/eval.yaml new file mode 100644 index 00000000000..16b76b2f443 --- /dev/null +++ b/eng/harness-evaluation/skills/change-tracking/eval.yaml @@ -0,0 +1,59 @@ +name: change-tracking +description: Evaluates change-tracking guidance against EF Core snapshot and entry-state code. +type: capability +defaults: + runs: 5 + timeout: 30m + model: claude-sonnet-5 + judge_model: gpt-5.6-terra + executor: copilot-sdk +stimuli: + - name: diagnose-temporary-value-snapshot-transition + prompt: > + Before analyzing files, use the available repository skill that applies to this task. Inspect + src/EFCore/ChangeTracking/Internal/SnapshotFactoryFactory.cs, + + src/EFCore/ChangeTracking/Internal/InternalEntityEntry.cs, and + + test/EFCore.Tests/ChangeTracking/TemporaryValuesTest.cs. A proposed change causes a temporary property to lose its + + modified state when an Added entry becomes Modified. Write analysis.md identifying the controlling code path, + + the shared-identity edge case that must remain safe, and the focused regression test you would add. Do not edit + source. + constraints: + max_turns: 80 + max_tokens: 2000000 + max_duration: 30m + tags: + repo-path: + - src/EFCore/ChangeTracking/Internal/SnapshotFactoryFactory.cs + - src/EFCore/ChangeTracking/Internal/InternalEntityEntry.cs + - test/EFCore.Tests/ChangeTracking/TemporaryValuesTest.cs + rubric: + - The analysis distinguishes snapshot comparison from state-transition handling in InternalEntityEntry. + - It explicitly considers SharedIdentityEntry when Added and Deleted entries can represent the same identity. + - The proposed test is located in TemporaryValuesTest.cs and verifies observable property/state behavior. + graders: + - type: skill-invocation + config: + required: + - change-tracking + - type: file-matches + config: + path: analysis.md + pattern: (InternalEntityEntry|SetEntityState)[\s\S]*(TemporaryValuesTest|temporary) + - type: prompt + config: + scoring: binary + threshold: 1 + evidence: + - trajectory + - repo + output_delivery: workspace +scoring: + weights: + skill-invocation: 0.3 + file-matches: 0.3 + prompt: 0.4 + threshold: 0.75 diff --git a/eng/harness-evaluation/skills/cosmos-provider/eval.yaml b/eng/harness-evaluation/skills/cosmos-provider/eval.yaml new file mode 100644 index 00000000000..07638f0010a --- /dev/null +++ b/eng/harness-evaluation/skills/cosmos-provider/eval.yaml @@ -0,0 +1,65 @@ +name: cosmos-provider +description: Evaluates Cosmos provider guidance against EF Core ReadItem translation. +type: capability +defaults: + runs: 5 + timeout: 25m + model: claude-sonnet-5 + judge_model: gpt-5.6-terra + executor: copilot-sdk +stimuli: + - name: explain-readitem-discriminator-selection + prompt: > + Before analyzing files, use the available repository skill that applies to this task. + + Inspect src/EFCore.Cosmos/Query/Internal/CosmosReadItemAndPartitionKeysExtractor.cs, + + test/EFCore.Cosmos.FunctionalTests/Query/ReadItemPartitionKeyQueryTestBase.cs, and + + test/EFCore.Cosmos.FunctionalTests/Query/ReadItemPartitionKeyQueryTest.cs. Write analysis.md explaining why + ReadItem_with_single_explicit_discriminator_mapping emits ReadItem, while + ReadItem_with_single_explicit_incorrect_discriminator_mapping and + ReadItem_with_single_explicit_parameterized_discriminator_mapping emit general queries. Identify the exact + extractor condition responsible, + explain why retaining each predicate is required for correctness, and outline the assertions that preserve all + three behaviors during a refactor. Do not edit source. Include the exact focused EFCore.Cosmos.FunctionalTests + command and state whether these tests require the emulator or a Linux-emulator condition. + constraints: + max_turns: 80 + max_tokens: 2000000 + max_duration: 25m + tags: + repo-path: + - src/EFCore.Cosmos/Query/Internal/CosmosReadItemAndPartitionKeysExtractor.cs + - test/EFCore.Cosmos.FunctionalTests/Query/ReadItemPartitionKeyQueryTestBase.cs + - test/EFCore.Cosmos.FunctionalTests/Query/ReadItemPartitionKeyQueryTest.cs + rubric: + - The analysis identifies the SqlConstantExpression discriminator condition in + CosmosReadItemAndPartitionKeysExtractor. + - It correctly explains the existing ReadItem result for the matching constant and general-query results for + incorrect and parameterized discriminator values. + - It preserves all three base/override assertions and includes the focused Cosmos functional-test command and + emulator constraints. + graders: + - type: skill-invocation + config: + required: + - cosmos-provider + - type: file-matches + config: + path: analysis.md + pattern: (?=[\s\S]*CosmosReadItemAndPartitionKeysExtractor)(?=[\s\S]*ReadItem)(?=[\s\S]*(EFCore\.Cosmos\.FunctionalTests|dotnet test))(?=[\s\S]*emulator)[\s\S]+ + - type: prompt + config: + scoring: binary + threshold: 1 + evidence: + - trajectory + - repo + output_delivery: workspace +scoring: + weights: + skill-invocation: 0.3 + file-matches: 0.3 + prompt: 0.4 + threshold: 0.75 diff --git a/eng/harness-evaluation/skills/make-custom-agent/eval.yaml b/eng/harness-evaluation/skills/make-custom-agent/eval.yaml new file mode 100644 index 00000000000..ee77d871c48 --- /dev/null +++ b/eng/harness-evaluation/skills/make-custom-agent/eval.yaml @@ -0,0 +1,59 @@ +name: make-custom-agent +description: Evaluates custom-agent authoring for EF Core API review. +type: capability +defaults: + runs: 5 + timeout: 10m + model: claude-sonnet-5 + judge_model: gpt-5.6-terra + executor: copilot-sdk +stimuli: + - name: create-an-efcore-api-review-agent + prompt: > + Before creating files, use the available repository skill that applies to this task. + + Inspect .github/workflows/api-review-baselines.yml, src/EFCore/EFCore.baseline.json, and + + .github/copilot-instructions.md. Create a declarative custom agent under .github/agents/ that reviews EF Core + ApiChief + + baseline deltas, explains compatibility impact, and points reviewers to evidence from the staged workflow and + baseline. + + Keep it read-only and do not create a skill or duplicate the root instructions. + constraints: + max_turns: 50 + max_tokens: 1000000 + max_duration: 10m + tags: + repo-path: + - .github/workflows/api-review-baselines.yml + - src/EFCore/EFCore.baseline.json + - .github/copilot-instructions.md + rubric: + - The output is a declarative agent in .github/agents with valid name and description frontmatter. + - Its responsibilities and allowed tools are specifically tied to EF Core ApiChief baseline review. + - It remains read-only, avoids secrets and internal URLs, and does not duplicate root repository instructions. + graders: + - type: skill-invocation + config: + required: + - make-custom-agent + - type: file-matches + config: + path: .github/agents/*.md + pattern: '(?m)^---[\s\S]*^name:[\s\S]*^description:' + - type: prompt + config: + scoring: binary + threshold: 1 + evidence: + - trajectory + - repo + output_delivery: workspace +scoring: + weights: + skill-invocation: 0.3 + file-matches: 0.3 + prompt: 0.4 + threshold: 0.75 diff --git a/eng/harness-evaluation/skills/make-github-actions-workflow/eval.yaml b/eng/harness-evaluation/skills/make-github-actions-workflow/eval.yaml new file mode 100644 index 00000000000..0a28ed3f5ff --- /dev/null +++ b/eng/harness-evaluation/skills/make-github-actions-workflow/eval.yaml @@ -0,0 +1,64 @@ +name: make-github-actions-workflow +description: Evaluates workflow authoring against EF Core's fork-safe PR automation. +type: capability +defaults: + runs: 5 + timeout: 5m + model: claude-sonnet-5 + judge_model: gpt-5.6-terra + executor: copilot-sdk +stimuli: + - name: extend-the-release-target-validation-workflow + prompt: > + Before changing files, use the available repository skill that applies to this task. + + Inspect .github/workflows/validate-pr-target-branch.yml and + + .github/workflows/label-and-milestone-issues.yml. Modify the staged validate-pr-target-branch.yml so an invalid + external + + PR targeting a release branch also receives one idempotent explanatory comment. Preserve its existing policy, keep + the + + pull_request_target path safe, preserve the existing immutable actions/github-script SHA reference, and do not + execute + + or check out fork code. Use a stable hidden HTML marker and verify the matching comment's bot author before updating + the existing bot comment across reruns. + constraints: &short_constraints + max_turns: 40 + max_tokens: 750000 + max_duration: 5m + tags: + repo-path: + - .github/workflows/validate-pr-target-branch.yml + - .github/workflows/label-and-milestone-issues.yml + rubric: + - The edited workflow is valid YAML and preserves the existing release-target validation behavior. + - Permissions are explicit and minimal, and pull_request_target never executes or checks out untrusted PR content. + - Commenting preserves the staged workflow's immutable actions/github-script SHA reference and is idempotent + across reruns without trusting a marker in a contributor-authored comment. + graders: + - type: skill-invocation + config: + required: + - make-github-actions-workflow + - type: file-matches + config: + path: .github/workflows/validate-pr-target-branch.yml + pattern: (?=[\s\S]*actions/github-script@)(?=[\s\S]*