diff --git a/.github/skills/add-community-extension/SKILL.md b/.github/skills/add-community-extension/SKILL.md deleted file mode 100644 index 179c11b3e2..0000000000 --- a/.github/skills/add-community-extension/SKILL.md +++ /dev/null @@ -1,174 +0,0 @@ ---- -name: add-community-extension -description: 'Add a community extension to the Spec Kit catalog from a GitHub issue submission. USE FOR: processing extension submission issues, validating catalog entries, updating catalog.community.json and docs/community/extensions.md, creating PRs. DO NOT USE FOR: creating new extensions from scratch, or first-party extension work.' -argument-hint: 'GitHub issue URL or number for the extension submission' ---- - -# Add Community Extension - -Process an extension submission issue and add or update it in the community catalog. - -## When to Use - -- A new `[Extension]` submission issue is filed -- An existing extension submits an update issue (new version, changed metadata) -- You need to add or update a community extension in `extensions/catalog.community.json` and `docs/community/extensions.md` - -## Procedure - -### 1. Fetch the submission issue - -Read the GitHub issue to extract all metadata: -- Extension ID, name, version, description, author -- Repository URL, download URL, homepage, documentation, changelog -- License, required spec-kit version, optional tool dependencies -- Number of commands and hooks -- Tags - -### 2. Validate against publishing rules - -Check **all** of the following (per `extensions/EXTENSION-PUBLISHING-GUIDE.md`): - -| Check | How | -|-------|-----| -| Repository exists and is public | Fetch the repository URL | -| `extension.yml` manifest present | Confirm in repo file listing | -| README.md present | Confirm in repo file listing | -| LICENSE file present | Confirm in repo file listing | -| GitHub release exists matching version | Check releases on the repo page | -| Download URL is accessible | Verify it follows `archive/refs/tags/vX.Y.Z.zip` pattern and release exists | -| Extension ID is lowercase-with-hyphens only | Regex: `^[a-z][a-z0-9-]*$` | -| Version follows semver | Format: `X.Y.Z` | -| Submission checklists are all checked | Confirm in issue body | - -### 3. Determine if this is an add or update - -Search `extensions/catalog.community.json` for the extension ID. - -- **Not found** → this is a **new addition**. Proceed to step 4. -- **Found** → this is an **update**. Proceed to step 4 but replace the existing entry in-place instead of inserting. - -### 4. Add or update `extensions/catalog.community.json` - -**New extension:** Insert the entry in **alphabetical order** by extension ID. - -**Update:** Replace the existing entry in-place. Update only the fields that changed (typically `version`, `download_url`, `description`, `provides`, `requires`, `tags`, `updated_at`). Preserve `created_at` and `downloads`/`stars` from the existing entry. - -Use the existing entries as the format template. Required fields: - -```json -{ - "": { - "name": "", - "id": "", - "description": "", - "author": "", - "version": "", - "download_url": "", - "repository": "", - "homepage": "", - "documentation": "", - "changelog": "", - "license": "", - "category": "", - "effect": "", - "requires": { - "speckit_version": "" - }, - "provides": { - "commands": , - "hooks": - }, - "tags": ["", ""], - "verified": false, - "downloads": 0, - "stars": 0, - "created_at": "T00:00:00Z", - "updated_at": "T00:00:00Z" - } -} -``` - -**Category** — free-form string; common values: `docs`, `code`, `process`, `integration`, `visibility` -**Effect** — one of: `read-only`, `read-write` - -If the extension has optional tool dependencies, add a `"tools"` array inside `"requires"`: - -```json -"tools": [{ "name": "", "required": false }] -``` - -Also update the top-level `"updated_at"` timestamp in the catalog. - -After editing, **validate the JSON** by running: - -```bash -python3 -c "import json; json.load(open('extensions/catalog.community.json')); print('Valid JSON')" -``` - -### 5. Add or update `docs/community/extensions.md` community extensions table - -**New extension:** Insert a new row into the `# Community Extensions` table in **alphabetical order** by extension name. - -**Update:** Find the existing row and update the description or other changed fields in-place. - -Determine the category and effect from the extension's behavior: - -``` -| | | `` | | []() | -``` - -**Category** — free-form; common values: `docs`, `code`, `process`, `integration`, `visibility` -**Effect** — write canonical values `read-only` or `read-write` in `extension.yml` and `catalog.community.json`; use `Read-only`/`Read+Write` only for the docs table display - -### 6. Commit, push, and open PR - -Use `add-` for new extensions, `update-` for updates: - -```bash -# New extension -git checkout -b add--extension - -# Update -git checkout -b update--extension -``` - -```bash -git add extensions/catalog.community.json docs/community/extensions.md - -# New extension -git commit -m "Add extension to community catalog - -Add extension submitted by @ to: -- extensions/catalog.community.json (alphabetical order) -- docs/community/extensions.md community extensions table - -Closes #" - -# Update -git commit -m "Update extension to v - -Update extension submitted by @: -- extensions/catalog.community.json (version, download_url, etc.) -- docs/community/extensions.md community extensions table - -Closes #" - -git push origin -``` - -Then create a PR to `upstream` (`github/spec-kit`) with: -- **Title:** `Add extension to community catalog` (or `Update extension to v`) -- **Body:** Include validation summary, `Closes #`, and `cc @` -- **Head:** `:` -- **Base:** `main` - -## Common Pitfalls - -- **Alphabetical order matters** — entries must be sorted by ID in the JSON and by name in the docs table. -- **Don't forget the catalog `updated_at`** — the top-level timestamp in `catalog.community.json` must be refreshed. -- **Validate JSON after editing** — a trailing comma or missing brace will break the catalog. -- **Use `Closes` not `Fixes`** — `Closes #N` is the correct keyword for submission issues. -- **Match the proposed entry but verify** — the issue may include a proposed JSON block, but always validate field values against the actual repository state. -- **Preserve `created_at` on updates** — keep the original `created_at` value; only change `updated_at`. -- **Preserve `downloads` and `stars` on updates** — these reflect usage metrics and must not be reset. diff --git a/.github/workflows/add-community-bundle.md b/.github/workflows/add-community-bundle.md index a54a35f890..ced6b654fa 100644 --- a/.github/workflows/add-community-bundle.md +++ b/.github/workflows/add-community-bundle.md @@ -66,7 +66,30 @@ with `[Bundle]:`. If it does not, stop without commenting. ## Step 1 - Read and Parse the Issue -Read issue #${{ github.event.issue.number }} and extract these issue-form fields: +Read issue #${{ github.event.issue.number }}. + +### 1a. Detect issue format + +The issue **must** be submitted using the GitHub issue form template +(`Bundle Submission`). Before attempting to parse any fields, check whether +the issue body follows the expected form structure. + +**Form-format indicator:** The body must contain a heading `### Bundle ID` +(the first required field in the form template). If this heading is absent, the +issue was submitted with a free-form body instead of the issue form. + +If the issue is **not** in form format: +1. Add a comment explaining that the issue must be submitted using the + `Bundle Submission` issue form template. Include a link to the form: + `https://github.com/github/spec-kit/issues/new?template=bundle_submission.yml` +2. Add the `validation-failed` label +3. **Stop — do not proceed further** + +If the issue **is** in form format, continue to parse the fields below. + +### 1b. Parse form fields + +Extract these issue-form fields: | Field | Issue Form ID | Required | |-------|---------------|----------| diff --git a/.github/workflows/add-community-extension.md b/.github/workflows/add-community-extension.md index 0521e52100..64c74d1df3 100644 --- a/.github/workflows/add-community-extension.md +++ b/.github/workflows/add-community-extension.md @@ -64,6 +64,27 @@ If it does not, stop without commenting. Read issue #${{ github.event.issue.number }}. +### 1a. Detect issue format + +The issue **must** be submitted using the GitHub issue form template +(`Extension Submission`). Before attempting to parse any fields, check whether +the issue body follows the expected form structure. + +**Form-format indicator:** The body must contain a heading `### Extension ID` +(the first required field in the form template). If this heading is absent, the +issue was submitted with a free-form body instead of the issue form. + +If the issue is **not** in form format: +1. Add a comment explaining that the issue must be submitted using the + `Extension Submission` issue form template. Include a link to the form: + `https://github.com/github/spec-kit/issues/new?template=extension_submission.yml` +2. Add the `validation-failed` label +3. **Stop — do not proceed further** + +If the issue **is** in form format, continue to parse the fields below. + +### 1b. Parse form fields + Extract the following fields from the structured issue body (GitHub issue form fields): diff --git a/.github/workflows/add-community-preset.md b/.github/workflows/add-community-preset.md index 038fbbe1a1..95474887de 100644 --- a/.github/workflows/add-community-preset.md +++ b/.github/workflows/add-community-preset.md @@ -64,6 +64,27 @@ If it does not, stop without commenting. Read issue #${{ github.event.issue.number }}. +### 1a. Detect issue format + +The issue **must** be submitted using the GitHub issue form template +(`Preset Submission`). Before attempting to parse any fields, check whether +the issue body follows the expected form structure. + +**Form-format indicator:** The body must contain a heading `### Preset ID` +(the first required field in the form template). If this heading is absent, the +issue was submitted with a free-form body instead of the issue form. + +If the issue is **not** in form format: +1. Add a comment explaining that the issue must be submitted using the + `Preset Submission` issue form template. Include a link to the form: + `https://github.com/github/spec-kit/issues/new?template=preset_submission.yml` +2. Add the `validation-failed` label +3. **Stop — do not proceed further** + +If the issue **is** in form format, continue to parse the fields below. + +### 1b. Parse form fields + Extract the following fields from the structured issue body (GitHub issue form fields): diff --git a/.gitignore b/.gitignore index 954a502ce3..25848045b0 100644 --- a/.gitignore +++ b/.gitignore @@ -60,3 +60,6 @@ docs/dev .grok/ .specify/ specs/ +benchmarks/evaluator/results/ +benchmarks/evaluator/reports/ +benchmarks/reports/ diff --git a/extensions/catalog.community.json b/extensions/catalog.community.json index 4b48c3e5db..c7157324fd 100644 --- a/extensions/catalog.community.json +++ b/extensions/catalog.community.json @@ -1747,6 +1747,44 @@ "created_at": "2026-07-08T00:00:00Z", "updated_at": "2026-07-08T00:00:00Z" }, + "evaluator": { + "name": "Evaluator Contract", + "id": "evaluator", + "description": "Standard evaluator result contract for evidence, provenance, uncertainty, and recovery — a provider-neutral protocol for extensions that evaluate artifact quality between phases", + "author": "ElectroHire", + "version": "1.0.0", + "download_url": "https://github.com/electrohire/spec-kit-evaluator/releases/download/v1.0.0/evaluator.zip", + "repository": "https://github.com/electrohire/spec-kit-evaluator", + "homepage": "https://github.com/electrohire/spec-kit-evaluator", + "documentation": "https://github.com/electrohire/spec-kit-evaluator/blob/main/README.md", + "changelog": "https://github.com/electrohire/spec-kit-evaluator/blob/main/CHANGELOG.md", + "license": "MIT", + "category": "process", + "effect": "read-write", + "requires": { + "speckit_version": ">=1.0.0" + }, + "provides": { + "commands": 4, + "hooks": 4 + }, + "tags": [ + "evaluator", + "evidence", + "provenance", + "quality", + "governance", + "compliance", + "workflow", + "model-routing", + "portfolio" + ], + "verified": false, + "downloads": 0, + "stars": 0, + "created_at": "2026-09-02T00:00:00Z", + "updated_at": "2026-09-02T00:00:00Z" + }, "extensify": { "name": "Extensify", "id": "extensify", diff --git a/extensions/evaluator/.gitignore b/extensions/evaluator/.gitignore new file mode 100644 index 0000000000..65c5f35df3 --- /dev/null +++ b/extensions/evaluator/.gitignore @@ -0,0 +1,6 @@ +# Spec Kit Evaluator Contract Extension +__pycache__/ +*.pyc +*.pyo +.pytest_cache/ +.ruff_cache/ \ No newline at end of file diff --git a/extensions/evaluator/CHANGELOG.md b/extensions/evaluator/CHANGELOG.md new file mode 100644 index 0000000000..8c95d917c5 --- /dev/null +++ b/extensions/evaluator/CHANGELOG.md @@ -0,0 +1,17 @@ +# Changelog + +All notable changes to the Evaluator Contract extension will be documented in this file. + +## [1.0.0] - 2026-09-01 + +### Added +- Initial release of the Evaluator Contract extension +- JSON Schema for evaluator results (6 outcomes, 14 finding kinds, 5 evidence kinds, 6 uncertainty levels) +- 4 commands: `speckit.evaluator.run`, `.compose`, `.report`, `.route` +- 3 parity scripts: Python, Bash, PowerShell +- 4 lifecycle hooks: `after_specify`, `after_plan`, `after_tasks`, `after_implement` +- Model routing: recommends budget/standard/premium tier per phase +- Composition: strict/majority/optimistic strategies with contradiction detection +- Quick-start demo: `python examples/demo.py` +- Token-economic benchmark suite with Monte Carlo simulation +- 70 tests with 0 regressions against spec-kit test suite \ No newline at end of file diff --git a/extensions/evaluator/LICENSE b/extensions/evaluator/LICENSE new file mode 100644 index 0000000000..27ca4b7316 --- /dev/null +++ b/extensions/evaluator/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 ElectroHire + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/extensions/evaluator/README.md b/extensions/evaluator/README.md new file mode 100644 index 0000000000..da6e7f151c --- /dev/null +++ b/extensions/evaluator/README.md @@ -0,0 +1,166 @@ +# Evaluator Contract Extension + +Standard evaluator result contract for evidence, provenance, uncertainty, and recovery — a provider-neutral protocol for extensions that evaluate artifact quality between Spec-Driven Development phases. + +## Overview + +Spec Kit has a strong lifecycle and extensible hook system, but there is no common contract for extensions that evaluate the quality or trustworthiness of artifacts between phases. This extension defines that contract. + +The evaluator contract lets any extension: + +1. **Register** for one or more lifecycle events via hooks +2. **Receive** the relevant resolved artifacts plus stable source/provenance references +3. **Return** a versioned machine-readable result conforming to a shared schema +4. **Distinguish** observed evidence from generated assertions +5. **Preserve** contradictory findings instead of forcing false consensus +6. **Represent** insufficient evidence or unresolved uncertainty explicitly +7. **Request** a bounded next action: `pass`, `warn`, `iterate`, `clarify`, `gather_evidence`, or `block` +8. **Persist** enough compact state to survive pause/resume +9. **Compose** deterministically with other evaluators +10. **Remain** implementation-neutral: deterministic, model-backed, local, remote, private, paid, or hybrid + +## Installation + +```bash +specify extension add evaluator +``` + +Or for local development: + +```bash +specify extension add --dev /path/to/spec-kit/extensions/evaluator +``` + +## Commands + +### `/speckit.evaluator.run` + +Run an evaluator against one or more artifacts and produce a versioned machine-readable result. + +```bash +/speckit.evaluator.run phase=after_plan artifacts=spec.md,plan.md +``` + +Results are written to `.specify/extensions/evaluator/results/--.json`. + +### `/speckit.evaluator.compose` + +Compose multiple evaluator results at a lifecycle point with deterministic precedence. + +```bash +/speckit.evaluator.compose phase=after_plan strategy=strict +``` + +Composed results are written to `.specify/extensions/evaluator/results/composed--.json`. + +### `/speckit.evaluator.report` + +Render evaluator results as a human-readable report, CI annotation, or release gate. + +```bash +/speckit.evaluator.report phase=after_plan format=terminal +/speckit.evaluator.report phase=after_plan format=ci-annotation +/speckit.evaluator.report phase=after_plan format=gate +``` + +## Evaluator Result Schema + +The full JSON Schema is at `schemas/evaluator-result.schema.json`. Every evaluator result MUST conform to this schema. + +### Minimal Valid Result + +```json +{ + "schema_version": "1.0", + "evaluator": { + "id": "my-evaluator", + "version": "0.1.0" + }, + "phase": "after_plan", + "outcome": "pass", + "findings": [] +} +``` + +### Outcome Semantics + +| Outcome | Meaning | Workflow Effect | +|---------|---------|----------------| +| `pass` | All checks passed | Continue to next phase | +| `warn` | Issues found but not blocking | Continue with warnings | +| `iterate` | Issues require revisiting a prior phase | Return to target phase | +| `clarify` | Ambiguities need human resolution | Pause for human input | +| `gather_evidence` | Insufficient evidence | Pause for evidence collection | +| `block` | Hard blocker | Stop the workflow | + +### Evidence Kinds + +| Kind | Meaning | +|------|---------| +| `observed` | Directly observed from an artifact or command output | +| `inferred` | Logically derived from observed evidence | +| `asserted` | Claimed by a model or agent without direct observation | +| `contradicted` | Conflicts with other observed evidence | +| `unsupported` | No evidence found to support or refute | + +## Composition Strategies + +When multiple evaluators run at the same phase, their results are composed: + +| Strategy | Behavior | +|----------|----------| +| `strict` (default) | Most severe outcome wins | +| `majority` | Most common outcome wins; ties break toward severity | +| `optimistic` | Least severe outcome wins | + +## Hooks + +The extension registers hooks at key lifecycle points: + +- `after_specify` — Evaluate spec quality, evidence, and provenance +- `after_plan` — Evaluate plan assumptions, risks, and coverage +- `after_tasks` — Evaluate task completeness and traceability +- `after_implement` — Evaluate implementation against spec, plan, and tasks + +All hooks are optional (prompt before executing) with priority 20. + +## Writing an Evaluator + +To write an evaluator that conforms to this contract: + +1. Create an extension that depends on `evaluator` +2. Register hooks at the lifecycle points you want to evaluate +3. In your command, read the relevant artifacts +4. Produce a result JSON file conforming to `evaluator-result.schema.json` +5. Write it to `.specify/extensions/evaluator/results/` + +See `templates/evaluator-result-template.json` for a starting point. + +## Design Rules + +1. **Generated assertions MUST remain distinguishable from observed evidence.** +2. **Model self-attestation MUST NOT satisfy an evidence gate by itself.** +3. **Contradictions MUST be preserved**, not collapsed into a single synthesized answer. +4. **Insufficient evidence MUST be represented explicitly** rather than inventing certainty. +5. **Deterministic checks SHOULD run before probabilistic review** where appropriate. +6. **Higher-risk work MAY require evaluator independence** — the same model/family SHOULD NOT both generate and certify the result. + +## File Structure + +``` +.specify/extensions/evaluator/ +├── schemas/ +│ └── evaluator-result.schema.json # JSON Schema for evaluator results +├── templates/ +│ └── evaluator-result-template.json # Template for new evaluator results +├── results/ # Individual evaluator result files +│ ├── --.json +│ └── composed--.json +├── reports/ # Human-readable reports (markdown format) +│ └── report--.md +└── evaluators.yml # Registered evaluator configuration +``` + +## License + +MIT — see the [Spec Kit license](../../LICENSE). diff --git a/extensions/evaluator/commands/speckit.evaluator.compose.md b/extensions/evaluator/commands/speckit.evaluator.compose.md new file mode 100644 index 0000000000..e18d7f65a6 --- /dev/null +++ b/extensions/evaluator/commands/speckit.evaluator.compose.md @@ -0,0 +1,156 @@ +--- +description: "Compose multiple evaluator results at a lifecycle point with deterministic precedence" +scripts: + sh: ../../scripts/bash/compose-results.sh + ps: ../../scripts/powershell/compose-results.ps1 + py: ../../scripts/python/compose_results.py +--- + +# Evaluator Compose + +Compose multiple independent evaluator results at a single lifecycle point into one aggregate result with deterministic precedence. + +When multiple evaluators run at the same phase (e.g., a schema validator, a security scanner, and an epistemic checker all after `plan`), their results must be composed into a single actionable verdict. This command applies deterministic composition rules so the outcome is reproducible. + +## User Input + +```text +$ARGUMENTS +``` + +The user input specifies **which results to compose**. Accept: + +1. **Phase** — the lifecycle phase to compose results for (e.g., `phase=after_plan`). Required. +2. **Result files** — explicit list of result file paths. If not provided, discover all result files for the given phase from `.specify/extensions/evaluator/results/`. +3. **Composition strategy** — `strict` (default), `majority`, or `optimistic`. See Composition Strategies below. + +## Prerequisites + +- **Path safety (do this before any read or write)**: resolve the project root and the real, symlink-resolved path of `.specify/extensions/evaluator/results/` and every result file you touch. **Refuse and report — never follow —** if any path component is a symlink, or if the resolved path does not remain inside the project root. +- At least one result file MUST exist for the specified phase. If none exist, produce a composed result with `outcome: "pass"` and a note that no evaluators ran. +- Each result file MUST be valid JSON conforming to the evaluator result schema. Skip and report invalid files; do not compose invalid data. + +## Execution + +### 1. Collect Results + +Read all result files for the specified phase from `.specify/extensions/evaluator/results/`. Filter to files matching the pattern `--.json`. + +### 2. Validate Each Result + +For each result file: +1. Parse as JSON. +2. Validate against `.specify/extensions/evaluator/schemas/evaluator-result.schema.json`. +3. Skip invalid results and report them. + +### 3. Apply Composition Strategy + +#### `strict` (default) + +The most severe outcome wins. Precedence order (most severe first): + +1. `block` — any evaluator blocks → composed outcome is `block` +2. `gather_evidence` — any evaluator needs evidence → `gather_evidence` +3. `iterate` — any evaluator requests iteration → `iterate` +4. `clarify` — any evaluator needs clarification → `clarify` +5. `warn` — any evaluator warns → `warn` +6. `pass` — all evaluators pass → `pass` + +#### `majority` + +The outcome with the most evaluators supporting it wins. Ties break toward the more severe outcome (using strict precedence). + +#### `optimistic` + +The least severe outcome wins. Use only when evaluators are advisory and blocking is explicitly not desired. + +### 4. Merge Findings + +All findings from all evaluators are preserved in the composed result. Each finding retains its original `id`, `evaluator` origin, and all fields. Findings are ordered by: + +1. Severity (critical → high → medium → low → info) +2. Evaluator priority (lower first) +3. Original finding order within each evaluator + +Contradictory findings are **preserved, not collapsed**. If evaluator A says "REQ-014 is supported" and evaluator B says "REQ-014 is unsupported", both findings appear in the composed result with their respective evidence. + +### 5. Determine Next Action + +The composed `next_action` is derived from the composed outcome: + +| Composed Outcome | Next Action Kind | Target Phase | +|-----------------|-----------------|--------------| +| `pass` | `pass` | null | +| `warn` | `warn` | null | +| `iterate` | `iterate` | Most common `target_phase` among iterate findings | +| `clarify` | `clarify` | null | +| `gather_evidence` | `gather_evidence` | null | +| `block` | `block` | null | + +### 6. Write Composed Result + +Write to `.specify/extensions/evaluator/results/composed--.json`. + +### 7. Report + +Output a summary: +- The phase +- The composition strategy used +- The number of evaluator results composed +- The composed outcome +- Total findings by severity +- Any contradictory findings flagged +- The recommended next action +- The path to the composed result file + +## Composition Rules + +1. **Deterministic**: same inputs + same strategy = same composed result. +2. **Contradiction-preserving**: conflicting findings are both recorded, not resolved. +3. **Evidence-respecting**: `observed` evidence from one evaluator is not downgraded by another evaluator's `asserted` claim. +4. **State-isolated**: each evaluator's `state` object is preserved under its evaluator ID in the composed result's `evaluator_states` map. +5. **Priority-ordered**: when evaluators declare a `priority` (in their config), lower values run first and their findings appear first at equal severity. + +## Composed Result Format + +```json +{ + "schema_version": "1.0", + "composed": true, + "phase": "after_plan", + "composition_strategy": "strict", + "composed_outcome": "iterate", + "composed_summary": "2 evaluators ran: 1 pass, 1 iterate. 3 findings total.", + "evaluator_results": [ + { "evaluator_id": "schema-validate", "outcome": "pass", "findings_count": 0 }, + { "evaluator_id": "epistemic", "outcome": "iterate", "findings_count": 3 } + ], + "findings": [ + "... all findings from all evaluators, ordered by severity ..." + ], + "next_action": { + "kind": "iterate", + "target_phase": "plan", + "message": "2 of 2 evaluators completed. 1 requests iteration. See findings for details." + }, + "evaluator_states": { + "schema-validate": {}, + "epistemic": { "... opaque evaluator state ..." } + }, + "metadata": { + "timestamp": "", + "evaluator_count": 2, + "contradictory_findings": [ + { "finding_a": "EPI-001", "finding_b": "SCH-003", "subject": "REQ-014" } + ] + } +} +``` + +## Guardrails + +- Never modify individual evaluator result files — compose reads them, writes a new composed file. +- Never resolve contradictions by dropping findings — preserve both. +- Never change another evaluator's evidence classification. +- Never merge `state` objects across evaluators — keep them isolated under evaluator IDs. +- The composed result is a new artifact; it does not replace individual evaluator results. diff --git a/extensions/evaluator/commands/speckit.evaluator.report.md b/extensions/evaluator/commands/speckit.evaluator.report.md new file mode 100644 index 0000000000..91c1e0a345 --- /dev/null +++ b/extensions/evaluator/commands/speckit.evaluator.report.md @@ -0,0 +1,120 @@ +--- +description: "Render evaluator results as a human-readable report, CI annotation, or release gate" +--- + +# Evaluator Report + +Render evaluator results (individual or composed) into a human-readable report, CI annotation, or release gate decision. + +This command consumes evaluator result JSON files and produces output suitable for different consumers: developers reading in-terminal, CI systems parsing annotations, or release pipelines checking gates. + +## User Input + +```text +$ARGUMENTS +``` + +The user input specifies **what to report** and **how**. Accept: + +1. **Result files** — one or more evaluator result file paths, or a composed result path. If not provided, discover the latest composed result for the current phase, or the latest individual results. +2. **Format** — `terminal` (default), `markdown`, `json`, `ci-annotation`, or `gate`. See Output Formats below. +3. **Phase** — filter results to a specific phase. +4. **Severity threshold** — only show findings at or above this severity (`critical`, `high`, `medium`, `low`, `info`). Default: `low`. + +## Prerequisites + +- **Path safety (do this before any read or write)**: resolve the project root and the real, symlink-resolved path of `.specify/extensions/evaluator/results/` and every result file you touch. **Refuse and report — never follow —** if any path component is a symlink, or if the resolved path does not remain inside the project root. +- At least one result file MUST exist. If none exist, report "No evaluator results found" and exit. + +## Execution + +### 1. Load Results + +Read the specified result files (or discover them). Each must be valid JSON conforming to the evaluator result schema. + +### 2. Filter Findings + +Apply the severity threshold. Findings below the threshold are excluded from the report but counted in the summary. + +### 3. Render in Requested Format + +#### `terminal` (default) + +A color-coded terminal report: + +``` +═══════════════════════════════════════════════════════════ + EVALUATOR REPORT — after_plan +═══════════════════════════════════════════════════════════ + Outcome: ITERATE + Evaluators: 2 run, 1 passed, 1 requests iteration + Findings: 3 total (0 critical, 2 high, 1 medium) +─────────────────────────────────────────────────────────── + + [HIGH] EPI-001 — unsupported_claim + Subject: REQ-014 + Evidence: none (unsupported) + Recommendation: gather_evidence + Rationale: Claim presented as fact without supporting evidence. + + [HIGH] EPI-002 — contradiction + Subject: REQ-007 + Evidence: spec.md#REQ-007 (observed), constitution.md (observed) + Recommendation: clarify + Rationale: Requirement conflicts with constitution article IV. + + [MEDIUM] EPI-003 — ambiguous_requirement + Subject: REQ-022 + Recommendation: clarify + Rationale: Requirement uses undefined term "scalable". + +─────────────────────────────────────────────────────────── + Next Action: iterate → plan + "2 of 2 evaluators completed. 1 requests iteration." +═══════════════════════════════════════════════════════════ +``` + +#### `markdown` + +A Markdown document written to `.specify/extensions/evaluator/reports/report--.md`. Suitable for PR comments, issue bodies, or documentation. + +#### `json` + +The raw JSON result(s) printed to stdout. Suitable for piping to other tools. + +#### `ci-annotation` + +GitHub Actions workflow commands (`::warning::`, `::error::`) or GitLab CI annotations emitted to stdout. Format: + +``` +::error file=spec.md,line=14,title=EPI-001::[unsupported_claim] Claim presented as fact without supporting evidence +::warning file=plan.md,line=42,title=EPI-003::[ambiguous_requirement] Requirement uses undefined term "scalable" +``` + +Findings with severity `critical` or `high` use `::error::`; `medium` and below use `::warning::`. + +#### `gate` + +A release-gate decision. Exit code 0 if the composed outcome is `pass` or `warn`; exit code 1 for `iterate`, `clarify`, or `gather_evidence`; exit code 2 for `block`. Prints the outcome and summary to stdout. + +### 4. Write Report (markdown format only) + +For `markdown` format, write the report to `.specify/extensions/evaluator/reports/report--.md`. + +## Output Formats Summary + +| Format | Output | Use Case | +|--------|--------|----------| +| `terminal` | Color-coded stdout | Developer review in terminal | +| `markdown` | File + stdout path | PR comments, documentation | +| `json` | Raw JSON stdout | Piping to other tools | +| `ci-annotation` | Workflow commands stdout | CI/CD pipeline annotations | +| `gate` | Exit code + stdout | Release gates, pre-commit hooks | + +## Guardrails + +- Never modify result files — report reads them only. +- Never fabricate or summarize away findings — the report reflects exactly what the evaluators produced. +- For `ci-annotation` format, ensure file paths and line numbers are accurate — do not guess. +- For `gate` format, the exit code MUST be deterministic given the same input results. +- Reports are written under `.specify/extensions/evaluator/reports/` — never outside this directory. diff --git a/extensions/evaluator/commands/speckit.evaluator.route.md b/extensions/evaluator/commands/speckit.evaluator.route.md new file mode 100644 index 0000000000..69b05a3a6c --- /dev/null +++ b/extensions/evaluator/commands/speckit.evaluator.route.md @@ -0,0 +1,178 @@ +--- +description: "Recommend which model tier to use for the next SDD phase based on evaluator findings — enables the portfolio approach (budget for routine, premium for critical)" +--- + +# Evaluator Route + +Analyze evaluator findings and recommend which model tier to use for the next SDD phase. This is the mechanism that enables the **portfolio approach**: budget models for routine generation, standard models for review, premium models for critical decisions. + +The recommendation is based on: +- Finding severity distribution (critical/high findings → escalate) +- Evidence quality (unsupported claims → need premium reasoning) +- Phase risk profile (implement is higher risk than specify) +- Cost optimization (budget is sufficient when risk is low) + +## User Input + +```text +$ARGUMENTS +``` + +Accept: +1. **Phase** — the next SDD phase to route for (e.g., `phase=plan`, `phase=implement`). Required. +2. **Result files** — evaluator result files to base the recommendation on. If not provided, discover the latest composed result for the current phase. +3. **Budget constraint** — optional maximum USD budget for the next phase. If provided, the recommendation must stay within budget. + +## Prerequisites + +- **Path safety**: resolve `.specify/extensions/evaluator/results/` — refuse symlinks. +- At least one evaluator result or composed result MUST exist. If none, default to `budget` tier with a note that no evaluation data is available. + +## Execution + +### 1. Load Evaluation Results + +Read the latest composed result or individual evaluator results for the current phase. + +### 2. Assess Risk Profile + +Score the risk of the next phase based on evaluator findings: + +| Factor | Weight | How Measured | +|--------|--------|-------------| +| Critical findings | 40% | Count of `critical` severity findings | +| High findings | 30% | Count of `high` severity findings | +| Evidence gaps | 20% | Count of `unsupported_claim` + `missing_evidence` findings | +| Contradictions | 10% | Count of contradictory finding pairs | + +Risk score = weighted sum, normalized to 0.0–1.0. + +### 3. Determine Recommended Tier + +| Risk Score | Recommended Tier | Rationale | +|-----------|-----------------|-----------| +| 0.0–0.2 | `budget` | Low risk — budget models sufficient | +| 0.2–0.5 | `standard` | Moderate risk — standard quality needed | +| 0.5–0.8 | `premium` | High risk — premium reasoning required | +| 0.8–1.0 | `premium` + escalation | Critical risk — premium + human review | + +### 4. Apply Phase Risk Baseline + +Each SDD phase has an inherent risk baseline that shifts the threshold: + +| Phase | Baseline Risk | Effect | +|-------|--------------|--------| +| `specify` | 0.1 | Slightly lower bar for premium (spec quality matters) | +| `plan` | 0.15 | Moderate — design decisions are costly to undo | +| `tasks` | 0.05 | Lower — task breakdown is mechanical | +| `implement` | 0.2 | Higher — implementation errors are expensive | +| `analyze` | 0.1 | Moderate — cross-artifact analysis | +| `checklist` | 0.0 | Lowest — checklist generation is routine | +| `clarify` | 0.15 | Moderate — clarification needs precision | +| `constitution` | 0.2 | Higher — governance decisions are critical | +| `converge` | 0.15 | Moderate — convergence assessment | + +### 5. Apply Budget Constraint (if provided) + +If a budget constraint is specified, downgrade the recommendation if the estimated cost exceeds the budget: + +1. Calculate estimated tokens for the next phase at the recommended tier +2. Calculate estimated cost at that tier +3. If cost > budget, try the next lower tier +4. If no tier fits the budget, recommend `budget` with a warning + +### 6. Produce Model Routing Recommendation + +Output a `model_routing` block conforming to the evaluator result schema: + +```json +{ + "model_routing": { + "recommended_tier": "standard", + "reason": "2 high-severity findings and 1 evidence gap — standard quality recommended for plan phase", + "escalation_triggers": [ + { + "condition": "Any new critical finding", + "escalate_to": "premium" + }, + { + "condition": "More than 5 unsupported claims in next evaluation", + "escalate_to": "premium" + } + ], + "estimated_tokens": 12000, + "estimated_cost_usd": 0.22, + "tier_breakdown": { + "budget": { + "estimated_tokens": 18000, + "estimated_cost_usd": 0.01 + }, + "standard": { + "estimated_tokens": 12000, + "estimated_cost_usd": 0.22 + }, + "premium": { + "estimated_tokens": 10000, + "estimated_cost_usd": 0.90 + } + } + } +} +``` + +### 7. Report + +Output: +- The recommended tier and reason +- The risk score breakdown +- Cost comparison across all tiers +- Escalation triggers (conditions that would upgrade the recommendation) +- The estimated savings vs always using premium + +## Model Tier Pricing Reference + +| Tier | Input $/1M tok | Output $/1M tok | Best For | +|------|---------------|-----------------|----------| +| `budget` | $0.12–$0.25 | $0.50–$1.25 | Routine generation, drafts, bounded tasks | +| `standard` | $3.00 | $15.00 | Review, moderate-complexity work | +| `premium` | $15.00 | $75.00 | Critical decisions, security, governance | +| `portfolio` | ~$1.30 | ~$6.40 | Routed blend (80% budget, 15% standard, 5% premium) | + +## Portfolio Approach Rules + +1. **Default to budget.** Start every phase at the budget tier. Only escalate when evaluator findings justify it. +2. **Escalate on evidence.** Upgrade when findings show `critical` severity, `insufficient_evidence` uncertainty, or `contradiction` between evaluators. +3. **Downgrade when clean.** If the previous phase had zero high/critical findings, drop back to budget for the next phase. +4. **Never use premium for generation.** Premium models are for evaluation and decision-making, not for drafting specs or writing boilerplate code. +5. **Deterministic evaluators are free.** Schema validators, linters, and static analyzers cost near-zero tokens. Run them always, at every phase. +6. **Model-backed evaluators use budget tier.** Epistemic checks, semantic review, and coverage analysis run on budget models by default. Only escalate the evaluator itself when findings warrant it. + +## Guardrails + +- Never recommend premium for a phase with zero high/critical findings. +- Never recommend budget when there are unresolved `block` outcomes. +- Always show the cost comparison — let the human see what they're saving. +- The routing recommendation is advisory — the human operator always has final say. + +## Workflow Integration (Explicit Wiring) + +The evaluator's model routing recommendation can be wired into workflows using +the existing expression mechanism. This keeps the dataflow visible and +intentional — no engine changes needed. + +```yaml +steps: + - id: evaluate + type: command + command: speckit.evaluator.route + input: + phase: plan + + - id: implement + type: command + command: speckit.implement + model: "{{ steps.evaluate.output.model_routing.recommended_tier }}" +``` + +Each downstream step independently decides whether to use the recommendation. +Workflow authors can validate, transform, or ignore it as needed. diff --git a/extensions/evaluator/commands/speckit.evaluator.run.md b/extensions/evaluator/commands/speckit.evaluator.run.md new file mode 100644 index 0000000000..4ec20ad250 --- /dev/null +++ b/extensions/evaluator/commands/speckit.evaluator.run.md @@ -0,0 +1,162 @@ +--- +description: "Run an evaluator against one or more artifacts and produce a versioned machine-readable result conforming to the evaluator result contract" +scripts: + sh: ../../scripts/bash/compose-results.sh + ps: ../../scripts/powershell/compose-results.ps1 + py: ../../scripts/python/compose_results.py +--- + +# Evaluator Run + +Execute an evaluator against specified artifacts and produce a result conforming to the **evaluator result contract** defined in `.specify/extensions/evaluator/schemas/evaluator-result.schema.json`. + +This command is the execution entry point for any evaluator — deterministic linters, model-backed reviewers, security scanners, policy checkers, provenance verifiers, or custom governance checks. The evaluator receives artifact references and returns a versioned, machine-readable result that downstream composition and reporting can consume. + +## User Input + +```text +$ARGUMENTS +``` + +The user input specifies **what to evaluate** and **which evaluator(s) to run**. Accept: + +1. **Phase context** — the lifecycle phase this evaluation runs under (e.g., `phase=after_plan`). If not provided, infer from the hook event or ask. +2. **Artifact references** — one or more artifact paths to evaluate (e.g., `spec.md`, `plan.md`, `tasks.md`). If not provided, discover artifacts for the current phase from `.specify/` and the feature directory. +3. **Evaluator selection** — which evaluator(s) to run. If not provided, discover registered evaluators from `.specify/extensions/evaluator/` config. + +## Prerequisites + +- **Path safety (do this before any read or write)**: resolve the project root and the real, symlink-resolved path of `.specify/extensions/evaluator/` and every artifact you touch. **Refuse and report — never follow —** if any path component is a symlink, or if the resolved path does not remain inside the project root. +- The evaluator result schema MUST exist at `.specify/extensions/evaluator/schemas/evaluator-result.schema.json`. If missing, report the path and instruct the user to reinstall the evaluator extension. +- Each artifact to evaluate MUST exist and be readable. Report missing artifacts; do not fabricate evaluations for absent files. + +## Execution + +### 1. Load the Evaluator Contract + +Read the schema from `.specify/extensions/evaluator/schemas/evaluator-result.schema.json`. Every result produced MUST validate against this schema. + +### 2. Discover Evaluators + +Look for evaluator configurations in `.specify/extensions/evaluator/`. An evaluator is any extension that declares it produces evaluator results. Discovery order: + +1. Check `.specify/extensions/evaluator/evaluators.yml` for a list of registered evaluator IDs. +2. For each registered evaluator, locate its configuration. +3. If no evaluators are registered, produce a single result with `outcome: "pass"` and a note that no evaluators are configured. + +### 3. Run Each Evaluator + +For each discovered evaluator, in priority order (lower `priority` value first, default 10): + +1. **Deterministic evaluators first**: run deterministic checks (schema validation, linting, static analysis) before model-backed evaluators. +2. **Invoke the evaluator** with the artifact references. +3. **Collect the result** — it MUST be valid JSON conforming to the evaluator result schema. +4. **Validate the result** against the schema. If validation fails, wrap the raw output in an error finding and set `outcome: "block"`. + +### 4. Produce the Result + +Write the result to `.specify/extensions/evaluator/results/--.json`. + +Each result file MUST contain exactly one evaluator result object. The filename pattern is: + +``` +--.json +``` + +Example: `epistemic-after_plan-20260715T143022Z.json` + +### 5. Report + +Output a summary to the user: + +- The evaluator ID and version +- The phase evaluated +- The outcome (`pass`, `warn`, `iterate`, `clarify`, `gather_evidence`, `block`) +- The number of findings by severity +- The recommended next action +- The path to the result file + +## Evaluator Result Contract + +Every result MUST conform to this structure (see the schema for full details): + +```json +{ + "schema_version": "1.0", + "evaluator": { + "id": "", + "version": "" + }, + "phase": "", + "outcome": "pass|warn|iterate|clarify|gather_evidence|block", + "summary": "", + "findings": [ + { + "id": "", + "severity": "critical|high|medium|low|info", + "kind": "", + "subject": "", + "evidence_refs": [ + { + "ref": "", + "kind": "observed|inferred|asserted|contradicted|unsupported" + } + ], + "provenance_refs": ["#"], + "uncertainty": "none|low|medium|high|insufficient_evidence", + "recommended_action": "none|gather_evidence|clarify|revise|iterate|escalate|accept_risk|block" + } + ], + "next_action": { + "kind": "pass|warn|iterate|clarify|gather_evidence|block", + "target_phase": "", + "message": "" + }, + "metadata": { + "timestamp": "", + "duration_ms": 0, + "artifacts_evaluated": [""], + "deterministic": true + }, + "state": {} +} +``` + +### Outcome Semantics + +| Outcome | Meaning | Workflow Effect | +|---------|---------|----------------| +| `pass` | All checks passed; no issues found | Continue to next phase | +| `warn` | Issues found but not blocking | Continue with warnings recorded | +| `iterate` | Issues require revisiting a prior phase | Return to `target_phase` | +| `clarify` | Ambiguities need human resolution | Pause for human input | +| `gather_evidence` | Insufficient evidence to decide | Pause for evidence collection | +| `block` | Hard blocker; cannot proceed | Stop the workflow | + +### Evidence Kinds + +| Kind | Meaning | +|------|---------| +| `observed` | Directly observed from an artifact or command output | +| `inferred` | Logically derived from observed evidence | +| `asserted` | Claimed by a model or agent without direct observation | +| `contradicted` | Conflicts with other observed evidence | +| `unsupported` | No evidence found to support or refute | + +### Key Design Rules + +1. **Generated assertions MUST remain distinguishable from observed evidence.** A model saying "the test passed" is `asserted`; a command exit code 0 with captured stdout is `observed`. +2. **Model self-attestation MUST NOT satisfy an evidence gate by itself.** An evaluator cannot certify its own output as evidence. +3. **Contradictions MUST be preserved**, not collapsed into a single synthesized answer. Conflicting findings from different evaluators are both recorded. +4. **Insufficient evidence MUST be represented explicitly** (`uncertainty: "insufficient_evidence"`) rather than inventing certainty. +5. **Deterministic checks SHOULD run before probabilistic review** where appropriate. +6. **Higher-risk work MAY require evaluator independence** — the same model/family SHOULD NOT both generate and certify the result. + +## Guardrails + +- Never modify source files — write only under `.specify/extensions/evaluator/results/`. +- Never treat a model-generated assertion as observed evidence — always classify it as `asserted`. +- Never collapse contradictory findings — preserve both and let composition resolve. +- Never fabricate evidence references — if no evidence exists, mark it `unsupported`. +- Never overwrite an existing result file without confirmation (interactive) or appending a disambiguating suffix (automated). +- The `state` object is evaluator-defined opaque data for pause/resume — do not interpret or modify another evaluator's state. diff --git a/extensions/evaluator/extension.yml b/extensions/evaluator/extension.yml new file mode 100644 index 0000000000..4abb3947c6 --- /dev/null +++ b/extensions/evaluator/extension.yml @@ -0,0 +1,87 @@ +schema_version: "1.0" + +extension: + id: evaluator + name: "Evaluator Contract" + version: "1.0.0" + description: "Standard evaluator result contract for evidence, provenance, uncertainty, and recovery — a provider-neutral protocol for extensions that evaluate artifact quality between phases" + category: "process" + effect: "read-write" + author: ElectroHire + repository: https://github.com/electrohire/spec-kit-evaluator + license: MIT + homepage: https://github.com/electrohire/spec-kit-evaluator + +requires: + speckit_version: ">=1.0.0" + +provides: + commands: + - name: speckit.evaluator.run + file: commands/speckit.evaluator.run.md + description: "Run an evaluator against one or more artifacts and produce a versioned machine-readable result" + - name: speckit.evaluator.compose + file: commands/speckit.evaluator.compose.md + description: "Compose multiple evaluator results at a lifecycle point with deterministic precedence" + - name: speckit.evaluator.report + file: commands/speckit.evaluator.report.md + description: "Render evaluator results as a human-readable report, CI annotation, or release gate" + - name: speckit.evaluator.route + file: commands/speckit.evaluator.route.md + description: "Recommend which model tier to use for the next SDD phase based on evaluator findings — enables the portfolio approach" + + templates: + - name: evaluator-result-template + file: templates/evaluator-result-template.json + description: "Template for a single evaluator result" + + scripts: + - name: evaluator-compose + file: scripts/python/compose_results.py + description: "Compose multiple evaluator results with deterministic precedence" + runtimes: [python] + - name: evaluator-compose-sh + file: scripts/bash/compose-results.sh + description: "Compose multiple evaluator results (POSIX shell)" + runtimes: [bash] + - name: evaluator-compose-ps + file: scripts/powershell/compose-results.ps1 + description: "Compose multiple evaluator results (PowerShell)" + runtimes: [powershell] + +hooks: + after_specify: + - command: "speckit.evaluator.run" + priority: 20 + optional: true + prompt: "Run evaluators against the specification?" + description: "Evaluate spec quality, evidence, and provenance after specification" + after_plan: + - command: "speckit.evaluator.run" + priority: 20 + optional: true + prompt: "Run evaluators against the plan?" + description: "Evaluate plan assumptions, risks, and coverage after planning" + after_tasks: + - command: "speckit.evaluator.run" + priority: 20 + optional: true + prompt: "Run evaluators against the task breakdown?" + description: "Evaluate task completeness and traceability after task generation" + after_implement: + - command: "speckit.evaluator.run" + priority: 20 + optional: true + prompt: "Run evaluators against the implementation?" + description: "Evaluate implementation against spec, plan, and tasks" + +tags: + - "evaluator" + - "evidence" + - "provenance" + - "quality" + - "governance" + - "compliance" + - "workflow" + - "model-routing" + - "portfolio" diff --git a/extensions/evaluator/schemas/evaluator-result.schema.json b/extensions/evaluator/schemas/evaluator-result.schema.json new file mode 100644 index 0000000000..2cf52d7787 --- /dev/null +++ b/extensions/evaluator/schemas/evaluator-result.schema.json @@ -0,0 +1,318 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://spec-kit.dev/schemas/evaluator-result.schema.json", + "title": "Evaluator Result", + "description": "Standard evaluator result contract for evidence, provenance, uncertainty, and recovery. Provider-neutral protocol for extensions that evaluate artifact quality between Spec-Driven Development phases.", + "type": "object", + "required": ["schema_version", "evaluator", "phase", "outcome", "findings"], + "properties": { + "schema_version": { + "type": "string", + "description": "Version of the evaluator result schema", + "examples": ["1.0"] + }, + "evaluator": { + "type": "object", + "required": ["id", "version"], + "properties": { + "id": { + "type": "string", + "description": "Unique evaluator identifier (e.g., extension id)", + "examples": ["epistemic", "security-scan", "schema-validate"] + }, + "version": { + "type": "string", + "description": "Semantic version of the evaluator", + "examples": ["0.1.0"] + }, + "name": { + "type": "string", + "description": "Human-readable evaluator name" + }, + "url": { + "type": "string", + "format": "uri", + "description": "Evaluator homepage or documentation URL" + } + }, + "additionalProperties": false + }, + "phase": { + "type": "string", + "description": "Lifecycle phase when the evaluator ran", + "enum": [ + "after_specify", + "after_plan", + "after_tasks", + "after_implement", + "after_analyze", + "after_checklist", + "after_clarify", + "after_constitution", + "after_converge", + "after_taskstoissues" + ] + }, + "outcome": { + "type": "string", + "description": "Aggregate evaluator outcome", + "enum": ["pass", "warn", "iterate", "clarify", "gather_evidence", "block"] + }, + "summary": { + "type": "string", + "description": "One-paragraph human-readable summary of the evaluation", + "maxLength": 500 + }, + "findings": { + "type": "array", + "description": "Individual findings from the evaluation", + "minItems": 0, + "items": { + "type": "object", + "required": ["id", "severity", "kind", "subject"], + "properties": { + "id": { + "type": "string", + "description": "Unique finding identifier within this result", + "examples": ["EPI-001", "SEC-042", "SCH-007"] + }, + "severity": { + "type": "string", + "description": "Finding severity", + "enum": ["critical", "high", "medium", "low", "info"] + }, + "kind": { + "type": "string", + "description": "Classification of the finding", + "enum": [ + "unsupported_claim", + "contradiction", + "missing_evidence", + "ambiguous_requirement", + "unverified_assertion", + "provenance_gap", + "schema_violation", + "policy_violation", + "security_concern", + "coverage_gap", + "traceability_gap", + "risk_unaddressed", + "assumption_unvalidated", + "other" + ] + }, + "subject": { + "type": "string", + "description": "Identifier of the artifact element the finding relates to (e.g., REQ-014, T-003, §3.2)", + "examples": ["REQ-014", "T-003", "spec.md#authentication"] + }, + "description": { + "type": "string", + "description": "Human-readable description of the finding", + "maxLength": 500 + }, + "evidence_refs": { + "type": "array", + "description": "References to observed evidence supporting or contradicting the finding", + "items": { + "type": "object", + "required": ["ref", "kind"], + "properties": { + "ref": { + "type": "string", + "description": "Reference to the evidence (file path, URL, artifact identifier)" + }, + "kind": { + "type": "string", + "description": "Nature of the evidence", + "enum": ["observed", "inferred", "asserted", "contradicted", "unsupported"] + }, + "description": { + "type": "string", + "description": "Brief description of what the evidence shows" + } + }, + "additionalProperties": false + } + }, + "provenance_refs": { + "type": "array", + "description": "References to source artifacts the finding relates to", + "items": { + "type": "string" + }, + "examples": [["spec.md#REQ-014", "plan.md#data-model"]] + }, + "uncertainty": { + "type": "string", + "description": "Level of uncertainty about the finding", + "enum": ["none", "low", "medium", "high", "insufficient_evidence"] + }, + "recommended_action": { + "type": "string", + "description": "Recommended action for this specific finding", + "enum": [ + "none", + "gather_evidence", + "clarify", + "revise", + "iterate", + "escalate", + "accept_risk", + "block" + ] + }, + "rationale": { + "type": "string", + "description": "Brief rationale for the finding and recommendation", + "maxLength": 500 + } + }, + "additionalProperties": false + } + }, + "next_action": { + "type": "object", + "description": "Recommended next action for the workflow", + "required": ["kind"], + "properties": { + "kind": { + "type": "string", + "description": "Type of next action", + "enum": ["pass", "warn", "iterate", "clarify", "gather_evidence", "block"] + }, + "target_phase": { + "type": "string", + "description": "Target phase to iterate back to (for iterate actions)", + "enum": [ + "specify", + "plan", + "tasks", + "implement", + "analyze", + "checklist", + "clarify", + "constitution", + "converge" + ] + }, + "message": { + "type": "string", + "description": "Human-readable message about the next action", + "maxLength": 500 + } + }, + "additionalProperties": false + }, + "model_routing": { + "type": "object", + "description": "Model routing recommendation for the next SDD phase. Enables the portfolio approach: budget for routine work, standard for review, premium for critical decisions.", + "properties": { + "recommended_tier": { + "type": "string", + "enum": ["budget", "standard", "premium", "portfolio"], + "description": "Recommended model tier for the next phase" + }, + "reason": { + "type": "string", + "description": "Why this tier is recommended (e.g., 'low risk, budget sufficient' or 'critical security finding, escalate to premium')", + "maxLength": 300 + }, + "escalation_triggers": { + "type": "array", + "description": "Conditions that would trigger escalation to a higher tier", + "items": { + "type": "object", + "required": ["condition", "escalate_to"], + "properties": { + "condition": { + "type": "string", + "description": "Condition that triggers escalation" + }, + "escalate_to": { + "type": "string", + "enum": ["standard", "premium"] + } + }, + "additionalProperties": false + } + }, + "estimated_tokens": { + "type": "integer", + "description": "Estimated tokens for the next phase at this tier", + "minimum": 0 + }, + "estimated_cost_usd": { + "type": "number", + "description": "Estimated USD cost for the next phase at this tier", + "minimum": 0 + }, + "tier_breakdown": { + "type": "object", + "description": "Cost/token comparison across all tiers for the next phase", + "properties": { + "budget": { + "type": "object", + "properties": { + "estimated_tokens": {"type": "integer"}, + "estimated_cost_usd": {"type": "number"} + } + }, + "standard": { + "type": "object", + "properties": { + "estimated_tokens": {"type": "integer"}, + "estimated_cost_usd": {"type": "number"} + } + }, + "premium": { + "type": "object", + "properties": { + "estimated_tokens": {"type": "integer"}, + "estimated_cost_usd": {"type": "number"} + } + } + } + } + }, + "additionalProperties": false + }, + "metadata": { + "type": "object", + "description": "Additional metadata about the evaluation run", + "properties": { + "timestamp": { + "type": "string", + "format": "date-time", + "description": "ISO 8601 timestamp of the evaluation" + }, + "duration_ms": { + "type": "integer", + "description": "Evaluation duration in milliseconds" + }, + "artifacts_evaluated": { + "type": "array", + "description": "List of artifacts that were evaluated", + "items": { + "type": "string" + } + }, + "model": { + "type": "string", + "description": "AI model used for the evaluation, if applicable" + }, + "deterministic": { + "type": "boolean", + "description": "Whether the evaluator is deterministic (true) or model-backed (false)" + } + }, + "additionalProperties": true + }, + "state": { + "type": "object", + "description": "Compact state for pause/resume — evaluator-defined opaque object", + "additionalProperties": true + } + }, + "additionalProperties": false +} diff --git a/extensions/evaluator/scripts/bash/compose-results.sh b/extensions/evaluator/scripts/bash/compose-results.sh new file mode 100644 index 0000000000..2c3a2e4d5d --- /dev/null +++ b/extensions/evaluator/scripts/bash/compose-results.sh @@ -0,0 +1,159 @@ +#!/usr/bin/env bash +# Compose multiple evaluator results with deterministic precedence. +# +# Usage: +# compose-results.sh --results-dir --phase [--strategy strict|majority|optimistic] [--output ] +# +# Reads evaluator result JSON files from a results directory and produces a +# composed result. Requires `jq` for JSON processing. + +set -euo pipefail + +RESULTS_DIR="" +PHASE="" +STRATEGY="strict" +OUTPUT="" + +usage() { + cat < --phase [--strategy strict|majority|optimistic] [--output ] + +Compose multiple evaluator results with deterministic precedence. + +Options: + --results-dir Directory containing evaluator result JSON files. + --phase Lifecycle phase to compose results for (e.g., after_plan). + --strategy Composition strategy: strict (default), majority, or optimistic. + --output Write composed result to this file instead of stdout. +EOF + exit 1 +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --results-dir) RESULTS_DIR="$2"; shift 2 ;; + --phase) PHASE="$2"; shift 2 ;; + --strategy) STRATEGY="$2"; shift 2 ;; + --output) OUTPUT="$2"; shift 2 ;; + *) usage ;; + esac +done + +if [[ -z "$RESULTS_DIR" || -z "$PHASE" ]]; then + echo "Error: --results-dir and --phase are required." >&2 + usage +fi + +if [[ ! -d "$RESULTS_DIR" ]]; then + echo "Error: results directory not found: $RESULTS_DIR" >&2 + exit 1 +fi + +# Check for jq +if ! command -v jq &>/dev/null; then + echo "Error: jq is required but not installed." >&2 + exit 1 +fi + +TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ") + +# Collect result files for the phase (exclude previously composed files) +# Portable to bash 3.2+ (macOS): avoid mapfile (bash 4+) +RESULT_FILES=() +while IFS= read -r f; do + RESULT_FILES+=("$f") +done < <(find "$RESULTS_DIR" -maxdepth 1 -name "*-${PHASE}-*.json" ! -name "composed-*" | sort) + +if [[ ${#RESULT_FILES[@]} -eq 0 ]]; then + # No results found — produce empty composed result + COMPOSED=$(jq -n \ + --arg phase "$PHASE" \ + --arg strategy "$STRATEGY" \ + --arg ts "$TIMESTAMP" \ + '{ + schema_version: "1.0", + composed: true, + phase: $phase, + composition_strategy: $strategy, + composed_outcome: "pass", + composed_summary: "No evaluator results found for this phase.", + evaluator_results: [], + findings: [], + next_action: { kind: "pass", target_phase: null, message: "No evaluator results found." }, + evaluator_states: {}, + metadata: { + timestamp: $ts, + evaluator_count: 0, + contradictory_findings: [] + } + }') +else + # Build a jq filter that merges all result files + # Strategy: read all files into an array, then apply composition logic + JQ_FILTER='def severity_order($s): + if $s == "critical" then 0 + elif $s == "high" then 1 + elif $s == "medium" then 2 + elif $s == "low" then 3 + else 4 end; + + def resolve_outcome(outcomes; strategy): + if strategy == "optimistic" then + if outcomes | index("pass") then "pass" + elif outcomes | index("warn") then "warn" + elif outcomes | index("clarify") then "clarify" + elif outcomes | index("iterate") then "iterate" + elif outcomes | index("gather_evidence") then "gather_evidence" + else "block" end + elif strategy == "majority" then + (outcomes | group_by(.) | sort_by([-(length), + if .[0] == "block" then 0 + elif .[0] == "gather_evidence" then 1 + elif .[0] == "iterate" then 2 + elif .[0] == "clarify" then 3 + elif .[0] == "warn" then 4 + else 5 end]) | .[0][0]) + else + if outcomes | index("block") then "block" + elif outcomes | index("gather_evidence") then "gather_evidence" + elif outcomes | index("iterate") then "iterate" + elif outcomes | index("clarify") then "clarify" + elif outcomes | index("warn") then "warn" + else "pass" end + end; + + . as $results + | ($results | map(.outcome)) as $outcomes + | ($results | map(.findings // []) | flatten) as $all_findings + | resolve_outcome($outcomes; $STRATEGY) as $composed_outcome + | { + schema_version: "1.0", + composed: true, + phase: $PHASE, + composition_strategy: $STRATEGY, + composed_outcome: $composed_outcome, + composed_summary: "\($results | length) evaluator(s) ran. \($all_findings | length) finding(s) total.", + evaluator_results: $results | map({ evaluator_id: .evaluator.id, outcome: .outcome, findings_count: (.findings // [] | length) }), + findings: $all_findings | sort_by(severity_order(.severity)), + next_action: { kind: $composed_outcome, target_phase: null, message: "Composed outcome: \($composed_outcome)." }, + evaluator_states: $results | map({ key: .evaluator.id, value: (.state // {}) }) | from_entries, + metadata: { + timestamp: $TIMESTAMP, + evaluator_count: $results | length, + contradictory_findings: [] + } + }' + + COMPOSED=$(for f in "${RESULT_FILES[@]}"; do cat "$f"; done | jq -s "$JQ_FILTER" \ + --arg PHASE "$PHASE" \ + --arg STRATEGY "$STRATEGY" \ + --arg TIMESTAMP "$TIMESTAMP") +fi + +if [[ -n "$OUTPUT" ]]; then + mkdir -p "$(dirname "$OUTPUT")" + echo "$COMPOSED" > "$OUTPUT" + echo "Composed result written to $OUTPUT" +else + echo "$COMPOSED" +fi diff --git a/extensions/evaluator/scripts/powershell/compose-results.ps1 b/extensions/evaluator/scripts/powershell/compose-results.ps1 new file mode 100644 index 0000000000..dc9f011ad3 --- /dev/null +++ b/extensions/evaluator/scripts/powershell/compose-results.ps1 @@ -0,0 +1,183 @@ +# Compose multiple evaluator results with deterministic precedence. +# +# Usage: +# .\compose-results.ps1 -ResultsDir -Phase [-Strategy strict|majority|optimistic] [-Output ] +# +# Reads evaluator result JSON files from a results directory and produces a +# composed result. + +param( + [Parameter(Mandatory=$true)] + [string]$ResultsDir, + + [Parameter(Mandatory=$true)] + [string]$Phase, + + [ValidateSet("strict", "majority", "optimistic")] + [string]$Strategy = "strict", + + [string]$Output +) + +$ErrorActionPreference = "Stop" + +if (-not (Test-Path $ResultsDir -PathType Container)) { + Write-Error "Results directory not found: $ResultsDir" + exit 1 +} + +$timestamp = (Get-Date).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ") + +# Collect result files for the phase (exclude previously composed files) +$resultFiles = Get-ChildItem -Path $ResultsDir -Filter "*-$Phase-*.json" | + Where-Object { $_.Name -notlike "composed-*" } | + Sort-Object Name + +if ($resultFiles.Count -eq 0) { + $composed = @{ + schema_version = "1.0" + composed = $true + phase = $Phase + composition_strategy = $Strategy + composed_outcome = "pass" + composed_summary = "No evaluator results found for this phase." + evaluator_results = @() + findings = @() + next_action = @{ + kind = "pass" + target_phase = $null + message = "No evaluator results found." + } + evaluator_states = @{} + metadata = @{ + timestamp = $timestamp + evaluator_count = 0 + contradictory_findings = @() + } + } +} else { + $allResults = @() + $allFindings = @() + $evaluatorSummaries = @() + $outcomes = @() + $evaluatorStates = @{} + + $severityOrder = @{ + critical = 0 + high = 1 + medium = 2 + low = 3 + info = 4 + } + + foreach ($file in $resultFiles) { + try { + $data = Get-Content -Path $file.FullName -Raw | ConvertFrom-Json + } catch { + Write-Warning "Skipping invalid result file: $($file.Name): $_" + continue + } + + if (-not $data.PSObject.Properties["schema_version"] -or + -not $data.PSObject.Properties["evaluator"] -or + -not $data.PSObject.Properties["outcome"] -or + -not $data.PSObject.Properties["findings"]) { + Write-Warning "Skipping $($file.Name): missing required keys" + continue + } + + $allResults += $data + $outcomes += $data.outcome + + $evaluatorSummaries += @{ + evaluator_id = $data.evaluator.id + outcome = $data.outcome + findings_count = if ($data.findings) { $data.findings.Count } else { 0 } + } + + if ($data.findings) { + foreach ($finding in $data.findings) { + $finding | Add-Member -NotePropertyName "_evaluator_id" -NotePropertyValue $data.evaluator.id -Force + $allFindings += $finding + } + } + + if ($data.PSObject.Properties["state"]) { + $evaluatorStates[$data.evaluator.id] = $data.state + } + } + + # Sort findings by severity + $allFindings = $allFindings | Sort-Object { + $sev = if ($_.PSObject.Properties["severity"]) { $_.severity } else { "info" } + if ($severityOrder.ContainsKey($sev)) { $severityOrder[$sev] } else { 99 } + }, { if ($_.PSObject.Properties["id"]) { $_.id } else { "" } } + + # Resolve composed outcome + function Resolve-Outcome { + param([string[]]$Outcomes, [string]$Strategy) + + $precedence = @("block", "gather_evidence", "iterate", "clarify", "warn", "pass") + + switch ($Strategy) { + "optimistic" { + for ($i = $precedence.Count - 1; $i -ge 0; $i--) { + if ($Outcomes -contains $precedence[$i]) { return $precedence[$i] } + } + return "pass" + } + "majority" { + $grouped = $Outcomes | Group-Object | Sort-Object Count -Descending + $maxCount = $grouped[0].Count + $tied = $grouped | Where-Object { $_.Count -eq $maxCount } | ForEach-Object { $_.Name } + foreach ($c in $precedence) { + if ($tied -contains $c) { return $c } + } + return $grouped[0].Name + } + default { + foreach ($c in $precedence) { + if ($Outcomes -contains $c) { return $c } + } + return "pass" + } + } + } + + $composedOutcome = Resolve-Outcome -Outcomes $outcomes -Strategy $Strategy + + $composed = @{ + schema_version = "1.0" + composed = $true + phase = $Phase + composition_strategy = $Strategy + composed_outcome = $composedOutcome + composed_summary = "$($allResults.Count) evaluator(s) ran. $($allFindings.Count) finding(s) total." + evaluator_results = $evaluatorSummaries + findings = $allFindings + next_action = @{ + kind = $composedOutcome + target_phase = $null + message = "Composed outcome: $composedOutcome." + } + evaluator_states = $evaluatorStates + metadata = @{ + timestamp = $timestamp + evaluator_count = $allResults.Count + contradictory_findings = @() + } + } +} + +$json = $composed | ConvertTo-Json -Depth 10 + +if ($Output) { + $parent = Split-Path $Output -Parent + if ($parent -and -not (Test-Path $parent)) { + New-Item -ItemType Directory -Path $parent -Force | Out-Null + } + $json | Set-Content -Path $Output -Encoding UTF8 + Write-Host "Composed result written to $Output" +} else { + Write-Output $json +} diff --git a/extensions/evaluator/scripts/python/compose_results.py b/extensions/evaluator/scripts/python/compose_results.py new file mode 100644 index 0000000000..c1a148c962 --- /dev/null +++ b/extensions/evaluator/scripts/python/compose_results.py @@ -0,0 +1,397 @@ +#!/usr/bin/env python3 +"""Compose multiple evaluator results with deterministic precedence. + +Reads evaluator result JSON files from a results directory, validates them +against the evaluator result schema, and produces a composed result with +deterministic outcome resolution. + +Usage: + compose_results.py --results-dir --phase [--strategy strict|majority|optimistic] [--output ] [--json] + +Output: + A composed evaluator result written to stdout (--json) or to the specified + output file. Exit code 0 on success, 1 on error. +""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + + +# -- Outcome precedence for strict composition (most severe first) ---------- +_STRICT_PRECEDENCE = [ + "block", + "gather_evidence", + "iterate", + "clarify", + "warn", + "pass", +] + +_SEVERITY_ORDER = { + "critical": 0, + "high": 1, + "medium": 2, + "low": 3, + "info": 4, +} + + +def _resolve_outcome_strict(outcomes: list[str]) -> str: + """Return the most severe outcome from the list.""" + for candidate in _STRICT_PRECEDENCE: + if candidate in outcomes: + return candidate + return "pass" + + +def _resolve_outcome_majority(outcomes: list[str]) -> str: + """Return the outcome with the most evaluators supporting it. + + Ties break toward the more severe outcome (strict precedence). + """ + counts: dict[str, int] = {} + for o in outcomes: + counts[o] = counts.get(o, 0) + 1 + max_count = max(counts.values()) + tied = [o for o, c in counts.items() if c == max_count] + if len(tied) == 1: + return tied[0] + return _resolve_outcome_strict(tied) + + +def _resolve_outcome_optimistic(outcomes: list[str]) -> str: + """Return the least severe outcome.""" + for candidate in reversed(_STRICT_PRECEDENCE): + if candidate in outcomes: + return candidate + return "pass" + + +def _resolve_outcome(outcomes: list[str], strategy: str) -> str: + if not outcomes: + return "pass" + if strategy == "majority": + return _resolve_outcome_majority(outcomes) + if strategy == "optimistic": + return _resolve_outcome_optimistic(outcomes) + return _resolve_outcome_strict(outcomes) + + +def _outcome_to_next_action(outcome: str, iterate_phase: str | None) -> dict[str, Any]: + """Derive the next_action block from the composed outcome.""" + kind = outcome + target_phase = None + if outcome == "iterate": + target_phase = iterate_phase + return { + "kind": kind, + "target_phase": target_phase, + "message": f"Composed outcome: {outcome}.", + } + + +def _severity_sort_key(finding: dict[str, Any]) -> tuple[int, str]: + return (_SEVERITY_ORDER.get(finding.get("severity", "info"), 99), finding.get("id", "")) + + +def _detect_contradictions(findings: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Detect pairs of findings that contradict each other on the same subject.""" + by_subject: dict[str, list[dict[str, Any]]] = {} + for f in findings: + subject = f.get("subject", "") + by_subject.setdefault(subject, []).append(f) + + contradictions: list[dict[str, Any]] = [] + for subject, group in by_subject.items(): + if len(group) < 2: + continue + kinds = {f.get("kind") for f in group} + # Contradiction: one says "supported" / passes, another says "unsupported" / fails + has_positive = any(k in ("pass", "observed") for k in kinds) + has_negative = any( + k in ("unsupported_claim", "contradiction", "missing_evidence", "unverified_assertion") + for k in kinds + ) + if has_positive and has_negative: + contradictions.append( + { + "subject": subject, + "finding_ids": [f["id"] for f in group], + "description": f"Conflicting findings on subject '{subject}'", + } + ) + return contradictions + + +def _load_result_file(filepath: Path) -> dict[str, Any] | None: + """Load and validate a single evaluator result file. + + Returns the parsed result dict, or None if the file is invalid. + """ + try: + with open(filepath, encoding="utf-8") as fh: + data = json.load(fh) + except (json.JSONDecodeError, OSError) as exc: + print(f"Warning: skipping invalid result file {filepath}: {exc}", file=sys.stderr) + return None + + # Basic structural validation (full schema validation is done by the + # command template; this is a lightweight check for the script). + required = ["schema_version", "evaluator", "phase", "outcome", "findings"] + for key in required: + if key not in data: + print(f"Warning: skipping {filepath}: missing required key '{key}'", file=sys.stderr) + return None + + if not isinstance(data.get("findings"), list): + print(f"Warning: skipping {filepath}: 'findings' is not an array", file=sys.stderr) + return None + + return data + + +def _merge_model_routing( + results: list[dict[str, Any]], composed_outcome: str +) -> dict[str, Any] | None: + """Merge model routing recommendations from multiple evaluators. + + When multiple evaluators provide model_routing, the most conservative + (highest tier) recommendation wins. If no evaluator provides routing, + derive one from the composed outcome and findings. + """ + routings = [r.get("model_routing") for r in results if r.get("model_routing")] + if not routings: + return None + + # Collect all recommended tiers + tiers = [mr["recommended_tier"] for mr in routings] + tier_precedence = {"premium": 3, "standard": 2, "budget": 1, "portfolio": 2} + + # Most conservative (highest tier) wins + best_tier = max(tiers, key=lambda t: tier_precedence.get(t, 0)) + + # Merge escalation triggers from all evaluators + all_triggers = [] + for mr in routings: + all_triggers.extend(mr.get("escalation_triggers", [])) + + # Merge tier breakdowns (take max estimates) + merged_breakdown: dict[str, dict[str, Any]] = {} + for tier_key in ("budget", "standard", "premium"): + estimates = [ + mr.get("tier_breakdown", {}).get(tier_key, {}) + for mr in routings + if mr.get("tier_breakdown", {}).get(tier_key) + ] + if estimates: + merged_breakdown[tier_key] = { + "estimated_tokens": max(e.get("estimated_tokens", 0) for e in estimates), + "estimated_cost_usd": max(e.get("estimated_cost_usd", 0) for e in estimates), + } + + # Build reason from the evaluator that recommended the winning tier + winning_routing = next( + (mr for mr in routings if mr["recommended_tier"] == best_tier), + routings[0], + ) + + return { + "recommended_tier": best_tier, + "reason": f"[Composed from {len(routings)} evaluator(s)] {winning_routing.get('reason', '')}", + "escalation_triggers": all_triggers[:5], # Cap at 5 + "estimated_tokens": winning_routing.get("estimated_tokens", 0), + "estimated_cost_usd": winning_routing.get("estimated_cost_usd", 0), + "tier_breakdown": merged_breakdown if merged_breakdown else None, + } + + +def compose_results( + results_dir: Path, + phase: str, + strategy: str = "strict", +) -> dict[str, Any]: + """Compose all evaluator results for a phase into one aggregate result. + + Args: + results_dir: Directory containing evaluator result JSON files. + phase: Lifecycle phase to filter results by. + strategy: Composition strategy ('strict', 'majority', or 'optimistic'). + + Returns: + A composed result dict. + """ + if not results_dir.is_dir(): + return _empty_composed(phase, strategy, "No results directory found.") + + # Discover result files for the phase + result_files = sorted(results_dir.glob(f"*-{phase}-*.json")) + # Exclude previously composed files + result_files = [f for f in result_files if not f.name.startswith("composed-")] + + if not result_files: + return _empty_composed(phase, strategy, "No evaluator results found for this phase.") + + # Load and validate all results + results: list[dict[str, Any]] = [] + evaluator_summaries: list[dict[str, Any]] = [] + for fp in result_files: + data = _load_result_file(fp) + if data is None: + continue + results.append(data) + evaluator_summaries.append( + { + "evaluator_id": data["evaluator"]["id"], + "outcome": data["outcome"], + "findings_count": len(data.get("findings", [])), + } + ) + + if not results: + return _empty_composed(phase, strategy, "No valid evaluator results found.") + + # Collect all findings + all_findings: list[dict[str, Any]] = [] + for r in results: + for f in r.get("findings", []): + # Tag each finding with its evaluator origin + f_with_origin = dict(f) + f_with_origin["_evaluator_id"] = r["evaluator"]["id"] + all_findings.append(f_with_origin) + + # Sort findings by severity, then by ID + all_findings.sort(key=_severity_sort_key) + + # Resolve composed outcome + outcomes = [r["outcome"] for r in results] + composed_outcome = _resolve_outcome(outcomes, strategy) + + # Determine iterate target phase + iterate_phases = [ + r.get("next_action", {}).get("target_phase") + for r in results + if r["outcome"] == "iterate" and r.get("next_action", {}).get("target_phase") + ] + most_common_iterate = max(set(iterate_phases), key=iterate_phases.count) if iterate_phases else None + + # Detect contradictions + contradictions = _detect_contradictions(all_findings) + + # Merge model routing recommendations + model_routing = _merge_model_routing(results, composed_outcome) + + # Collect evaluator states + evaluator_states: dict[str, Any] = {} + for r in results: + eid = r["evaluator"]["id"] + if "state" in r: + evaluator_states[eid] = r["state"] + + # Build composed result + composed = { + "schema_version": "1.0", + "composed": True, + "phase": phase, + "composition_strategy": strategy, + "composed_outcome": composed_outcome, + "composed_summary": ( + f"{len(results)} evaluator(s) ran. " + f"Outcomes: {', '.join(f'{s['evaluator_id']}={s['outcome']}' for s in evaluator_summaries)}. " + f"{len(all_findings)} finding(s) total." + ), + "evaluator_results": evaluator_summaries, + "findings": all_findings, + "next_action": _outcome_to_next_action(composed_outcome, most_common_iterate), + "model_routing": model_routing, + "evaluator_states": evaluator_states, + "metadata": { + "timestamp": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "evaluator_count": len(results), + "contradictory_findings": contradictions, + }, + } + + return composed + + +def _empty_composed(phase: str, strategy: str, message: str) -> dict[str, Any]: + return { + "schema_version": "1.0", + "composed": True, + "phase": phase, + "composition_strategy": strategy, + "composed_outcome": "pass", + "composed_summary": message, + "evaluator_results": [], + "findings": [], + "next_action": {"kind": "pass", "target_phase": None, "message": message}, + "evaluator_states": {}, + "metadata": { + "timestamp": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "evaluator_count": 0, + "contradictory_findings": [], + }, + } + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Compose multiple evaluator results with deterministic precedence." + ) + parser.add_argument( + "--results-dir", + required=True, + type=Path, + help="Directory containing evaluator result JSON files.", + ) + parser.add_argument( + "--phase", + required=True, + help="Lifecycle phase to compose results for (e.g., after_plan).", + ) + parser.add_argument( + "--strategy", + choices=["strict", "majority", "optimistic"], + default="strict", + help="Composition strategy (default: strict).", + ) + parser.add_argument( + "--output", + type=Path, + default=None, + help="Write composed result to this file instead of stdout.", + ) + parser.add_argument( + "--json", + action="store_true", + default=False, + help="Output raw JSON to stdout (default when --output is not specified).", + ) + + args = parser.parse_args() + + composed = compose_results( + results_dir=args.results_dir, + phase=args.phase, + strategy=args.strategy, + ) + + output_json = json.dumps(composed, indent=2, ensure_ascii=False) + + if args.output: + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(output_json + "\n", encoding="utf-8") + print(f"Composed result written to {args.output}") + else: + print(output_json) + + +if __name__ == "__main__": + main() diff --git a/extensions/evaluator/templates/evaluator-result-template.json b/extensions/evaluator/templates/evaluator-result-template.json new file mode 100644 index 0000000000..f54981de5d --- /dev/null +++ b/extensions/evaluator/templates/evaluator-result-template.json @@ -0,0 +1,45 @@ +{ + "schema_version": "1.0", + "evaluator": { + "id": "", + "version": "0.1.0", + "name": "", + "url": "" + }, + "phase": "after_specify", + "outcome": "pass", + "summary": "", + "findings": [ + { + "id": "", + "severity": "medium", + "kind": "unsupported_claim", + "subject": "", + "description": "", + "evidence_refs": [ + { + "ref": "", + "kind": "observed", + "description": "" + } + ], + "provenance_refs": ["#"], + "uncertainty": "low", + "recommended_action": "gather_evidence", + "rationale": "" + } + ], + "next_action": { + "kind": "pass", + "target_phase": null, + "message": "" + }, + "metadata": { + "timestamp": "", + "duration_ms": 0, + "artifacts_evaluated": [""], + "model": null, + "deterministic": true + }, + "state": {} +}