diff --git a/docs/docs/usage/dbt-pr-review.md b/docs/docs/usage/dbt-pr-review.md index 2c2a19afe5..9a01afd6a0 100644 --- a/docs/docs/usage/dbt-pr-review.md +++ b/docs/docs/usage/dbt-pr-review.md @@ -20,7 +20,7 @@ a model's opinion: - **column-lineage / DAG blast radius** — which downstream models a change breaks - **query equivalence** — whether a "refactor" provably returns the same rows -- **PII classification** — columns that newly expose sensitive data +- **PII classification** — PII columns present in a touched model are flagged; acknowledgement is a follow-up - **A–F grade + anti-patterns** — readability, correctness, warehouse-cost issues !!! warning "The bot posts a COMMENT review — never a formal GitHub *Approve*" @@ -115,15 +115,19 @@ Options: | `--severity ` | Minimum severity to surface: `critical`, `warning`, `suggestion`. | | `--post` | Post the verdict to the GitHub PR (uses `GITHUB_TOKEN` + the Actions event). | | `--no-ai` | Disable the advisory LLM reviewer lane (no model calls / cost) — deterministic-only. | +| `--ai-model ` | Explicit model for the advisory reviewer lane; overrides `ALTIMATE_REVIEW_AI_MODEL` and `aiModel` in `.altimate/review.yml`. | +| `--ai-reasoning ` | AI reviewer reasoning level (`none`, `minimal`, `low`, `medium`, or `high`); overrides `ALTIMATE_REVIEW_AI_REASONING` and `aiReasoningEffort` in `.altimate/review.yml`. | +| `--ai-timeout ` | AI reviewer deadline (10–1800 seconds); overrides `ALTIMATE_REVIEW_AI_TIMEOUT_SECONDS` and `aiTimeoutSeconds` in `.altimate/review.yml`. | +| `--ai-max-output-tokens ` | AI reviewer output budget (512–32768 tokens); overrides `aiMaxOutputTokens` in `.altimate/review.yml`. | | `--explain-tier` | Emit the classifier's tier-reason list on the verdict envelope so you can see why a diff was rated `trivial`, `lite`, or `full`. Reasons already surface in the PR comment for `full`-tier runs — this flag adds them to `trivial`/`lite` for debugging. | | `--force-tier ` | **[EXPERIMENTAL / bench debug]** Bypass the classifier and force `trivial` / `lite` / `full`. The verdict envelope carries `tierForced: true` and the classifier's original decision for audit. | | `--json` / `--output ` | Emit the verdict envelope as JSON. | > **Full vs lint-only.** With a compiled `manifest.json` present, the reviewer -> proves lineage and equivalence exactly. Without it (or without a warehouse) it -> runs **lint-only** and conservatively *warns* on changes it cannot prove safe — -> clearly labeled, never mistaken for a full verdict. Run `dbt compile` first for -> the full verdict. +> can resolve the dbt graph. Without it, it runs **lint-only** and conservatively +> *warns* on changes it cannot prove safe — clearly labeled, never mistaken for a +> full verdict. Run `dbt compile` first; the review separately identifies missing +> catalog or compiled-SQL artifacts that reduce fidelity. !!! question "Stuck in lint-only mode? It is **not** an API-key problem." The deterministic engine (lineage, equivalence, PII, grade) runs fully @@ -140,6 +144,14 @@ Options: on the CLI), or set `manifestPath:` in `.altimate/review.yml`. - **Freshness.** A stale manifest that predates the changed models can't resolve them. Run `dbt compile` (or `dbt build`) to regenerate it before reviewing. + - **Column metadata.** Run `dbt docs generate` so `target/catalog.json` + supplies real column types for lineage and PII analysis. + - **Base compiled SQL.** In CI, compile the base ref into + `target-base/compiled`: + `git worktree add --detach ../dbt-review-base "$(git merge-base origin/ )" && (cd ../dbt-review-base && dbt deps && dbt compile --target-path ..//target-base)`, + where `` is the same commit you pass to `--head` (the PR head SHA in CI; `HEAD` when reviewing the checked-out branch). + Compile the merge-base (fork point), not the base tip: the review diffs against the merge-base, so base-only commits must not leak into `target-base/compiled`. + Without `target-base/compiled`, equivalence is undecidable and the review says so. - **Working directory.** Run the review from the dbt project root so the relative manifest path resolves. @@ -184,10 +196,34 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - with: { fetch-depth: 0 } - # Produce target/manifest.json for the full verdict (adapter-specific). - - run: pip install dbt-core dbt-bigquery && dbt deps && dbt compile - - uses: AltimateAI/altimate-code/github/review@v0.8.5 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha }} # compile the PR head, not the synthetic merge commit + # Produce manifest/catalog plus compiled SQL for both sides (adapter-specific). + - name: Build dbt review artifacts + # Non-fatal so the review still runs (lint-only, naming the missing + # artifacts) when compile or docs generate fails; keep a separate dbt + # build check if compile failures must fail the PR. + continue-on-error: true + env: + DBT_PROFILES_DIR: ${{ github.workspace }} + PR_BASE_REF: ${{ github.event.pull_request.base.ref }} + PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: | + pip install dbt-core dbt-bigquery + dbt deps + dbt compile + dbt docs generate || echo "docs generate skipped — lineage/PII run without catalog column types" + # Compile the fork point (merge-base), which is what the review compares against. + git fetch --no-tags origin "+refs/heads/${PR_BASE_REF}:refs/remotes/origin/${PR_BASE_REF}" + MERGE_BASE=$(git merge-base "origin/${PR_BASE_REF}" "${PR_HEAD_SHA}") + git worktree add --detach ../dbt-review-base "${MERGE_BASE}" + ( + cd ../dbt-review-base + dbt deps + dbt compile --target-path "${{ github.workspace }}/target-base" + ) + - uses: AltimateAI/altimate-code/github/review@v0.10.0 # exact pin; bump when the release with the gateway inputs ships with: mode: comment # `gate` to block merges manifest_path: target/manifest.json @@ -197,31 +233,103 @@ jobs: altimate_instance: ${{ secrets.ALTIMATE_INSTANCE }} # …or bring your own: model: anthropic/claude-sonnet-4-6 # model_api_key: ${{ secrets.ANTHROPIC_API_KEY }} + # …or use the default free gateway model: + # altimate_gateway_key: ${{ secrets.ALTIMATE_GATEWAY_KEY }} + # altimate_gateway_url: https://gateway.example.com # only for a self-hosted gateway; defaults to https://altimate-gateway.onealtimate.com ``` -### Model & credentials for the advisory lane +Without `target-base/compiled`, base-vs-head equivalence is undecidable; the +review reports that explicitly rather than presenting the run as lint-only. + +### Choosing the AI reviewer's model + +The deterministic engine (lineage, equivalence, PII, grade, and lint) needs no +model or model credentials. The optional advisory lane always makes its model +choice explicit in headless runs. Action routes have this precedence: +**A (`altimate_api_key`) > B (`model` + `model_api_key`) > C +(`altimate_gateway_key`) > none**. With no selected route and no local +`aiModel`, the lane is skipped. -The deterministic engine (lineage, equivalence, PII, grade, lint — the only layer -that can **block**) runs entirely from the compiled artifacts and needs **no model -or credentials**. The optional layer-3 **LLM reviewer** does — supply it one of two -ways, or neither (it self-disables, leaving a deterministic-only review): +Route A uses the hosted Altimate backend and defaults to +`altimate-backend/altimate-default`. `ai_model` can override that model within +the route: + +```yaml +with: + altimate_api_key: ${{ secrets.ALTIMATE_API_KEY }} + altimate_instance: ${{ secrets.ALTIMATE_INSTANCE }} + # ai_model: altimate-backend/altimate-default +``` -| Route | Action inputs | Result | -|-------|---------------|--------| -| **Hosted altimate model** | `altimate_api_key` + `altimate_instance` (+ optional `altimate_url`) | uses the altimate-hosted default model | -| **Bring-your-own** | `model` (e.g. `anthropic/claude-sonnet-4-6`) + `model_api_key` | uses your provider/model | +Route B is an explicit bring-your-own model choice; both inputs are required: -Always pass keys as repo **secrets**. Warehouse credentials are consumed by the -`dbt compile` step (via your `profiles.yml`), **not** by the review step. A -complete, copy-paste workflow lives at +```yaml +with: + model: anthropic/claude-sonnet-4-6 + model_api_key: ${{ secrets.ANTHROPIC_API_KEY }} +``` + +Route C configures the OpenAI-compatible Altimate gateway. It defaults to the +free `altimate-base` model; use `ai_model: altimate-gateway/altimate-pro` to +select the pro model. The gateway URL defaults to the production gateway, +`https://altimate-gateway.onealtimate.com`; set `altimate_gateway_url` only +for a self-hosted gateway (it must use HTTPS). The key is issued per +repository by Altimate on request; it has a daily budget and does not expire. +Do not use a self-serve gateway key from the `altimate` CLI here: those keys +expire after seven days and carry only a one-time starter credit, so the +review lane would start reporting `error` within days. + +```yaml +with: + altimate_gateway_key: ${{ secrets.ALTIMATE_GATEWAY_KEY }} + # altimate_gateway_url: https://gateway.example.com # self-hosted only + # ai_model: altimate-gateway/altimate-pro + # ai_reasoning: none + # ai_timeout_seconds: 900 +``` + +Reasoning models such as altimate-base think for two to three minutes before answering; a review takes three to five minutes and needs 6K+ output tokens. To trade depth for speed, set `aiReasoningEffort: none`; the gateway then answers without a thinking phase (roughly 30 s for a typical PR instead of 3–5 min). The gateway route defaults to a 900 s timeout. Set +`aiTimeoutSeconds` and `aiMaxOutputTokens` in `.altimate/review.yml`, or use +`--ai-timeout` and `--ai-max-output-tokens` for one run. The timeout also honors +`ALTIMATE_REVIEW_AI_TIMEOUT_SECONDS`, with precedence **flag > environment > +repository config > changed-file default**. + +Always pass keys as repository **secrets**. Warehouse credentials are consumed +by the `dbt compile` step (via `profiles.yml`), not by the review step. A full +workflow lives at [`github/review/examples/altimate-ingestion.yml`](https://github.com/AltimateAI/altimate-code/blob/main/github/review/examples/altimate-ingestion.yml). +For local headless use, model precedence is **flag > environment > repository +config**: + +```yaml +# .altimate/review.yml +aiModel: altimate-gateway/altimate-base +``` + +```bash +altimate review --ai-model altimate-gateway/altimate-base +ALTIMATE_REVIEW_AI_MODEL=altimate-gateway/altimate-base altimate review +``` + +The selected provider must also be configured in Altimate Code. The in-session +`dbt_pr_review` tool honors `aiModel` when it is set; otherwise it uses the +session's current model. The headless CLI never silently borrows a chat model. +The AI lane never affects the verdict: its findings remain advisory and are +clamped below `critical`. + Re-pushing commits updates the same summary comment in place; fixed findings are dropped on the next run. `--post` targets **GitHub** PRs (it reads `GITHUB_TOKEN` / `GITHUB_REPOSITORY` and posts via the GitHub API). On other platforms (e.g. GitLab), run with `--json`/`--output` and post the verdict using that platform's own API — native GitLab posting is not yet built in. +The summary comment is updated in place only when the reviewer can tell which +comment is its own. With the default Actions token that is automatic. If you post +with a **GitHub App installation token**, the API cannot reveal the App's identity, +so set `ALTIMATE_REVIEW_BOT_LOGIN=[bot]`; otherwise each run creates a +new summary comment. The reviewer never adopts a marker comment from an unknown bot. + ## Configuration — `.altimate/review.yml` Per-repo configuration, the analogue of an `AGENTS.md`. Tune the rubric, choose @@ -232,6 +340,10 @@ mode: comment # comment | gate severityThreshold: suggestion manifestPath: target/manifest.json dialect: snowflake +aiModel: altimate-gateway/altimate-base # optional explicit advisory model +aiReasoningEffort: none # optional; none|minimal turn thinking off for altimate-base +aiTimeoutSeconds: 300 # optional; 10–1800 seconds, otherwise uses changed-file default +aiMaxOutputTokens: 8192 # 512..32768; includes reasoning tokens reviewers: [] # empty = risk-tier defaults; or pin lanes dataDiff: # OFF by default — see "Data-diff in CI" below enabled: false @@ -303,7 +415,7 @@ In GitHub Actions, supply the connection from a secret — both sides of the dif run against the **same** warehouse (base-compiled vs head-compiled SQL): ```yaml - - uses: AltimateAI/altimate-code/github/review@v0.8.5 + - uses: AltimateAI/altimate-code/github/review@v0.10.0 # exact pin; bump when the release with the gateway inputs ships with: mode: comment manifest_path: target/manifest.json diff --git a/docs/internal/2026-09-02-telemetry-analysis.md b/docs/internal/2026-09-02-telemetry-analysis.md new file mode 100644 index 0000000000..706db4f7c5 --- /dev/null +++ b/docs/internal/2026-09-02-telemetry-analysis.md @@ -0,0 +1,157 @@ +# Telemetry Analysis 2026-09-02 — How Are We Doing + +**Verdict: the CLI cohort is steady and small; the datamates channel, which is nine in ten observed machine IDs, is failing most of its tasks.** The `completed` label on datamates fell from 57% of outcomes in mid-July to 20% in the (partial) week of Aug 30, while recorded errors rose from 22% to 60%. The Console rate limit on the free default model, first seen Aug 12, accounts for roughly half of the decline; sessions that were never rate-limited still fell from 57% to 31%. Everything else in this report matters less than fixing that channel. + +Source: Azure Log Analytics `altimate-code-os` (`AppEvents`). Window **2026-08-19T00:00Z → 2026-09-02T00:00Z** (half-open, 14d, "now") vs **2026-08-05 → 2026-08-19** ("prior"); weekly series use `startofweek` (Sunday) UTC over 63 days. Unless stated, rows are restricted to release builds (`AppVersion` strict semver). Counts are distinct `Properties.machine_id` ("machine IDs", not people) or distinct `SessionId`; outcome percentages are over `agent_outcome` events. Queries are in the appendix file. Supersedes the 2026-08-05 fix list; its status is in §9. + +## 1. Who is using it + +Observed machine IDs, release builds, 14d. Rows are **not exclusive**: 8 IDs carry both `datamates` and `cli` events, and the side-channel sources ride on the same IDs. Unique union = 2,046. + +| Segment | Machine IDs | Generated | Completed a task | Note | +|---|---:|---:|---:|---| +| datamates (dbt Power User extension) | 1,457 | 673 | **124** | 88% seen on one day only | +| cli humans | 166 | 60 | 49 | 30 active ≥3 days, 9 active ≥10 days | +| docker / user / config_rule / poweruser | 147 | 0 | 0 | side-channel events on IDs above | +| **internal fleet on `0.7.3`** | 198 | 166 | — | ours; Aug 27–29, one project, fresh ID per run | +| **CI review runners on `0.9.3` / `0.8.3`** | 198 | 0 | — | headless dbt PR review in customers' CI | +| Headline (naive dashboard) | 2,057 | | | prior fortnight 2,478 incl. the 690-ID shs-dx-it fleet | + +Excluding the fleet and CI rows: **~1,630 IDs now vs ~1,790 prior (−7%)**; with 88% one-day IDs on datamates the week-to-week noise is larger than that, so I read it as no trend. Weekly cli IDs that generate (weeks of Jul 19, Jul 26, Aug 16, Aug 23, Aug 30): 48 / 69 / 46 / 41 / 43. Weekly datamates IDs that generate (Jul 19 → Aug 30): 357 / 456 / 412 / 337 / 379 / 338 / 223 (partial). Flat. + +- OS (IDs with a `session_start`): datamates 605 win32 / 434 darwin / 195 linux; cli 27 / 23 / 22. +- Versions: 0.9.7 (707 IDs), 0.9.5 (630), 0.9.6 (362). **0.10.0 was published to npm today** (`latest`) and has not reached users (1 internal event). +- Providers by IDs with a session: `opencode/big-pickle` 991 (48% of all IDs), `altimate-backend` 190, `openrouter` 54, `github-copilot-enterprise` 22, `amazon-bedrock` 22, `openai` 19, `anthropic` 18. + +## 2. P0 — the datamates channel is failing most tasks + +### 2.1 Weekly decomposition + +`completed` share of `agent_outcome` events on `source=datamates`, with three controls: excluding the `abandoned` label (which is mislabelled, see §4), excluding sessions that ever hit the rate limit, and the gateway (`altimate-backend`) cohort which never uses `big-pickle`. + +| Week of | n | completed | error | completed excl. abandoned | non-rate-limited sessions: completed / error | big-pickle: completed / error | gateway: completed / error | cli completed | +|---|---:|---:|---:|---:|---:|---:|---:|---:| +| Jul 19 | 825 | 57.2% | 22.7% | 71.6% | 57.2 / 22.7 | 58.7 / 18.4 | 64.1 / 20.2 | 76.8% | +| Jul 26 | 1,209 | 53.3% | 17.0% | 75.9% | 53.3 / 17.0 | 56.2 / 11.2 | 59.3 / 13.9 | 77.2% | +| Aug 2 | 1,128 | 52.6% | 15.1% | 77.7% | 50.6 / 16.1 | 56.5 / 5.9 | 51.3 / 41.7 | 71.0% | +| Aug 9 | 784 | 38.5% | 34.8% | 52.5% | 40.4 / 36.5 | 42.1 / 27.6 | 31.1 / 57.4 | 75.9% | +| Aug 16 | 603 | 24.4% | 54.7% | 30.8% | 37.1 / 32.9 | 18.0 / 56.8 | 51.8 / 35.1 | 74.4% | +| Aug 23 | 643 | 25.2% | 52.1% | 32.6% | 33.6 / 35.6 | 23.6 / 51.4 | 41.9 / 41.0 | 76.8% | +| Aug 30 (3 days, preliminary) | 319 | **20.1%** | **59.9%** | 25.1% | 30.8 / 37.9 | 15.8 / 62.1 | 47.8 / 37.0 | 73.2% | + +What the table supports: +- The collapse is not an artifact of the `abandoned` label (71.6% → 25.1% with it removed) and not a partial-week effect (weeks of Aug 16 and Aug 23 are full). +- Rate-limited sessions explain about half: sessions never rate-limited went 57% → 31% completed, 23% → 38% error. The rest is a broader rise in provider errors on datamates that I have not decomposed further; the error taxonomy is in §2.3. +- The cli cohort (`source=cli`, mostly BYO-key providers) is flat at 73–77% across the same weeks, so this is not a CLI-wide regression. +- Gateway completion has bounced between 31% and 64% since Aug 2; its errors are chronic (§2.2), not new. + +### 2.2 Mechanism one: `big-pickle` rate limiting + +- `APIError: Error from provider (Console): Rate limit exceeded` — **298 of the 975 datamates IDs on `big-pickle` (31%)** in 14 days; both the `error` event and the `agent_outcome.reason` give the same 298. It is the #1 error reason by a factor of four. +- Sessions that hit it (436 with any outcome): **410 error, 14 completed, 7 abandoned, 5 aborted.** 146 of them (33%) never got a generation through. Median time from `session_start` to the first hit: 86 s; 21% under 10 s. +- The rate is 5–13 hits per 100 generations in every UTC hour. That rules out a simple peak-hour pattern; it does not distinguish per-account throttling from a continuously saturated shared ceiling. Provider-side quota data would. +- Datamates IDs are seen on one day 88% of the time whether rate-limited or not, so churn from this is not separately measurable. +- First-session outcome for a datamates ID (554 IDs with an outcome): **93 completed, 346 error, 64 aborted, 51 abandoned** = 17% completed. +- Code (`packages/opencode/src/session/processor.ts:1187-1213`, `session/retry.ts:11`): on 429 the CLI retries the **same model** up to 5 times with backoff, then surfaces the error. There is no failover, no distinction between a transient 429 and a hard quota, and no message telling the user what to do. `big-pickle` becomes the default only when no credential exists (`provider/provider.ts:2082-2107`), which is exactly the fresh-datamates case. Unchanged in 0.10.0. + +### 2.3 Mechanism two: chronic gateway `Not authenticated` and the rest of the error tail + +`APIError: Forbidden: {"detail":"Not authenticated"}` on `altimate-backend / altimate-default`: **62 datamates IDs in 14 days**, one in three of the 190 gateway IDs, and none of the 62 completed a task later in the window. It is not new: 25–36 IDs every week since mid-July on every version from 0.9.1 to 0.10.0. Hypothesis, not proven: a stale or revoked key. Code: the provider takes a static API key from `~/.altimate/altimate.json`, falling back to the TUI auth store (`provider/provider.ts:335-378`); there is no expiry check, refresh, or credential validation on 401/403 during chat (the Anthropic plugin at `altimate/plugin/anthropic.ts:80-106` checks expiry and refreshes proactively; this provider has nothing analogous). The raw `{"detail":…}` reaches the user because `provider/error.ts` does not read FastAPI's `detail`. + +Other datamates error reasons this fortnight (IDs): Google Vertex location missing 10; `"undefined/chat/completions" cannot be parsed as a URL` 8 (flagged Aug 5, still present); `Forbidden: request was blocked by a gateway or proxy` 8; AWS credential failures 6+4; `Failed to process error response` 10; `Service Unavailable` 5. + +### 2.4 Bedrock: 22 IDs, 0 completed + +35 outcomes by cause: bundling break `config6.parseKnownFiles is not a function` 8 outcomes / 8 IDs; bad or expired security token 8 / 5; no credentials found 6 / 4; account not enabled for the model 2 / 2; bad model id 2 / 1; non-error 2. The bundling bug (flagged Aug 5) is the largest single cause but only a third of the IDs; the rest is credential setup. Code: `fromNodeProviderChain` at `provider/provider.ts:505`; `@aws-sdk/credential-providers` is bundled rather than externalized (`script/build.ts:338-362`). No fix between 0.9.7 and 0.10.0. A credential preflight with a clear message would cover both halves. + +## 3. P0 — telemetry sends content the published policy says it never sends + +`docs/docs/reference/telemetry.md:151-163` promises no SQL, no file paths, no tool arguments, and that error messages are "scrubbed of file paths before sending." Production rows on 0.9.5–0.9.7 contradict all three: + +- `core_failure.error_message` carries raw absolute paths with home-directory usernames (`File not found: /Users//…`, `C:\Users\\…`): 297 of 1,072 rows on 0.9.7 (53 IDs); `masked_args` the same on 33 IDs. +- `masked_args` preserves SQL text and arbitrary unquoted strings by design: the tests expect `SELECT * FROM users` and table names to survive (`packages/opencode/test/telemetry/telemetry.test.ts:1996`). `sql_execute_failure.masked_sql` likewise carries full column names. + +Code: `Telemetry.maskString` (`altimate/telemetry/index.ts:1414-1436`) redacts API keys, bearer tokens, emails and internal hosts, and `maskArgs` applies it recursively — but the sanitizer has no rule for paths, identifiers, or free text, so everything else passes. Every standard tool's error reaches it via `tool/tool.ts:131-227`. A real path scrubber exists (`redactPaths()` in `altimate/tools/sample-setup.ts:273-289`) but only for model-visible output. The last commits to `maskString` predate July 30. This is a gap between policy and code, not a decision. + +Fix shape: stop sending `masked_args` values (send an allow-listed structural signature — `input_signature` already exists); strip `$HOME` and identifiers from `error_message`; audit `masked_sql`; decide what to do about rows already in Azure. + +## 4. P0 — measurement: what the dashboards are getting wrong + +1. **Internal fleets carry release version numbers.** 198 darwin IDs ran `AppVersion=0.7.3` on Aug 27–29 from one internal project (the same `project_id` runs `0.0.0-*` dev builds and `0.10.0`), one session each, providers `genlocal` / `google-vertex*`. Those provider IDs are not in the shipped defaults, so this is an internal config, not a customer. They pass the semver filter and were 10% of the headline machine count, and they produced two of the fortnight's "top errors" that no user saw (`Timed out opening DuckDB database`, 48 IDs; `temperature and top_p cannot both be specified`, 24). The fix is a `run_context=internal|ci|interactive` dimension, not faking the version. +2. **Customer CI runs the headless review on pinned versions.** 177 IDs on 0.9.3 and 21 on 0.8.3 emit only `native_call` (`altimate_core.check / column_lineage / review_ai_prompt / equivalence …`), a fresh ID and a `first_launch` per run. An exact action ref (`@v0.9.3`) installs exactly that version (`github/review/action.yml:68`); only non-semver refs resolve `latest`. The marker that arms `first_launch` is written by the shell installer (`install:504`) on every run. The CI telemetry gate is deliberately not keyed on `CI`/`GITHUB_ACTIONS` (`telemetry/index.ts:50-64`) because CI review is a product surface — right call, but install and WAU counts need a CI dimension. Effect: 67 of 160 "new cli installs" this fortnight are CI runs. Estimated headless review invocations per week (distinct IDs emitting `altimate_core.review_ai_prompt`, a proxy): 16 → 26 → 34 → 73 → 54 → **89** — growing, and invisible in `review_run` (27 events, none from these IDs) until customers move their pins. +3. **Dev builds dwarf production.** Non-release events this fortnight: **1.27M events, 5,621 IDs, 30,272 sessions** — 78% of raw volume (the `0.0.0-worktree-unsloth_integration` fleet alone is 4,541 IDs). Release builds: 365K events, 2,057 IDs. +4. **`abandoned` is misclassified.** `session/prompt.ts:1861-1868`: `abandoned` = zero cost **and** zero tool calls. Cost is 0 for `github-copilot-enterprise`, `alibaba-token-plan`, `opencode`, `altimate-backend`, so on those providers any text-only answer is booked as `abandoned` (reason hard-coded to `no_tools_invoked`). 732 cli "abandoned" outcomes on 34 IDs this fortnight are mostly this; every completion rate here understates the true answer rate on the free and gateway paths. +5. **The Aug 5 ACP P0 was a test artifact.** Every `source=acp` row in 90 days is `cli_version=local` / `provider_id=test` (286 CI IDs). Closing it. +6. datamates machine-ID churn is unchanged: 1,284 of 1,464 IDs (88%) seen one day; median sessions per ID = 1. + +## 5. Activation, now instrumented + +Fresh `first_launch` on cli 0.9.5–0.9.7 this fortnight: 102 IDs. The stages below are *reach* counts, not an ordered funnel (`model_picker_shown` also fires outside onboarding): + +| Reached | IDs | +|---|---:| +| first_launch | 102 | +| onboarding_started | 12 | +| model_picker_shown | 13 | +| provider_selected | 8 | +| onboarding_completed | 6 (5 abandoned at `provider_setup`) | +| any session_start | 33 | +| any generation | 28 | +| any completed task | 19 | + +69 of 102 never start a session; 45 of those emit only `native_call` (headless lint/safety/grade from scripts or CI), so interactive activation is roughly 33 of ~57. Retention with proper exposure: of the **52 installs at least 7 days old**, 15 ever started a session, 9 were active on days 1–6, **5 were active on day 7 or later** (10% of installs; a third of those who started). The onboarding flow is seen by one in eight fresh installs. + +## 6. P1 — reliability and product + +- **Warehouse drivers: fixed in 0.10.0, not yet in users' hands.** `driver not installed. Run: npm install …` still hit snowflake 12 IDs, databricks 4, duckdb 4, bigquery 3, pg 2 (Aug 5: 23/13/–/13/3). Root cause (`import()` inside the compiled binary resolving against bunfs) is fixed by #1122 (`packages/drivers/src/resolve.ts`). It only helps IDs that upgrade — next item. +- **Datamates self-upgrade fails for half the machines and sticks.** 211 datamates IDs attempted an upgrade (233 events): 110 succeeded only, **100 failed only**, 1 both — 47% failure, all `Upgrade failed for curl (exit code 1)`. Of 105 IDs (all sources) with a failed upgrade, 10 later reached 0.9.7. cli upgrades: 30 ok / 4 error. Code: the message means the `altimate.sh/install` script itself exited 1 (`installation/index.ts:175,186-217`); method detection (`installation/index.ts:262-270`) classifies anything under `.altimate/bin` as a curl install by design, and the telemetry `method` field collapses curl/yarn/pnpm/scoop/choco/unknown to `other` (`:436-440`). The extension-managed binary lives in that path, so the auto-updater (`cli/upgrade.ts:138-163`) runs the curl script against it. Cheapest containment: the extension sets `ALTIMATE_CLI_DISABLE_AUTOUPDATE=1` (or `autoupdate: "notify"`, `cli/upgrade.ts:94,149`) and owns its binary; durable fix is an installer-owner marker instead of path guessing. +- **Tools with at least one observed execute-time error, by IDs** (call-level rates in brackets; schema-validation failures happen before the error wrapper at `tool/tool.ts:276` and are not captured): `schema_inspect` 43 of 57 IDs (155 of 1,435 calls, 11%; `No warehouse configured` 17 IDs, drivers 10, `Schema ? does not exist` 7, `Invalid table ID` 6); `sql_explain` 12 of 12 (14 of 18 calls); `sql_fix` 5 of 5 (6 of 6); `altimate_core_semantics` 5 of 6 (6 of 7); `webfetch` 17 of 28 (56 of 204, mostly a hallucinated `altimate.ai/config.json`). `sql_explain` and `sql_fix` are registered unconditionally (`tool/registry.ts:417-419`) even with no warehouse. +- **Runaway emitters.** `memory_operation` (58K) + `memory_injection` (52K) = **30% of release telemetry from ~35 IDs**; injection fires on every agentic step (`session/prompt.ts:1447`), operations come from the memory tools (48,587 project-scope update writes across 835 sessions). `filetime_drift` still 22K events / 651 IDs; `file/time.ts:81-89` untouched since Aug 5. Buffer is 200 events, FIFO drop, no per-type cap (`telemetry/index.ts:68-69, 1800-1806`); overflow on 11 IDs. One ID is 17% of all release events. +- **Review.** 27 interactive `review_run`s on 9 IDs. 20 show `verdict=COMMENT` against `ideal_verdict=REQUEST_CHANGES`: that is comment mode doing what it promises (`review/verdict.ts:71-73`; default in `github/review/action.yml`). Findings per run: **full-tier runs median 26 (p90 31, one outlier at 3,946); degraded runs (17 of 27) median 114, p90 212.** `severity_threshold` defaults to `suggestion` and nothing caps count (`review/orchestrate.ts:1416-1420` dedupes only). The degraded path is the noisy one; fix its rules or cap per file before touching the global threshold. +- **Ripgrep.** `JSON record exceeded 65536 bytes` 16 IDs (from 84), all pre-0.9.7; fixed by #1094 (16 MiB cap, bad record skipped). `RipgrepDownloadFailedError` 4 IDs. +- **Permission friction.** bash user-denied on 47 IDs, rule-denied 31; `task` denied by customer rule 232 times on 5 IDs. The rule-dump message is still `[{?:?,?:?,?:?},…]` (824 events). +- **`invalid` tool.** 76 calls / 37 IDs; the attempted name is in `params.tool` (`session/llm.ts:265-284`) and never reaches telemetry. +- **MCP.** `dbt` server errors 10 IDs, `azure-devops` 9, `github` 7 (was 99). +- **Cost** is still 0 for `opencode`, `github-copilot-enterprise`, `alibaba-token-plan`, `altimate-backend`. + +## 7. What is working + +- **Windows grep is fixed.** `? is not recognized as an internal or external command`: 63 IDs prior fortnight → **0** (#1074, 0.9.5). +- **`task` tool**: model-resolution and `promptOps` failures gone; residual errors are customers' own deny rules. +- **cli completion** steady at 73–77%. Sessions that compact complete at 73% vs 65% for those that do not. Median completed task: 4.1 min, 9 tool calls. +- **Onboarding funnel events** live on 0.9.5+; `upgrade_attempted.error` populated; `general` intent share 55% → 48%. +- **Skills.** `dbt-troubleshoot` 292 IDs (dominant), `dbt-develop` 35, `query-optimize` 28, `dbt-test` 13, `sql-review` 13. Datamates intent is overwhelmingly `debug_dbt` (756 IDs). +- **Review in CI is growing** (~89 invocations last week) even though we do not count it. `self-hosted error: no healthy upstream` (51 IDs Aug 5–18) cleared. + +## 8. Recommended order of work + +1. **Contain the rate limit on `big-pickle`.** Distinguish hard quota from a transient 429; stop the five futile retries on hard quota; show a clear message with the two ways out (connect the gateway, or BYO key); add a circuit breaker per session. Fail over automatically only to a model the user already has credentials for, or with explicit consent. Track datamates completion weekly with the §2.1 decomposition. +2. **Gateway auth.** Instrument the 401/403 path (which credential source, key age), validate the key once on failure and prompt a re-connect, parse FastAPI `detail`. 62 IDs, none recover. +3. **Telemetry policy.** Stop sending `masked_args` values; strip `$HOME` and identifiers from `error_message`; audit `masked_sql`; decide on the existing rows. Then re-verify against `docs/docs/reference/telemetry.md`. +4. **Datamates upgrade.** Extension sets `ALTIMATE_CLI_DISABLE_AUTOUPDATE=1` now; installer-owner marker next. This is also how 0.10.0's driver fix reaches datamates. +5. **Bedrock.** Externalize or fix the AWS credential bundle; add a credential preflight with an actionable message. +6. **Measurement.** `run_context` dimension (internal / CI / interactive) kept with the real version; `ci=true` under `GITHUB_ACTIONS` excluded from install/WAU counts; record the attempted name on `invalid`; make `abandoned` independent of cost. +7. **Volume.** Delete `filetime_drift`; sample `memory_injection` or move it off the per-step path; per-type buffer cap. +8. **Review noise.** Fix the degraded-path rules or cap per file/category; count headless runs once pins move. + +## 9. Status of the 2026-08-05 fix list + +| Aug 5 item | Now | +|---|---| +| P0-0 CI pollution (`provider_id=test`) | gate shipped in 0.9.5; pinned-version CI runners and semver-tagged internal fleets bypass it (§4) | +| P0-1 Windows grep cmd.exe | **FIXED** | +| P0-2 `task` 73% failure | **FIXED** (residual = customer deny rules) | +| P0-3 acp 0% completion | **was a test artifact** (§4.5); closing | +| P0-4 builder completion | cli ~75% (ok); datamates 20% (**worse**, §2) | +| P0-5 drivers not bundled | fixed in 0.10.0 (#1122); awaiting upgrades | +| P0-6 PII masker shreds diagnostics | swung the other way: sends paths, SQL and args (§3); rule dumps still 824 events | +| P1-1 `filetime_drift` noise | **NOT DONE** (22K events) | +| P1-2 buffer overflow / runaway emitters | **WORSE** (memory_* now 30% of volume) | +| P1-3 upgrade diagnostics | error text populated; method still `other`; 47% fail on datamates | +| P1-4 bash permission friction | unchanged | +| P1-6 GitHub MCP | 7 IDs (was 99) — improved | +| P1-7 `schema_inspect` / `sql_explain` errors | unchanged | + +## Appendix + +All KQL used, with the batch/query name that produced each figure, is in `docs/internal/2026-09-02-telemetry-queries.md`. Queried 2026-09-02 23:30Z – 2026-09-03 01:30Z. Rate-limit sessions = any `error` event whose message has `Rate limit exceeded`; gateway cohort = `session_start.provider_id == "altimate-backend"`; big-pickle cohort = `provider_id == "opencode"`. diff --git a/docs/internal/2026-09-02-telemetry-queries.md b/docs/internal/2026-09-02-telemetry-queries.md new file mode 100644 index 0000000000..204def9690 --- /dev/null +++ b/docs/internal/2026-09-02-telemetry-queries.md @@ -0,0 +1,1665 @@ +# Telemetry queries — 2026-09-02 analysis + +Workspace `b511e30e-4b93-4093-98a5-b80fc4718111` (`altimate-code-os`), table `AppEvents`. Every query below was run with timespan P45D on 2026-09-02/03 via the Log Analytics REST data plane. Each shares the same prelude (window lets, strict-semver `rel` flag, `mid`/`src` extends). Batch prefixes: b = first sweep, c = property fixes + anomalies, d = fleet/CI/funnel, e = native_call/upgrade/provider, f = rate limit/CI review, g = reconciliation after review. + +## b00_samples + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Cur | summarize n=count(), p=any(tostring(Properties)) by Name | order by n desc | take 60 +``` + +## b01_scale + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +All | extend test = tostring(Properties.provider_id)=="test" or tostring(Properties.cli_version)=="local" | summarize events=count(), machines=dcount(mid), sessions=dcount(SessionId) by win, rel, test | order by win, events desc +``` + +## b02_versions + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Cur | summarize machines=dcount(mid), sessions=dcount(SessionId), events=count(), first=min(TimeGenerated) by AppVersion | order by machines desc +``` + +## b03_source + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Rel | summarize machines=dcount(mid), sessions=dcount(SessionId), events=count() by win, src | order by src, win +``` + +## b04_os + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Rel | where Name=="session_start" | summarize machines=dcount(mid), sessions=dcount(SessionId) by win, os=tostring(Properties.os), src | order by src, os, win +``` + +## b05_daily + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +AppEvents | where TimeGenerated > ago(45d) | where AppVersion matches regex @"^[0-9]+\.[0-9]+\.[0-9]+$" | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source) | summarize machines=dcount(mid), sessions=dcount(SessionId) by day=bin(TimeGenerated,1d), src | order by day asc, src +``` + +## b06_events + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Rel | summarize events=count(), machines=dcount(mid) by Name, win | order by Name, win +``` + +## b07_outcome + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Rel | where Name=="agent_outcome" | summarize n=count(), machines=dcount(mid) by win, agent=tostring(Properties.agent), outcome=tostring(Properties.outcome) | order by agent, win, outcome +``` + +## b08_outcome_src + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Cur | where Name=="agent_outcome" | summarize n=count(), machines=dcount(mid) by src, outcome=tostring(Properties.outcome) | order by src, outcome +``` + +## b09_err_reasons + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Cur | where Name=="agent_outcome" and tostring(Properties.outcome)=="error" | summarize n=count(), machines=dcount(mid) by reason=substring(tostring(Properties.error_message),0,140), src | order by machines desc | take 40 +``` + +## b10_core_failure + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Rel | where Name=="core_failure" | summarize n=count(), machines=dcount(mid) by win, tool=tostring(Properties.tool), class=tostring(Properties.error_class) | order by machines desc | take 60 +``` + +## b11_core_failure_msgs + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Cur | where Name=="core_failure" | summarize n=count(), machines=dcount(mid) by tool=tostring(Properties.tool), msg=substring(tostring(Properties.error_message),0,150) | order by machines desc | take 50 +``` + +## b12_task + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Cur | where Name=="tool_call" and tostring(Properties.tool_name)=="task" | summarize n=count(), machines=dcount(mid) by success=tostring(Properties.success), err=substring(tostring(Properties.error_message),0,120) | order by n desc | take 20 +``` + +## b13_warehouse + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Cur | where Name in ("warehouse_query","schema_inspect","sql_execute_failure") | summarize n=count(), errs=countif(tostring(Properties.success)=="false" or Name=="sql_execute_failure"), machines=dcount(mid) by Name, wh=tostring(Properties.warehouse_type) | order by n desc +``` + +## b13b_npm + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Cur | where tostring(Properties) has "npm install" | summarize n=count(), machines=dcount(mid) by Name, msg=extract(@"(npm install [^\s\\\"]+(?: [^\s\\\"]+)?)", 1, tostring(Properties)) | order by machines desc | take 20 +``` + +## b14_upgrade + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Rel | where Name=="upgrade_attempted" | summarize n=count(), machines=dcount(mid) by win, status=tostring(Properties.status), method=tostring(Properties.method), err=substring(tostring(Properties.error_message),0,100) | order by win, n desc +``` + +## b15_permission + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Rel | where Name=="permission_denied" | summarize n=count(), machines=dcount(mid) by win, tool=tostring(Properties.tool) | order by machines desc | take 20 +``` + +## b16_skills + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Rel | where Name has "skill" | summarize n=count(), machines=dcount(mid) by win, Name, skill=tostring(Properties.skill_name) | order by machines desc | take 40 +``` + +## b17_onboarding + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Rel | where Name startswith "onboarding" or Name in ("model_picker_shown","provider_selected","scan_gate_shown","first_launch") | summarize n=count(), machines=dcount(mid) by win, Name | order by Name, win +``` + +## b18_review + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Rel | where Name startswith "review" | summarize n=count(), machines=dcount(mid), p=any(tostring(Properties)) by win, Name | order by Name, win +``` + +## b19_filetime + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Rel | where Name=="filetime_drift" | summarize n=count(), machines=dcount(mid) by win, ahead=tostring(Properties.mtime_ahead) +``` + +## b20_overflow + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Cur | where tostring(Properties) has "BufferOverflow" | summarize n=count(), machines=dcount(mid) by Name +``` + +## b21_mcp + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Cur | where Name=="mcp_server_status" | summarize n=count(), machines=dcount(mid) by server=tostring(Properties.server_name), status=tostring(Properties.status) | order by machines desc | take 25 +``` + +## b22_ratelimit + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Rel | where tostring(Properties) has "Rate limit" | summarize n=count(), machines=dcount(mid) by day=bin(TimeGenerated,1d), src | order by day asc +``` + +## b23_tokens + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Cur | where Name=="generation" or Name=="llm_generation" or Name has "generation" | summarize n=count(), machines=dcount(mid), tin=sum(todouble(Measurements.tokens_input)), tout=sum(todouble(Measurements.tokens_output)), cost=sum(todouble(Measurements.cost)) by Name, provider=tostring(Properties.provider_id), model=tostring(Properties.model_id) | order by n desc | take 40 +``` + +## b24_toolcalls + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Cur | where Name=="tool_call" | summarize n=count(), errs=countif(tostring(Properties.success)=="false"), machines=dcount(mid) by tool=tostring(Properties.tool_name) | order by n desc | take 50 +``` + +## b25_retention + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Rel | where src=="cli" | summarize first=min(TimeGenerated), last=max(TimeGenerated), days=dcount(bin(TimeGenerated,1d)) by mid | summarize machines=count(), new_in_cur=countif(first>=W0), active_cur=countif(last>=W0), retained_prev_to_cur=countif(first=W0), multi_day=countif(days>=2), d5plus=countif(days>=5) +``` + +## b26_datamates_churn + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Cur | where src=="datamates" | summarize days=dcount(bin(TimeGenerated,1d)) by mid | summarize machines=count(), one_day=countif(days==1), two_plus=countif(days>=2), five_plus=countif(days>=5) +``` + +## b27_top_machines + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Cur | summarize n=count() by mid, src | top 10 by n | extend share=round(100.0*n/toscalar(Cur|count),1) +``` + +## c01_devbuilds + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +All | where not(rel) and win=="cur" | summarize machines=dcount(mid), sessions=dcount(SessionId), events=count(), providers=make_set(tostring(Properties.provider_id),5), oss=make_set(tostring(Properties.os),4) by ver=extract(@"^(0\.0\.0-[a-z]+)", 1, AppVersion), src | order by events desc | take 25 +``` + +## c01b_devbuilds_ver + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +All | where not(rel) and win=="cur" | summarize machines=dcount(mid), sessions=dcount(SessionId), events=count() by AppVersion | order by machines desc | take 15 +``` + +## c02_versions + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Cur | summarize machines=dcount(mid), sessions=dcount(SessionId), events=count(), firstSeen=min(TimeGenerated) by AppVersion | order by machines desc +``` + +## c03_emptysrc + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Rel | where src=="" | summarize machines=dcount(mid), sessions=dcount(SessionId), events=count() by win, AppVersion, os=tostring(Properties.os) | order by machines desc | take 20 +``` + +## c04_sources45 + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +AppEvents | where TimeGenerated > ago(45d) | where AppVersion matches regex @"^[0-9]+\.[0-9]+\.[0-9]+$" | summarize machines=dcount(tostring(Properties.machine_id)), events=count(), lastSeen=max(TimeGenerated) by src=tostring(Properties.source) +``` + +## c05_err_reasons + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Cur | where Name=="agent_outcome" and tostring(Properties.outcome)=="error" | summarize n=count(), machines=dcount(mid) by src, cls=tostring(Properties.error_class), reason=substring(tostring(Properties.reason),0,140) | order by machines desc | take 40 +``` + +## c06_datamates_outcome + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Cur | where Name=="agent_outcome" and src=="datamates" | summarize n=count(), machines=dcount(mid) by AppVersion, outcome=tostring(Properties.outcome) | order by AppVersion, outcome +``` + +## c07_core_failure + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Rel | where Name=="core_failure" | summarize n=count(), machines=dcount(mid) by win, tool=tostring(Properties.tool_name), class=tostring(Properties.error_class) | order by machines desc | take 60 +``` + +## c08_task + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Rel | where Name=="tool_call" and tostring(Properties.tool_name)=="task" | summarize n=count(), machines=dcount(mid) by win, status=tostring(Properties.status), err=substring(tostring(Properties.error_message),0,120) | order by win, n desc | take 20 +``` + +## c09_toolcalls + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Cur | where Name=="tool_call" | summarize n=count(), errs=countif(tostring(Properties.status)!="success"), machines=dcount(mid), err_machines=dcountif(mid, tostring(Properties.status)!="success") by tool=tostring(Properties.tool_name) | order by n desc | take 60 +``` + +## c10_permission + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Rel | where Name=="permission_denied" | summarize n=count(), machines=dcount(mid) by win, tool=tostring(Properties.tool_name), src | order by machines desc | take 25 +``` + +## c11_upgrade + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Rel | where Name=="upgrade_attempted" | summarize n=count(), machines=dcount(mid) by win, status=tostring(Properties.status), method=tostring(Properties.method), err=substring(tostring(Properties.error),0,100), fromv=tostring(Properties.from_version), tov=tostring(Properties.to_version) | order by win, machines desc | take 40 +``` + +## c12_error_event + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Rel | where Name=="error" | summarize n=count(), machines=dcount(mid) by win, src, ename=tostring(Properties.error_name), ctx=tostring(Properties.context), msg=substring(tostring(Properties.error_message),0,100) | order by machines desc | take 40 +``` + +## c13_grep + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Rel | where Name=="core_failure" and tostring(Properties.tool_name) in ("grep","glob") | summarize n=count(), machines=dcount(mid) by win, msg=substring(tostring(Properties.error_message),0,110) | order by machines desc | take 20 +``` + +## c14_pii_paths + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Rel | where Name=="core_failure" | extend m=tostring(Properties.error_message) | summarize n=count(), machines=dcount(mid), leaked=countif(m matches regex @"(/Users/|/home/|C:\\Users\\|D:\\)"), leaked_machines=dcountif(mid, m matches regex @"(/Users/|/home/|C:\\Users\\|D:\\)"), masked=countif(m has "?") by win, AppVersion | order by AppVersion +``` + +## c15_funnel + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +let FL = Cur | where Name=="first_launch" and src=="cli" and AppVersion in ("0.9.5","0.9.6","0.9.7","0.10.0") | distinct mid; Cur | where mid in (FL) | summarize fl=dcountif(mid,Name=="first_launch"), onb_start=dcountif(mid,Name=="onboarding_started"), picker=dcountif(mid,Name=="model_picker_shown"), provider=dcountif(mid,Name=="provider_selected"), scan=dcountif(mid,Name=="scan_gate_shown"), onb_done=dcountif(mid,Name=="onboarding_completed"), abandoned=dcountif(mid,Name=="onboarding_abandoned"), first_prompt=dcountif(mid,Name=="first_prompt_sent"), sess=dcountif(mid,Name=="session_start"), gen=dcountif(mid,Name=="generation"), outcome=dcountif(mid,Name=="agent_outcome"), completed=dcountif(mid,Name=="agent_outcome" and tostring(Properties.outcome)=="completed"), returned=dcountif(mid, Name=="session_start" and TimeGenerated > W1 - 1d) +``` + +## c15b_funnel_ver + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Rel | where Name=="first_launch" | summarize machines=dcount(mid) by win, AppVersion, src, upg=tostring(Properties.is_upgrade) | order by machines desc | take 30 +``` + +## c16_retention + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Rel | where src=="cli" | summarize firstT=min(TimeGenerated), lastT=max(TimeGenerated), days=dcount(bin(TimeGenerated,1d)) by mid | summarize machines=count(), new_in_cur=countif(firstT>=W0), active_cur=countif(lastT>=W0), retained_prev_to_cur=countif(firstT=W0), churned=countif(firstT=2), d5plus=countif(days>=5) +``` + +## c16b_retention_all + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Rel | summarize firstT=min(TimeGenerated), lastT=max(TimeGenerated), days=dcount(bin(TimeGenerated,1d)) by mid, src | summarize machines=count(), new_in_cur=countif(firstT>=W0), retained_prev_to_cur=countif(firstT=W0), churned=countif(firstT=2), d5plus=countif(days>=5) by src +``` + +## c17_weekly + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +AppEvents | where TimeGenerated > ago(63d) | where AppVersion matches regex @"^[0-9]+\.[0-9]+\.[0-9]+$" | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), pid=tostring(Properties.provider_id) | where not(pid startswith "shs-dx-it") | summarize machines=dcount(mid), sessions=dcount(SessionId), gen_machines=dcountif(mid, Name=="generation") by wk=startofweek(TimeGenerated), src | order by wk asc, src +``` + +## c18_finish + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Rel | where Name=="generation" | summarize n=count(), machines=dcount(mid) by win, fr=tostring(Properties.finish_reason) | order by win, n desc +``` + +## c19_measurements + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Cur | where Name in ("session_end","generation","agent_outcome","tool_call","compaction_triggered") | summarize any(tostring(Measurements)) by Name +``` + +## c20_intent + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Rel | where Name=="task_classified" | summarize n=count(), machines=dcount(mid) by win, intent=tostring(Properties.intent) | order by intent, win +``` + +## c21_wh_connect + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Rel | where Name=="warehouse_connect" | summarize n=count(), machines=dcount(mid) by win, wh=tostring(Properties.warehouse_type), ok=tostring(Properties.success), cat=tostring(Properties.error_category), err=substring(tostring(Properties.error),0,90) | order by machines desc | take 30 +``` + +## c23_drivers + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Cur | extend m=tostring(Properties) | where m has "driver not installed" or m has "npm install" | summarize n=count(), machines=dcount(mid) by Name, drv=extract(@"npm install ([@a-z/\-\.]+)", 1, m) | order by machines desc +``` + +## c24_compaction + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Rel | where Name=="compaction_triggered" | summarize n=count(), machines=dcount(mid), sessions=dcount(SessionId) by win, trig=tostring(Properties.trigger) | order by win +``` + +## c25_doom + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Rel | where Name=="doom_loop_detected" | summarize n=count(), machines=dcount(mid) by win, tool=tostring(Properties.tool_name) +``` + +## c26_top_machines + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +let T = Cur | summarize n=count() by mid | top 12 by n; Cur | where mid in (T) | summarize n=count(), sessions=dcount(SessionId), ver=any(AppVersion), src=any(src), prov=make_set(tostring(Properties.provider_id),3), model=make_set(tostring(Properties.model_id),3), os=any(tostring(Properties.os)), wh=make_set(tostring(Properties.warehouse_type),3), agents=make_set(tostring(Properties.agent),4), skills=make_set(tostring(Properties.skill_name),4) by mid | order by n desc +``` + +## c27_invalid + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Cur | where Name=="tool_call" and tostring(Properties.tool_name)=="invalid" | summarize n=count(), machines=dcount(mid) by p=substring(tostring(Properties),0,300) | order by n desc | take 8 +``` + +## c28_review + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Rel | where Name=="review_run" | summarize n=count(), machines=dcount(mid) by win, status=tostring(Properties.status), verdict=tostring(Properties.verdict), ideal=tostring(Properties.ideal_verdict), tier=tostring(Properties.tier), degraded=tostring(Properties.degraded), inv=tostring(Properties.invocation) | order by win, n desc +``` + +## c29_session_len + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Cur | where Name=="session_end" | extend d=todouble(Measurements.duration_ms), t=todouble(Measurements.turn_count), tc=todouble(Measurements.tool_calls) | summarize n=count(), p50_dur_min=percentile(d,50)/60000, p90_dur_min=percentile(d,90)/60000, p50_turns=percentile(t,50), p90_turns=percentile(t,90), p50_tools=percentile(tc,50) by src +``` + +## c30_ratelimit_effect + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +let RL = Cur | where tostring(Properties) has "Rate limit exceeded" | distinct mid; Cur | where src=="datamates" | summarize days=dcount(bin(TimeGenerated,1d)), gens=countif(Name=="generation"), completed=countif(Name=="agent_outcome" and tostring(Properties.outcome)=="completed") by mid, rl=mid in (RL) | summarize machines=count(), multi_day=countif(days>=2), avg_gens=avg(gens), any_completed=countif(completed>0) by rl +``` + +## c31_bigpickle_err + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Cur | where tostring(Properties.provider_id)=="opencode" | where Name in ("agent_outcome","error") | summarize n=count(), machines=dcount(mid) by Name, o=tostring(Properties.outcome), ename=tostring(Properties.error_name), msg=substring(tostring(Properties.error_message),0,90) | order by machines desc | take 15 +``` + +## c32_provider_machines + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Rel | where Name=="session_start" | summarize machines=dcount(mid), sessions=dcount(SessionId) by win, prov=tostring(Properties.provider_id) | order by machines desc | take 40 +``` + +## d01_acp + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +AppEvents | where TimeGenerated > ago(75d) | where AppVersion matches regex @"^[0-9]+\.[0-9]+\.[0-9]+$" | where tostring(Properties.source)=="acp" | summarize machines=dcount(tostring(Properties.machine_id)), sessions=dcount(SessionId), events=count() by wk=startofweek(TimeGenerated), AppVersion | order by wk asc +``` + +## d01b_acp_any + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +AppEvents | where TimeGenerated > ago(45d) | extend p=tostring(Properties) | where p has "acp" | summarize n=count(), machines=dcount(tostring(Properties.machine_id)) by AppVersion, src=tostring(Properties.source), Name | order by n desc | take 15 +``` + +## d02_v073 + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Cur | where AppVersion=="0.7.3" | summarize machines=dcount(mid), sessions=dcount(SessionId), events=count(), projects=dcount(tostring(Properties.project_id)), prov=make_set(tostring(Properties.provider_id),4), wh=make_set(tostring(Properties.warehouse_type),4) by day=bin(TimeGenerated,1d) | order by day asc +``` + +## d02b_v073_names + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Cur | where AppVersion=="0.7.3" | summarize n=count(), machines=dcount(mid) by Name, os=tostring(Properties.os) | order by n desc | take 20 +``` + +## d02c_v073_project + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Cur | where AppVersion=="0.7.3" | summarize n=count(), machines=dcount(mid), sessions=dcount(SessionId) by pid=tostring(Properties.project_id) | order by machines desc | take 8 +``` + +## d03_v093 + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Cur | where AppVersion=="0.9.3" | summarize n=count(), machines=dcount(mid) by Name, src, upg=tostring(Properties.is_upgrade) | order by machines desc | take 20 +``` + +## d03b_v093_daily + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Rel | where AppVersion=="0.9.3" and Name=="first_launch" | summarize machines=dcount(mid) by day=bin(TimeGenerated,1d), os=tostring(Properties.os) | order by day asc +``` + +## d04_temp_top_p + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Rel | extend p=tostring(Properties) | where p has "cannot both be specified" | summarize n=count(), machines=dcount(mid) by win, Name, AppVersion, src | order by machines desc +``` + +## d04b_temp_top_p_prov + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +let S = Rel | extend p=tostring(Properties) | where p has "cannot both be specified" | distinct SessionId; Rel | where SessionId in (S) and Name=="session_start" | summarize machines=dcount(mid) by prov=tostring(Properties.provider_id), model=tostring(Properties.model_id), AppVersion | order by machines desc +``` + +## d05_fl_nosession + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +let FL = Cur | where Name=="first_launch" and src=="cli" and AppVersion in ("0.9.5","0.9.6","0.9.7") | distinct mid; let S = Cur | where Name=="session_start" | distinct mid; Cur | where mid in (FL) and not(mid in (S)) | summarize n=count(), machines=dcount(mid) by Name | order by machines desc +``` + +## d05b_fl_nosession_os + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +let FL = Cur | where Name=="first_launch" and src=="cli" and AppVersion in ("0.9.5","0.9.6","0.9.7") | distinct mid; let S = Cur | where Name=="session_start" | distinct mid; Cur | where mid in (FL) and Name=="first_launch" | summarize machines=dcount(mid), no_session=dcountif(mid, not(mid in (S))) by os=tostring(Properties.os), upg=tostring(Properties.is_upgrade) +``` + +## d05c_fl_props + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Cur | where Name=="first_launch" | take 3 | project Properties, Measurements +``` + +## d06_datamates_daily + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Cur | where src=="datamates" | summarize machines=dcount(mid), gen=dcountif(mid,Name=="generation"), completed=dcountif(mid,Name=="agent_outcome" and tostring(Properties.outcome)=="completed"), errored=dcountif(mid,Name=="agent_outcome" and tostring(Properties.outcome)=="error"), rl=dcountif(mid, tostring(Properties) has "Rate limit exceeded"), notauth=dcountif(mid, tostring(Properties) has "Not authenticated") by day=bin(TimeGenerated,1d) | order by day asc +``` + +## d07_notauth + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +let S = Cur | extend p=tostring(Properties) | where p has "Not authenticated" | distinct SessionId; Cur | where SessionId in (S) and Name=="session_start" | summarize machines=dcount(mid), sessions=dcount(SessionId) by prov=tostring(Properties.provider_id), model=tostring(Properties.model_id), AppVersion, src | order by machines desc | take 10 +``` + +## d08_schema_inspect + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Cur | where Name=="core_failure" and tostring(Properties.tool_name)=="schema_inspect" | summarize n=count(), machines=dcount(mid) by cls=tostring(Properties.error_class), msg=substring(tostring(Properties.error_message),0,120) | order by machines desc | take 12 +``` + +## d09_webfetch + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Cur | where Name=="core_failure" and tostring(Properties.tool_name)=="webfetch" | summarize n=count(), machines=dcount(mid) by msg=substring(tostring(Properties.error_message),0,120) | order by machines desc | take 10 +``` + +## d10_memory_emitters + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Cur | where Name in ("memory_operation","memory_injection","native_call","filetime_drift","sql_pre_validation") | summarize n=count() by Name, mid | summarize machines=count(), total=sum(n), top1=max(n), top3=sum(iff(n>5000,n,0)), machines_over_5k=countif(n>5000) by Name +``` + +## d10b_memory_ops + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Cur | where Name=="memory_operation" | summarize n=count(), machines=dcount(mid), sessions=dcount(SessionId) by op=tostring(Properties.operation), scope=tostring(Properties.scope), upd=tostring(Properties.is_update) | order by n desc +``` + +## d11_genlocal + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Cur | where tostring(Properties.provider_id)=="genlocal" or tostring(Properties.model_id)=="altimate-base" | summarize n=count(), machines=dcount(mid) by Name, AppVersion, src, o=tostring(Properties.outcome) | order by machines desc | take 15 +``` + +## d12_bigpickle + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +let BP = Cur | where Name=="session_start" and tostring(Properties.provider_id)=="opencode" | distinct mid; let RL = Cur | extend p=tostring(Properties) | where p has "Rate limit exceeded" | distinct mid; Cur | where mid in (BP) | summarize machines=dcount(mid), rl=dcountif(mid, mid in (RL)), gen=dcountif(mid,Name=="generation"), completed=dcountif(mid, Name=="agent_outcome" and tostring(Properties.outcome)=="completed"), errored=dcountif(mid, Name=="agent_outcome" and tostring(Properties.outcome)=="error") by src +``` + +## d12b_bigpickle_by_model + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Cur | where Name=="session_start" and tostring(Properties.provider_id)=="opencode" | summarize machines=dcount(mid), sessions=dcount(SessionId) by model=tostring(Properties.model_id), src | order by machines desc +``` + +## d13_sessions_zero_gen + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +let G = Cur | where Name=="generation" | distinct SessionId; Cur | where Name=="session_start" | summarize sessions=dcount(SessionId), no_gen=dcountif(SessionId, not(SessionId in (G))), machines=dcount(mid), machines_no_gen_only=dcount(mid) - dcountif(mid, SessionId in (G)) by src +``` + +## d14_pii_sample + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Cur | where Name=="core_failure" and AppVersion=="0.9.7" | extend m=tostring(Properties.error_message) | where m matches regex @"(/Users/|/home/|C:\\Users\\)" | summarize n=count(), machines=dcount(mid), sample=any(substring(m,0,160)), args=any(substring(tostring(Properties.masked_args),0,160)) by tool=tostring(Properties.tool_name), cls=tostring(Properties.error_class) | order by machines desc +``` + +## d15_outcome_dur + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Cur | where Name=="agent_outcome" and tostring(Properties.outcome)=="completed" | extend d=todouble(Measurements.duration_ms)/60000, tc=todouble(Measurements.tool_calls), g=todouble(Measurements.generations) | summarize n=count(), p50_min=round(percentile(d,50),1), p90_min=round(percentile(d,90),1), p50_tools=percentile(tc,50), p90_tools=percentile(tc,90), p50_gens=percentile(g,50) by src +``` + +## d18_event_diff + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Rel | summarize cur=countif(win=="cur"), prev=countif(win=="prev"), cur_m=dcountif(mid,win=="cur"), prev_m=dcountif(mid,win=="prev") by Name | where cur==0 or prev==0 | order by cur desc, prev desc +``` + +## d19_review_cats + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Cur | where Name=="review_run" | extend bc=parse_json(tostring(Properties.by_category)) | summarize runs=count(), machines=dcount(mid), lineage=sum(toint(bc.lineage_breakage)), semantic=sum(toint(bc.semantic_change)), pii=sum(toint(bc.pii_exposure)), cost=sum(toint(bc.warehouse_cost)), sqlq=sum(toint(bc.sql_quality)), sqlc=sum(toint(bc.sql_correctness)), join_risk=sum(toint(bc.join_risk)), fanout=sum(toint(bc.fanout)), contract=sum(toint(bc.contract_violation)), degraded=countif(tostring(Properties.degraded)=="true") +``` + +## d22_env + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Cur | where Name=="environment_census" | summarize machines=dcount(mid) by dbt=tostring(Properties.dbt_detected), wh=tostring(Properties.warehouse_types), src | order by machines desc | take 15 +``` + +## d22b_wh_census + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Cur | where Name=="warehouse_census" | summarize machines=dcount(mid) by wh=tostring(Properties.warehouse_types) | order by machines desc | take 12 +``` + +## d24_cli_core + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +let C = Cur | where src=="cli" | summarize days=dcount(bin(TimeGenerated,1d)), gens=countif(Name=="generation") by mid | where days>=3 and gens>0; Cur | where mid in (C) and Name=="session_start" | summarize machines=dcount(mid), sessions=dcount(SessionId) by prov=tostring(Properties.provider_id), AppVersion, os=tostring(Properties.os) | order by machines desc | take 25 +``` + +## d24b_cli_core_count + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Cur | where src=="cli" | summarize days=dcount(bin(TimeGenerated,1d)), gens=countif(Name=="generation"), completed=countif(Name=="agent_outcome" and tostring(Properties.outcome)=="completed") by mid | summarize total=count(), gen_any=countif(gens>0), d2=countif(days>=2 and gens>0), d3=countif(days>=3 and gens>0), d5=countif(days>=5 and gens>0), d10=countif(days>=10 and gens>0), completed_any=countif(completed>0) +``` + +## d24c_dm_core_count + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Cur | where src=="datamates" | summarize days=dcount(bin(TimeGenerated,1d)), gens=countif(Name=="generation"), completed=countif(Name=="agent_outcome" and tostring(Properties.outcome)=="completed") by mid | summarize total=count(), gen_any=countif(gens>0), d2=countif(days>=2 and gens>0), d3=countif(days>=3 and gens>0), d5=countif(days>=5 and gens>0), completed_any=countif(completed>0) +``` + +## d25_sql_exec_fail + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Cur | where Name=="sql_execute_failure" | summarize n=count(), machines=dcount(mid) by wh=tostring(Properties.warehouse_type), msg=substring(tostring(Properties.error_message),0,100) | order by machines desc | take 20 +``` + +## d26_duckdb_timeout + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Cur | extend p=tostring(Properties) | where p has "Timed out opening DuckDB" | summarize n=count(), machines=dcount(mid) by AppVersion, src, os=tostring(Properties.os), Name | order by n desc +``` + +## d27_abandon_reason + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Cur | where Name=="agent_outcome" and tostring(Properties.outcome) in ("abandoned","aborted") | summarize n=count(), machines=dcount(mid) by src, o=tostring(Properties.outcome), reason=substring(tostring(Properties.reason),0,80), ft=tostring(Properties.final_tool) | order by machines desc | take 20 +``` + +## d28_task_signal + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Rel | where Name=="task_outcome_signal" | summarize n=count(), machines=dcount(mid) by win, sig=tostring(Properties.signal) | order by win, n desc +``` + +## d29_first_prompt + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Rel | where Name in ("first_prompt_sent","activation_menu_shown","activation_job_selected","environment_scan_completed","big_pickle_choice","gateway_device_code_issued","gateway_auth_failed") | summarize n=count(), machines=dcount(mid), p=any(substring(tostring(Properties),0,200)) by win, Name | order by Name, win +``` + +## e01_native_093 + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Cur | where AppVersion=="0.9.3" and Name=="native_call" | summarize n=count(), machines=dcount(mid) by method=tostring(Properties.method), status=tostring(Properties.status), src | order by n desc | take 15 +``` + +## e02_native_nosession + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +let S = Cur | where Name=="session_start" | distinct mid; Cur | where Name=="native_call" and not(mid in (S)) | summarize n=count(), machines=dcount(mid) by AppVersion, method=tostring(Properties.method), src | order by machines desc | take 20 +``` + +## e03_native_methods + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Cur | where Name=="native_call" | summarize n=count(), machines=dcount(mid), errs=countif(tostring(Properties.status)!="success") by method=tostring(Properties.method) | order by machines desc | take 25 +``` + +## e04_fleet_project_history + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +AppEvents | where TimeGenerated > ago(90d) | where tostring(Properties.project_id)=="faf13d3e2eb7aa9ddd80dc82357926b0126f1f8f" | summarize machines=dcount(tostring(Properties.machine_id)), events=count(), vers=make_set(AppVersion,5), node=make_set(tostring(Properties.node_version),3), arch=make_set(tostring(Properties.arch),3) by wk=startofweek(TimeGenerated) | order by wk asc +``` + +## e05_v093_machines_history + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +let M = AppEvents | where TimeGenerated between (datetime(2026-08-27) .. datetime(2026-09-02)) and AppVersion=="0.9.3" and Name=="first_launch" | distinct mid=tostring(Properties.machine_id); AppEvents | where TimeGenerated > ago(30d) | where tostring(Properties.machine_id) in (M) | summarize n=count(), machines=dcount(tostring(Properties.machine_id)) by Name, AppVersion, pid=tostring(Properties.project_id), os=tostring(Properties.os) | order by n desc | take 15 +``` + +## e06_upgrade_by_src + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Cur | where Name=="upgrade_attempted" | summarize n=count(), machines=dcount(mid) by src, status=tostring(Properties.status), method=tostring(Properties.method) | order by src, status +``` + +## e07_upgrade_success_followup + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +let U = Cur | where Name=="upgrade_attempted" and tostring(Properties.status)=="error" | distinct mid; Cur | where mid in (U) | summarize maxv=max(AppVersion), minv=min(AppVersion), later_ok=countif(Name=="upgrade_attempted" and tostring(Properties.status)=="success") by mid | summarize machines=count(), eventually_upgraded=countif(later_ok>0 or maxv=="0.9.7") +``` + +## e08_ripgrep + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Rel | where Name=="core_failure" and tostring(Properties.error_message) has "Ripgrep" | summarize n=count(), machines=dcount(mid) by win, msg=substring(tostring(Properties.error_message),0,60), AppVersion | order by machines desc +``` + +## e09_dm_version_daily + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Cur | where src=="datamates" and Name=="session_start" | summarize machines=dcount(mid) by day=bin(TimeGenerated,1d), AppVersion | order by day asc, AppVersion +``` + +## e10_cli_new_by_week + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +AppEvents | where TimeGenerated > ago(63d) | where AppVersion matches regex @"^[0-9]+\.[0-9]+\.[0-9]+$" | where Name=="first_launch" | summarize installs=dcount(tostring(Properties.machine_id)) by wk=startofweek(TimeGenerated), src=tostring(Properties.source), upg=tostring(Properties.is_upgrade) | order by wk asc +``` + +## e11_cli_completed_share_by_provider + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +let SS = Cur | where Name=="session_start" | project SessionId, prov=tostring(Properties.provider_id); Cur | where Name=="agent_outcome" | join kind=inner SS on SessionId | summarize n=count(), completed=countif(tostring(Properties.outcome)=="completed"), error=countif(tostring(Properties.outcome)=="error"), machines=dcount(mid) by prov | where n>=20 | extend completion=round(100.0*completed/n,1), err=round(100.0*error/n,1) | order by machines desc +``` + +## e12_masked_args_pii + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Cur | where Name=="core_failure" | extend a=tostring(Properties.masked_args) | summarize n=count(), machines=dcount(mid), with_home=countif(a matches regex @"(/Users/[^/]+|/home/[^/]+|C:\\Users\\[^\\]+)"), with_home_m=dcountif(mid, a matches regex @"(/Users/[^/]+|/home/[^/]+|C:\\Users\\[^\\]+)") by AppVersion | where n>20 | order by AppVersion +``` + +## e13_toolchain + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Cur | where Name=="tool_chain_outcome" | summarize n=count(), machines=dcount(mid) by src, had_errors=tostring(Properties.had_errors), fo=tostring(Properties.final_outcome) | order by n desc +``` + +## e14_generation_dur + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Cur | where Name=="generation" | extend d=todouble(Measurements.duration_ms)/1000 | summarize n=count(), p50_s=round(percentile(d,50),1), p90_s=round(percentile(d,90),1), cache_read_share=round(100.0*sum(todouble(Measurements.tokens_cache_read))/sum(todouble(Measurements.tokens_input_total)),1) by prov=tostring(Properties.provider_id) | where n>300 | order by n desc +``` + +## e15_sessions_per_machine + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Cur | where Name=="session_start" | summarize s=dcount(SessionId) by mid, src | summarize machines=count(), p50=percentile(s,50), p90=percentile(s,90), one=countif(s==1) by src +``` + +## e16_intent_by_src + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Cur | where Name=="task_classified" | summarize machines=dcount(mid), n=count() by src, intent=tostring(Properties.intent) | order by src, machines desc +``` + +## e17_completion_trend + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +AppEvents | where TimeGenerated > ago(63d) | where AppVersion matches regex @"^[0-9]+\.[0-9]+\.[0-9]+$" | where Name=="agent_outcome" | extend src=tostring(Properties.source), o=tostring(Properties.outcome) | where src in ("cli","datamates") | summarize n=count(), completion=round(100.0*countif(o=="completed")/count(),1), error=round(100.0*countif(o=="error")/count(),1), machines=dcount(tostring(Properties.machine_id)) by wk=startofweek(TimeGenerated), src | order by wk asc, src +``` + +## f01_rl_hourly + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Cur | where src=="datamates" | summarize gens=countif(Name=="generation"), rl=countif(Name=="error" and tostring(Properties.error_message) has "Rate limit exceeded"), rl_machines=dcountif(mid, Name=="error" and tostring(Properties.error_message) has "Rate limit exceeded"), machines=dcount(mid) by h=hourofday(TimeGenerated) | extend rl_per_100gen=round(100.0*rl/gens,1) | order by h asc +``` + +## f02_rl_sessions + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +let RLS = Cur | where Name=="error" and tostring(Properties.error_message) has "Rate limit exceeded" | distinct SessionId; Cur | where SessionId in (RLS) | summarize gens=countif(Name=="generation"), outcome=anyif(tostring(Properties.outcome), Name=="agent_outcome"), tools=countif(Name=="tool_call") by SessionId | summarize sessions=count(), zero_gen=countif(gens==0), p50_gens=percentile(gens,50), completed=countif(outcome=="completed"), errored=countif(outcome=="error"), no_outcome=countif(isempty(outcome)) +``` + +## f03_rl_first_gen + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +let RLS = Cur | where Name=="error" and tostring(Properties.error_message) has "Rate limit exceeded" | summarize rl_t=min(TimeGenerated) by SessionId; Cur | where Name=="session_start" | join kind=inner RLS on SessionId | extend secs=datetime_diff("second", rl_t, TimeGenerated) | summarize sessions=count(), p50_secs_to_rl=percentile(secs,50), p90=percentile(secs,90), under_10s=countif(secs<10) +``` + +## f04_bedrock + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +let BS = Cur | where Name=="session_start" and tostring(Properties.provider_id)=="amazon-bedrock" | project SessionId, AppVersion, src, mid; Cur | where Name=="agent_outcome" | join kind=inner BS on SessionId | summarize n=count(), machines=dcount(mid) by AppVersion, src, o=tostring(Properties.outcome), reason=substring(tostring(Properties.reason),0,90) | order by machines desc +``` + +## f05_ci_review_daily + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +AppEvents | where TimeGenerated > ago(63d) | where Name=="native_call" and tostring(Properties.method) in ("altimate_core.review_ai_prompt","altimate_core.review_lexical_scan") | summarize runs=dcount(tostring(Properties.machine_id)), events=count() by wk=startofweek(TimeGenerated), AppVersion | order by wk asc +``` + +## f06_ci_review_sessionless + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +let S = Cur | where Name=="session_start" | distinct mid; Cur | where Name=="native_call" and tostring(Properties.method)=="altimate_core.review_ai_prompt" | summarize machines=dcount(mid), sessionless=dcountif(mid, not(mid in (S))), with_review_run=dcountif(mid, mid in (toscalar(Cur | where Name=="review_run" | summarize make_set(mid)))) by AppVersion +``` + +## f07_notauth_recovery + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +let NA = Cur | where Name=="error" and tostring(Properties.error_message) has "Not authenticated" | summarize first_na=min(TimeGenerated) by mid; Cur | where Name=="agent_outcome" | join kind=inner NA on mid | summarize later_completed=dcountif(mid, tostring(Properties.outcome)=="completed" and TimeGenerated>first_na), machines=dcount(mid) +``` + +## f08_dm_first_session_outcome + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +let FS = Cur | where src=="datamates" and Name=="session_start" | summarize arg_min(TimeGenerated, SessionId) by mid | project mid, SessionId; Cur | where Name=="agent_outcome" | join kind=inner FS on SessionId | summarize machines=dcount(mid) by o=tostring(Properties.outcome), cls=tostring(Properties.error_class) | order by machines desc +``` + +## f09_win_share + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Cur | where Name=="session_start" and src in ("cli","datamates") | summarize machines=dcount(mid) by src, os=tostring(Properties.os) | order by src, machines desc +``` + +## f10_events_share + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Cur | summarize n=count() by Name | extend share=round(100.0*n/toscalar(Cur|count),1) | top 8 by n +``` + +## f11_dm_rl_repeat + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +let RL = Cur | where Name=="error" and tostring(Properties.error_message) has "Rate limit exceeded" | summarize days=dcount(bin(TimeGenerated,1d)), n=count() by mid; RL | summarize machines=count(), one_day=countif(days==1), two_plus=countif(days>=2), p50_hits=percentile(n,50), p90_hits=percentile(n,90) +``` + +## f12_gateway_users + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Cur | where Name=="session_start" and tostring(Properties.provider_id)=="altimate-backend" | summarize machines=dcount(mid), sessions=dcount(SessionId) by src, AppVersion | order by machines desc +``` + +## f13_compaction_sessions + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +let C = Cur | where Name=="compaction_triggered" | distinct SessionId; Cur | where Name=="agent_outcome" | summarize n=count(), completed=countif(tostring(Properties.outcome)=="completed"), errors=countif(tostring(Properties.outcome)=="error") by compacted=SessionId in (C) | extend completion=round(100.0*completed/n,1) +``` + +## g01_dm_weekly_decomp + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +let R = AppEvents | where TimeGenerated > ago(63d) | where AppVersion matches regex @"^[0-9]+\.[0-9]+\.[0-9]+$" | extend src=tostring(Properties.source), mid=tostring(Properties.machine_id); +let RLS = R | where Name=="error" and tostring(Properties.error_message) has "Rate limit exceeded" | distinct SessionId; +let SS = R | where Name=="session_start" | summarize prov=any(tostring(Properties.provider_id)) by SessionId; +R | where Name=="agent_outcome" and src=="datamates" | extend o=tostring(Properties.outcome), rl=SessionId in (RLS) | join kind=leftouter SS on SessionId +| summarize n=count(), completed=countif(o=="completed"), error=countif(o=="error"), abandoned=countif(o=="abandoned"), aborted=countif(o=="aborted"), rl_sessions=dcountif(SessionId, rl), rl_errors=countif(rl and o=="error"), bp=countif(prov=="opencode"), bp_completed=countif(prov=="opencode" and o=="completed"), bp_error=countif(prov=="opencode" and o=="error"), gw=countif(prov=="altimate-backend"), gw_completed=countif(prov=="altimate-backend" and o=="completed"), gw_error=countif(prov=="altimate-backend" and o=="error"), nonrl=countif(not(rl)), nonrl_completed=countif(not(rl) and o=="completed"), nonrl_error=countif(not(rl) and o=="error"), machines=dcount(mid) by wk=startofweek(TimeGenerated) +| extend comp_pct=round(100.0*completed/n,1), err_pct=round(100.0*error/n,1), comp_excl_ab=round(100.0*completed/(completed+error),1), nonrl_comp=round(100.0*nonrl_completed/nonrl,1), nonrl_err=round(100.0*nonrl_error/nonrl,1), bp_comp=round(100.0*bp_completed/bp,1), bp_err=round(100.0*bp_error/bp,1), gw_comp=round(100.0*gw_completed/gw,1), gw_err=round(100.0*gw_error/gw,1) +| order by wk asc +``` + +## g02_rl_sessions_full + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +let RLS = Cur | where Name=="error" and tostring(Properties.error_message) has "Rate limit exceeded" | distinct SessionId; Cur | where Name=="agent_outcome" and SessionId in (RLS) | summarize n=count(), sessions=dcount(SessionId) by o=tostring(Properties.outcome) +``` + +## g02b_rl_machines + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +let A = Cur | where Name=="error" and tostring(Properties.error_message) has "Rate limit exceeded" | distinct mid; let B = Cur | where Name=="agent_outcome" and tostring(Properties.reason) has "Rate limit exceeded" | distinct mid; Cur | where Name=="session_start" and src=="datamates" | summarize dm=dcount(mid), err_event=dcountif(mid, mid in (A)), outcome_reason=dcountif(mid, mid in (B)), bp=dcountif(mid, tostring(Properties.provider_id)=="opencode"), bp_rl=dcountif(mid, tostring(Properties.provider_id)=="opencode" and mid in (A)) +``` + +## g03_first_session_full + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +let FS = Cur | where src=="datamates" and Name=="session_start" | summarize arg_min(TimeGenerated, SessionId) by mid | project mid, SessionId; Cur | where Name=="agent_outcome" | join kind=inner FS on SessionId | summarize machines=dcount(mid) by o=tostring(Properties.outcome) | order by machines desc +``` + +## g04_upgrade_cohort + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Cur | where Name=="upgrade_attempted" and src=="datamates" | summarize ok=countif(tostring(Properties.status)=="success"), err=countif(tostring(Properties.status)=="error"), n=count() by mid | summarize machines=count(), events=sum(n), any_ok=countif(ok>0), any_err=countif(err>0), err_only=countif(err>0 and ok==0), ok_only=countif(ok>0 and err==0), both=countif(ok>0 and err>0) +``` + +## g05_review_per_run + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +Cur | where Name=="review_run" | extend bc=parse_json(tostring(Properties.by_category)) | extend total=toint(bc.lineage_breakage)+toint(bc.semantic_change)+toint(bc.contract_violation)+toint(bc.pii_exposure)+toint(bc.materialization)+toint(bc.warehouse_cost)+toint(bc.test_coverage)+toint(bc.sql_quality)+toint(bc.idempotency)+toint(bc.freshness)+toint(bc.join_risk)+toint(bc.fanout)+toint(bc.dedup)+toint(bc.sql_correctness) | summarize runs=count(), sum_total=sum(total), p50=percentile(total,50), p90=percentile(total,90), max=max(total), degraded_p50=percentileif(total,50,tostring(Properties.degraded)=="true"), full_p50=percentileif(total,50,tostring(Properties.degraded)=="false") +``` + +## g06_d7_retention + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +let FL = Cur | where Name=="first_launch" and src=="cli" and AppVersion in ("0.9.5","0.9.6","0.9.7") and TimeGenerated < W1 - 7d | summarize fl=min(TimeGenerated) by mid; let S = Cur | where Name=="session_start" | distinct mid; FL | join kind=leftouter (Cur | where Name in ("session_start","generation") | project mid, t=TimeGenerated) on mid | summarize interactive=max(iff(isnotempty(t),1,0)), d1=max(iff(t >= fl + 1d and t < fl + 7d,1,0)), d7=max(iff(t >= fl + 7d,1,0)) by mid | summarize installs=count(), any_session=countif(interactive==1), active_d1_to_d6=countif(d1==1), active_d7plus=countif(d7==1) +``` + +## g07_bedrock_causes + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +let BS = Cur | where Name=="session_start" and tostring(Properties.provider_id)=="amazon-bedrock" | distinct SessionId; Cur | where Name=="agent_outcome" and SessionId in (BS) | extend r=tostring(Properties.reason) | extend cause=case(r has "parseKnownFiles","bundling", r has "SigV4 authentication requires","no_credentials", r has "security token","bad_or_expired_token", r has "model identifier","bad_model_id", r has "use case" or r has "verified","account_not_enabled", tostring(Properties.outcome)!="error","non_error", "other") | summarize outcomes=count(), machines=dcount(mid) by cause | order by outcomes desc +``` + +## g08_population_union + +```kql +let W0=datetime(2026-08-19); let W1=datetime(2026-09-02); let P0=datetime(2026-08-05); +let RX = @"^[0-9]+\.[0-9]+\.[0-9]+$"; +let All = AppEvents | where TimeGenerated between (P0 .. W1) | extend mid=tostring(Properties.machine_id), src=tostring(Properties.source), win=iff(TimeGenerated>=W0,"cur","prev"), rel=AppVersion matches regex RX; +let Rel = All | where rel; +let Cur = Rel | where win=="cur"; +let Prev = Rel | where win=="prev"; +let Fleet = Cur | where AppVersion in ("0.7.3","0.9.3","0.8.3") | distinct mid; Cur | summarize srcs=make_set(src) by mid | extend fleet=mid in (Fleet), has_dm=set_has_element(srcs,"datamates"), has_cli=set_has_element(srcs,"cli") | summarize total=count(), fleet_ids=countif(fleet), dm_only=countif(not(fleet) and has_dm and not(has_cli)), cli_only=countif(not(fleet) and has_cli and not(has_dm)), dm_and_cli=countif(not(fleet) and has_dm and has_cli), neither=countif(not(fleet) and not(has_dm) and not(has_cli)) +``` diff --git a/docs/internal/2026-09-03-dbt-pr-review-deep-dive.md b/docs/internal/2026-09-03-dbt-pr-review-deep-dive.md new file mode 100644 index 0000000000..800c45762c --- /dev/null +++ b/docs/internal/2026-09-03-dbt-pr-review-deep-dive.md @@ -0,0 +1,149 @@ +# dbt PR Review — state, experience, and a plan for the AI + feedback layer (2026-09-03) + +Inputs: main @ `1caa234ff9`; Azure telemetry (45 d); GitHub mining of the three repos that run the review; a clean end-to-end run on `jaffle_shop_duckdb` with five injected changes; competitor research (Kilo Code, CodeRabbit, Greptile, Graphite, Recce). Supporting notes: `code-map-review.md`, `telemetry-review-notes.md`, `dogfood-notes.md`, `e2e-notes.md`, `research-feedback-loops.md` (session scratchpad; key content is folded in below). + +## 0. Correction to the 2026-09-02 telemetry report + +The "growing customer CI review usage" line was **our own repos**. The 177 headless machines on 0.9.3 are `AltimateAI/altimate-ingestion` (private) — its `dbt-pr-review.yml` pins `--version 0.9.3`, 176 runs since Aug 19 on PRs by our engineers and dependabot. The 0.8.3 runs are `dbt-pr-review-demo`. Public GitHub code search finds **zero third-party workflows** using `altimate-code/github/review` (one of ours, `altimate-bigquery-demo`, uses the separate `altimate-code-actions@v0`). External usage of the review is 11 interactive machine IDs in 45 days. `project_id` is empty on every review event, so telemetry could not have told us this. + +What that means: the review has a real dogfood corpus (30 reviewed PRs, 853 findings, engineers who did not react once) and no external users yet. That corpus is the asset. + +## 1. What the product is today + +Three layers, already built, all funnelling through `reviewPullRequest()` (`packages/opencode/src/altimate/review/run.ts:209`): + +1. **Deterministic engine** (Rust core via `Dispatcher`): equivalence, column lineage / blast radius, PII, grade, AST lint. Documented as the only layer that can produce `critical` and block. +2. **Deterministic catalog**: regex + 1,000-rule self-verifying catalog over raw model + diff; dbt-specific signals. Documented as capped at `warning`; in fact it emits `critical` in places and three of its warnings can block (see §4, Codex finding 1). +3. **LLM reviewer — already exists.** `ai-review.ts` is a transport: system prompt and response parser live in the compiled core (`altimate_core.review_ai_prompt` / `review_ai_parse`); one-shot `LLM.stream`, ≤20 files × 6,000 chars, 60 s timeout, findings clamped to `warning`, and `computeIdealVerdict` excludes `evidence.tool==="ai-review"` from the block count (`verdict.ts:63-65`). Runs only on lite/full tiers and only if a model is configured (`action.yml:129-163`). + +So "it's just deterministic" is what users *see*, because the AI layer is invisible when it runs and silent when it fails. There is **no feedback mechanism of any kind**: no reaction or reply handling, no suppression store, no false-positive marking. `applyOverride()` (`verdict.ts:258`) exists with zero callers. + +## 2. What we found + +### 2.1 Quality of the proofs (e2e, 0.10.0, jaffle_shop_duckdb, five injected changes) +| Injected change | Result | +|---|---| +| Semantic filter change | Caught precisely by engine equivalence **only with `catalog.json`**; otherwise "could not be decided" | +| `not_null` test removed | Caught every run, best-worded finding | +| New model with `SELECT *` + `DATEDIFF` | Partial (suggestion-level), plus a false `unbalanced_quote (possible injection breakout)` | +| Inconsistent column rename breaking two consumers | **Best catch**: critical, names the column and both consumers — only with `catalog.json` | +| Pure column reorder (safe refactor) | **False positive**: "NOT row-equivalent"; the comparator is positional | +| (not injected) pre-existing PII columns | critical `pii_exposure` on every touch, by design (`orchestrate.ts ~1021`); docs say "newly expose" | + +The engine is genuinely good when it has artifacts. Whether it has them is decided by undocumented steps. + +### 2.2 Experience and operations +| # | Issue | Evidence | Root cause | +|---|---|---|---| +| 1 | Reviews run lint-only most of the time | 57% of dogfood PRs, 72% of interactive `review_run` `degraded=true`; the demo PR meant to show "safe refactor approved" shows the hedge instead | Docs and examples never say to run `dbt docs generate` (catalog) or compile the **base** into `target-base/compiled`; dogfood runs `dbt parse` only. `grep target-base` hits only prose. | +| 2 | "Lint-only run — no manifest/warehouse" banner lies | e2e: manifest present, most lanes high-confidence, banner still shown | `degraded = runDegraded \|\| findings.some(f => f.degraded)` (`orchestrate.ts:1424`, `format.ts:45`) — one undecidable finding flips the whole-run banner | +| 3 | AI lane fails silently | 18% of CI runs call `review_ai_prompt` and never `review_ai_parse`; e2e default run attempted a dead local endpoint every time, docs say "skipped" | Every failure path returns `[]` with a log (`ai-review.ts:88,130,174`); orchestrator discards even that; 60 s timeout vs ~240 KB prompt | +| 4 | Wrong base ref, and one review posted to the wrong PR | PR #1320 (base `deployment`) → posted to #1319 (base `main`, same head) | `defaultBaseRef` (`git.ts:143-154`) walks `origin/main→master`, never reads the event's `pull_request.base.ref`; PR number does come from the event, so the posting side needs a repro from the dogfood workflow | +| 5 | One rule is half of all findings | `test_coverage` "new model has no uniqueness/grain test" = 427 of 853; repeated 9–14× across sibling PRs; never acted on | `missingGrainTestLane` fires per added mart/intermediate model (`orchestrate.ts:735-762`), one finding per model, no collapse | +| 6 | Inline comments duplicate on every re-run | code | Only the summary is upserted by marker; `pulls.createReview` always creates (`post-github.ts:65-107`) | +| 7 | No org attribution in telemetry | `project_id` "" on 100% of review events | `Telemetry.setContext` is only called from the interactive session loop (`session/prompt.ts:662`); headless review never sets it | +| 8 | Coverage trap | 8 of 10 unreviewed dogfood PRs touched non-Snowflake packages | workflow path filter; product cannot see it, but the summary could report "N changed dbt files outside the review scope" | +| 9 | Review takes 6–14 minutes in CI | p50 362 s telemetry / 591 s workflow, p90 ~10–14 min | dbt install + parse dominate; the review itself is ~5 s locally | +| 10 | Interactive path fails on small models | `altimate-code run "Review…"`: 79 K-token builder prompt vs 65 K window | no fallback agent; the `reviewer` agent exists but is not chosen | +| 11 | Two products disagree on confidence | `altimate-code-actions@v0` calls an unverified CTE inline "✅ Safe"; the CLI hedges the same class | separate codebases, separate posture | + +### 2.3 Engagement +Zero reactions on 30 bot comments. One explicit reply in 30 PRs (#1286: "Thanks — addressing all four…", fixed 2 of 4). 100% of interactive `review_post_outcome` are `not_requested`. We cannot tell whether anyone reads it, because there is nothing to click and nothing that records what happened to a finding afterwards. + +## 3. How others close the loop + +| Tool | Signal | Store | Applied | Suppression | +|---|---|---|---|---| +| Kilo Code | replies/feedback on past reviews, batch-analysed (opt-in "Code Review Memory", Jun 2026) | `REVIEW.md` in the repo, read from the **base** branch so a PR cannot rewrite its own policy; >10 K chars truncated | prompt guidance on every review; memory run proposes a PR you merge | manual via REVIEW.md ("files to skip", severity calibration) | +| CodeRabbit | `@coderabbitai` replies + inferred | hosted "Learnings" DB with dashboard, scopes, 0–30 d approval delay, PR quarantine until merge | retrieved per review | learning that says "don't flag X" | +| Greptile | 👍/👎, replies, addressed-vs-ignored commits | hosted Memory | **automatic**: comment types suppressed after repeated ignores; rule suggestions after ~10 PRs; security/logic never suppressed | `ignorePatterns`, scoped rules with "Last Applied" | +| Graphite Diamond | up/downvote, accepted suggestions | offline eval datasets | model/prompt iteration, not per-team | manual natural-language Exclusions | +| Recce (dbt) | check approve/reject | preset checks per project | run every PR | deterministic: skip non-data PRs, drop no-data-impact findings | + +The pattern that fits us: Kilo's **version-controlled, human-approved repo file** as the store (auditable, matches the signed-envelope posture, no server-side PII), Greptile's **cheap signals** (reactions, replies, addressed-vs-ignored) as input, and Recce's **deterministic de-noising** where a rule is structurally wrong. Our differentiator is that most findings come from named deterministic rules with stable `ruleKey`s, so feedback can attach to a rule, not to prose — and aggregate across customers. + +## 4. What the Codex plan review changed (2026-09-03) + +Codex reviewed the first draft of this plan against the code and found two things the draft, the README and the public docs all get wrong today: + +1. **"Only the deterministic engine blocks" is false.** `dbt-patterns.ts:165` emits `critical` (e.g. CROSS JOIN) and the rule catalog has critical rules (`rule-catalog.ts:289`, `select-into`). Any three non-AI warnings also trip `REQUEST_CHANGES` (`verdict.ts:52`); the only exclusion is `evidence.tool === "ai-review"`. The regex/catalog layer blocks directly and cumulatively. Either the docs change or the verdict gets an explicit provenance allow-list (`verdictEligible`) instead of an AI-only deny-list, with a test of one catalog critical plus three catalog warnings. +2. **Review policy is controlled by the PR under review.** `reviewPullRequest()` loads `.altimate/review.yml` from the checkout, i.e. the PR head (`run.ts:238`). A PR can rewrite `reviewers`, `exclude`, `rubric.blockOn`, thresholds and `ai`; a bogus non-empty `reviewers` list disables most lanes (`orchestrate.ts:1173`). Governance must load from an immutable base commit, policy changes must be reported in the PR, and the effective-policy hash signed into the envelope. This precedes any suppression store. + +Sizing corrections accepted: base-ref handling has to pick one baseline semantics (merge-base SHA everywhere, or base-tip with two-dot diff) and compile that exact SHA, since `git diff base...head` and `git show base:file` disagree once the target branch moves (`git.ts:35,100`); gate mode has no lifecycle (a clean rerun does not clear the earlier `REQUEST_CHANGES` review, and `applyOverride()` cannot update a failed check) so a single owned check-run should be the gate's source of truth; the grain-test rule should be grouped in presentation, not merged into one finding (`Finding.file` is required and per-model outcomes matter later); a workflow skipped by `paths:` cannot post a scope warning, so coverage needs an always-on job; compiling the base ref inside the composite action runs PR-controlled macros and `docs generate` touches the warehouse, so that stays in an opt-in reference workflow; `altimate-ingestion` should pin an exact current release with automated bumps, not `latest`. + +Design corrections accepted for later phases: appending `REVIEW.md` to the AI user message is verdict-safe but not prompt-safe (the core prompt declares all PR content untrusted; repo guidance is an instruction and needs a defined lower-authority slot in the core prompt, loaded from a base blob SHA, with adversarial tests); GitHub suggestion blocks need a typed replacement/range field the core parser does not have (probe of `altimate-core` 0.7.0 discards unknown fields) so they are a core release, not a TS patch; feedback needs a **low-cardinality shipped `ruleId` + `origin` + `ruleVersion`** because today's `ruleKey` is stripped by the `Finding` schema (`finding.ts:65`) and some keys embed lint text, relation names or AI titles; addressed-vs-ignored needs a durable per-run snapshot (run id, SHAs, every surfaced finding id) because the upserted summary destroys history and only line-anchored findings become individual comments (`format.ts:146`); suppression must be three operations (presentation mute, false-positive suppression for non-verdict-eligible rules only, authorized risk acceptance that preserves the ideal verdict) and the blocking list is the full default rubric (`rubric.ts:39`: lineage, contract, PII, semantic change, join risk, fanout, SQL correctness); ignored findings rank, they never propose suppression; the store is hybrid (repo files for approved policy, minimal pseudonymous observations server-side via the GitHub App, or signed run manifests if server storage is rejected); the CLI and action never pass `prTitle`/`prBody` (`cli/cmd/review.ts:84`) so intent checking is absent in CI, and the AI lane uses generic `Provider.defaultModel()` rather than an explicit selector, which is how the e2e run hit a dead local endpoint; and the metrics need stable repo/PR/run ids plus eligibility flags before "per PR" or "per rule" rates mean anything (`review_run` measures an invocation and its timer excludes CI wall time). + +## 5. Plan (revised) + +### Phase 0A — correctness and trust (before anything user-visible) +1. **Verdict provenance allow-list.** Findings carry `origin: engine | catalog | ai`; only `engine` findings are `verdictEligible` unless the rubric opts a catalog rule in. Test: one catalog critical + three catalog warnings → COMMENT. Update README and public docs to match whatever is decided. +2. **Policy from the base.** Load `.altimate/review.yml` from the resolved base commit (`git show :.altimate/review.yml`), diff it against the head copy, list policy changes in the summary, sign `policySha` and the effective rubric hash into the envelope. +3. **One baseline semantics.** Resolve `baseSha = merge-base(base, head)` once in `git.ts`; use it for `diff`, `show`, and as the SHA the base-compile step must build; take `base.ref`/`head.sha` from `GITHUB_EVENT_PATH`, never guess `origin/main`. +4. **Gate lifecycle.** Publish one owned check-run per PR as the gate; update it on reruns; wire `applyOverride()` behind maintainer authorization with a real HMAC key, prior-envelope verification, and an audit record (actor, reason, time, SHAs, prior verdict). + +### Phase 0B — capabilities, status, telemetry (the branch in flight, `fix/dbt-pr-review-ci-experience`) +5. Capability fields on the envelope (manifest, catalog, head/base compiled coverage per changed model, data-diff, AI status) rendered honestly; the "Lint-only" banner only when no manifest resolved; undecidable count separate. +6. Artifact hints with the exact command (`dbt docs generate`; compile the base SHA into `target-base/`). +7. AI lane status `ok | skipped | timeout | error` with reason, in the comment and in `review_run` (`ai_status`, `ai_findings`, eligible/attempted, duration, prompt size, model id, prompt version). Explicit AI-lane model selector with a disabled state; no fallback to whatever `defaultModel()` finds. Instrument prompt size before raising the timeout. +8. Pass `prTitle`/`prBody` from the event into `reviewPullRequest()` so the AI lane can do intent checks in CI. +9. `project_id` (and a privacy-safe stable repo/PR/run id) on headless review telemetry. +10. Docs and both example workflows: `dbt compile` head, compile the base SHA into `target-base/compiled` in a separate worktree, `dbt docs generate`; state plainly what is undecidable without them; fix the PII "newly expose" sentence. + +### Phase 0C — presentation and rollout +11. **Make the comment readable.** Evidence from `altimate-ingestion` PR #1375: 406 lines, 130 findings, seven models each with two near-identical paragraphs (fan-out, "could not be proven equivalent"), five grain columns each with a 60-word paragraph, and the one contextual finding worth reading (an unused CTE that means a documented gate may not apply, an AI-lane finding) buried at position 15. Deterministic fixes, presentation only, findings stay atomic: (a) `groupKey` on the repetitive lanes so fan-out, undecidable equivalence and grain `not_null` render as one item each with the members' specifics; (b) fold any severity section past 12 items into `
`, never critical; (c) a "Read first" block of up to three items (critical, then AI contextual, then high-confidence ungrouped warnings) when a review has eight or more findings; (d) an incremental line on re-run, "Since last review: N fixed · M new · K unchanged", from a hidden finding-id block in the previous sticky comment. Section headers show "54 findings · 9 items" so volume stays honest. +12. Inline comments deduped by ``; previous bot reviews dismissed or reconciled on rerun. +13. Always-on coverage job (or no `paths:` filter) so unreviewed dbt files are reported. +14. `altimate-ingestion`: pin the current release with automated bumps; widen to the Databricks package; measure on real volume. + +### Phase 1 — the AI layer earns its place (2–3 weeks) +- Hosted altimate model on by default in our own CI; measure `ai_status=ok` share, AI findings per completed invocation, and engineer touches. +- Core prompt release: a defined, lower-authority "repository review guidance" slot; guidance loaded from the base blob SHA, ≤10 K chars; cannot change output/verdict/grounding rules; adversarial test corpus (delimiter closure, fake finding markers, prompt-leak, poisoned feedback text). Only then read `.altimate/REVIEW.md`. +- Feed the AI lane changed schema/test files, not only model contexts. +- **What the AI layer does for readability, and what it must not do.** It writes the three-to-five-line executive summary at the top of the comment: what this PR changes in plain language, which of the deterministic findings are one change repeated N times, and the one or two things a human should look at. Every sentence cites finding ids; it adds no findings of its own in that block and never touches the verdict. It also proposes the "Read first" ordering when the deterministic heuristic ties. It does not rewrite the deterministic finding text (that stays reproducible and signed). The grouping and folding in Phase 0C do most of the work without a model; the AI summary is the layer that makes a 54-warning comment readable in ten seconds. +- Judge on the dogfood corpus: replay the 30 reviewed PRs with AI on/off; grader model calibrated against a blinded human sample and an injection corpus; ship if true-positive share ≥70% and duplicates ≤10%. +- Suggestion blocks deferred until the core parser has a typed replacement/range field validated against the right side of the diff. + +### Phase 2 — the feedback loop (after Phase 1 data) +- **Identity first**: shipped `ruleId` + `origin` + `ruleVersion` on every finding (low cardinality, no model/column/path/relation names); `finding.id` stays the repo-local occurrence fingerprint. Per-run snapshot (run id, base/head SHA, capability state, every surfaced finding id/rule id) posted as a hidden block or stored server-side. +- **Signals**: 👍/👎 on inline comments; `@altimate-code-agent feedback false-positive|helpful|accept-risk ` for line-less findings; addressed-vs-ignored derived from snapshots (ranking only). +- **Operations**: presentation mute; false-positive suppression only for non-verdict-eligible rules, scoped by path glob + rule + reason + actor + expiry + policy SHA; authorized risk acceptance that keeps the finding and ideal verdict but overrides enforcement. +- **Store**: approved policy in `.altimate/review.yml` + `.altimate/REVIEW.md` on the base branch; observations server-side via the GitHub App (pseudonymous repo/PR/finding ids, rule id, signal, actor role, SHA, time) with retention controls, or signed run manifests if server storage is rejected. +- **Incorporation**: `altimate review learn` (also `@altimate-code-agent learn`) drafts policy edits and opens a PR; proposals require explicit structured false-positive signals from maintainers, minimum distinct PRs and users, a negative-rate denominator, recency, and unchanged rule version; shadow mode until calibrated; every proposed glob shows its corpus-wide blast radius. +- **Telemetry**: `review_feedback` and `review_finding_outcome` keyed by public rule id; aggregated across repos this is the dbt rule-quality dataset nobody else has. + +### Phase 3 — the agent on top +Replies on a finding open an altimate-code session with the finding, compiled SQL and lineage as context ("explain", "fix", "diff the data" via the opt-in warehouse lane); the `reviewer` agent, not `builder`, is the default for review conversations so it starts on a 65 K-window model. The GitHub App already exists; the missing piece is finding-scoped context. + +### Metrics (once Phase 0B/2 identities exist) +- Lint-only share of CI invocations < 20% (from 57%). Definition: over a trailing 4-week window, among `review_run` events with `invocation=cli` and `empty_scope=false` (at least one reviewable model file), the share with `lint_only=true` (no changed model resolved against a manifest). Both fields are recorded on the event by this PR; `degraded` merges empty scope and is not used. Per-finding `degraded` / `undecidableFindings` are excluded so undecidable equivalence cannot move this number. The 57% baseline was measured on the 30 dogfood PRs from the posted comment banner (which used the old combined flag), so the first post-fix reading resets the baseline. +- AI `status=ok` ≥ 90% of eligible invocations (from ≤ 82%). +- Findings per completed invocation p50 ≤ 8 (from 11); top rule ≤ 10% of findings (from 50%). +- Reaction or structured feedback on ≥ 20% of reviewed PRs (from 0–3%). +- Addressed rate per public rule id, from snapshots. +- CI wall time p50 < 4 min, from job data, not `review_run.duration_ms`. + +## 6. Decisions for Anand +1. Resolve the blocking invariant: enforce "only engine blocks" in code, or rewrite the docs/README to say catalog rules block too. Recommendation: enforce, with a rubric opt-in for named catalog rules. +2. Hosted altimate model as the default AI lane in the action (we pay per review) vs BYO key only. Recommendation: hosted by default with a per-repo cap; the AI layer is otherwise invisible. +3. Feedback store: hybrid (repo policy + server observations via the GitHub App) vs repo-only. Recommendation: hybrid; repo-only cannot see reactions or cross-PR thresholds. +4. Reconcile `altimate-code-actions@v0` (says "Safe" without proof) with this product: fold in or retire. +5. Widen `altimate-ingestion` to the Databricks package now and move it from the 0.9.3 pin to an exact pin of the current release with automated bumps (Dependabot/Renovate on the `--version` line), so Phase 1 measures real volume without floating `latest`. Same policy as Phase 0C item 14. +6. Budget a core release (prompt guidance slot, typed suggestion payload, `ruleId` in engine output) — Phases 1–2 depend on it. + +## 7. Deferred from PR #1241's automated review (eighth wave, all P2, none affect the verdict) +- Signature versioning so a sticky comment from before the field existed does not report "review settings changed" once (`verdict.ts`). +- Ignore an inherited `HEAD_REF` environment variable when no custom head was selected (`action.yml`). +- Skip base-artifact probing for renames whose old SQL is never read (`run.ts`). +- Treat a valid zero-model manifest as available when a PR deletes the project's last model (`orchestrate.ts`). +- Make the compiled resolver use the artifact directories the fidelity probe accepted when an integration supplies a custom `getContent` (`run.ts`). +- Check core parse success before reporting the AI lane as `ok` (`ai-review.ts`). + +## 7a. Gateway route status (2026-09-04) +The gateway now has a production hostname (staging is deprecated). The action's `altimate_gateway_url` defaults to it and stays overridable for a self-hosted gateway; the tracker-leak check allowlists that one public hostname and still flags the apex and every other subdomain. **Resolved 2026-09-04:** after the rename the host briefly served the certificate issued for the staging name, so verifying HTTPS clients refused the production name. A certificate for the production name was issued the same morning; TLS verification, `/register` and `/v1/*` now succeed from outside, and a certbot deploy hook reloads the containerised nginx on renewal. + +**Live measurement (2026-09-04, `altimate-base`, thinking on):** the model reasons for two to three minutes before its first text token (~25 tok/s). A 12-file prompt took 155 s and was truncated by the 4K output cap before gateway PR 15 lifted it; afterwards it completed in 170 s (2,241 reasoning + 1,030 text tokens). A three-change jaffle-shop review with `--ai-timeout 300 --ai-max-output-tokens 8192` finished in 168 s (2,808 prompt, 4,464 completion of which 4,170 reasoning) and produced two true-positive advisory findings the deterministic lanes did not name: a returned-order filter applied to order metrics but not to lifetime value, and a model whose plural name promises per-gap intervals while computing one span. Counts moved 3/4/6 → 3/5/7, "Read first" and the verdict were unchanged. The earlier 68 s timeout with zero findings was an under-provisioned default, not a gateway fault; the gateway route now defaults to 900 s and the ceiling is 1800 s. Gateway PR 18 (per-request `reasoning_effort` pass-through) went live on prod the same evening via the release branch. **Paired comparison, same project and changes:** `aiReasoningEffort: none` finished the AI call in 13 s with 276 completion tokens (0 reasoning) versus 168 s and 4,464 (4,170 reasoning) with thinking on; both runs produced two true-positive advisory findings, different from each other rather than degraded (reasoning-off caught filter-before-grouping turning a customer's zero into NULL, and `select *` propagating PII into a new model, which the deterministic PII lane does not trace). Read first, verdict and counts were identical. The Route C default stays thinking-on pending more pairs; `ai_reasoning: none` is documented as the speed setting. Note: the gateway deploys from `codex/altimate-base-gateway-release`, not `main`. + +## 8. Not verified +- The wrong-PR posting in #1320: the base-ref bug is confirmed; PR-number resolution reads the event correctly, so the misdirection needs a repro against the dogfood workflow's exact trigger. +- The AI layer's output quality at scale: two live runs (four findings, all true positives, see 7a); no precision/recall measurement across many PRs yet, so no basis yet for flipping the Route C reasoning default. +- The positional equivalence comparator lives in the core; reproduced, not fixed. diff --git a/github/review/action.yml b/github/review/action.yml index 48c369b4e6..609b172e44 100644 --- a/github/review/action.yml +++ b/github/review/action.yml @@ -1,5 +1,5 @@ name: "altimate dbt PR review" -description: "Layered AI code review for dbt/SQL pull requests — a signed verdict backed by lineage, query equivalence, PII, and grade, plus advisory LLM comments." +description: "Layered AI code review for dbt/SQL pull requests — a signed verdict backed by lineage, query equivalence, PII, and grade, plus an AI reviewer outcome with advisory findings." branding: icon: "git-pull-request" color: "orange" @@ -18,10 +18,10 @@ inputs: required: false default: "suggestion" base: - description: "Base git ref to diff against. Defaults to the merge-base with origin/main." + description: "Base git ref to diff against. On pull_request events, defaults to origin/." required: false head: - description: "Head git ref. Defaults to the checked-out PR head." + description: "Head git ref. On pull_request events, defaults to the PR head SHA." required: false post: description: "Post the verdict to the PR (summary comment + inline review)." @@ -44,12 +44,28 @@ inputs: description: "Altimate API base URL." required: false default: "https://api.myaltimate.com" + altimate_gateway_key: + description: "Altimate gateway API key → uses the free `altimate-base` model for the advisory lane. Use a repo secret." + required: false + altimate_gateway_url: + description: "HTTPS Altimate gateway base URL. Defaults to the production gateway; override only for a self-hosted gateway." + required: false + default: "https://altimate-gateway.onealtimate.com" model: description: "Bring-your-own advisory model as 'provider/model' (e.g. anthropic/claude-sonnet-4-6). Used when `altimate_api_key` is not set." required: false model_api_key: description: "API key for `model`'s provider (e.g. an Anthropic key for an anthropic/* model). Use a repo secret." required: false + ai_model: + description: "Optional provider/model override within the selected Altimate action route." + required: false + ai_reasoning: + description: "Optional reasoning level for the advisory reviewer: none, minimal, low, medium or high. Unset leaves the model default (altimate-base thinks before answering)." + required: false + ai_timeout_seconds: + description: "Optional AI reviewer timeout in seconds (10..1800). The Altimate gateway route defaults to 900 s." + required: false runs: using: "composite" @@ -112,20 +128,70 @@ runs: shell: bash run: echo "$HOME/.altimate/bin" >> $GITHUB_PATH - # Configure the OPTIONAL advisory LLM lane. Two mutually-exclusive routes: + - name: Fetch pull request base ref + if: ${{ github.event_name == 'pull_request' }} + shell: bash + env: + PR_BASE_REF: ${{ github.event.pull_request.base.ref }} + PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} + IN_HEAD: ${{ inputs.head }} + run: | + set -euo pipefail + git fetch --no-tags origin "+refs/heads/${PR_BASE_REF}:refs/remotes/origin/${PR_BASE_REF}" + if [[ -n "${IN_HEAD:-}" ]]; then + if [[ "$IN_HEAD" == -* ]]; then + echo "::error::Custom head must not start with '-': $IN_HEAD" >&2 + exit 1 + fi + if git rev-parse --verify --quiet "$IN_HEAD^{commit}" >/dev/null; then + echo "HEAD_REF=$IN_HEAD" >> "$GITHUB_ENV" + elif git fetch --no-tags origin -- "$IN_HEAD"; then + git update-ref refs/altimate/review-head FETCH_HEAD + echo "HEAD_REF=refs/altimate/review-head" >> "$GITHUB_ENV" + else + echo "Unable to fetch custom head '$IN_HEAD' from origin" >&2 + exit 1 + fi + else + git fetch --no-tags origin "$PR_HEAD_SHA" + fi + if [[ "$(git rev-parse --is-shallow-repository)" == "true" ]]; then + git fetch --no-tags --unshallow origin + fi + + # Configure the OPTIONAL advisory LLM lane. Route precedence is: # A) altimate_api_key (+ altimate_instance) → hosted altimate model - # B) model (+ model_api_key) → bring-your-own provider - # Neither set → the LLM lane self-disables and the review is deterministic-only. + # B) model + model_api_key → bring-your-own provider + # C) altimate_gateway_key → free altimate-base gateway model + # No route → the LLM lane self-disables and the review is deterministic-only. - name: Configure advisory reviewer model + credentials shell: bash env: IN_ALT_KEY: ${{ inputs.altimate_api_key }} IN_ALT_INSTANCE: ${{ inputs.altimate_instance }} IN_ALT_URL: ${{ inputs.altimate_url }} + IN_GATEWAY_KEY: ${{ inputs.altimate_gateway_key }} + IN_GATEWAY_URL: ${{ inputs.altimate_gateway_url }} IN_MODEL: ${{ inputs.model }} IN_MODEL_API_KEY: ${{ inputs.model_api_key }} + IN_AI_MODEL: ${{ inputs.ai_model }} + IN_AI_REASONING: ${{ inputs.ai_reasoning }} + IN_AI_TIMEOUT_SECONDS: ${{ inputs.ai_timeout_seconds }} run: | set -euo pipefail + AI_REASONING="${IN_AI_REASONING:-}" + if [[ -n "$AI_REASONING" && ! "$AI_REASONING" =~ ^(none|minimal|low|medium|high)$ ]]; then + echo "::error::ai_reasoning must be one of: none, minimal, low, medium or high." + exit 1 + fi + AI_TIMEOUT_SECONDS="${IN_AI_TIMEOUT_SECONDS:-}" + if [[ -n "$AI_TIMEOUT_SECONDS" ]]; then + if [[ ! "$AI_TIMEOUT_SECONDS" =~ ^[0-9]+$ ]] || + (( 10#$AI_TIMEOUT_SECONDS < 10 || 10#$AI_TIMEOUT_SECONDS > 1800 )); then + echo "::error::ai_timeout_seconds must be an integer between 10 and 1800." + exit 1 + fi + fi if [[ -n "${IN_ALT_KEY:-}" ]]; then # Route A — hosted altimate model (altimate-backend). if [[ -z "${IN_ALT_INSTANCE:-}" ]]; then @@ -142,26 +208,50 @@ runs: --arg inst "$IN_ALT_INSTANCE" \ '{altimateUrl:$url, altimateInstanceName:$inst, altimateApiKey:$ENV.IN_ALT_KEY}' \ > "$HOME/.altimate/altimate.json" - echo "Advisory lane: hosted altimate model (tenant: $IN_ALT_INSTANCE)." - elif [[ -n "${IN_MODEL:-}" ]]; then + AI_MODEL="${IN_AI_MODEL:-altimate-backend/altimate-default}" + echo "ALTIMATE_ACTION_AI_MODEL=$AI_MODEL" >> "$GITHUB_ENV" + echo "Advisory lane: hosted altimate model ($AI_MODEL; tenant: $IN_ALT_INSTANCE)." + elif [[ -n "${IN_MODEL:-}" && -n "${IN_MODEL_API_KEY:-}" ]]; then # Route B — bring-your-own provider/model via inline opencode config. PROVIDER="${IN_MODEL%%/*}" - if [[ -n "${IN_MODEL_API_KEY:-}" ]]; then - CONTENT=$(jq -nc --arg m "$IN_MODEL" --arg p "$PROVIDER" --arg k "$IN_MODEL_API_KEY" \ - '{model:$m, provider: {($p): {options: {apiKey:$k}}}}') - else - CONTENT=$(jq -nc --arg m "$IN_MODEL" '{model:$m}') - fi + CONTENT=$(jq -nc --arg m "$IN_MODEL" --arg p "$PROVIDER" --arg k "$IN_MODEL_API_KEY" \ + '{model:$m, provider: {($p): {options: {apiKey:$k}}}}') echo "OPENCODE_CONFIG_CONTENT=$CONTENT" >> "$GITHUB_ENV" + echo "ALTIMATE_ACTION_AI_MODEL=$IN_MODEL" >> "$GITHUB_ENV" echo "Advisory lane: bring-your-own model ($IN_MODEL)." - elif [[ -n "${IN_MODEL_API_KEY:-}" ]]; then - # Guard a likely misconfiguration: a key with no model would be silently - # ignored and the lane would disable without explanation. - echo "::error::model_api_key is set but model is empty — set \`model\` (e.g. anthropic/claude-sonnet-4-6) or use altimate_api_key." + elif [[ -n "${IN_GATEWAY_KEY:-}" ]]; then + # Route C — Altimate's OpenAI-compatible LiteLLM gateway. + if [[ -z "${IN_GATEWAY_URL:-}" ]]; then + echo "::error::altimate_gateway_key is set but altimate_gateway_url is empty — provide the HTTPS gateway base URL." + exit 1 + fi + if [[ ! "$IN_GATEWAY_URL" =~ ^https://[^[:space:]]+$ ]]; then + echo "::error::altimate_gateway_url must be an HTTPS URL." + exit 1 + fi + GATEWAY_BASE_URL="${IN_GATEWAY_URL%/}" + GATEWAY_BASE_URL="${GATEWAY_BASE_URL%/v1}/v1" + CONTENT=$(jq -nc \ + --arg base_url "$GATEWAY_BASE_URL" \ + '{provider: {"altimate-gateway": {npm: "@ai-sdk/openai-compatible", name: "Altimate Gateway", options: {baseURL:$base_url, apiKey:$ENV.IN_GATEWAY_KEY}, models: {"altimate-base": {name:"altimate-base"}, "altimate-pro": {name:"altimate-pro"}}}}}') + echo "OPENCODE_CONFIG_CONTENT=$CONTENT" >> "$GITHUB_ENV" + AI_MODEL="${IN_AI_MODEL:-altimate-gateway/altimate-base}" + # matches the gateway's LiteLLM request_timeout (900 s); raise both together + AI_TIMEOUT_SECONDS="${AI_TIMEOUT_SECONDS:-900}" + echo "ALTIMATE_ACTION_AI_MODEL=$AI_MODEL" >> "$GITHUB_ENV" + echo "Advisory lane: Altimate gateway model ($AI_MODEL)." + elif [[ -n "${IN_MODEL:-}" || -n "${IN_MODEL_API_KEY:-}" ]]; then + echo "::error::model and model_api_key must be set together for the bring-your-own route." exit 1 else echo "Advisory lane: no model/credentials provided — running deterministic-only." fi + if [[ -n "$AI_REASONING" ]]; then + echo "ALTIMATE_ACTION_AI_REASONING=$AI_REASONING" >> "$GITHUB_ENV" + fi + if [[ -n "$AI_TIMEOUT_SECONDS" ]]; then + echo "ALTIMATE_ACTION_AI_TIMEOUT_SECONDS=$AI_TIMEOUT_SECONDS" >> "$GITHUB_ENV" + fi - name: Run dbt PR review shell: bash @@ -175,13 +265,40 @@ runs: IN_BASE: ${{ inputs.base }} IN_HEAD: ${{ inputs.head }} IN_POST: ${{ inputs.post }} + EVENT_NAME: ${{ github.event_name }} + PR_BASE_REF: ${{ github.event.pull_request.base.ref }} + PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} GITHUB_TOKEN: ${{ github.token }} GITHUB_REPOSITORY: ${{ github.repository }} GITHUB_EVENT_PATH: ${{ github.event_path }} ALTIMATE_REVIEW_SIGNING_KEY: ${{ inputs.signing_key }} run: | args=(--mode "$IN_MODE" --manifest "$IN_MANIFEST" --severity "$IN_SEVERITY") - [[ -n "$IN_BASE" ]] && args+=(--base "$IN_BASE") - [[ -n "$IN_HEAD" ]] && args+=(--head "$IN_HEAD") + if [[ -n "$IN_BASE" ]]; then + args+=(--base "$IN_BASE") + elif [[ "$EVENT_NAME" == "pull_request" && -n "$PR_BASE_REF" ]]; then + # Use the fork point for both the file list and old content: base-only + # changes after it must never be attributed to this pull request. + if MERGE_BASE=$(git merge-base "origin/$PR_BASE_REF" "${HEAD_REF:-$PR_HEAD_SHA}"); then + args+=(--base "$MERGE_BASE") + else + echo "::warning::Could not compute pull request merge-base; falling back to origin/$PR_BASE_REF" + args+=(--base "origin/$PR_BASE_REF") + fi + fi + if [[ "$EVENT_NAME" == "pull_request" && -n "${HEAD_REF:-$PR_HEAD_SHA}" ]]; then + args+=(--head "${HEAD_REF:-$PR_HEAD_SHA}") + elif [[ -n "$IN_HEAD" ]]; then + args+=(--head "$IN_HEAD") + fi [[ "$IN_POST" == "true" ]] && args+=(--post) + if [[ -n "${ALTIMATE_ACTION_AI_MODEL:-}" ]]; then + args+=(--ai-model "$ALTIMATE_ACTION_AI_MODEL") + fi + if [[ -n "${ALTIMATE_ACTION_AI_REASONING:-}" ]]; then + args+=(--ai-reasoning "$ALTIMATE_ACTION_AI_REASONING") + fi + if [[ -n "${ALTIMATE_ACTION_AI_TIMEOUT_SECONDS:-}" ]]; then + args+=(--ai-timeout "$ALTIMATE_ACTION_AI_TIMEOUT_SECONDS") + fi altimate review "${args[@]}" diff --git a/github/review/examples/altimate-ingestion.yml b/github/review/examples/altimate-ingestion.yml index dc52867909..69c48d15b2 100644 --- a/github/review/examples/altimate-ingestion.yml +++ b/github/review/examples/altimate-ingestion.yml @@ -2,9 +2,9 @@ # Copy to the target repo as `.github/workflows/dbt-pr-review.yml` and adjust the # dbt adapter + profiles/secrets to match the warehouse. # -# Flow: checkout (full history) → dbt deps/compile (+ docs generate) → altimate review. +# Flow: checkout (full history) → compile head + base and generate docs → altimate review. # The review step itself needs NO warehouse access — it runs the deterministic -# engine over the compiled artifacts (target/manifest.json + target/catalog.json). +# engine over target/manifest.json, target/catalog.json, and both compiled-SQL trees. name: dbt PR review on: @@ -26,6 +26,10 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 # REQUIRED: the merge-base diff needs full history + # Check out the PR head, not the synthetic merge commit the event + # defaults to: the head compile must match the commit the review + # compares against, or base-only changes leak into target/compiled. + ref: ${{ github.event.pull_request.head.sha }} - uses: actions/setup-python@v5 with: @@ -35,19 +39,28 @@ jobs: - name: Install dbt run: pip install "dbt-snowflake>=1.8,<2.0" - # Compile to produce target/manifest.json. `dbt docs generate` additionally - # produces target/catalog.json (real warehouse column types) — strongly - # recommended so column-lineage breakage and proven equivalence fire at full - # fidelity rather than degraded (manifest-only) mode. + # Compile HEAD to produce target/manifest.json + target/compiled, generate + # target/catalog.json for real warehouse column types, and compile the PR + # base into target-base/compiled for base-vs-head equivalence. # # Warehouse creds are consumed HERE (by dbt), via env_var() in profiles.yml. # They are NOT needed by the review step. # - # SECURITY: `dbt compile` executes the PR's Jinja/macros against the - # warehouse (e.g. a malicious macro calling run_query()). Use a - # LEAST-PRIVILEGE, read-only warehouse role for these credentials so a - # hostile PR can't read or mutate production data at compile time. - - name: dbt deps + compile + # SECURITY: `dbt compile` and `dbt docs generate` both execute the PR's + # Jinja/macros against the warehouse (`run_query()` runs during compile, + # not only during `dbt run`). A read-only role still lets a hostile macro + # READ data. Point these credentials at an isolated CI database with + # sanitized/synthetic data and a role scoped to it, never at production. + # Keep the trigger on `pull_request` (not `pull_request_target`): PRs from + # forks then receive no secrets, so this step fails. `continue-on-error` + # lets the review step still run for such PRs; with a read-only fork + # token it cannot post, so it prints the summary to the job log instead. + # Trade-off: `continue-on-error` also makes a genuine compile failure on + # a same-repo PR non-fatal here (the review then runs lint-only and says + # which artifacts are missing). Keep a separate `dbt build`/`dbt compile` + # check job if compile failures must fail the PR. + - name: Build dbt review artifacts + continue-on-error: true env: DBT_PROFILES_DIR: ${{ github.workspace }} SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }} @@ -56,15 +69,31 @@ jobs: SNOWFLAKE_ROLE: ${{ secrets.SNOWFLAKE_ROLE }} SNOWFLAKE_WAREHOUSE: ${{ secrets.SNOWFLAKE_WAREHOUSE }} SNOWFLAKE_DATABASE: ${{ secrets.SNOWFLAKE_DATABASE }} + # Route event data through env, never straight into the script. + PR_BASE_REF: ${{ github.event.pull_request.base.ref }} + PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} run: | dbt deps dbt compile - dbt docs generate || echo "docs generate skipped — review runs in degraded (manifest-only) mode" + dbt docs generate || echo "docs generate skipped — lineage/PII run without catalog column types" + # Compile the fork point (merge-base), not the base tip: the review + # compares against the merge-base, so base-only commits made after the + # fork must not appear in target-base/compiled either. + git fetch --no-tags origin "+refs/heads/${PR_BASE_REF}:refs/remotes/origin/${PR_BASE_REF}" + MERGE_BASE=$(git merge-base "origin/${PR_BASE_REF}" "${PR_HEAD_SHA}") + git worktree add --detach ../dbt-review-base "${MERGE_BASE}" + ( + cd ../dbt-review-base + dbt deps + dbt compile --target-path "${{ github.workspace }}/target-base" + ) - name: altimate dbt PR review - # Pin to the altimate-code release that ships `altimate review` - # (the first release cut after the dbt-pr-review feature merges). - uses: AltimateAI/altimate-code/github/review@v0.8.5 + # Exact pin to the current release; bump it with Dependabot/Renovate. + # The `altimate_gateway_*` and `ai_model` inputs ship in the release + # after v0.10.0 — on v0.10.0 they are ignored with a warning, so move + # the pin forward when that release is out if you use Route C. + uses: AltimateAI/altimate-code/github/review@v0.10.0 with: mode: comment # start non-blocking; switch to `gate` once trusted manifest_path: target/manifest.json @@ -79,7 +108,11 @@ jobs: # model: anthropic/claude-sonnet-4-6 # model_api_key: ${{ secrets.ANTHROPIC_API_KEY }} # - # Omit both routes entirely to run a deterministic-only review (no AI comments). + # Route C — default free gateway model (use instead of Routes A/B): + # altimate_gateway_key: ${{ secrets.ALTIMATE_GATEWAY_KEY }} + # altimate_gateway_url: https://gateway.example.com # only for a self-hosted gateway; defaults to https://altimate-gateway.onealtimate.com + # + # Omit all routes entirely to run a deterministic-only review (no AI comments). # Optional: sign the verdict envelope (tamper-evident, reproducible). signing_key: ${{ secrets.ALTIMATE_REVIEW_SIGNING_KEY }} diff --git a/packages/opencode/src/altimate/review/ai-review.ts b/packages/opencode/src/altimate/review/ai-review.ts index 1bd10a99e8..e865f59981 100644 --- a/packages/opencode/src/altimate/review/ai-review.ts +++ b/packages/opencode/src/altimate/review/ai-review.ts @@ -7,10 +7,15 @@ import { MessageID, SessionID } from "@/session/schema" import { Log } from "@/altimate/util/log" import { Dispatcher } from "../native" import { type Finding, type ReviewCategory, type Severity, makeFinding } from "./finding" +import { NO_MODEL_REASON, type AiReviewStatus } from "./verdict" +import { DEFAULT_AI_MAX_OUTPUT_TOKENS, type AiReasoningEffort } from "./config" const log = Log.create({ service: "ai-review" }) -const AI_TIMEOUT_MS = 60_000 +/** Skip reason when the change touched no model SQL the AI lane can read. */ +export const NO_MODEL_FILES_REASON = + "no changed model files (the AI lane reads model SQL only; schema, macro, snapshot and test changes are covered by the deterministic lanes)" + const MAX_DIFF_CHARS = 6_000 // per file, keep the prompt bounded const MAX_FILES = 20 @@ -28,8 +33,33 @@ export interface AiReviewInput { files: AiReviewFile[] /** Deterministic engine findings — grounding the AI must NOT duplicate. */ grounding: Finding[] + /** Explicit provider/model for this advisory lane. */ + model?: string + /** Allow the interactive tool to fall back to the current session model. */ + allowSessionModel: boolean + /** Active provider/model supplied by the interactive tool context. */ + sessionModel?: string prTitle?: string prBody?: string + /** Override the review deadline (primarily for tests). */ + timeoutMs?: number + /** Total output budget, including reasoning tokens. */ + maxOutputTokens?: number + /** Per-request model reasoning level. Unset preserves the model default. */ + reasoningEffort?: AiReasoningEffort +} + +export interface AiReviewResult { + findings: Finding[] + status: AiReviewStatus + reason?: string + /** Effective provider/model used by the advisory lane. */ + model?: string + durationMs?: number + promptChars?: number + promptTokens?: number + completionTokens?: number + reasoningTokens?: number } /** @@ -45,6 +75,105 @@ function truncate(s: string | undefined, n: number): string { return s.length > n ? s.slice(0, n) + "\n… (truncated)" : s } +function errorReason(err: unknown): string { + const name = + err instanceof Error ? (err.name && err.name !== "Error" ? err.name : err.constructor.name || "Error") : "Error" + const message = err instanceof Error ? err.message : String(err) + const raw = message && message !== name ? `${name}: ${message}` : name + return redactReason(raw, 120) +} + +/** Every reason that can reach a PR comment passes through here: URLs, API + * keys and bearer tokens are masked and whitespace collapsed before the cut. */ +function redactReason(raw: string, max: number): string { + const clean = raw + .replace(/\b[a-z][a-z0-9+.-]*:\/\/\S+/gi, "") + .replace(/sk-(?:ant-)?[A-Za-z0-9_-]{20,}/g, "sk-***") + .replace(/Bearer\s+[A-Za-z0-9._-]{20,}/gi, "Bearer ***") + .replace(/\s+/g, " ") + .trim() + return truncateAtWord(clean, max) +} + +/** Cut at the last word boundary before `max` and mark the cut, so a rendered + * reason never ends mid-word ("…able to ac"). */ +function truncateAtWord(s: string, max: number): string { + if (s.length <= max) return s + const cut = s.lastIndexOf(" ", max - 1) + return `${s.slice(0, cut > max / 2 ? cut : max - 1).trimEnd()}…` +} + +function noModelError(err: unknown): boolean { + if (!(err instanceof Error)) return false + if (err instanceof Provider.ModelNotFoundError || err instanceof Provider.NoModelsError) return true + return err.message === "no providers found" || err.message === "no models found" +} + +interface AiUsage { + promptTokens?: number + completionTokens?: number + reasoningTokens?: number +} + +function tokenCount(value: unknown): number | undefined { + return typeof value === "number" && Number.isFinite(value) && Number.isInteger(value) && value >= 0 + ? value + : undefined +} + +function findTokenCount(value: unknown, keys: ReadonlySet, seen = new Set()): number | undefined { + if (!value || typeof value !== "object" || seen.has(value)) return undefined + seen.add(value) + for (const [key, item] of Object.entries(value)) { + if (keys.has(key)) { + const count = tokenCount(item) + if (count !== undefined) return count + } + } + for (const item of Object.values(value)) { + const count = findTokenCount(item, keys, seen) + if (count !== undefined) return count + } + return undefined +} + +function readUsage(usage: unknown, providerMetadata: unknown): AiUsage { + const normalized = usage && typeof usage === "object" ? (usage as Record) : undefined + const details = + normalized?.outputTokenDetails && typeof normalized.outputTokenDetails === "object" + ? (normalized.outputTokenDetails as Record) + : undefined + const sources = { usage, providerMetadata } + return { + promptTokens: + tokenCount(normalized?.inputTokens) ?? + findTokenCount(sources, new Set(["promptTokens", "prompt_tokens", "input_tokens"])), + completionTokens: + tokenCount(normalized?.outputTokens) ?? + findTokenCount(sources, new Set(["completionTokens", "completion_tokens", "output_tokens"])), + reasoningTokens: + tokenCount(details?.reasoningTokens) ?? + tokenCount(normalized?.reasoningTokens) ?? + findTokenCount(sources, new Set(["reasoningTokens", "reasoning_tokens"])), + } +} + +function mergeUsage(current: AiUsage, next: AiUsage): AiUsage { + return { + promptTokens: next.promptTokens ?? current.promptTokens, + completionTokens: next.completionTokens ?? current.completionTokens, + reasoningTokens: next.reasoningTokens ?? current.reasoningTokens, + } +} + +function isLengthFinishReason(reason: unknown): boolean { + return reason === "length" || reason === "max_tokens" || reason === "max_output_tokens" +} + +function suggestedFindingCount(text: string): number { + return text.match(/["']file["']\s*:/g)?.length ?? 0 +} + /** Assemble the user message (mechanical formatting — not IP). */ function buildUserMessage(input: AiReviewInput): string { const parts: string[] = [] @@ -71,24 +200,110 @@ function buildUserMessage(input: AiReviewInput): string { } /** - * Run the LLM reviewer lane. Returns advisory findings (severity ≤ warning, - * clamped by core), or [] if no model / core is available or the call fails — - * a review must never crash because the AI layer is unavailable. + * Run the LLM reviewer lane. Findings are advisory (severity ≤ warning, + * clamped by core), and failures are returned as status rather than thrown — a + * review must never crash because the AI layer is unavailable. */ -export async function runAiReview(input: AiReviewInput): Promise { +export async function runAiReview(input: AiReviewInput): Promise { + const startedAt = Date.now() + let effectiveModel: string | undefined + let promptChars = 0 + let usage: AiUsage = {} + const finish = (result: AiReviewResult): AiReviewResult => ({ + ...result, + ...(effectiveModel ? { model: effectiveModel } : {}), + durationMs: Math.max(0, Date.now() - startedAt), + promptChars, + ...(usage.promptTokens !== undefined ? { promptTokens: usage.promptTokens } : {}), + ...(usage.completionTokens !== undefined ? { completionTokens: usage.completionTokens } : {}), + ...(usage.reasoningTokens !== undefined ? { reasoningTokens: usage.reasoningTokens } : {}), + }) + if (!input.model && !input.allowSessionModel) { + return finish({ findings: [], status: "skipped", reason: NO_MODEL_REASON }) + } + const files = input.files.filter((f) => f.status !== "deleted" && (f.diff || f.sql)) - if (!files.length) return [] + // The orchestrator hands this lane model files only. A schema-, macro-, + // snapshot- or test-only change is still reviewed deterministically, so the + // reason must say what the lane covers rather than deny the files exist. + if (!files.length) return finish({ findings: [], status: "skipped", reason: NO_MODEL_FILES_REASON }) + const userMessage = buildUserMessage({ ...input, files }) + promptChars = userMessage.length + const aiTimeoutMs = + input.timeoutMs ?? Math.min(300, 120 + 4 * Math.min(files.length, MAX_FILES)) * 1_000 + const maxOutputTokens = input.maxOutputTokens ?? DEFAULT_AI_MAX_OUTPUT_TOKENS + const timeoutReason = `timed out after ${aiTimeoutMs / 1000}s (raise aiTimeoutSeconds / --ai-timeout)` const controller = new AbortController() - const timeout = setTimeout(() => controller.abort(), AI_TIMEOUT_MS) + const timeout = setTimeout(() => controller.abort(), aiTimeoutMs) + const setupTimedOut = Symbol("setupTimedOut") + const abortPromise = new Promise((resolve) => { + controller.signal.addEventListener("abort", () => resolve(setupTimedOut), { once: true }) + }) + let streamAborted = false + let finishReason: unknown + let rawFinishReason: unknown + let providerMetadata: unknown try { - // Prompt comes from the compiled core, not this file. - const promptRes = await Dispatcher.call("altimate_core.review_ai_prompt", {}) - const system = ((promptRes.data ?? {}) as Record).prompt as string | undefined - if (!system) return [] + const setup = (async () => { + let model: Awaited> + if (input.model) { + const parsed = Provider.parseModel(input.model) + if (!parsed.providerID.length || !parsed.modelID.length || /\s/.test(input.model)) { + return { modelError: "Error: expected provider/model" } + } + effectiveModel = input.model + try { + model = await Provider.getModel(parsed.providerID, parsed.modelID) + } catch (err) { + return { modelError: errorReason(err) } + } + } else if (input.sessionModel) { + const parsed = Provider.parseModel(input.sessionModel) + if (!parsed.providerID.length || !parsed.modelID.length || /\s/.test(input.sessionModel)) { + const defaultModel = await Provider.defaultModel() + model = await Provider.getModel(defaultModel.providerID, defaultModel.modelID) + } else { + effectiveModel = input.sessionModel + try { + model = await Provider.getModel(parsed.providerID, parsed.modelID) + } catch (err) { + return { modelError: errorReason(err) } + } + } + } else { + const defaultModel = await Provider.defaultModel() + model = await Provider.getModel(defaultModel.providerID, defaultModel.modelID) + } + effectiveModel = `${model.providerID}/${model.id}` - const defaultModel = await Provider.defaultModel() - const model = await Provider.getModel(defaultModel.providerID, defaultModel.modelID) + // Prompt comes from the compiled core, not this file. + const promptRes = await Dispatcher.call("altimate_core.review_ai_prompt", {}) + // A core that threw is a broken reviewer, not an intentional omission: + // surface it as an error so CI does not read it as "skipped by design". + if (promptRes.success === false) return { promptError: promptRes.error ?? "core prompt failed" } + const system = ((promptRes.data ?? {}) as Record).prompt as string | undefined + if (!system) return undefined + return { system, model } + })() + const setupResult = await Promise.race([setup, abortPromise]) + if (setupResult === setupTimedOut) return finish({ findings: [], status: "timeout", reason: timeoutReason }) + if (!setupResult) return finish({ findings: [], status: "skipped", reason: "reviewer prompt unavailable" }) + if ("promptError" in setupResult) { + return finish({ + findings: [], + status: "error", + reason: `reviewer prompt failed: ${redactReason(setupResult.promptError ?? "core prompt failed", 160)}`, + }) + } + if ("modelError" in setupResult) { + return finish({ + findings: [], + status: "error", + reason: `configured AI model not available: ${input.model ?? input.sessionModel} — ${setupResult.modelError}`, + }) + } + const { system, model } = setupResult const agent: Agent.Info = { name: "dbt-ai-reviewer", @@ -108,34 +323,110 @@ export async function runAiReview(input: AiReviewInput): Promise { model: { providerID: model.providerID, modelID: model.id }, } - const stream = await LLM.stream({ - agent, - user, - system: [system], - small: false, - tools: {}, - model, - abort: controller.signal, - sessionID: user.sessionID, - retries: 1, - messages: [{ role: "user", content: buildUserMessage({ ...input, files }) }], - }) - for await (const _ of stream.fullStream) { - // drain to avoid SDK hangs + const streamResult = await Promise.race([ + LLM.stream({ + agent, + user, + system: [system], + small: false, + tools: {}, + model, + abort: controller.signal, + sessionID: user.sessionID, + retries: 1, + messages: [{ role: "user", content: userMessage }], + // Reasoning models spend from the same budget before emitting the JSON + // array, so the advisory lane must reserve enough for both phases. + maxOutputTokens, + reasoningEffort: input.reasoningEffort, + }), + abortPromise, + ]) + if (streamResult === setupTimedOut || controller.signal.aborted) { + return finish({ findings: [], status: "timeout", reason: timeoutReason }) + } + const stream = streamResult + const drain = (async () => { + for await (const event of stream.fullStream) { + // drain to avoid SDK hangs + if (event.type === "abort") streamAborted = true + if (event.type === "error") throw event.error + if (event.type === "finish-step") { + finishReason = event.finishReason ?? finishReason + rawFinishReason = event.rawFinishReason ?? rawFinishReason + providerMetadata = event.providerMetadata ?? providerMetadata + usage = mergeUsage(usage, readUsage(event.usage, providerMetadata)) + } + if (event.type === "finish") { + finishReason = event.finishReason ?? finishReason + rawFinishReason = event.rawFinishReason ?? rawFinishReason + usage = mergeUsage(usage, readUsage(event.totalUsage, providerMetadata)) + } + } + })() + const drainResult = await Promise.race([drain, abortPromise]) + if (drainResult === setupTimedOut || controller.signal.aborted || streamAborted) { + return finish({ findings: [], status: "timeout", reason: timeoutReason }) + } + const resultDetails = await Promise.race([ + Promise.all([ + Promise.resolve(stream.text), + Promise.resolve(stream.finishReason), + Promise.resolve(stream.rawFinishReason), + Promise.resolve(stream.totalUsage), + Promise.resolve(stream.providerMetadata), + ]), + abortPromise, + ]) + if (resultDetails === setupTimedOut || controller.signal.aborted) { + return finish({ findings: [], status: "timeout", reason: timeoutReason }) + } + const [text, streamFinishReason, streamRawFinishReason, streamUsage, streamProviderMetadata] = resultDetails + finishReason = streamFinishReason ?? finishReason + rawFinishReason = streamRawFinishReason ?? rawFinishReason + providerMetadata = streamProviderMetadata ?? providerMetadata + usage = mergeUsage(usage, readUsage(streamUsage, providerMetadata)) + const outputTruncated = isLengthFinishReason(finishReason) || isLengthFinishReason(rawFinishReason) + const truncated = (): AiReviewResult => + finish({ + findings: [], + status: "error", + reason: `output truncated at ${usage.completionTokens ?? maxOutputTokens} tokens (raise aiMaxOutputTokens)`, + }) + if (!text?.trim()) { + return outputTruncated ? truncated() : finish({ findings: [], status: "error", reason: "empty response" }) } - const text = await Promise.resolve(stream.text).catch((err: unknown) => { - log.error("ai review stream failed", { error: err }) - return undefined - }) - if (!text) return [] // Parse + clamp in core (the prompt-injection-resistant, advisory-only // contract). Returns already-validated, severity-clamped, file-checked items. - const parseRes = await Dispatcher.call("altimate_core.review_ai_parse", { - text, - valid_files: files.map((f) => f.path), - }) - const parsed = (((parseRes.data ?? {}) as Record).findings as any[]) ?? [] + let parseResult: Awaited> | typeof setupTimedOut + try { + parseResult = await Promise.race([ + Dispatcher.call("altimate_core.review_ai_parse", { + text, + valid_files: files.map((f) => f.path), + }), + abortPromise, + ]) + } catch (err) { + if (outputTruncated) return truncated() + throw err + } + if (parseResult === setupTimedOut || controller.signal.aborted) { + return finish({ findings: [], status: "timeout", reason: timeoutReason }) + } + const parseRes = parseResult + if (parseRes.success === false) { + if (outputTruncated) return truncated() + throw new Error(parseRes.error ?? "AI response parse failed") + } + const parsedValue = ((parseRes.data ?? {}) as Record).findings + if (!Array.isArray(parsedValue)) { + if (outputTruncated) return truncated() + throw new Error("AI response parse failed") + } + const parsed = parsedValue as any[] + if (outputTruncated && parsed.length < suggestedFindingCount(text)) return truncated() const byFile = new Map(files.map((f) => [f.path, f])) const out: Finding[] = [] @@ -169,11 +460,15 @@ export async function runAiReview(input: AiReviewInput): Promise { log.warn("skipping malformed ai finding", { error: err }) } } - log.info("ai review complete", { findings: out.length }) - return out + log.info("ai review complete", { findings: out.length, ...usage }) + return finish({ findings: out, status: "ok" }) } catch (err) { log.error("ai review failed", { error: err }) - return [] + if (noModelError(err)) return finish({ findings: [], status: "skipped", reason: NO_MODEL_REASON }) + if (controller.signal.aborted || streamAborted || (err as { name?: unknown } | undefined)?.name === "AbortError") { + return finish({ findings: [], status: "timeout", reason: timeoutReason }) + } + return finish({ findings: [], status: "error", reason: errorReason(err) }) } finally { clearTimeout(timeout) } diff --git a/packages/opencode/src/altimate/review/compiled.ts b/packages/opencode/src/altimate/review/compiled.ts index b804a0f33e..c3d16888a5 100644 --- a/packages/opencode/src/altimate/review/compiled.ts +++ b/packages/opencode/src/altimate/review/compiled.ts @@ -40,9 +40,11 @@ export interface CompiledResolverOptions { /** dbt project root (the dir containing `dbt_project.yml`). */ cwd: string projectName?: string - /** Directory holding HEAD-side compiled SQL (relative to cwd). */ + /** BASE-side dbt project name. Defaults to projectName. */ + baseProjectName?: string + /** Directory holding HEAD-side compiled SQL (relative to cwd, or absolute). */ headDir?: string - /** Directory holding BASE-side compiled SQL (relative to cwd). */ + /** Directory holding BASE-side compiled SQL (relative to cwd, or absolute). */ baseDir?: string /** Prefix within the repo-relative file path that maps to `cwd`. * For a monorepo where the dbt project lives at `packages/dbt/`, callers @@ -58,7 +60,8 @@ export interface CompiledResolverOptions { * undefined when no compiled artifact is present. */ export function makeCompiledResolver(opts: CompiledResolverOptions) { - const project = opts.projectName + const headProject = opts.projectName + const baseProject = opts.baseProjectName ?? headProject const headDir = opts.headDir ?? "target/compiled" const baseDir = opts.baseDir ?? "target-base/compiled" // Normalise the prefix so both "" and "." mean "no prefix". Match against @@ -72,6 +75,7 @@ export function makeCompiledResolver(opts: CompiledResolverOptions) { : "" return async (file: string, side: "old" | "new"): Promise => { + const project = side === "new" ? headProject : baseProject if (!project) return undefined // When the dbt project sits inside a subdir of the repo, `file` (from // `git diff --name-status`) is repo-root relative and always uses @@ -80,7 +84,8 @@ export function makeCompiledResolver(opts: CompiledResolverOptions) { // this the compiled resolver silently misses in monorepo layouts. const rel = prefix && (file === prefix || file.startsWith(prefix + "/")) ? file.slice(prefix.length + 1) : file const root = side === "new" ? headDir : baseDir - const compiledRoot = path.join(opts.cwd, root, project) + const rootAbs = path.isAbsolute(root) ? root : path.join(opts.cwd, root) + const compiledRoot = path.join(rootAbs, project) // Shared realpath containment check — matches makeContentResolver's // symlink-safe read so a future tweak to the containment logic can't // leave one call site behind (cubic + kilo suggestion). diff --git a/packages/opencode/src/altimate/review/config.ts b/packages/opencode/src/altimate/review/config.ts index 793121c510..8d587ce633 100644 --- a/packages/opencode/src/altimate/review/config.ts +++ b/packages/opencode/src/altimate/review/config.ts @@ -3,9 +3,14 @@ import path from "node:path" import { promises as fs } from "node:fs" import YAML from "yaml" import { Rubric, DEFAULT_RUBRIC } from "./rubric" -import { ReviewMode } from "./verdict" +import { ReviewMode, AiReasoningEffort } from "./verdict" import { Severity } from "./finding" +export const DEFAULT_AI_MAX_OUTPUT_TOKENS = 8_192 +// Defined next to the envelope schema so the accepted values and the signed +// envelope cannot drift; re-exported here for config/CLI callers. +export { AI_REASONING_EFFORTS, AiReasoningEffort } from "./verdict" + /** * Per-repo review configuration, read from `.altimate/review.yml` (the * analogue of Cloudflare's AGENTS.md). Lets each team tune the rubric, choose @@ -26,6 +31,20 @@ export const ReviewConfig = z.object({ dialect: z.string().default(""), /** Enable the advisory LLM reviewer lane (needs a configured model). */ ai: z.boolean().default(true), + /** + * provider/model for the advisory reviewer lane, e.g. + * altimate-gateway/altimate-base; when unset the headless CLI skips the lane + * and the in-session tool uses the session's model. + */ + // Model ids may themselves contain `/` (for example OpenRouter ids); the + // provider parser deliberately splits only on the first slash. + aiModel: z.string().regex(/^[^\s/]+\/\S+$/, "provider/model").optional(), + /** Per-request reasoning level. Unset leaves the model default unchanged. */ + aiReasoningEffort: AiReasoningEffort.optional(), + /** Advisory reviewer deadline. Unset uses the changed-file formula. */ + aiTimeoutSeconds: z.number().int().min(10).max(1800).optional(), + /** Total output budget, including reasoning tokens for reasoning models. */ + aiMaxOutputTokens: z.number().int().min(512).max(32_768).default(DEFAULT_AI_MAX_OUTPUT_TOKENS), /** * Data-diff: actually run base-vs-head against the warehouse (core DataParity) * and report row/value deltas. OPT-IN — it costs warehouse compute and needs a diff --git a/packages/opencode/src/altimate/review/dbt-patterns.ts b/packages/opencode/src/altimate/review/dbt-patterns.ts index 1ed240b083..069374f8fd 100644 --- a/packages/opencode/src/altimate/review/dbt-patterns.ts +++ b/packages/opencode/src/altimate/review/dbt-patterns.ts @@ -1572,6 +1572,7 @@ export function detectSchemaYmlPatterns( file: file.path, model: g.model, column: g.column, + groupKey: `grain_not_null:${g.model}`, confidence: "high", evidence: { tool: "dbt-patterns", diff --git a/packages/opencode/src/altimate/review/diff-filter.ts b/packages/opencode/src/altimate/review/diff-filter.ts index 4079fb6409..a05b46fbe7 100644 --- a/packages/opencode/src/altimate/review/diff-filter.ts +++ b/packages/opencode/src/altimate/review/diff-filter.ts @@ -34,12 +34,18 @@ const SKIP_FILE_PATTERNS = [ /** Extensions worth reviewing for a dbt project. */ const REVIEWABLE_EXT = [".sql", ".yml", ".yaml", ".csv", ".py"] +/** True when a path has a dbt source extension, before any path exclusions. */ +export function hasReviewableDbtExtension(path: string): boolean { + const p = path.replace(/\\/g, "/") + return REVIEWABLE_EXT.some((ext) => p.toLowerCase().endsWith(ext)) +} + /** True when a changed path should be reviewed (not a build artifact). */ export function shouldReview(path: string): boolean { const p = path.replace(/\\/g, "/") if (SKIP_DIR_PATTERNS.some((re) => re.test(p))) return false if (SKIP_FILE_PATTERNS.some((re) => re.test(p))) return false - return REVIEWABLE_EXT.some((ext) => p.toLowerCase().endsWith(ext)) + return hasReviewableDbtExtension(p) } export type DbtFileKind = diff --git a/packages/opencode/src/altimate/review/finding.ts b/packages/opencode/src/altimate/review/finding.ts index c9fb1d42e9..0ee52b1534 100644 --- a/packages/opencode/src/altimate/review/finding.ts +++ b/packages/opencode/src/altimate/review/finding.ts @@ -80,8 +80,11 @@ export const Finding = z.object({ model: z.string().optional(), /** Column the finding concerns, when applicable. */ column: z.string().optional(), + /** Stable presentation key for collapsing related findings in summaries. */ + groupKey: z.string().optional(), confidence: Confidence.default("high"), - /** True when produced without a manifest/warehouse (lint-only degraded run). */ + /** True when analysis ran without a manifest/warehouse (lint-only; blast radius unverified), + * or deterministic analysis could not decide (undecidable equivalence). */ degraded: z.boolean().default(false), evidence: Evidence.optional(), }) diff --git a/packages/opencode/src/altimate/review/format.ts b/packages/opencode/src/altimate/review/format.ts index 0befe5883c..fe4d21b1ae 100644 --- a/packages/opencode/src/altimate/review/format.ts +++ b/packages/opencode/src/altimate/review/format.ts @@ -9,6 +9,14 @@ import { type VerdictEnvelope } from "./verdict" export const REVIEW_MARKER = "" +export interface FindingDelta { + noLongerSurfaced: number + new: number + unchanged: number + reviewSettingsChanged?: boolean + analysisScopeChanged?: { from: VerdictEnvelope["tier"]; to: VerdictEnvelope["tier"] } +} + const SEVERITY_EMOJI: Record = { critical: "🛑", warning: "⚠️", @@ -21,6 +29,14 @@ const VERDICT_LABEL: Record = { REQUEST_CHANGES: "🛑 Changes requested", } +function codeSpan(value: string): string { + const runs = value.match(/`+/g) + const maxRun = runs ? Math.max(...runs.map((run) => run.length)) : 0 + const fence = "`".repeat(maxRun + 1) + const pad = /^`|`$/.test(value) ? " " : "" + return `${fence}${pad}${value}${pad}${fence}` +} + /** One-line headline used at the top of the summary and as the check title. */ export function verdictHeadline(env: VerdictEnvelope): string { const { critical, warning, suggestion } = env.summary @@ -37,13 +53,67 @@ export function verdictHeadline(env: VerdictEnvelope): string { } /** Full PR/MR summary comment body (markdown), prefixed with the dedup marker. */ -export function renderSummary(env: VerdictEnvelope): string { +export function renderSummary(env: VerdictEnvelope, delta?: FindingDelta): string { const lines: string[] = [REVIEW_MARKER, "", `## ${verdictHeadline(env)}`, ""] - if (env.summary.degraded) { + if (delta) { + const changeNote = delta.reviewSettingsChanged + ? " (review settings changed)" + : delta.analysisScopeChanged + ? ` (analysis scope changed: ${delta.analysisScopeChanged.from} → ${delta.analysisScopeChanged.to})` + : "" + lines.push( + `**Since last review:** ${delta.noLongerSurfaced} no longer surfaced · ${delta.new} new · ${delta.unchanged} unchanged` + + changeNote, + "", + ) + } + + const readFirst = selectReadFirst(env.findings) + if (readFirst.length) { + lines.push("**Read first**", "") + for (const finding of readFirst) { + lines.push(`- **${finding.title}** ${codeSpan(finding.file)}`) + } + lines.push("") + } + + if (env.summary.lintOnly ?? env.summary.degraded) { + lines.push( + "> ⚙️ Lint-only run — no changed model resolved against a dbt manifest (missing manifest, or the changed models are not in it). Run `dbt compile` on this branch so lineage/equivalence can run.", + "", + ) + } + + if (env.summary.emptyScope) { + if ( + env.summary.emptyScopeReason === "all_excluded" && + env.summary.emptyScopeFileCount !== undefined + ) { + lines.push( + `> ⚙️ Nothing to review — all ${env.summary.emptyScopeFileCount} changed dbt files are excluded by the review configuration (\`exclude\` globs)`, + "", + ) + } else { + lines.push("> ⚙️ Nothing to review — no dbt model, schema, or macro files changed in this diff.", "") + } + } + + const undecidableFindings = + env.summary.undecidableFindings ?? env.findings.filter((finding) => finding.degraded).length + if (undecidableFindings > 0) { lines.push( - "> ⚙️ **Lint-only run** — no dbt manifest/warehouse was available, so lineage, equivalence and", - "> data-impact checks were skipped. Wire `manifest_path` (and optionally warehouse creds) for the full verdict.", + `> ℹ️ ${undecidableFindings} finding${undecidableFindings === 1 ? "" : "s"} could not be decided — compiled SQL missing for base or head, unsupported SQL for this dialect, or no schema — see each finding.`, + "", + ) + } + + if (env.summary.artifactHints?.length) { + lines.push( + // Which analysis degrades depends on the artifact (compiled base SQL + // affects equivalence only; manifest/catalog affect lineage and impact), + // so the banner stays neutral and each finding names its own gap. + `> 🧩 Missing artifacts: ${env.summary.artifactHints.join(" · ")} — some analyses run at reduced fidelity; affected findings say which`, "", ) } @@ -57,21 +127,7 @@ export function renderSummary(env: VerdictEnvelope): string { // the signed envelope's tierReasons[]; the summary shows the first 8. if (env.tierReasons && env.tierReasons.length) { const RENDER_CAP = 8 - // Pick an inline-code-span fence longer than any backtick run inside `r` - // so a path like `packages/…/foo`bar`.sql` cannot terminate the span - // (cubic-review P3). - const shown = env.tierReasons - .slice(0, RENDER_CAP) - .map((r) => { - const runs = r.match(/`+/g) - const maxRun = runs ? Math.max(...runs.map((run) => run.length)) : 0 - const fence = "`".repeat(maxRun + 1) - // If the reason itself starts/ends with a backtick, pad with a space so - // the leading/trailing backtick isn't glued to the fence. - const pad = /^`|`$/.test(r) ? " " : "" - return `${fence}${pad}${r}${pad}${fence}` - }) - .join(", ") + const shown = env.tierReasons.slice(0, RENDER_CAP).map(codeSpan).join(", ") const overflow = env.tierReasons.length > RENDER_CAP ? ` (+${env.tierReasons.length - RENDER_CAP} more in verdict envelope)` @@ -79,19 +135,26 @@ export function renderSummary(env: VerdictEnvelope): string { lines.push(`> 🧭 **Tier: ${env.tier}** — ${shown}${overflow}`, "") } - if (!env.findings.length) { + if (!env.findings.length && !env.summary.emptyScope) { lines.push("No issues found in the changed dbt models. 🎉", "") } else { const grouped = groupBySeverity(env.findings) for (const sev of ["critical", "warning", "suggestion"] as const) { const items = grouped[sev] if (!items.length) continue - lines.push(`### ${SEVERITY_EMOJI[sev]} ${capitalize(sev)} (${items.length})`, "") - for (const f of items) { - const loc = f.file + (f.startLine ? `:${f.startLine}` : "") - lines.push( - `- **${f.title}** \n ${oneLine(f.body)} \n \`${loc}\`${f.degraded ? " · _unverified_" : ""} · ${f.category}`, - ) + const summaryGroups = groupForSummary(items) + const sectionCount = + summaryGroups.length === items.length + ? `${items.length}` + : `${items.length} findings · ${summaryGroups.length} items` + lines.push(`### ${SEVERITY_EMOJI[sev]} ${capitalize(sev)} (${sectionCount})`, "") + + const renderedItems = summaryGroups.map((group) => renderSummaryGroup(group, env.summary.artifactHints)) + const fold = sev !== "critical" && renderedItems.length > 12 + lines.push(...renderedItems.slice(0, fold ? 12 : renderedItems.length)) + if (fold) { + const remainder = renderedItems.slice(12) + lines.push("", "
", `${remainder.length} more …`, "", ...remainder, "", "
") } lines.push("") } @@ -105,12 +168,36 @@ export function renderSummary(env: VerdictEnvelope): string { ) } + if (env.summary.aiReview) { + const ai = env.summary.aiReview + // Name the model and configured reasoning on every status so a reader can tell what ran. + const details = [ai.model, ai.reasoningEffort ? `reasoning: ${ai.reasoningEffort}` : undefined].filter(Boolean) + const context = details.length ? ` (${details.join(", ")})` : "" + const duration = ai.durationMs === undefined ? "" : ` · ${Math.round(ai.durationMs / 1_000)}s` + if (ai.status === "ok") { + lines.push( + `🤖 AI reviewer${context}: ${ai.findings} advisory finding${ai.findings === 1 ? "" : "s"}${duration}`, + "", + ) + } else if (ai.status === "skipped") { + lines.push(`🤖 AI reviewer${context}: skipped${ai.reason ? ` — ${ai.reason}` : ""}${duration}`, "") + } else if (ai.status === "timeout") { + lines.push(`🤖 AI reviewer${context}: ${ai.reason ?? "timed out"}${duration}`, "") + } else { + lines.push(`🤖 AI reviewer${context}: error${ai.reason ? ` — ${ai.reason}` : ""}${duration}`, "") + } + } + lines.push( "---", `altimate dbt-pr-review · verdict \`${env.verdict}\`` + (env.signature ? ` · signed \`${env.signature.slice(0, 18)}…\`` : "") + (env.manifestHash ? ` · manifest \`${env.manifestHash.slice(0, 10)}\`` : "") + "", + "", + ``, + ...(env.policySignature ? [``] : []), + ``, ) return lines.join("\n") } @@ -140,6 +227,155 @@ function groupBySeverity(findings: Finding[]): Record { return out } +const MISSING_GRAIN_TITLE_FAMILY = "has no uniqueness/grain test" + +function summaryGroupKey(finding: Finding): string | undefined { + if (finding.groupKey) return `group:${finding.groupKey}` + if (finding.title.toLowerCase().includes(MISSING_GRAIN_TITLE_FAMILY)) { + return `title:${MISSING_GRAIN_TITLE_FAMILY}` + } + return undefined +} + +/** Group only the human summary; the envelope and inline comments remain atomic. */ +function groupForSummary(findings: Finding[]): Finding[][] { + const groups: Finding[][] = [] + const groupIndexes = new Map() + for (const finding of findings) { + const key = summaryGroupKey(finding) + if (!key) { + groups.push([finding]) + continue + } + const existing = groupIndexes.get(key) + if (existing === undefined) { + groupIndexes.set(key, groups.length) + groups.push([finding]) + } else { + groups[existing].push(finding) + } + } + return groups +} + +function selectReadFirst(findings: Finding[]): Finding[] { + if (findings.length < 8) return [] + + const selected: Finding[] = [] + const selectedIds = new Set() + const add = (finding: Finding) => { + if (selected.length < 3 && !selectedIds.has(finding.id)) { + selected.push(finding) + selectedIds.add(finding.id) + } + } + + for (const finding of findings) if (finding.severity === "critical") add(finding) + for (const finding of findings) if (finding.evidence?.tool === "ai-review") add(finding) + + const repetitiveWarningIds = new Set( + groupForSummary(findings.filter((finding) => finding.severity === "warning")) + .filter((group) => group.length >= 3) + .flatMap((group) => group.map((finding) => finding.id)), + ) + for (const finding of findings) { + if (finding.severity === "warning" && finding.confidence === "high" && !repetitiveWarningIds.has(finding.id)) { + add(finding) + } + } + return selected +} + +function titleFamily(finding: Finding): string { + const modelPrefix = finding.model ? `${finding.model}: ` : "" + return modelPrefix && finding.title.startsWith(modelPrefix) + ? finding.title.slice(modelPrefix.length) + : finding.title +} + +function groupedTitle(findings: Finding[]): string { + const family = titleFamily(findings[0]) + const newModel = /^new model has\s+(.+)$/i.exec(family) + if (newModel) return `${findings.length} new models have ${newModel[1]}` + return `${findings.length} findings: ${family}` +} + +function renderSummaryGroup(findings: Finding[], artifactHints?: string[]): string { + if (findings.length > 1) return renderGroupedFinding(findings, artifactHints) + const finding = findings[0] + const loc = finding.file + (finding.startLine ? `:${finding.startLine}` : "") + return ( + `- **${finding.title}** \n ${oneLine(finding.body)} \n ` + + `${codeSpan(loc)}${finding.degraded ? " · _unverified_" : ""} · ${finding.category}` + ) +} + +function renderGroupedFinding(findings: Finding[], artifactHints?: string[]): string { + const subjects = findings.map((finding) => codeSpan(finding.model ?? finding.file)).join(", ") + const categories = [...new Set(findings.map((finding) => finding.category))].join(", ") + const files = [...new Set(findings.map((finding) => finding.file))] + const locations = files.slice(0, 3).map(codeSpan).join(", ") + const more = files.length > 3 ? ` · +${files.length - 3} more` : "" + const unverified = findings.some((finding) => finding.degraded) ? " · _unverified_" : "" + const metadata = ` \n ${locations}${more}${unverified}` + + if (findings[0].groupKey === "lineage_fanout") { + const members = findings + .map((finding) => { + const subject = codeSpan(finding.model ?? finding.file) + const result = finding.evidence?.result + if (!result || typeof result !== "object") return subject + const impact = result as Record + if ( + typeof impact.directCount !== "number" || + typeof impact.transitiveCount !== "number" || + typeof impact.testCount !== "number" + ) { + return subject + } + return `${subject} (${impact.directCount} direct/${impact.transitiveCount} transitive, +${impact.testCount} tests)` + }) + .join(", ") + return `- **Downstream fan-out on ${findings.length} models** (informational) — ${members}${metadata}` + } + + if (findings[0].groupKey === "equivalence_undecided") { + const flatBody = oneLine(findings[0].body) + const cause = /equivalence could not be decided\s*\(([^)]+)\)/i.exec(flatBody)?.[1]?.trim() ?? flatBody + const sentence = /[.!?]$/.test(cause) ? cause : `${cause}.` + const remedy = artifactHints?.some((hint) => /\bcompiled\b/i.test(hint)) + ? "Fix once: compile base and head (see missing-artifact line)." + : "Undecidable with the available artifacts — unsupported SQL for this dialect or missing schema; verify with a data-diff." + return ( + `- **Equivalence could not be decided for ${findings.length} models** — ${sentence} ` + + `${remedy} Models: ${subjects}${metadata}` + ) + } + + if (findings[0].groupKey?.startsWith("grain_not_null:")) { + const model = findings[0].model ?? findings[0].groupKey.slice("grain_not_null:".length) + const columns = [ + ...new Set(findings.map((finding) => finding.column).filter((column): column is string => !!column)), + ] + .map(codeSpan) + .join(", ") + const flatBody = oneLine(findings[0].body) + const remediationStart = flatBody.lastIndexOf(" Add ") + let remediation = + remediationStart >= 0 + ? flatBody.slice(remediationStart + 1) + : "Add `not_null` coverage to each listed grain column." + if (findings[0].column) remediation = remediation.replace(codeSpan(findings[0].column), "each listed column") + return ( + `- **${codeSpan(model)}: grain columns without \`not_null\`** — ${columns || subjects} · ${categories} \n ` + + remediation + + metadata + ) + } + + return `- **${groupedTitle(findings)}** — ${subjects} · ${categories}${metadata}` +} + function capitalize(s: string): string { return s.charAt(0).toUpperCase() + s.slice(1) } diff --git a/packages/opencode/src/altimate/review/git.ts b/packages/opencode/src/altimate/review/git.ts index 1e73fb7327..c270ee488e 100644 --- a/packages/opencode/src/altimate/review/git.ts +++ b/packages/opencode/src/altimate/review/git.ts @@ -139,18 +139,34 @@ export async function gitRepoRoot(cwd: string): Promise { } } -/** Resolve a sensible default base ref (merge-base with origin/main/master). */ -export async function defaultBaseRef(cwd: string): Promise { +/** Resolve a sensible default base ref from the PR event or main/master. */ +export async function defaultBaseRef(cwd: string, head = "HEAD"): Promise { + const eventPath = process.env.GITHUB_EVENT_PATH + if (eventPath) { + try { + const event = JSON.parse(await fs.readFile(eventPath, "utf8")) + const ref = event?.pull_request?.base?.ref + if (typeof ref === "string" && ref) { + const candidate = `origin/${ref}` + const mb = (await git(["merge-base", head, candidate], cwd)).trim() + if (mb) return mb + } + } catch { + // Invalid/missing event or unavailable remote ref — use the normal fallbacks. + } + } + for (const candidate of ["origin/main", "origin/master", "main", "master"]) { try { - const mb = (await git(["merge-base", "HEAD", candidate], cwd)).trim() + const mb = (await git(["merge-base", head, candidate], cwd)).trim() if (mb) return mb } catch { // try next } } - // Fall back to the previous commit. - return "HEAD~1" + // Fall back to the selected head's parent so a custom `--head` is never + // compared against an unrelated checkout commit. + return `${head}~1` } /** Compute a short hash of the manifest file for the verdict envelope. */ diff --git a/packages/opencode/src/altimate/review/orchestrate.ts b/packages/opencode/src/altimate/review/orchestrate.ts index d0d89143a4..ce5f21a284 100644 --- a/packages/opencode/src/altimate/review/orchestrate.ts +++ b/packages/opencode/src/altimate/review/orchestrate.ts @@ -8,13 +8,21 @@ import { dedupe, SEVERITY_ORDER, } from "./finding" -import { type ChangedFile, filterChangedFiles } from "./diff-filter" +import { type ChangedFile, classifyDbtFile, filterChangedFiles } from "./diff-filter" import { classifyPR, compilePathTokenResolver, TIER_LANES } from "./risk-tier" import { type Rubric, exclusionReason, clampSeverity } from "./rubric" -import { type ReviewConfig } from "./config" -import { type ReviewMode, type VerdictEnvelope, buildEnvelope, signEnvelope } from "./verdict" +import { type AiReasoningEffort, type ReviewConfig } from "./config" +import { + type AiReviewSummary, + type ReviewMode, + type VerdictEnvelope, + NO_MODEL_REASON, + buildEnvelope, + makeReviewPolicySignature, + signEnvelope, +} from "./verdict" import { detectModelPatterns, detectSchemaYmlPatterns, splitDiff } from "./dbt-patterns" -import { type AiReviewInput } from "./ai-review" +import type { AiReviewInput, AiReviewResult } from "./ai-review" /** * The deterministic review recipe. @@ -77,6 +85,8 @@ function fnTokensOf(text: string): string[] { /** Impact-analysis result, normalized. */ export interface ImpactResult { hasManifest: boolean + /** True only when the requested model resolved to a manifest node. */ + resolved: boolean /** SAFE | LOW | MEDIUM | HIGH | BREAKING (from the DAG walk). */ severity: "SAFE" | "LOW" | "MEDIUM" | "HIGH" | "BREAKING" | "UNKNOWN" directCount: number @@ -186,9 +196,22 @@ export interface OrchestrateInput { * Optional LLM reviewer lane. Injected (not imported) so the orchestrator * stays pure/unit-testable: production wires `runAiReview` (harness LLM); * tests pass a fake or omit it (the lane is then skipped). Receives the - * deterministic findings as grounding and returns ADVISORY findings only. + * deterministic findings as grounding and returns ADVISORY findings plus the + * lane status. */ - aiReview?: (input: AiReviewInput) => Promise + aiReview?: (input: AiReviewInput) => Promise + /** Explicit provider/model for the advisory lane, when configured. */ + aiModel?: string + /** Per-request reasoning level for the advisory lane, when configured. */ + aiReasoningEffort?: AiReasoningEffort + /** Whether this caller may fall back to its current session model. */ + allowSessionModel?: boolean + /** Active provider/model supplied by the interactive tool context. */ + sessionModel?: string + /** Effective advisory reviewer deadline resolved by the entry point. */ + aiTimeoutMs?: number + /** Effective advisory reviewer output budget resolved by the entry point. */ + aiMaxOutputTokens?: number /** PR metadata passed to the AI reviewer for intent checking. */ prTitle?: string prBody?: string @@ -198,6 +221,8 @@ export interface OrchestrateInput { forceTier?: "trivial" | "lite" | "full" /** Manifest was stale relative to change-affecting files (see run.ts::detectStaleManifest). */ staleManifest?: boolean + /** Missing dbt build artifacts detected by the entry point. */ + artifactHints?: string[] } /** Derive the dbt model name from a model file path. */ @@ -300,6 +325,7 @@ function lineageBreakageLane(file: ChangedFile & { kind: string }, impact: Impac (degraded ? "\n\n_Lint-only: no manifest, blast radius unverified._" : ""), file: file.path, model, + groupKey: "lineage_fanout", confidence: degraded ? "unknown" : "high", degraded, evidence: { tool: "impact_analysis", result: impact }, @@ -339,6 +365,7 @@ async function semanticChangeLane( ` (no schema, or unsupported SQL). Treat as a potential behavior change and verify with a data-diff.`, file: file.path, model, + groupKey: "equivalence_undecided", confidence: "unknown", degraded: true, evidence: { tool: "altimate_core.equivalence", result: { decided: false } }, @@ -755,6 +782,7 @@ async function missingGrainTestLane(ctx: ModelContext, runner: ReviewRunner): Pr `Add a uniqueness test on the model's grain so the contract is enforced in CI.`, file: file.path, model, + groupKey: "missing_grain_test", confidence: "medium", evidence: { tool: "dbt.manifest", result: { hasGrainTest: false } }, ruleKey: "test_coverage:missing-grain-test", @@ -1056,6 +1084,7 @@ interface ModelContext { } export async function runReview(input: OrchestrateInput): Promise { + const changedDbtFileCount = input.changedFiles.filter((file) => classifyDbtFile(file.path) !== "other").length const reviewable = filterChangedFiles(input.changedFiles, input.rubric.exclusions.excludeGlobs) const dialect = input.config.dialect const getContent = input.getContent @@ -1066,10 +1095,6 @@ export async function runReview(input: OrchestrateInput): Promise f.kind === "model_sql" || f.kind === "python_model") const ctxByPath = new Map() - let anyManifest = false - if (input.runner.manifestAvailable) { - anyManifest = await input.runner.manifestAvailable().catch(() => false) - } await Promise.all( modelFiles.map(async (file) => { const model = modelNameFromPath(file.path) @@ -1085,16 +1110,28 @@ export async function runReview(input: OrchestrateInput): Promise 0 ? !anyManifest : reviewable.length === 0 + const nonDeletedModelContexts = [...ctxByPath.values()].filter((ctx) => ctx.file.status !== "deleted") + const anyResolved = nonDeletedModelContexts.some((ctx) => ctx.impact.resolved) + let manifestAvailable = [...ctxByPath.values()].some((ctx) => ctx.impact.hasManifest) + if (modelFiles.length > 0 && nonDeletedModelContexts.length === 0 && input.runner.manifestAvailable) { + manifestAvailable = await input.runner.manifestAvailable().catch(() => false) + } + const lintOnly = + modelFiles.length > 0 && (nonDeletedModelContexts.length > 0 ? !anyResolved : !manifestAvailable) + const emptyScope = reviewable.length === 0 + const emptyScopeReason = !emptyScope + ? undefined + : changedDbtFileCount === 0 + ? "no_dbt_files" + : "all_excluded" // High-risk path tokens are user-configured (billing/pci/patient/etc.) — // the reviewer core carries no default list. `undefined` when no @@ -1150,6 +1187,24 @@ export async function runReview(input: OrchestrateInput): Promise ({ path: ctx.file.path, status: ctx.file.status, @@ -1399,17 +1462,57 @@ export async function runReview(input: OrchestrateInput): Promise SEVERITY_ORDER[b.severity] - SEVERITY_ORDER[a.severity] || a.file.localeCompare(b.file)) - const degraded = runDegraded || findings.some((f) => f.degraded) + if (aiReviewSummary) { + aiReviewSummary = { + ...aiReviewSummary, + findings: findings.filter((f) => f.evidence?.tool === "ai-review").length, + } + } + const envelope = buildEnvelope({ findings, tier, mode: input.mode, + policySignature, rubric: input.rubric, engine: { core: input.coreVersion, model: input.modelVersion, cliVersion: input.cliVersion }, manifestHash: input.manifestHash, staleManifest: input.staleManifest, generatedAt: input.generatedAt, - degraded, + lintOnly, + emptyScope, + emptyScopeReason, + emptyScopeFileCount: emptyScopeReason === "all_excluded" ? changedDbtFileCount : undefined, + artifactHints: input.artifactHints, + aiReview: aiReviewSummary, // Include tierReasons whenever `--explain-tier` / `--force-tier` is set, // when a riskTierPathTokens config error was caught above, OR when the // classifier lands on `full` tier — a naturally-full run is a customer- diff --git a/packages/opencode/src/altimate/review/post-github.ts b/packages/opencode/src/altimate/review/post-github.ts index 9ef43286f5..250fc87e77 100644 --- a/packages/opencode/src/altimate/review/post-github.ts +++ b/packages/opencode/src/altimate/review/post-github.ts @@ -1,7 +1,13 @@ import { Octokit } from "@octokit/rest" import { promises as fs } from "node:fs" import { type VerdictEnvelope, VCS_EVENT } from "./verdict" -import { renderSummary, inlineComments, REVIEW_MARKER, verdictHeadline } from "./format" +import { + renderSummary, + inlineComments, + REVIEW_MARKER, + verdictHeadline, + type FindingDelta, +} from "./format" /** * Post a verdict envelope to a GitHub pull request: an upserted summary comment @@ -56,22 +62,103 @@ export interface PostResult { postError?: string } -export async function postGitHubReview(env: VerdictEnvelope, target: GitHubTarget): Promise { - const octo = new Octokit({ auth: target.token }) +function lastLineMarker(body: string, pattern: RegExp): string | undefined { + let value: string | undefined + for (const match of body.matchAll(pattern)) value = match[1] + return value +} + +/** Parse the finding fingerprint block appended to an Altimate summary. */ +export function parseFindingIds(body: string | null | undefined): Set | undefined { + if (!body) return undefined + const value = lastLineMarker(body, /^[ \t]*\r?$/gm) + if (value === undefined) return undefined + return new Set( + value + .split(",") + .map((id) => id.trim()) + .filter(Boolean), + ) +} + +function parsePolicySignature(body: string | null | undefined): string | undefined { + if (!body) return undefined + return lastLineMarker(body, /^[ \t]*\r?$/gm) +} + +function parseTier(body: string | null | undefined): VerdictEnvelope["tier"] | undefined { + if (!body) return undefined + return lastLineMarker(body, /^[ \t]*\r?$/gm) as + | VerdictEnvelope["tier"] + | undefined +} + +/** Compare the prior sticky comment's finding ids with the current envelope. */ +export function computeFindingDelta( + previousBody: string | null | undefined, + current: VerdictEnvelope, +): FindingDelta | undefined { + const previousIds = parseFindingIds(previousBody) + if (!previousIds) return undefined + const currentIds = new Set(current.findings.map((finding) => finding.id)) + if (previousIds.size === 0 && currentIds.size === 0) return undefined + const previousPolicySignature = parsePolicySignature(previousBody) + const previousTier = parseTier(previousBody) + const reviewSettingsChanged = + previousPolicySignature && current.policySignature && previousPolicySignature !== current.policySignature + ? true + : undefined + const reviewSettingsUnchanged = + previousPolicySignature !== undefined && + current.policySignature !== undefined && + previousPolicySignature === current.policySignature + return { + noLongerSurfaced: [...previousIds].filter((id) => !currentIds.has(id)).length, + new: [...currentIds].filter((id) => !previousIds.has(id)).length, + unchanged: [...currentIds].filter((id) => previousIds.has(id)).length, + reviewSettingsChanged, + analysisScopeChanged: + reviewSettingsUnchanged && previousTier && previousTier !== current.tier + ? { from: previousTier, to: current.tier } + : undefined, + } +} + +export async function postGitHubReview( + env: VerdictEnvelope, + target: GitHubTarget, + octo: Octokit = new Octokit({ auth: target.token }), +): Promise { const { owner, repo, prNumber } = target const result: PostResult = { inlineFellBack: false } // 1. Upsert the summary comment (dedup by marker). Paginate ALL comments — // on a busy PR the prior marker comment can be past the first page, and // missing it would post a duplicate summary on every rerun. - const summary = renderSummary(env) + // Identity precedence: explicit override > token's own user > Actions bot. + // A GitHub App installation token resolves neither `GET /user` nor `GET /app` + // (the latter needs an app JWT), so App users must name their bot login + // (`[bot]`) via ALTIMATE_REVIEW_BOT_LOGIN or every rerun would + // post a fresh summary. Arbitrary Bot authors are deliberately not trusted. + let authenticatedLogin = process.env.ALTIMATE_REVIEW_BOT_LOGIN?.trim() || undefined + if (!authenticatedLogin) { + try { + authenticatedLogin = (await octo.rest.users.getAuthenticated()).data.login + } catch { + // Fall through to the Actions bot fallback below. + } + } const existing = await octo.paginate(octo.rest.issues.listComments, { owner, repo, issue_number: prNumber, per_page: 100, }) - const prior = existing.find((c) => c.body?.includes(REVIEW_MARKER)) + const hasMarker = (comment: (typeof existing)[number]) => comment.body?.includes(REVIEW_MARKER) + const prior = authenticatedLogin + ? existing.find((comment) => hasMarker(comment) && comment.user?.login === authenticatedLogin) + : existing.find((comment) => hasMarker(comment) && comment.user?.login === "github-actions[bot]") + const summary = renderSummary(env, computeFindingDelta(prior?.body, env)) if (prior) { const r = await octo.rest.issues.updateComment({ owner, repo, comment_id: prior.id, body: summary }) result.summaryCommentId = r.data.id diff --git a/packages/opencode/src/altimate/review/run.ts b/packages/opencode/src/altimate/review/run.ts index 9e8db4cb7b..41010f854e 100644 --- a/packages/opencode/src/altimate/review/run.ts +++ b/packages/opencode/src/altimate/review/run.ts @@ -1,7 +1,7 @@ import path from "node:path" -import { readFile, realpath, stat, access } from "node:fs/promises" +import { readFile, readdir, realpath, stat, access } from "node:fs/promises" import { Installation } from "../../installation" -import { loadReviewConfig, resolveRubric } from "./config" +import { loadReviewConfig, resolveRubric, type AiReasoningEffort } from "./config" import type { Severity } from "./finding" import { collectChangedFiles, makeContentResolver, defaultBaseRef, gitRepoRoot, manifestHash } from "./git" import { makeCompiledResolver, dbtProjectName } from "./compiled" @@ -10,7 +10,7 @@ import { createDispatcherRunner } from "./runner" import { runReview } from "./orchestrate" import { runAiReview } from "./ai-review" import type { ReviewMode, VerdictEnvelope } from "./verdict" -import type { ChangedFile } from "./diff-filter" +import { filterChangedFiles, type ChangedFile } from "./diff-filter" /** * End-to-end review entry point: load `.altimate/review.yml`, collect the diff, @@ -42,6 +42,18 @@ export interface ReviewPullRequestOptions { cliVersion?: string /** Disable the LLM reviewer lane (default: enabled; self-degrades if no model). */ noAi?: boolean + /** Override the advisory reviewer provider/model from config. */ + aiModel?: string + /** Override the advisory reviewer reasoning level from config. */ + aiReasoningEffort?: AiReasoningEffort + /** Override the advisory reviewer deadline. */ + aiTimeoutMs?: number + /** Override the advisory reviewer output budget. */ + aiMaxOutputTokens?: number + /** Allow falling back to the current session model (default: false). */ + allowSessionModel?: boolean + /** Active provider/model supplied by an interactive tool invocation. */ + sessionModel?: string /** PR metadata for the AI reviewer's intent check. */ prTitle?: string prBody?: string @@ -127,6 +139,132 @@ async function autoDiscoverManifest(cwd: string): Promise<{ path: string; projec } } +interface CompiledArtifactDirs { + headDir: string + baseDir: string +} + +async function compiledArtifactDirs(manifestAbs: string, dbtRoot: string): Promise { + const manifestDir = path.dirname(manifestAbs) + const headDir = path.join(manifestDir, "compiled") + const siblingBaseDir = path.join(`${manifestDir}-base`, "compiled") + let baseDir = path.join(dbtRoot, "target-base", "compiled") + try { + await access(siblingBaseDir) + baseDir = siblingBaseDir + } catch { + /* keep the conventional target-base/compiled fallback */ + } + return { headDir, baseDir } +} + +function artifactDirLabel(dir: string, dbtRoot: string): string { + return path.relative(dbtRoot, dir).split(path.sep).join("/") || path.basename(dir) +} + +async function resolveBaseProjectName( + baseDir: string, + projectName: string, + dbtRoot: string, +): Promise { + const baseRoot = path.isAbsolute(baseDir) ? baseDir : path.join(dbtRoot, baseDir) + try { + if ((await stat(path.join(baseRoot, projectName))).isDirectory()) return projectName + } catch { + /* look for a renamed base project below */ + } + + try { + const directories = (await readdir(baseRoot, { withFileTypes: true })).filter((entry) => entry.isDirectory()) + if (directories.length === 1) return directories[0]!.name + if (directories.length > 1) return undefined + } catch { + /* keep the head project name when the base compiled directory is absent */ + } + return projectName +} + +/** Report missing dbt artifacts only when the manifest itself exists. */ +export async function detectArtifactHints( + manifestAbs: string, + dbtRoot: string, + changedModels: Array> = [], + projectName?: string, + pathPrefix?: string, + artifactDirs?: CompiledArtifactDirs, + baseProjectName?: string, +): Promise { + if (changedModels.length === 0 || changedModels.every((file) => file.status === "deleted")) return [] + + try { + await access(manifestAbs) + } catch { + return [] + } + + const hints: string[] = [] + try { + const catalog = JSON.parse(await readFile(path.join(path.dirname(manifestAbs), "catalog.json"), "utf8")) + const nonEmptyObject = (value: unknown): value is Record => + value !== null && typeof value === "object" && !Array.isArray(value) && Object.keys(value).length > 0 + const hasUsableEntry = (entries: unknown) => + nonEmptyObject(entries) && + Object.values(entries).some((entry) => nonEmptyObject(entry) && nonEmptyObject(entry.columns)) + if (!hasUsableEntry(catalog?.nodes) && !hasUsableEntry(catalog?.sources)) { + hints.push("catalog.json unreadable or empty (regenerate with `dbt docs generate`)") + } + } catch (error) { + if ((error as { code?: string }).code === "ENOENT") { + hints.push("catalog.json (run `dbt docs generate`)") + } else { + hints.push("catalog.json unreadable or empty (regenerate with `dbt docs generate`)") + } + } + + if (projectName === undefined) { + hints.push( + "dbt project name not resolved — no readable dbt_project.yml next to the manifest, so compiled SQL cannot be located", + ) + return hints + } + + const { headDir, baseDir } = artifactDirs ?? (await compiledArtifactDirs(manifestAbs, dbtRoot)) + const resolvedBaseProjectName = + baseProjectName ?? (await resolveBaseProjectName(baseDir, projectName, dbtRoot)) + if (resolvedBaseProjectName === undefined) { + hints.push( + `${artifactDirLabel(baseDir, dbtRoot)} has several project directories; expected \`${projectName}\``, + ) + } + const getCompiled = makeCompiledResolver({ + cwd: dbtRoot, + projectName, + baseProjectName: resolvedBaseProjectName, + pathPrefix, + headDir, + baseDir, + }) + const baseModels = changedModels.filter((file) => file.status !== "added" && file.status !== "deleted") + const headModels = changedModels.filter((file) => file.status !== "deleted") + const [missingBase, missingHead] = await Promise.all([ + resolvedBaseProjectName === undefined + ? Promise.resolve(0) + : Promise.all(baseModels.map((file) => getCompiled(file.oldPath ?? file.path, "old"))).then( + (contents) => contents.filter((content) => content === undefined || !content.trim()).length, + ), + Promise.all(headModels.map((file) => getCompiled(file.path, "new"))).then( + (contents) => contents.filter((content) => content === undefined || !content.trim()).length, + ), + ]) + if (missingBase > 0) { + hints.push(`${artifactDirLabel(baseDir, dbtRoot)} missing for ${missingBase} changed model(s) (compile the base ref)`) + } + if (missingHead > 0) { + hints.push(`${artifactDirLabel(headDir, dbtRoot)} missing for ${missingHead} changed model(s) (run \`dbt compile\` for the head)`) + } + return hints +} + /** Whether a repo-relative path is one whose modification could invalidate * the compiled manifest — dbt source (SQL, YAML, Python models, seed CSV, * docs markdown blocks) or top-level dbt config. `README.md` at repo root, @@ -211,6 +349,15 @@ export async function reviewPullRequest(opts: ReviewPullRequestOptions): Promise if (opts.manifestPath) config.manifestPath = opts.manifestPath if (opts.mode) config.mode = opts.mode if (opts.severityThreshold) config.severityThreshold = opts.severityThreshold + // Treat --no-ai as an effective config override so the policy signature + // cannot depend on a model that this invocation will never use. + if (opts.noAi) config.ai = false + const aiModel = opts.aiModel ?? config.aiModel + const aiReasoningEffort = opts.aiReasoningEffort ?? config.aiReasoningEffort + const aiTimeoutMs = + opts.aiTimeoutMs ?? (config.aiTimeoutSeconds === undefined ? undefined : config.aiTimeoutSeconds * 1_000) + const aiMaxOutputTokens = opts.aiMaxOutputTokens ?? config.aiMaxOutputTokens + const allowSessionModel = opts.allowSessionModel ?? false const rubric = resolveRubric(config) // Only resolve a base ref if we actually need git (to collect changed files @@ -218,8 +365,9 @@ export async function reviewPullRequest(opts: ReviewPullRequestOptions): Promise // `getContent` (e.g. a non-git CI integration) must not be forced through a // git lookup that can fail when there's no usable history. const needGit = !opts.changedFiles || !opts.getContent - const base = opts.base ?? (needGit ? await defaultBaseRef(opts.cwd) : "") - const changedFiles = opts.changedFiles ?? (await collectChangedFiles({ base, head: opts.head, cwd: opts.cwd })) + const base = opts.base ?? (needGit ? await defaultBaseRef(opts.cwd, opts.head || "HEAD") : "") + const changedFiles = + opts.changedFiles ?? (await collectChangedFiles({ base, head: opts.head || undefined, cwd: opts.cwd })) // Resolve the repo top-level once; used to root working-tree FS reads, the // stale-manifest existence check, and the compiled-SQL resolver's path // mapping. Falls back to opts.cwd when we couldn't resolve it (non-git or @@ -232,7 +380,7 @@ export async function reviewPullRequest(opts: ReviewPullRequestOptions): Promise changedFiles.filter((f) => f.status === "renamed" && f.oldPath).map((f) => [f.path, f.oldPath as string]), ) const getContent = - opts.getContent ?? makeContentResolver({ base, head: opts.head, cwd: opts.cwd, renames, gitRoot }) + opts.getContent ?? makeContentResolver({ base, head: opts.head || undefined, cwd: opts.cwd, renames, gitRoot }) // Resolve the manifest against the PROJECT being reviewed (cwd), not the // binary's process.cwd() — otherwise a relative path silently misses when the @@ -325,9 +473,26 @@ export async function reviewPullRequest(opts: ReviewPullRequestOptions): Promise /* keep original values on realpath failure */ } const pathPrefix = path.relative(gitRootReal, dbtRootReal) + const changedModels = filterChangedFiles(changedFiles, rubric.exclusions.excludeGlobs).filter( + (file) => file.kind === "model_sql" || file.kind === "python_model", + ) + const manifestReal = await realpath(manifestAbs).catch(() => manifestAbs) + const artifactDirs = await compiledArtifactDirs(manifestReal, dbtRootReal) + const baseProjectName = projectName + ? await resolveBaseProjectName(artifactDirs.baseDir, projectName, dbtRootReal) + : undefined + const artifactHints = await detectArtifactHints( + manifestAbs, + dbtRootReal, + changedModels, + projectName, + pathPrefix, + artifactDirs, + baseProjectName, + ) const getCompiled = opts.getContent ? undefined - : makeCompiledResolver({ cwd: dbtRootReal, projectName, pathPrefix }) + : makeCompiledResolver({ cwd: dbtRootReal, projectName, baseProjectName, pathPrefix, ...artifactDirs }) return runReview({ changedFiles, @@ -347,11 +512,21 @@ export async function reviewPullRequest(opts: ReviewPullRequestOptions): Promise // so a missing default silently regresses envelope provenance for tool- // path verdicts (cubic-review PR #1041). cliVersion: opts.cliVersion ?? Installation.VERSION, - aiReview: opts.noAi || config.ai === false ? undefined : runAiReview, + aiReview: + opts.noAi || config.ai === false + ? async () => ({ findings: [], status: "skipped" as const, reason: "disabled by configuration" }) + : runAiReview, + aiModel, + aiReasoningEffort, + aiTimeoutMs, + aiMaxOutputTokens, + allowSessionModel, + sessionModel: opts.sessionModel, prTitle: opts.prTitle, prBody: opts.prBody, explainTier: opts.explainTier, forceTier: opts.forceTier, staleManifest, + artifactHints, }) } diff --git a/packages/opencode/src/altimate/review/runner.ts b/packages/opencode/src/altimate/review/runner.ts index eec3732d81..473480160b 100644 --- a/packages/opencode/src/altimate/review/runner.ts +++ b/packages/opencode/src/altimate/review/runner.ts @@ -268,11 +268,18 @@ export function createDispatcherRunner(opts: DispatcherRunnerOptions): ReviewRun async impact(model: string): Promise { const mf = await loadManifest() if (!mf.ok) { - return { hasManifest: false, severity: "UNKNOWN", directCount: 0, transitiveCount: 0, testCount: 0 } + return { + hasManifest: false, + resolved: false, + severity: "UNKNOWN", + directCount: 0, + transitiveCount: 0, + testCount: 0, + } } const target = mf.byName.get(model) ?? [...mf.models.values()].find((m) => m.name.endsWith(`.${model}`)) if (!target) { - return { hasManifest: true, severity: "SAFE", directCount: 0, transitiveCount: 0, testCount: 0 } + return { hasManifest: true, resolved: false, severity: "SAFE", directCount: 0, transitiveCount: 0, testCount: 0 } } const direct = new Set(mf.children.get(target.unique_id) ?? []) const all = new Set(direct) @@ -298,6 +305,7 @@ export function createDispatcherRunner(opts: DispatcherRunnerOptions): ReviewRun const severity = total === 0 ? "SAFE" : total <= 3 ? "LOW" : total <= 10 ? "MEDIUM" : "HIGH" return { hasManifest: true, + resolved: true, severity, directCount: direct.size, transitiveCount: transitive.length, diff --git a/packages/opencode/src/altimate/review/telemetry.ts b/packages/opencode/src/altimate/review/telemetry.ts index fc20cc3b17..9be32711af 100644 --- a/packages/opencode/src/altimate/review/telemetry.ts +++ b/packages/opencode/src/altimate/review/telemetry.ts @@ -7,6 +7,7 @@ // Caller attribution needs no code: neither event declares a `source` field, so the envelope's // process-level `source` (from Flag.ALTIMATE_CLI_CLIENT) passes through untouched. A caller that // exports that variable is attributed automatically; one that does not reports `cli`. +import { createHash } from "node:crypto" import { Telemetry } from "../telemetry" import { ReviewCategory, type Finding } from "./finding" import type { VerdictEnvelope } from "./verdict" @@ -14,6 +15,32 @@ import type { PostResult } from "./post-github" export type ReviewInvocation = "cli" | "tool" +const KNOWN_AI_MODEL_PROVIDERS = new Set([ + "altimate-backend", + "altimate-gateway", + "openai", + "anthropic", + "google", + "google-vertex", + "google-vertex-anthropic", + "amazon-bedrock", + "github-copilot", + "github-copilot-enterprise", + "openrouter", + "opencode", + "opencode-go", + "azure", + "mistral", + "groq", + "deepseek", + "xai", +]) + +function telemetryAiModel(model: string | undefined): string | undefined { + if (model === undefined || KNOWN_AI_MODEL_PROVIDERS.has(model.split("/", 1)[0]!)) return model + return `custom/${createHash("sha256").update(model).digest("hex").slice(0, 8)}` +} + /** * Count surfaced findings by category, zero-filled across the whole enum. * @@ -109,7 +136,19 @@ export function emitReviewRun(input: { tier: env.tier, // Optional in the schema and explicitly invalid as `false`, so normalise rather than copy. tier_forced: env.tierForced === true, + // Compatibility field: degraded covers either run-level reduced scope, + // never an individual undecidable finding. degraded: env.summary.degraded, + lint_only: env.summary.lintOnly ?? (env.summary.degraded && !env.summary.emptyScope), + empty_scope: env.summary.emptyScope ?? false, + undecidable_findings: + env.summary.undecidableFindings ?? env.findings.filter((finding) => finding.degraded).length, + ai_status: env.summary.aiReview?.status, + ai_model: telemetryAiModel(env.summary.aiReview?.model), + ai_findings: env.summary.aiReview?.findings ?? 0, + ai_duration_ms: env.summary.aiReview?.durationMs, + ai_prompt_chars: env.summary.aiReview?.promptChars, + ai_reasoning_tokens: env.summary.aiReview?.reasoningTokens, stale_manifest: env.staleManifest === true, critical: env.summary.critical, warning: env.summary.warning, @@ -130,7 +169,7 @@ export function emitReviewRun(input: { * once-ness with a latch plus a `finally`; see cli/cmd/review.ts. */ export function emitReviewPostOutcome(input: { - outcome: "not_requested" | "not_attempted" | "target_unresolved" | "full" | "partial" | "summary_failed" + outcome: "not_requested" | "not_attempted" | "target_unresolved" | "full" | "partial" | "summary_failed" | "forbidden" durationMs: number sessionID: string }): void { diff --git a/packages/opencode/src/altimate/review/verdict.ts b/packages/opencode/src/altimate/review/verdict.ts index fe7441bb81..a82f936142 100644 --- a/packages/opencode/src/altimate/review/verdict.ts +++ b/packages/opencode/src/altimate/review/verdict.ts @@ -78,6 +78,117 @@ export function applyMode(verdict: Verdict, mode: ReviewMode): Verdict { export const RiskTier = z.enum(["trivial", "lite", "full"]) export type RiskTier = z.infer +export const EmptyScopeReason = z.enum(["no_dbt_files", "all_excluded"]) +export type EmptyScopeReason = z.infer + +export const AiReviewStatus = z.enum(["ok", "skipped", "timeout", "error"]) +export type AiReviewStatus = z.infer +export const NO_MODEL_REASON = + "no AI model configured (set aiModel in .altimate/review.yml, --ai-model, or the action's model inputs)" + +/** + * Reasoning levels the advisory lane may request per run. Single source of + * truth for config, CLI and the envelope; config.ts re-exports it because it + * already depends on this module. + */ +export const AI_REASONING_EFFORTS = ["none", "minimal", "low", "medium", "high"] as const +export const AiReasoningEffort = z.enum(AI_REASONING_EFFORTS) +export type AiReasoningEffort = z.infer + +export const AiReviewSummary = z.object({ + status: AiReviewStatus, + reason: z.string().optional(), + findings: z.number().int().nonnegative(), + /** Effective provider/model used by the advisory lane. */ + model: z.string().optional(), + /** Per-request reasoning level used by the advisory lane. */ + reasoningEffort: AiReasoningEffort.optional(), + durationMs: z.number().int().nonnegative().optional(), + promptChars: z.number().int().nonnegative().optional(), + promptTokens: z.number().int().nonnegative().optional(), + completionTokens: z.number().int().nonnegative().optional(), + reasoningTokens: z.number().int().nonnegative().optional(), +}) +export type AiReviewSummary = z.infer + +export interface ReviewPolicySignatureInput { + severityThreshold: Severity + enabledReviewers: string[] + dialect: string + rubric: Rubric + /** Whether the advisory AI lane is effectively enabled for this run. */ + aiEnabled: boolean + /** Explicit advisory model, or `session` when the active session is used. */ + aiModel?: string + dataDiff: { + enabled: boolean + warehouse: string + } +} + +function normalizePolicyValue(value: unknown): unknown { + if (Array.isArray(value)) return value.map(normalizePolicyValue) + if (value && typeof value === "object") { + return Object.fromEntries( + Object.entries(value) + .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)) + .map(([key, item]) => [key, normalizePolicyValue(item)]), + ) + } + return value +} + +/** Compact fingerprint for the settings that determine which findings surface. */ +export function makeReviewPolicySignature(input: ReviewPolicySignatureInput): string { + const { excludeGlobs, ...booleanExclusions } = input.rubric.exclusions + const enabledExclusions = Object.entries(booleanExclusions) + .filter(([, enabled]) => enabled) + .map(([name]) => name) + .sort() + const body = JSON.stringify( + normalizePolicyValue({ + severityThreshold: input.severityThreshold, + enabledReviewers: [...new Set(input.enabledReviewers)].sort(), + dialect: input.dialect, + exclusions: { + excludeGlobs: [...new Set(excludeGlobs)].sort(), + enabled: enabledExclusions, + }, + rubric: { + version: input.rubric.version, + blockOn: [...new Set(input.rubric.blockOn)].sort(), + warningPatternThreshold: input.rubric.warningPatternThreshold, + thresholds: input.rubric.thresholds, + }, + ai: input.aiEnabled ? { model: input.aiModel } : "ai:off", + dataDiff: input.dataDiff, + }), + ) + return createHash("sha256").update(body).digest("hex").slice(0, 16) +} + +const ReviewSummary = z.object({ + critical: z.number().int().nonnegative(), + warning: z.number().int().nonnegative(), + suggestion: z.number().int().nonnegative(), + /** Compatibility flag for lintOnly or emptyScope. Never reflects individual undecidable findings. */ + degraded: z.boolean(), + /** True when no changed model resolved against a dbt manifest. */ + lintOnly: z.boolean().optional(), + /** True when the diff contains no reviewable dbt files. */ + emptyScope: z.boolean().optional(), + /** Why the review scope is empty. */ + emptyScopeReason: EmptyScopeReason.optional(), + /** Number of changed dbt files removed by path/configuration filters. */ + emptyScopeFileCount: z.number().int().positive().optional(), + /** Surfaced findings whose deterministic analysis could not decide. */ + undecidableFindings: z.number().int().nonnegative().optional(), + /** Missing dbt artifacts that reduce lineage/equivalence fidelity. */ + artifactHints: z.array(z.string()).optional(), + /** Advisory AI lane outcome, when that lane applied. */ + aiReview: AiReviewSummary.optional(), +}) + export const EngineVersions = z.object({ reviewer: z.string().default("dbt-pr-review/1"), core: z.string().optional(), @@ -96,6 +207,8 @@ export const VerdictEnvelope = z.object({ idealVerdict: Verdict, mode: ReviewMode, tier: RiskTier, + /** Fingerprint of the user-configured review settings used for this run. */ + policySignature: z.string().optional(), /** G1 — reasons the classifier assigned this tier (only when --explain-tier). */ tierReasons: z.array(z.string()).optional(), /** G2 — true when --force-tier bypassed the classifier. Included in signature so @@ -104,13 +217,7 @@ export const VerdictEnvelope = z.object({ /** The classifier's original tier before --force-tier overrode it (G2). */ tierClassified: RiskTier.optional(), findings: z.array(Finding), - summary: z.object({ - critical: z.number().int().nonnegative(), - warning: z.number().int().nonnegative(), - suggestion: z.number().int().nonnegative(), - /** True when the review ran without a manifest/warehouse (lint-only). */ - degraded: z.boolean(), - }), + summary: ReviewSummary, engine: EngineVersions, /** Hash of the dbt manifest the verdict was computed against, when present. */ manifestHash: z.string().optional(), @@ -169,7 +276,19 @@ export interface BuildEnvelopeInput { engine?: Partial manifestHash?: string generatedAt?: string + /** Run-level lint-only flag. */ + lintOnly?: boolean + /** Run-level empty-review-scope flag. */ + emptyScope?: boolean + /** Why the review scope is empty. */ + emptyScopeReason?: EmptyScopeReason + /** Number of changed dbt files removed by path/configuration filters. */ + emptyScopeFileCount?: number + /** Compatibility input alias for lintOnly. */ degraded?: boolean + artifactHints?: string[] + aiReview?: AiReviewSummary + policySignature?: string /** G1 — classifier reasons for the tier (only surfaced when explainTier=true). */ tierReasons?: string[] /** G2 — set when --force-tier was applied. */ @@ -180,10 +299,30 @@ export interface BuildEnvelopeInput { staleManifest?: boolean } -function summarize(findings: Finding[], degraded: boolean): VerdictEnvelope["summary"] { +function summarize( + findings: Finding[], + lintOnly: boolean, + emptyScope: boolean | undefined, + emptyScopeReason: EmptyScopeReason | undefined, + emptyScopeFileCount: number | undefined, + artifactHints: string[], + aiReview?: AiReviewSummary, +): VerdictEnvelope["summary"] { const tally: Record = { critical: 0, warning: 0, suggestion: 0 } for (const f of findings) tally[f.severity]++ - return { critical: tally.critical, warning: tally.warning, suggestion: tally.suggestion, degraded } + return { + critical: tally.critical, + warning: tally.warning, + suggestion: tally.suggestion, + degraded: lintOnly || emptyScope === true, + lintOnly, + emptyScope, + emptyScopeReason, + emptyScopeFileCount, + undecidableFindings: findings.filter((f) => f.degraded).length, + artifactHints, + aiReview, + } } /** Assemble the verdict envelope (unsigned). Call signEnvelope to sign it. */ @@ -191,18 +330,27 @@ export function buildEnvelope(input: BuildEnvelopeInput): VerdictEnvelope { const rubric = input.rubric ?? DEFAULT_RUBRIC const ideal = computeIdealVerdict(input.findings, rubric) const verdict = applyMode(ideal, input.mode) - const degraded = input.degraded ?? input.findings.some((f) => f.degraded) + const lintOnly = input.lintOnly ?? input.degraded ?? false return VerdictEnvelope.parse({ version: "1", verdict, idealVerdict: ideal, mode: input.mode, tier: input.tier, + policySignature: input.policySignature, tierReasons: input.tierReasons, tierForced: input.tierForced, tierClassified: input.tierClassified, findings: input.findings, - summary: summarize(input.findings, degraded), + summary: summarize( + input.findings, + lintOnly, + input.emptyScope, + input.emptyScopeReason, + input.emptyScopeFileCount, + input.artifactHints ?? [], + input.aiReview, + ), engine: EngineVersions.parse(input.engine ?? {}), manifestHash: input.manifestHash, staleManifest: input.staleManifest ? true : undefined, diff --git a/packages/opencode/src/altimate/telemetry/index.ts b/packages/opencode/src/altimate/telemetry/index.ts index a036aa42b9..0e9f090e22 100644 --- a/packages/opencode/src/altimate/telemetry/index.ts +++ b/packages/opencode/src/altimate/telemetry/index.ts @@ -1129,10 +1129,26 @@ export namespace Telemetry { mode?: string tier?: string tier_forced?: boolean - /** The envelope's fidelity flag: no reviewable files, no usable manifest for the changed - * models, OR a surfaced finding whose engine analysis was undecidable. It does NOT mean - * merely "no warehouse". */ + /** Compatibility run-level reduced-scope flag (`lint_only` or `empty_scope`). */ degraded?: boolean + /** True when reviewable models exist but no dbt manifest was available. */ + lint_only?: boolean + /** True when no reviewable dbt files remain after filtering. */ + empty_scope?: boolean + /** Count of surfaced findings whose deterministic analysis could not decide. */ + undecidable_findings?: number + /** Advisory AI reviewer outcome when the lane applied. */ + ai_status?: "ok" | "skipped" | "timeout" | "error" + /** Effective provider/model used by the advisory AI reviewer. */ + ai_model?: string + /** Surfaced advisory AI findings after filtering and deduplication. */ + ai_findings?: number + /** Wall-clock duration of the advisory AI lane. */ + ai_duration_ms?: number + /** Character count of the advisory review prompt. */ + ai_prompt_chars?: number + /** Provider-reported reasoning tokens, when available. */ + ai_reasoning_tokens?: number stale_manifest?: boolean critical?: number warning?: number @@ -1155,7 +1171,14 @@ export namespace Telemetry { * completed review and the post attempt (a bad `--output` path, a stdout write error). * Emitted from the caller's `finally` so a completed review always carries exactly one * post outcome. */ - outcome: "not_requested" | "not_attempted" | "target_unresolved" | "full" | "partial" | "summary_failed" + outcome: + | "not_requested" + | "not_attempted" + | "target_unresolved" + | "full" + | "partial" + | "summary_failed" + | "forbidden" duration_ms: number } // altimate_change end diff --git a/packages/opencode/src/altimate/tools/dbt-pr-review.ts b/packages/opencode/src/altimate/tools/dbt-pr-review.ts index 16ce290242..1bfb4d57d8 100644 --- a/packages/opencode/src/altimate/tools/dbt-pr-review.ts +++ b/packages/opencode/src/altimate/tools/dbt-pr-review.ts @@ -1,12 +1,27 @@ import z from "zod" import { Tool } from "../../tool/tool" import { Instance } from "../../project/instance" +import { MessageV2 } from "../../session/message-v2" import { reviewPullRequest } from "../review/run" // altimate_change — review feature telemetry import { emitReviewRun } from "../review/telemetry" import { renderSummary, verdictHeadline } from "../review/format" import { ReviewMode } from "../review/verdict" +export async function sessionModelFromContext( + ctx: Pick, +): Promise { + let message: ReturnType + try { + message = MessageV2.get({ sessionID: ctx.sessionID, messageID: ctx.messageID }) + } catch { + return undefined + } + if (message?.info.role !== "assistant") return undefined + if (!message.info.providerID || !message.info.modelID) return undefined + return `${message.info.providerID}/${message.info.modelID}` +} + /** * dbt_pr_review — run the full deterministic dbt PR review and return a signed * verdict envelope. The reviewer agent calls this to produce a review backed by @@ -43,6 +58,7 @@ export const DbtPrReviewTool = Tool.define("dbt_pr_review", { const startedAt = Date.now() let env try { + const sessionModel = await sessionModelFromContext(ctx) env = await reviewPullRequest({ cwd, base: args.base, @@ -50,6 +66,8 @@ export const DbtPrReviewTool = Tool.define("dbt_pr_review", { manifestPath: args.manifest_path, mode: args.mode, modelVersion: ctx.agent, + allowSessionModel: true, + sessionModel, }) } catch (err) { emitReviewRun({ diff --git a/packages/opencode/src/cli/cmd/review.ts b/packages/opencode/src/cli/cmd/review.ts index 717588c3dd..641f7cfbd1 100644 --- a/packages/opencode/src/cli/cmd/review.ts +++ b/packages/opencode/src/cli/cmd/review.ts @@ -3,6 +3,8 @@ import { UI } from "../ui" import { cmd } from "./cmd" import { bootstrap } from "../bootstrap" import { Installation } from "../../installation" +import { Instance } from "../../project/instance" +import { Telemetry } from "../../altimate/telemetry" import { reviewPullRequest } from "../../altimate/review/run" import { renderSummary } from "../../altimate/review/format" import { postGitHubReview, resolveGitHubTarget } from "../../altimate/review/post-github" @@ -10,6 +12,87 @@ import { postGitHubReview, resolveGitHubTarget } from "../../altimate/review/pos import { classifyPostOutcome, emitReviewPostOutcome, emitReviewRun } from "../../altimate/review/telemetry" import type { ReviewMode } from "../../altimate/review/verdict" import type { Severity } from "../../altimate/review/finding" +import { + AI_REASONING_EFFORTS, + AiReasoningEffort, + type AiReasoningEffort as AiReasoningEffortValue, +} from "../../altimate/review/config" + +const MAX_GITHUB_PR_BODY_CHARS = 4_000 + +function nonBlank(value: string | undefined): string | undefined { + const trimmed = value?.trim() + return trimmed || undefined +} + +function boundedInteger(value: unknown, name: string, min: number, max: number): number | undefined { + if (value === undefined || value === null || value === "") return undefined + const parsed = typeof value === "number" ? value : Number(value) + if (!Number.isInteger(parsed) || parsed < min || parsed > max) { + throw new Error(`${name} must be an integer between ${min} and ${max}`) + } + return parsed +} + +function parseAiReasoningEffort(value: string | undefined): AiReasoningEffortValue | undefined { + const normalized = nonBlank(value) + if (normalized === undefined) return undefined + const parsed = AiReasoningEffort.safeParse(normalized) + if (!parsed.success) { + throw new Error( + `--ai-reasoning / ALTIMATE_REVIEW_AI_REASONING must be one of: ${AI_REASONING_EFFORTS.join(", ")}`, + ) + } + return parsed.data +} + +function requestStatus(err: unknown): number | undefined { + const value = err as { status?: unknown; response?: { status?: unknown } } | undefined + const status = value?.status ?? value?.response?.status + return typeof status === "number" ? status : undefined +} + +async function readGitHubPullRequestMetadata(): Promise<{ prTitle?: string; prBody?: string }> { + const eventPath = process.env.GITHUB_EVENT_PATH + if (!eventPath) return {} + try { + const event = JSON.parse(await fs.readFile(eventPath, "utf8")) as { + pull_request?: { title?: unknown; body?: unknown } + } + const title = event.pull_request?.title + const body = event.pull_request?.body + return { + prTitle: typeof title === "string" ? title : undefined, + prBody: typeof body === "string" ? body.slice(0, MAX_GITHUB_PR_BODY_CHARS) : undefined, + } + } catch { + return {} + } +} + +/** True when the GitHub event is a pull request from a fork: the job then runs with a + * read-only token and cannot post, so a 401/403 on --post is expected rather than a + * misconfiguration. Anything else (same-repo PR, missing event) is NOT tolerated. */ +async function isForkPullRequestEvent(): Promise { + const eventPath = process.env.GITHUB_EVENT_PATH + if (!eventPath) return false + try { + const event = JSON.parse(await fs.readFile(eventPath, "utf8")) as { + pull_request?: { + head?: { repo?: { fork?: unknown; full_name?: unknown } } + base?: { repo?: { full_name?: unknown } } + } + } + const head = event.pull_request?.head?.repo + const base = event.pull_request?.base?.repo + if (head?.fork === true) return true + return ( + typeof head?.full_name === "string" && typeof base?.full_name === "string" && head.full_name !== base.full_name + ) + } catch { + return false + } +} /** * `altimate review` — run the dbt PR review locally or in CI. @@ -56,6 +139,23 @@ export const ReviewCommand = cmd({ default: false, describe: "disable the advisory LLM reviewer lane (no model calls / cost)", }) + .option("ai-model", { + type: "string", + describe: "provider/model for the advisory LLM reviewer lane (overrides config)", + }) + .option("ai-reasoning", { + type: "string", + choices: AI_REASONING_EFFORTS, + describe: "AI reviewer reasoning level (overrides environment and config)", + }) + .option("ai-timeout", { + type: "number", + describe: "AI reviewer timeout in seconds (10..1800; overrides environment and config)", + }) + .option("ai-max-output-tokens", { + type: "number", + describe: "AI reviewer output budget (512..32768; overrides config)", + }) .option("explain-tier", { type: "boolean", default: false, @@ -69,6 +169,22 @@ export const ReviewCommand = cmd({ .option("cwd", { type: "string", describe: "project directory (default: current dir)" }), async handler(args) { const cwd = (args.cwd as string) || process.cwd() + const aiTimeoutSeconds = boundedInteger( + args.aiTimeout ?? nonBlank(process.env.ALTIMATE_REVIEW_AI_TIMEOUT_SECONDS), + "--ai-timeout / ALTIMATE_REVIEW_AI_TIMEOUT_SECONDS", + 10, + 1800, + ) + const aiReasoningEffort = parseAiReasoningEffort( + nonBlank(args.aiReasoning as string | undefined) ?? nonBlank(process.env.ALTIMATE_REVIEW_AI_REASONING), + ) + const aiMaxOutputTokens = boundedInteger( + args.aiMaxOutputTokens, + "--ai-max-output-tokens", + 512, + 32_768, + ) + const prMetadata = await readGitHubPullRequestMetadata() if (args.forceTier) { process.stderr.write( `⚠️ --force-tier=${args.forceTier} is EXPERIMENTAL (bench / debug only). ` + @@ -76,6 +192,7 @@ export const ReviewCommand = cmd({ ) } await bootstrap(cwd, async () => { + Telemetry.setContext({ sessionId: "", projectId: Instance.project?.id ?? "" }) // altimate_change — time the engine only. Output writing and posting happen after this and // must not be counted as review latency, nor turn a computed review into a failed one. const startedAt = Date.now() @@ -91,8 +208,16 @@ export const ReviewCommand = cmd({ // With `boolean-negation: false` above, `--no-ai` binds to `noAi` and // the historical `--ai=false` programmatic path stays supported. noAi: args.noAi === true || args.ai === false, + aiModel: + nonBlank(args.aiModel as string | undefined) ?? nonBlank(process.env.ALTIMATE_REVIEW_AI_MODEL), + aiReasoningEffort, + aiTimeoutMs: aiTimeoutSeconds === undefined ? undefined : aiTimeoutSeconds * 1_000, + aiMaxOutputTokens, + allowSessionModel: false, explainTier: args.explainTier === true, forceTier: args.forceTier as "trivial" | "lite" | "full" | undefined, + prTitle: prMetadata.prTitle, + prBody: prMetadata.prBody, // Stamp the CLI version into engine.cliVersion so an auditor can // reconstruct which policy version generated a stored verdict long // after the binary that ran it is gone. @@ -160,18 +285,30 @@ export const ReviewCommand = cmd({ r = await postGitHubReview(env, target) } catch (err) { // A throw here means the summary comment itself failed; nothing was published. - emitPostOnce("summary_failed", postDuration()) - throw err + const status = requestStatus(err) + // Only a fork PR (read-only token) may swallow an auth failure; on a same-repo PR + // a 401/403 means a bad token or missing `pull-requests: write` and must fail. + if ((status === 401 || status === 403) && (await isForkPullRequestEvent())) { + emitPostOnce("forbidden", postDuration()) + UI.println(`could not post the review: ${status} (fork pull request, read-only token); printing summary instead`) + // Keep stdout machine-readable under --json: the human summary goes to stderr. + if (args.json) UI.println(renderSummary(env)) + } else { + emitPostOnce("summary_failed", postDuration()) + throw err + } } - emitPostOnce(classifyPostOutcome(r), postDuration()) - const where = `${target.owner}/${target.repo}#${target.prNumber}` - if (r.postError) { - UI.println(`⚠️ Posted the summary comment to ${where}, but the review event failed: ${r.postError}`) - } else { - UI.println( - `Posted review to ${where}` + - (r.inlineFellBack ? " (inline comments fell back to summary-only)" : ""), - ) + if (r) { + emitPostOnce(classifyPostOutcome(r), postDuration()) + const where = `${target.owner}/${target.repo}#${target.prNumber}` + if (r.postError) { + UI.println(`⚠️ Posted the summary comment to ${where}, but the review event failed: ${r.postError}`) + } else { + UI.println( + `Posted review to ${where}` + + (r.inlineFellBack ? " (inline comments fell back to summary-only)" : ""), + ) + } } } } diff --git a/packages/opencode/src/session/llm.ts b/packages/opencode/src/session/llm.ts index 4f85b10c5b..6dbec12edc 100644 --- a/packages/opencode/src/session/llm.ts +++ b/packages/opencode/src/session/llm.ts @@ -57,6 +57,12 @@ export namespace LLM { tools: Record retries?: number toolChoice?: "auto" | "required" | "none" + // altimate_change start — explicit request controls for callers such as the dbt PR review's + // AI lane, whose reasoning models spend most of the default budget thinking before answering + maxOutputTokens?: number + /** Provider reasoning level for this request; unset preserves the model default. */ + reasoningEffort?: string + // altimate_change end } export type StreamOutput = StreamTextResult @@ -144,6 +150,9 @@ export namespace LLM { if (isCodex) { options.instructions = SystemPrompt.instructions() } + // altimate_change start — let advisory callers override reasoning for one request + if (input.reasoningEffort !== undefined) options.reasoningEffort = input.reasoningEffort + // altimate_change end // altimate_change start — pass maxOutputTokens INTO chat.params hook so plugins // (codex, github-copilot, third-party) can override it. Upstream PRs #21220 + #21225 @@ -167,9 +176,10 @@ export namespace LLM { topP: input.agent.topP ?? ProviderTransform.topP(input.model), topK: ProviderTransform.topK(input.model), maxOutputTokens: - isCodex || provider.id.includes("github-copilot") + input.maxOutputTokens ?? + (isCodex || provider.id.includes("github-copilot") ? undefined - : ProviderTransform.maxOutputTokens(input.model), + : ProviderTransform.maxOutputTokens(input.model)), options, }, ) diff --git a/packages/opencode/test/altimate/review-ai.test.ts b/packages/opencode/test/altimate/review-ai.test.ts new file mode 100644 index 0000000000..2960feb2ea --- /dev/null +++ b/packages/opencode/test/altimate/review-ai.test.ts @@ -0,0 +1,513 @@ +import { afterEach, describe, expect, mock, spyOn, test } from "bun:test" +import { Provider } from "@/provider/provider" +import { LLM } from "@/session/llm" +import { MessageV2 } from "@/session/message-v2" +import { Dispatcher } from "@/altimate/native" +import { runAiReview, type AiReviewFile } from "@/altimate/review/ai-review" +import { NO_MODEL_REASON } from "@/altimate/review/verdict" +import { sessionModelFromContext } from "@/altimate/tools/dbt-pr-review" + +afterEach(() => mock.restore()) + +function stubModelAndPrompt(parseResult: any = { data: { findings: [] } }) { + let parseCalls = 0 + spyOn(Provider as any, "defaultModel").mockImplementation(async () => ({ + providerID: "test-provider", + modelID: "test-model", + })) + spyOn(Provider as any, "getModel").mockImplementation(async () => ({ + providerID: "test-provider", + id: "test-model", + modelID: "test-model", + })) + spyOn(Dispatcher as any, "call").mockImplementation(async (method: string) => { + if (method === "altimate_core.review_ai_prompt") return { data: { prompt: "Review the change." } } + if (method === "altimate_core.review_ai_parse") { + parseCalls++ + return parseResult + } + throw new Error(`unexpected dispatcher method: ${method}`) + }) + return () => parseCalls +} + +function reviewFile(index: number): AiReviewFile { + return { + path: `models/model_${index}.sql`, + status: "modified", + model: `model_${index}`, + diff: "+select 1\n", + } +} + +describe("runAiReview model selection", () => { + test("skips without consulting the session model when no explicit model is configured", async () => { + const defaultModel = spyOn(Provider as any, "defaultModel") + const getModel = spyOn(Provider as any, "getModel") + const dispatcher = spyOn(Dispatcher as any, "call") + + const result = await runAiReview({ + files: [reviewFile(0)], + grounding: [], + allowSessionModel: false, + }) + + expect(result).toMatchObject({ findings: [], status: "skipped", reason: NO_MODEL_REASON, promptChars: 0 }) + expect(result.durationMs).toBeGreaterThanOrEqual(0) + expect(defaultModel).not.toHaveBeenCalled() + expect(getModel).not.toHaveBeenCalled() + expect(dispatcher).not.toHaveBeenCalled() + }) + + test("returns an error naming an unavailable configured model without falling back", async () => { + const defaultModel = spyOn(Provider as any, "defaultModel") + const getModel = spyOn(Provider as any, "getModel").mockRejectedValue(new Error("provider is not configured")) + const dispatcher = spyOn(Dispatcher as any, "call") + const stream = spyOn(LLM as any, "stream") + + const result = await runAiReview({ + files: [reviewFile(0)], + grounding: [], + model: "openrouter/openai/gpt-5", + allowSessionModel: false, + }) + + expect(result).toMatchObject({ + findings: [], + status: "error", + reason: + "configured AI model not available: openrouter/openai/gpt-5 — Error: provider is not configured", + model: "openrouter/openai/gpt-5", + }) + expect(getModel).toHaveBeenCalledWith("openrouter", "openai/gpt-5") + expect(defaultModel).not.toHaveBeenCalled() + expect(dispatcher).not.toHaveBeenCalled() + expect(stream).not.toHaveBeenCalled() + }) + + test("uses the active session model without consulting the provider default", async () => { + const defaultModel = spyOn(Provider as any, "defaultModel") + const getModel = spyOn(Provider as any, "getModel").mockResolvedValue({ + providerID: "openrouter", + id: "openai/gpt-5", + modelID: "openai/gpt-5", + }) + spyOn(Dispatcher as any, "call").mockResolvedValue({ data: {} }) + + const result = await runAiReview({ + files: [reviewFile(0)], + grounding: [], + allowSessionModel: true, + sessionModel: "openrouter/openai/gpt-5", + }) + + expect(result).toMatchObject({ + findings: [], + status: "skipped", + reason: "reviewer prompt unavailable", + model: "openrouter/openai/gpt-5", + }) + expect(getModel).toHaveBeenCalledWith("openrouter", "openai/gpt-5") + expect(defaultModel).not.toHaveBeenCalled() + }) + + test("reports a core that failed to build the prompt as an error, not a skip", async () => { + const defaultModel = spyOn(Provider as any, "defaultModel") + spyOn(Provider as any, "getModel").mockResolvedValue({ + providerID: "openrouter", + id: "openai/gpt-5", + modelID: "openai/gpt-5", + }) + // The native handler catches its own exception and reports it this way. + // The message may carry internal URLs; it must be redacted like every + // other reason that reaches a PR comment. + spyOn(Dispatcher as any, "call").mockResolvedValue({ + success: false, + data: {}, + error: "prompt template missing at https://core.internal/prompts/review?token=abc123", + }) + + const result = await runAiReview({ + files: [reviewFile(0)], + grounding: [], + allowSessionModel: true, + sessionModel: "openrouter/openai/gpt-5", + }) + + expect(result).toMatchObject({ + findings: [], + status: "error", + reason: "reviewer prompt failed: prompt template missing at ", + model: "openrouter/openai/gpt-5", + }) + expect(defaultModel).not.toHaveBeenCalled() + }) + + test("reads the active assistant model from the invoking tool context", async () => { + const getMessage = spyOn(MessageV2 as any, "get").mockReturnValue({ + info: { + role: "assistant", + providerID: "openrouter", + modelID: "openai/gpt-5", + }, + }) + + const model = await sessionModelFromContext({ sessionID: "session-id", messageID: "message-id" } as any) + + expect(model).toBe("openrouter/openai/gpt-5") + expect(getMessage).toHaveBeenCalledWith({ sessionID: "session-id", messageID: "message-id" }) + }) +}) + +describe("runAiReview stream handling", () => { + test("returns timeout when pre-stream setup never resolves", async () => { + spyOn(Provider as any, "defaultModel").mockResolvedValue({ + providerID: "test-provider", + modelID: "test-model", + }) + spyOn(Provider as any, "getModel").mockResolvedValue({ + providerID: "test-provider", + id: "test-model", + modelID: "test-model", + }) + spyOn(Dispatcher as any, "call").mockImplementation( + (() => new Promise(() => {})) as any, + ) + const stream = spyOn(LLM as any, "stream") + + const result = await runAiReview({ + files: [reviewFile(0)], + grounding: [], + allowSessionModel: true, + timeoutMs: 5, + }) + + expect(result).toMatchObject({ + findings: [], + status: "timeout", + reason: "timed out after 0.005s (raise aiTimeoutSeconds / --ai-timeout)", + model: "test-provider/test-model", + }) + expect(stream).not.toHaveBeenCalled() + }) + + test("caps the timeout to the files included in the prompt", async () => { + stubModelAndPrompt() + const nativeSetTimeout = globalThis.setTimeout + const delays: number[] = [] + spyOn(globalThis as any, "setTimeout").mockImplementation( + ((callback: (...args: any[]) => void, delay?: number, ...args: any[]) => { + delays.push(Number(delay)) + return nativeSetTimeout(callback, delay, ...args) + }) as any, + ) + const stream = spyOn(LLM as any, "stream").mockImplementation(async () => ({ + fullStream: { + async *[Symbol.asyncIterator]() {}, + }, + text: Promise.resolve("[]"), + })) + + const result = await runAiReview({ + files: Array.from({ length: 25 }, (_, index) => reviewFile(index)), + grounding: [], + allowSessionModel: true, + }) + + expect(result).toMatchObject({ findings: [], status: "ok", model: "test-provider/test-model" }) + const request = stream.mock.calls[0][0] as { messages: Array<{ content: string }> } + expect(result.promptChars).toBe(request.messages[0]!.content.length) + expect(delays).toContain(200_000) + expect(stream.mock.calls[0][0]).toMatchObject({ maxOutputTokens: 8_192 }) + }) + + test("passes an explicit output budget and reasoning level to the LLM stream", async () => { + stubModelAndPrompt() + const stream = spyOn(LLM as any, "stream").mockImplementation(async () => ({ + fullStream: { + async *[Symbol.asyncIterator]() {}, + }, + text: Promise.resolve("[]"), + })) + + await runAiReview({ + files: [reviewFile(0)], + grounding: [], + allowSessionModel: true, + maxOutputTokens: 12_288, + reasoningEffort: "minimal", + }) + + expect(stream.mock.calls[0][0]).toMatchObject({ + maxOutputTokens: 12_288, + reasoningEffort: "minimal", + }) + }) + + test("reports provider usage, including reasoning tokens from provider metadata", async () => { + stubModelAndPrompt() + spyOn(LLM as any, "stream").mockImplementation(async () => ({ + fullStream: { + async *[Symbol.asyncIterator]() { + yield { + type: "finish-step", + finishReason: "stop", + rawFinishReason: "stop", + usage: { inputTokens: 321, outputTokens: 144 }, + providerMetadata: { + openai: { usage: { completion_tokens_details: { reasoning_tokens: 89 } } }, + }, + } + yield { + type: "finish", + finishReason: "stop", + rawFinishReason: "stop", + totalUsage: { inputTokens: 321, outputTokens: 144 }, + } + }, + }, + text: Promise.resolve("[]"), + })) + + const result = await runAiReview({ files: [reviewFile(0)], grounding: [], allowSessionModel: true }) + + expect(result).toMatchObject({ + status: "ok", + promptTokens: 321, + completionTokens: 144, + reasoningTokens: 89, + }) + }) + + test("reports a truncation error instead of partial findings", async () => { + const parseCalls = stubModelAndPrompt() + spyOn(LLM as any, "stream").mockImplementation(async () => ({ + fullStream: { + async *[Symbol.asyncIterator]() { + yield { + type: "finish", + finishReason: "length", + rawFinishReason: "max_tokens", + totalUsage: { inputTokens: 500, outputTokens: 4_096 }, + } + }, + }, + text: Promise.resolve('[{"file":"models/model_0.sql","title":"partial","body":"partial"}]'), + })) + + const result = await runAiReview({ files: [reviewFile(0)], grounding: [], allowSessionModel: true }) + + expect(result).toMatchObject({ + findings: [], + status: "error", + reason: "output truncated at 4096 tokens (raise aiMaxOutputTokens)", + completionTokens: 4_096, + }) + expect(parseCalls()).toBe(1) + }) + + test("reports max_tokens truncation when the core parser fails", async () => { + const parseCalls = stubModelAndPrompt({ success: false, data: {}, error: "invalid JSON" }) + spyOn(LLM as any, "stream").mockImplementation(async () => ({ + fullStream: { + async *[Symbol.asyncIterator]() { + yield { + type: "finish-step", + finishReason: "other", + rawFinishReason: "max_tokens", + usage: {}, + } + }, + }, + text: Promise.resolve("["), + })) + + const result = await runAiReview({ + files: [reviewFile(0)], + grounding: [], + allowSessionModel: true, + maxOutputTokens: 6_144, + }) + + expect(result).toMatchObject({ + findings: [], + status: "error", + reason: "output truncated at 6144 tokens (raise aiMaxOutputTokens)", + }) + expect(parseCalls()).toBe(1) + }) + + test("treats whitespace-only model output as an empty response", async () => { + const parseCalls = stubModelAndPrompt() + spyOn(LLM as any, "stream").mockImplementation(async () => ({ + fullStream: { + async *[Symbol.asyncIterator]() {}, + }, + text: Promise.resolve(" \n\t "), + })) + + const result = await runAiReview({ files: [reviewFile(0)], grounding: [], allowSessionModel: true }) + + expect(result).toMatchObject({ + findings: [], + status: "error", + reason: "empty response", + model: "test-provider/test-model", + }) + expect(parseCalls()).toBe(0) + }) + + test("classifies typed provider model errors as an unconfigured model", async () => { + spyOn(Provider as any, "defaultModel").mockImplementation(async () => ({ + providerID: "test-provider", + modelID: "missing-model", + })) + spyOn(Provider as any, "getModel").mockRejectedValue( + new Provider.ModelNotFoundError({ + providerID: "test-provider" as any, + modelID: "missing-model" as any, + suggestions: [], + }), + ) + spyOn(Dispatcher as any, "call").mockImplementation(async (method: string) => { + if (method === "altimate_core.review_ai_prompt") return { data: { prompt: "Review the change." } } + throw new Error(`unexpected dispatcher method: ${method}`) + }) + const stream = spyOn(LLM as any, "stream") + + const result = await runAiReview({ files: [reviewFile(0)], grounding: [], allowSessionModel: true }) + + expect(result).toMatchObject({ findings: [], status: "skipped", reason: NO_MODEL_REASON }) + expect(stream).not.toHaveBeenCalled() + }) + + test("returns timeout without parsing partial text when the signal aborts before the stream resolves", async () => { + const parseCalls = stubModelAndPrompt() + let fireTimeout: (() => void) | undefined + let signal: AbortSignal | undefined + spyOn(globalThis as any, "setTimeout").mockImplementation(((callback: () => void) => { + fireTimeout = callback + return 1 + }) as any) + + let resolveText: (value: string) => void = () => {} + const text = new Promise((resolve) => { + resolveText = resolve + }) + spyOn(LLM as any, "stream").mockImplementation(async (input: { abort: AbortSignal }) => { + signal = input.abort + return { + fullStream: { + async *[Symbol.asyncIterator]() { + fireTimeout?.() + resolveText('[{"file":"models/model_0.sql","title":"partial","body":"partial"}]') + }, + }, + text, + } + }) + + const result = await runAiReview({ files: [reviewFile(0)], grounding: [], allowSessionModel: true }) + + expect(signal?.aborted).toBe(true) + expect(result).toMatchObject({ + findings: [], + status: "timeout", + reason: "timed out after 124s (raise aiTimeoutSeconds / --ai-timeout)", + model: "test-provider/test-model", + }) + expect(parseCalls()).toBe(0) + }) + + test("returns timeout without reading text when fullStream stalls", async () => { + const parseCalls = stubModelAndPrompt() + let fireTimeout: (() => void) | undefined + let textRead = false + spyOn(globalThis as any, "setTimeout").mockImplementation(((callback: () => void) => { + fireTimeout = callback + return 1 + }) as any) + + spyOn(LLM as any, "stream").mockImplementation(async () => ({ + fullStream: { + async *[Symbol.asyncIterator]() { + yield { type: "text-delta", text: "partial" } + fireTimeout?.() + await new Promise(() => {}) + }, + }, + get text() { + textRead = true + return Promise.resolve("[]") + }, + })) + + const result = await runAiReview({ files: [reviewFile(0)], grounding: [], allowSessionModel: true }) + + expect(result).toMatchObject({ + findings: [], + status: "timeout", + reason: "timed out after 124s (raise aiTimeoutSeconds / --ai-timeout)", + model: "test-provider/test-model", + }) + expect(textRead).toBe(false) + expect(parseCalls()).toBe(0) + }) + + test("returns timeout without parsing when stream.text never settles after fullStream ends", async () => { + const parseCalls = stubModelAndPrompt() + let fireTimeout: (() => void) | undefined + spyOn(globalThis as any, "setTimeout").mockImplementation(((callback: () => void) => { + fireTimeout = callback + return 1 + }) as any) + + spyOn(LLM as any, "stream").mockImplementation(async () => ({ + fullStream: { + async *[Symbol.asyncIterator]() {}, + }, + get text() { + fireTimeout?.() + return new Promise(() => {}) + }, + })) + + const result = await runAiReview({ files: [reviewFile(0)], grounding: [], allowSessionModel: true }) + + expect(result).toMatchObject({ + findings: [], + status: "timeout", + reason: "timed out after 124s (raise aiTimeoutSeconds / --ai-timeout)", + model: "test-provider/test-model", + }) + expect(parseCalls()).toBe(0) + }) + + test("returns a sanitised error without parsing partial text when the stream emits an error event", async () => { + const parseCalls = stubModelAndPrompt() + spyOn(LLM as any, "stream").mockImplementation(async () => ({ + fullStream: { + async *[Symbol.asyncIterator]() { + yield { type: "text-delta", text: "partial" } + yield { + type: "error", + error: new Error( + "upstream failed at https://provider.example/v1 using sk-abcdefghijklmnopqrstuvwxyz", + ), + } + }, + }, + text: Promise.resolve('[{"file":"models/model_0.sql","title":"partial","body":"partial"}]'), + })) + + const result = await runAiReview({ files: [reviewFile(0)], grounding: [], allowSessionModel: true }) + + expect(result).toMatchObject({ + findings: [], + status: "error", + reason: "Error: upstream failed at using sk-***", + model: "test-provider/test-model", + }) + expect(parseCalls()).toBe(0) + }) +}) diff --git a/packages/opencode/test/altimate/review-ci.test.ts b/packages/opencode/test/altimate/review-ci.test.ts index 7f82d9caff..713b035973 100644 --- a/packages/opencode/test/altimate/review-ci.test.ts +++ b/packages/opencode/test/altimate/review-ci.test.ts @@ -1,17 +1,47 @@ -import { describe, test, expect, afterEach } from "bun:test" +import { $ } from "bun" +import { describe, test, expect, afterEach, mock, spyOn } from "bun:test" +import path from "node:path" import { resolveGitHubTarget } from "../../src/altimate/review/post-github" +import * as PostGitHub from "../../src/altimate/review/post-github" +import { defaultBaseRef } from "../../src/altimate/review/git" +import * as ReviewRun from "../../src/altimate/review/run" import { ReviewCommand } from "../../src/cli/cmd/review" import { buildReviewSchemaContext } from "../../src/altimate/review/schema-context" +import { Telemetry } from "../../src/altimate/telemetry" +import { Provider } from "../../src/provider/provider" +import { ModelID, ProviderID } from "../../src/provider/schema" +import { Instance } from "../../src/project/instance" +import { buildEnvelope } from "../../src/altimate/review/verdict" +import { makeFinding } from "../../src/altimate/review/finding" +import { REVIEW_MARKER } from "../../src/altimate/review/format" +import { tmpdir } from "../fixture/fixture" +import YAML from "yaml" -const ENV_KEYS = ["GITHUB_TOKEN", "GH_TOKEN", "GITHUB_REPOSITORY", "GITHUB_EVENT_PATH", "ALTIMATE_PR_NUMBER"] +const ENV_KEYS = [ + "GITHUB_TOKEN", + "GH_TOKEN", + "GITHUB_REPOSITORY", + "GITHUB_EVENT_PATH", + "ALTIMATE_PR_NUMBER", + "ALTIMATE_REVIEW_AI_MODEL", + "ALTIMATE_REVIEW_AI_REASONING", + "ALTIMATE_REVIEW_AI_TIMEOUT_SECONDS", + "OPENCODE_CONFIG_CONTENT", +] const saved: Record = {} for (const k of ENV_KEYS) saved[k] = process.env[k] +const savedExitCode = process.exitCode afterEach(() => { for (const k of ENV_KEYS) { if (saved[k] === undefined) delete process.env[k] else process.env[k] = saved[k] } + mock.restore() + Telemetry.setContext({ sessionId: "", projectId: "" }) + // Bun retains a previously assigned numeric exit code when it is reset to + // undefined, so explicitly restore the successful process default. + process.exitCode = savedExitCode ?? 0 }) describe("review CLI command", () => { @@ -56,6 +86,529 @@ describe("review CLI command", () => { expect((explicitFalse as any)[f.camel]).toBe(false) } }) + + test("sets project context before the CLI emits review_run", async () => { + await using tmp = await tmpdir({ git: true }) + await Bun.write(path.join(tmp.path, "README.md"), "before\n") + await $`git add README.md`.cwd(tmp.path).quiet() + await $`git commit -m fixture`.cwd(tmp.path).quiet() + await Bun.write(path.join(tmp.path, "README.md"), "after\n") + + let projectIdAtRun = "" + spyOn(process.stdout, "write").mockImplementation(() => true) + spyOn(Telemetry, "track").mockImplementation((event) => { + if (event.type === "review_run" && event.invocation === "cli") { + projectIdAtRun = Telemetry.getContext().projectId + } + }) + + await (ReviewCommand.handler as any)({ + cwd: tmp.path, + base: "HEAD", + mode: "comment", + post: false, + json: true, + noAi: true, + explainTier: false, + }) + + expect(projectIdAtRun).not.toBe("") + }) + + test("passes pull request title and capped body from the GitHub event", async () => { + await using tmp = await tmpdir({ git: true }) + const eventPath = path.join(tmp.path, "event.json") + const body = "intent ".repeat(700) + await Bun.write(eventPath, JSON.stringify({ pull_request: { title: "Keep customer grain", body } })) + process.env.GITHUB_EVENT_PATH = eventPath + + const review = spyOn(ReviewRun, "reviewPullRequest").mockResolvedValue( + buildEnvelope({ findings: [], tier: "trivial", mode: "comment" }), + ) + spyOn(process.stdout, "write").mockImplementation(() => true) + + await (ReviewCommand.handler as any)({ + cwd: tmp.path, + base: "HEAD", + mode: "comment", + post: false, + json: true, + noAi: true, + explainTier: false, + }) + + expect(review).toHaveBeenCalledTimes(1) + expect(review.mock.calls[0][0]).toMatchObject({ + prTitle: "Keep customer grain", + prBody: body.slice(0, 4_000), + }) + }) + + test("passes the AI model flag ahead of the environment and disables session fallback", async () => { + await using tmp = await tmpdir({ git: true }) + process.env.ALTIMATE_REVIEW_AI_MODEL = "environment/model" + const review = spyOn(ReviewRun, "reviewPullRequest").mockResolvedValue( + buildEnvelope({ findings: [], tier: "trivial", mode: "comment" }), + ) + spyOn(process.stdout, "write").mockImplementation(() => true) + + await (ReviewCommand.handler as any)({ + cwd: tmp.path, + base: "HEAD", + mode: "comment", + post: false, + json: true, + noAi: false, + aiModel: "flag/model", + explainTier: false, + }) + expect(review.mock.calls[0][0]).toMatchObject({ aiModel: "flag/model", allowSessionModel: false }) + + review.mockClear() + await (ReviewCommand.handler as any)({ + cwd: tmp.path, + base: "HEAD", + mode: "comment", + post: false, + json: true, + noAi: false, + explainTier: false, + }) + expect(review.mock.calls[0][0]).toMatchObject({ aiModel: "environment/model", allowSessionModel: false }) + + review.mockClear() + delete process.env.ALTIMATE_REVIEW_AI_MODEL + await (ReviewCommand.handler as any)({ + cwd: tmp.path, + base: "HEAD", + mode: "comment", + post: false, + json: true, + noAi: false, + aiModel: " \t ", + explainTier: false, + }) + expect(review.mock.calls[0][0]).toMatchObject({ aiModel: undefined, allowSessionModel: false }) + + review.mockClear() + process.env.ALTIMATE_REVIEW_AI_MODEL = " \n " + await (ReviewCommand.handler as any)({ + cwd: tmp.path, + base: "HEAD", + mode: "comment", + post: false, + json: true, + noAi: false, + explainTier: false, + }) + expect(review.mock.calls[0][0]).toMatchObject({ aiModel: undefined, allowSessionModel: false }) + }) + + test("passes AI timeout precedence and the explicit output budget", async () => { + await using tmp = await tmpdir({ git: true }) + process.env.ALTIMATE_REVIEW_AI_TIMEOUT_SECONDS = "240" + const review = spyOn(ReviewRun, "reviewPullRequest").mockResolvedValue( + buildEnvelope({ findings: [], tier: "trivial", mode: "comment" }), + ) + spyOn(process.stdout, "write").mockImplementation(() => true) + + const baseArgs = { + cwd: tmp.path, + base: "HEAD", + mode: "comment", + post: false, + json: true, + noAi: false, + explainTier: false, + } + await (ReviewCommand.handler as any)({ + ...baseArgs, + aiTimeout: 180, + aiMaxOutputTokens: 12_288, + }) + expect(review.mock.calls[0][0]).toMatchObject({ + aiTimeoutMs: 180_000, + aiMaxOutputTokens: 12_288, + }) + + review.mockClear() + await (ReviewCommand.handler as any)(baseArgs) + expect(review.mock.calls[0][0]).toMatchObject({ aiTimeoutMs: 240_000 }) + expect(review.mock.calls[0][0].aiMaxOutputTokens).toBeUndefined() + + review.mockClear() + delete process.env.ALTIMATE_REVIEW_AI_TIMEOUT_SECONDS + await (ReviewCommand.handler as any)(baseArgs) + expect(review.mock.calls[0][0].aiTimeoutMs).toBeUndefined() + }) + + test("passes AI reasoning with flag, environment, config fallback precedence and validates values", async () => { + await using tmp = await tmpdir({ git: true }) + process.env.ALTIMATE_REVIEW_AI_REASONING = "low" + const review = spyOn(ReviewRun, "reviewPullRequest").mockResolvedValue( + buildEnvelope({ findings: [], tier: "trivial", mode: "comment" }), + ) + spyOn(process.stdout, "write").mockImplementation(() => true) + const baseArgs = { + cwd: tmp.path, + base: "HEAD", + mode: "comment", + post: false, + json: true, + noAi: false, + explainTier: false, + } + + await (ReviewCommand.handler as any)({ ...baseArgs, aiReasoning: "high" }) + expect(review.mock.calls[0][0].aiReasoningEffort).toBe("high") + + review.mockClear() + await (ReviewCommand.handler as any)({ ...baseArgs, aiReasoning: " \t " }) + expect(review.mock.calls[0][0].aiReasoningEffort).toBe("low") + + review.mockClear() + process.env.ALTIMATE_REVIEW_AI_REASONING = " \n " + await (ReviewCommand.handler as any)(baseArgs) + // Undefined delegates to reviewPullRequest's repository-config fallback. + expect(review.mock.calls[0][0].aiReasoningEffort).toBeUndefined() + + await expect((ReviewCommand.handler as any)({ ...baseArgs, aiReasoning: "maximum" })).rejects.toThrow( + "--ai-reasoning / ALTIMATE_REVIEW_AI_REASONING must be one of: none, minimal, low, medium, high", + ) + }) + + test("prints the summary and exits successfully when GitHub rejects posting with 403", async () => { + await using tmp = await tmpdir({ git: true }) + for (const k of ENV_KEYS) delete process.env[k] + process.env.GITHUB_TOKEN = "token" + process.env.GITHUB_REPOSITORY = "owner/repo" + process.env.ALTIMATE_PR_NUMBER = "7" + process.exitCode = undefined + // A fork pull request: the token is read-only, so a 403 on post is expected. + const eventPath = path.join(tmp.path, "event.json") + await Bun.write( + eventPath, + JSON.stringify({ + pull_request: { head: { repo: { fork: true, full_name: "someone/repo" } }, base: { repo: { full_name: "owner/repo" } } }, + }), + ) + process.env.GITHUB_EVENT_PATH = eventPath + + const env = buildEnvelope({ findings: [], tier: "trivial", mode: "comment" }) + spyOn(ReviewRun, "reviewPullRequest").mockResolvedValue(env) + spyOn(PostGitHub, "postGitHubReview").mockRejectedValue( + Object.assign(new Error("Resource not accessible by integration"), { status: 403 }), + ) + const stdout: string[] = [] + const stderr: string[] = [] + const events: Telemetry.Event[] = [] + spyOn(process.stdout, "write").mockImplementation(((chunk: unknown) => { + stdout.push(String(chunk)) + return true + }) as any) + spyOn(process.stderr, "write").mockImplementation(((chunk: unknown) => { + stderr.push(String(chunk)) + return true + }) as any) + spyOn(Telemetry, "track").mockImplementation((event) => events.push(event)) + + await (ReviewCommand.handler as any)({ + cwd: tmp.path, + mode: "comment", + post: true, + json: true, + noAi: true, + explainTier: false, + }) + + expect(stderr.join("")).toContain("could not post the review: 403 (fork pull request, read-only token); printing summary instead") + // Under --json stdout stays machine-readable: the human summary goes to stderr. + expect(stderr.join("")).toContain(REVIEW_MARKER) + expect(stdout.join("")).not.toContain(REVIEW_MARKER) + expect(() => JSON.parse(stdout.join("").trim())).not.toThrow() + expect(events.filter((event) => event.type === "review_post_outcome")).toMatchObject([{ outcome: "forbidden" }]) + expect(process.exitCode ?? 0).toBe(0) + }) + + test("a 403 on a same-repository pull request is a misconfiguration and still fails", async () => { + await using tmp = await tmpdir({ git: true }) + for (const k of ENV_KEYS) delete process.env[k] + delete process.env.GITHUB_EVENT_PATH + process.env.GITHUB_TOKEN = "token" + process.env.GITHUB_REPOSITORY = "owner/repo" + process.env.ALTIMATE_PR_NUMBER = "7" + process.exitCode = undefined + + spyOn(ReviewRun, "reviewPullRequest").mockResolvedValue(buildEnvelope({ findings: [], tier: "trivial", mode: "comment" })) + spyOn(PostGitHub, "postGitHubReview").mockRejectedValue(Object.assign(new Error("Forbidden"), { status: 403 })) + spyOn(process.stdout, "write").mockImplementation(() => true) + spyOn(process.stderr, "write").mockImplementation(() => true) + const events: Telemetry.Event[] = [] + spyOn(Telemetry, "track").mockImplementation((event) => events.push(event)) + + await expect( + (ReviewCommand.handler as any)({ cwd: tmp.path, mode: "comment", post: true, json: false, noAi: true, explainTier: false }), + ).rejects.toThrow("Forbidden") + expect(events.filter((event) => event.type === "review_post_outcome")).toMatchObject([{ outcome: "summary_failed" }]) + }) + + test("retains the gate verdict exit code when forbidden posting falls back to stdout", async () => { + await using tmp = await tmpdir({ git: true }) + for (const k of ENV_KEYS) delete process.env[k] + process.env.GITHUB_TOKEN = "token" + process.env.GITHUB_REPOSITORY = "owner/repo" + process.env.ALTIMATE_PR_NUMBER = "7" + process.exitCode = undefined + + const blocking = makeFinding({ + severity: "critical", + category: "contract_violation", + title: "Breaking contract change", + body: "A contracted column was removed.", + file: "models/orders.sql", + ruleKey: "breaking-contract", + }) + const env = buildEnvelope({ findings: [blocking], tier: "full", mode: "gate" }) + expect(env.verdict).toBe("REQUEST_CHANGES") + const eventPath = path.join(tmp.path, "event.json") + await Bun.write( + eventPath, + JSON.stringify({ pull_request: { head: { repo: { fork: true, full_name: "someone/repo" } }, base: { repo: { full_name: "owner/repo" } } } }), + ) + process.env.GITHUB_EVENT_PATH = eventPath + spyOn(ReviewRun, "reviewPullRequest").mockResolvedValue(env) + spyOn(PostGitHub, "postGitHubReview").mockRejectedValue(Object.assign(new Error("Forbidden"), { status: 403 })) + spyOn(process.stdout, "write").mockImplementation(() => true) + spyOn(process.stderr, "write").mockImplementation(() => true) + + await (ReviewCommand.handler as any)({ + cwd: tmp.path, + mode: "gate", + post: true, + json: false, + noAi: true, + explainTier: false, + }) + + expect(Number(process.exitCode)).toBe(2) + }) +}) + +describe("reviewPullRequest head handling", () => { + test("treats an empty review head as omitted", async () => { + await using tmp = await tmpdir({ git: true }) + delete process.env.GITHUB_EVENT_PATH + await Bun.write(path.join(tmp.path, "README.md"), "before\n") + await $`git add README.md`.cwd(tmp.path).quiet() + await $`git commit -m fixture`.cwd(tmp.path).quiet() + await Bun.write(path.join(tmp.path, "README.md"), "after\n") + + const env = await ReviewRun.reviewPullRequest({ cwd: tmp.path, head: "", noAi: true }) + + expect(env.summary.emptyScope).toBe(true) + }) + + test("uses the pull request base ref from GITHUB_EVENT_PATH when it resolves", async () => { + await using tmp = await tmpdir({ git: true }) + await $`git update-ref refs/remotes/origin/release HEAD`.cwd(tmp.path).quiet() + const eventPath = path.join(tmp.path, "event.json") + await Bun.write(eventPath, JSON.stringify({ pull_request: { base: { ref: "release" } } })) + process.env.GITHUB_EVENT_PATH = eventPath + + const expected = (await $`git merge-base HEAD origin/release`.cwd(tmp.path).quiet().text()).trim() + expect(await defaultBaseRef(tmp.path)).toBe(expected) + }) + + test("computes the pull request fork point from a caller-supplied head", async () => { + await using tmp = await tmpdir({ git: true }) + const root = (await $`git rev-parse HEAD`.cwd(tmp.path).quiet().text()).trim() + + await Bun.write(path.join(tmp.path, "release.txt"), "release\n") + await $`git add release.txt`.cwd(tmp.path).quiet() + await $`git commit -m release`.cwd(tmp.path).quiet() + const release = (await $`git rev-parse HEAD`.cwd(tmp.path).quiet().text()).trim() + await $`git update-ref refs/remotes/origin/release ${release}`.cwd(tmp.path).quiet() + + await Bun.write(path.join(tmp.path, "current.txt"), "current\n") + await $`git add current.txt`.cwd(tmp.path).quiet() + await $`git commit -m current`.cwd(tmp.path).quiet() + await $`git update-ref refs/heads/custom-head ${root}`.cwd(tmp.path).quiet() + + const eventPath = path.join(tmp.path, "event.json") + await Bun.write(eventPath, JSON.stringify({ pull_request: { base: { ref: "release" } } })) + process.env.GITHUB_EVENT_PATH = eventPath + + expect(await defaultBaseRef(tmp.path)).toBe(release) + expect(await defaultBaseRef(tmp.path, "custom-head")).toBe(root) + }) + + test("the composite action fetches and derives the merge-base from its effective head", async () => { + const action = await Bun.file(path.resolve(import.meta.dir, "../../../../github/review/action.yml")).text() + expect(action).toContain('if [[ "$IN_HEAD" == -* ]]') + expect(action).toContain("echo \"::error::Custom head must not start with '-': $IN_HEAD\" >&2") + expect(action).toContain('git rev-parse --verify --quiet "$IN_HEAD^{commit}"') + expect(action).toContain('git fetch --no-tags origin -- "$IN_HEAD"') + expect(action).toContain("git update-ref refs/altimate/review-head FETCH_HEAD") + expect(action).toContain('echo "HEAD_REF=refs/altimate/review-head" >> "$GITHUB_ENV"') + expect(action).not.toContain("+refs/heads/${IN_HEAD}:refs/remotes/origin/${IN_HEAD}") + expect(action).toContain('echo "HEAD_REF=$IN_HEAD" >> "$GITHUB_ENV"') + expect(action).toContain('echo "Unable to fetch custom head \'$IN_HEAD\' from origin" >&2') + expect(action).toContain('git merge-base "origin/$PR_BASE_REF" "${HEAD_REF:-$PR_HEAD_SHA}"') + expect(action).toContain('args+=(--head "${HEAD_REF:-$PR_HEAD_SHA}")') + expect(action).toContain('args+=(--ai-timeout "$ALTIMATE_ACTION_AI_TIMEOUT_SECONDS")') + expect(action).toContain('args+=(--ai-reasoning "$ALTIMATE_ACTION_AI_REASONING")') + }) +}) + +describe("advisory model configuration", () => { + test("declares, validates, and exports the optional AI reasoning input without a default", async () => { + await using tmp = await tmpdir() + const actionText = await Bun.file(path.resolve(import.meta.dir, "../../../../github/review/action.yml")).text() + const action = YAML.parse(actionText) as { + inputs: Record + runs: { steps: Array<{ name?: string; run?: string }> } + } + expect(action.inputs.ai_reasoning).toEqual({ + description: + "Optional reasoning level for the advisory reviewer: none, minimal, low, medium or high. Unset leaves the model default (altimate-base thinks before answering).", + required: false, + }) + const script = action.runs.steps.find( + (step) => step.name === "Configure advisory reviewer model + credentials", + )?.run + expect(script).toBeString() + + const run = async (reasoning: string, index: string) => { + const githubEnv = path.join(tmp.path, `github-env-reasoning-${index}`) + const proc = Bun.spawn(["bash", "-c", script!], { + env: { + ...process.env, + HOME: tmp.path, + GITHUB_ENV: githubEnv, + IN_ALT_KEY: "", + IN_ALT_INSTANCE: "", + IN_ALT_URL: "", + IN_GATEWAY_KEY: "", + IN_GATEWAY_URL: "", + IN_MODEL: "", + IN_MODEL_API_KEY: "", + IN_AI_MODEL: "", + IN_AI_REASONING: reasoning, + IN_AI_TIMEOUT_SECONDS: "", + }, + stdout: "pipe", + stderr: "pipe", + }) + const [stdout, stderr, exitCode] = await Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + proc.exited, + ]) + return { githubEnv, stdout, stderr, exitCode } + } + + for (const value of ["none", "minimal", "low", "medium", "high"]) { + const result = await run(value, value) + expect(result.exitCode).toBe(0) + expect(await Bun.file(result.githubEnv).text()).toContain(`ALTIMATE_ACTION_AI_REASONING=${value}`) + } + + const invalid = await run("maximum", "invalid") + expect(invalid.exitCode).toBe(1) + expect(invalid.stdout + invalid.stderr).toContain( + "::error::ai_reasoning must be one of: none, minimal, low, medium or high.", + ) + }) + + test("normalizes gateway URLs before appending the OpenAI-compatible /v1 path", async () => { + await using tmp = await tmpdir() + const actionText = await Bun.file(path.resolve(import.meta.dir, "../../../../github/review/action.yml")).text() + const action = YAML.parse(actionText) as { runs: { steps: Array<{ name?: string; run?: string }> } } + const script = action.runs.steps.find( + (step) => step.name === "Configure advisory reviewer model + credentials", + )?.run + expect(script).toBeString() + + for (const [index, gatewayUrl] of [ + "https://gateway.example.com", + "https://gateway.example.com/", + "https://gateway.example.com/v1", + "https://gateway.example.com/v1/", + ].entries()) { + const githubEnv = path.join(tmp.path, `github-env-${index}`) + const proc = Bun.spawn(["bash", "-c", script!], { + env: { + ...process.env, + HOME: tmp.path, + GITHUB_ENV: githubEnv, + IN_ALT_KEY: "", + IN_ALT_INSTANCE: "", + IN_ALT_URL: "", + IN_GATEWAY_KEY: "gateway-key", + IN_GATEWAY_URL: gatewayUrl, + IN_MODEL: "", + IN_MODEL_API_KEY: "", + IN_AI_MODEL: "", + }, + stdout: "pipe", + stderr: "pipe", + }) + const [stdout, stderr, exitCode] = await Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + proc.exited, + ]) + expect({ stdout, stderr, exitCode }).toMatchObject({ exitCode: 0 }) + const configLine = (await Bun.file(githubEnv).text()) + .split("\n") + .find((line) => line.startsWith("OPENCODE_CONFIG_CONTENT=")) + expect(configLine).toBeString() + const config = JSON.parse(configLine!.slice("OPENCODE_CONFIG_CONTENT=".length)) + expect(config.provider["altimate-gateway"].options.baseURL).toBe("https://gateway.example.com/v1") + expect(await Bun.file(githubEnv).text()).toContain("ALTIMATE_ACTION_AI_TIMEOUT_SECONDS=900") + } + }) + + test("resolves an altimate-gateway provider from OPENCODE_CONFIG_CONTENT without network access", async () => { + await using tmp = await tmpdir() + process.env.OPENCODE_CONFIG_CONTENT = JSON.stringify({ + provider: { + "altimate-gateway": { + npm: "@ai-sdk/openai-compatible", + name: "Altimate Gateway", + options: { + baseURL: "https://gateway.example.com/v1", + apiKey: "test-gateway-key", + }, + models: { + "altimate-base": { name: "altimate-base" }, + "altimate-pro": { name: "altimate-pro" }, + }, + }, + }, + }) + const fetch = spyOn(globalThis as any, "fetch").mockRejectedValue(new Error("unexpected network request")) + + await Instance.provide({ + directory: tmp.path, + fn: async () => { + try { + const model = await Provider.getModel( + ProviderID.make("altimate-gateway"), + ModelID.make("altimate-base"), + ) + expect(String(model.providerID)).toBe("altimate-gateway") + expect(String(model.id)).toBe("altimate-base") + expect(model.api.npm).toBe("@ai-sdk/openai-compatible") + const providers = await Provider.list() + expect(providers["altimate-gateway"].options.baseURL).toBe("https://gateway.example.com/v1") + } finally { + await Instance.dispose() + } + }, + }) + expect(fetch).not.toHaveBeenCalled() + }) }) describe("resolveGitHubTarget", () => { diff --git a/packages/opencode/test/altimate/review-dbt-patterns.test.ts b/packages/opencode/test/altimate/review-dbt-patterns.test.ts index 5c5099dfab..2657830a5d 100644 --- a/packages/opencode/test/altimate/review-dbt-patterns.test.ts +++ b/packages/opencode/test/altimate/review-dbt-patterns.test.ts @@ -827,6 +827,7 @@ models: expect(gap!.severity).toBe("warning") expect(gap!.model).toBe("mrt_x") expect(gap!.column).toBe("price_start_time") + expect(gap!.groupKey).toBe("grain_not_null:mrt_x") // Contract is enforced → recommendation should point at `constraints:`. expect(gap!.body).toContain("constraints: [{type: not_null}]") // Non-gap columns must not appear as findings. diff --git a/packages/opencode/test/altimate/review-run-stale.test.ts b/packages/opencode/test/altimate/review-run-stale.test.ts index 61164e0c79..231db38ef2 100644 --- a/packages/opencode/test/altimate/review-run-stale.test.ts +++ b/packages/opencode/test/altimate/review-run-stale.test.ts @@ -1,5 +1,18 @@ import { describe, test, expect } from "bun:test" -import { isManifestAffecting } from "../../src/altimate/review/run" +import { promises as fs } from "node:fs" +import path from "node:path" +import { tmpdir } from "../fixture/fixture" +import { detectArtifactHints, isManifestAffecting, reviewPullRequest } from "../../src/altimate/review/run" +import { renderSummary } from "../../src/altimate/review/format" + +const USABLE_CATALOG = JSON.stringify({ + nodes: { + "model.analytics.fixture": { + metadata: { name: "fixture" }, + columns: { id: { name: "id", type: "integer" } }, + }, + }, +}) /** * Guards on `warnIfStale`'s changed-file filter. The stale warning gates on a @@ -58,3 +71,396 @@ describe("isManifestAffecting", () => { expect(isManifestAffecting(rel)).toBe(false) }) }) + +describe("detectArtifactHints", () => { + test("returns no hints when no models changed, even when the catalog is absent", async () => { + await using tmp = await tmpdir() + const manifest = path.join(tmp.path, "target", "manifest.json") + await fs.mkdir(path.dirname(manifest), { recursive: true }) + await fs.writeFile(manifest, "{}") + + expect(await detectArtifactHints(manifest, tmp.path)).toEqual([]) + }) + + test("distinguishes a missing catalog from an unreadable or empty catalog", async () => { + await using tmp = await tmpdir() + const target = path.join(tmp.path, "target") + const manifest = path.join(target, "manifest.json") + const changedModels = [{ path: "models/a.sql", status: "added" as const }] + await fs.mkdir(target, { recursive: true }) + await fs.writeFile(manifest, "{}") + + expect(await detectArtifactHints(manifest, tmp.path, changedModels, "analytics")).toContain( + "catalog.json (run `dbt docs generate`)", + ) + + for (const contents of ["{}", "{not json"]) { + await fs.writeFile(path.join(target, "catalog.json"), contents) + expect(await detectArtifactHints(manifest, tmp.path, changedModels, "analytics")).toContain( + "catalog.json unreadable or empty (regenerate with `dbt docs generate`)", + ) + } + }) + + test("treats catalog entries without populated columns as unusable", async () => { + await using tmp = await tmpdir() + const target = path.join(tmp.path, "target") + const manifest = path.join(target, "manifest.json") + const changedModels = [{ path: "models/a.sql", status: "added" as const }] + await fs.mkdir(target, { recursive: true }) + await fs.writeFile(manifest, "{}") + + for (const catalog of [ + { nodes: { "model.analytics.a": { metadata: { name: "a" } } } }, + { sources: { "source.analytics.raw": { columns: {} } } }, + ]) { + await fs.writeFile(path.join(target, "catalog.json"), JSON.stringify(catalog)) + expect(await detectArtifactHints(manifest, tmp.path, changedModels, "analytics")).toContain( + "catalog.json unreadable or empty (regenerate with `dbt docs generate`)", + ) + } + }) + + test("reports both missing compiled directories when the catalog exists", async () => { + await using tmp = await tmpdir() + const target = path.join(tmp.path, "target") + const manifest = path.join(target, "manifest.json") + await fs.mkdir(target, { recursive: true }) + await fs.writeFile(manifest, "{}") + await fs.writeFile(path.join(target, "catalog.json"), USABLE_CATALOG) + + expect( + await detectArtifactHints( + manifest, + tmp.path, + [{ path: "models/a.sql", status: "modified" }], + "analytics", + ), + ).toEqual([ + "target-base/compiled missing for 1 changed model(s) (compile the base ref)", + "target/compiled missing for 1 changed model(s) (run `dbt compile` for the head)", + ]) + }) + + test("reports changed models missing from a partially populated compiled directory", async () => { + await using tmp = await tmpdir() + const project = "analytics" + const target = path.join(tmp.path, "target") + const manifest = path.join(target, "manifest.json") + await fs.mkdir(path.join(target, "compiled", project, "models"), { recursive: true }) + await fs.mkdir(path.join(tmp.path, "target-base", "compiled", project, "models"), { recursive: true }) + await fs.writeFile(manifest, "{}") + await fs.writeFile(path.join(target, "catalog.json"), USABLE_CATALOG) + await fs.writeFile(path.join(target, "compiled", project, "models", "a.sql"), "select 1") + await fs.writeFile(path.join(tmp.path, "target-base", "compiled", project, "models", "a.sql"), "select 1") + await fs.writeFile(path.join(tmp.path, "target-base", "compiled", project, "models", "b.sql"), "select 1") + + expect( + await detectArtifactHints( + manifest, + tmp.path, + [ + { path: "models/a.sql", status: "modified" }, + { path: "models/b.sql", status: "modified" }, + ], + project, + ), + ).toEqual(["target/compiled missing for 1 changed model(s) (run `dbt compile` for the head)"]) + }) + + test("treats zero-byte and whitespace-only compiled model files as missing", async () => { + await using tmp = await tmpdir() + const project = "analytics" + const target = path.join(tmp.path, "target") + const manifest = path.join(target, "manifest.json") + const headModel = path.join(target, "compiled", project, "models", "a.sql") + const baseModel = path.join(tmp.path, "target-base", "compiled", project, "models", "a.sql") + await fs.mkdir(path.dirname(headModel), { recursive: true }) + await fs.mkdir(path.dirname(baseModel), { recursive: true }) + await fs.writeFile(manifest, "{}") + await fs.writeFile(path.join(target, "catalog.json"), USABLE_CATALOG) + await fs.writeFile(headModel, "") + await fs.writeFile(baseModel, " \n\t") + + expect( + await detectArtifactHints( + manifest, + tmp.path, + [{ path: "models/a.sql", status: "modified" }], + project, + ), + ).toEqual([ + "target-base/compiled missing for 1 changed model(s) (compile the base ref)", + "target/compiled missing for 1 changed model(s) (run `dbt compile` for the head)", + ]) + }) + + test("uses compiled SQL beside a custom manifest and prefers its sibling base directory", async () => { + await using tmp = await tmpdir() + const project = "analytics" + const build = path.join(tmp.path, "build") + const headModel = path.join(build, "compiled", project, "models", "a.sql") + const siblingBaseModel = path.join(tmp.path, "build-base", "compiled", project, "models", "a.sql") + const fallbackBaseModel = path.join(tmp.path, "target-base", "compiled", project, "models", "a.sql") + await fs.mkdir(path.dirname(headModel), { recursive: true }) + await fs.mkdir(path.dirname(siblingBaseModel), { recursive: true }) + await fs.mkdir(path.dirname(fallbackBaseModel), { recursive: true }) + await fs.writeFile(path.join(build, "manifest.json"), "{}") + await fs.writeFile(path.join(build, "catalog.json"), USABLE_CATALOG) + await fs.writeFile(headModel, "select 1") + await fs.writeFile(fallbackBaseModel, "select 1") + + const manifest = path.join(build, "manifest.json") + const changedModels = [{ path: "models/a.sql", status: "modified" as const }] + // The sibling `build-base/compiled` DIRECTORY already exists (created above), so it is + // preferred over `target-base/compiled` even though only the fallback holds `a.sql`. + // The hint therefore names the sibling; the fallback is never consulted once the + // sibling directory is present. + expect(await detectArtifactHints(manifest, tmp.path, changedModels, project)).toEqual([ + "build-base/compiled missing for 1 changed model(s) (compile the base ref)", + ]) + + await fs.writeFile(siblingBaseModel, "select 1") + expect(await detectArtifactHints(manifest, tmp.path, changedModels, project)).toEqual([]) + }) + + test("uses the sole base project directory and oldPath when the dbt project was renamed", async () => { + await using tmp = await tmpdir() + const target = path.join(tmp.path, "target") + const manifest = path.join(target, "manifest.json") + const headModel = path.join(target, "compiled", "new_analytics", "models", "new_name.sql") + const baseModel = path.join(tmp.path, "target-base", "compiled", "old_analytics", "models", "old_name.sql") + await fs.mkdir(path.dirname(headModel), { recursive: true }) + await fs.mkdir(path.dirname(baseModel), { recursive: true }) + await fs.writeFile(manifest, "{}") + await fs.writeFile(path.join(target, "catalog.json"), USABLE_CATALOG) + await fs.writeFile(headModel, "select 1") + await fs.writeFile(baseModel, "select 1") + + expect( + await detectArtifactHints( + manifest, + tmp.path, + [{ path: "models/new_name.sql", oldPath: "models/old_name.sql", status: "renamed" }], + "new_analytics", + ), + ).toEqual([]) + }) + + test("reports an ambiguous base project directory without assuming the head project name", async () => { + await using tmp = await tmpdir() + const project = "analytics" + const target = path.join(tmp.path, "target") + const manifest = path.join(target, "manifest.json") + const headModel = path.join(target, "compiled", project, "models", "a.sql") + const baseRoot = path.join(tmp.path, "target-base", "compiled") + await fs.mkdir(path.dirname(headModel), { recursive: true }) + await fs.mkdir(path.join(baseRoot, "old_analytics"), { recursive: true }) + await fs.mkdir(path.join(baseRoot, "legacy_analytics"), { recursive: true }) + await fs.writeFile(manifest, "{}") + await fs.writeFile(path.join(target, "catalog.json"), USABLE_CATALOG) + await fs.writeFile(headModel, "select 1") + + expect( + await detectArtifactHints( + manifest, + tmp.path, + [{ path: "models/a.sql", status: "modified" }], + project, + ), + ).toEqual(["target-base/compiled has several project directories; expected `analytics`"]) + }) + + test("does not request a catalog or compiled SQL for a deletion-only diff", async () => { + await using tmp = await tmpdir() + const target = path.join(tmp.path, "target") + const manifest = path.join(target, "manifest.json") + await fs.mkdir(target, { recursive: true }) + await fs.writeFile(manifest, "{}") + + expect( + await detectArtifactHints( + manifest, + tmp.path, + [{ path: "models/deleted.sql", status: "deleted" }], + "analytics", + ), + ).toEqual([]) + }) + + test("does not report artifacts when the manifest itself is absent", async () => { + await using tmp = await tmpdir() + expect( + await detectArtifactHints( + path.join(tmp.path, "target", "manifest.json"), + tmp.path, + [{ path: "models/a.sql", status: "modified" }], + "analytics", + ), + ).toEqual([]) + }) + + test("reports an unresolved project name once and keeps the catalog hint", async () => { + await using tmp = await tmpdir() + const manifest = path.join(tmp.path, "target", "manifest.json") + await fs.mkdir(path.dirname(manifest), { recursive: true }) + await fs.writeFile(manifest, "{}") + + expect( + await detectArtifactHints(manifest, tmp.path, [ + { path: "models/a.sql", status: "modified" }, + { path: "models/b.sql", status: "modified" }, + ]), + ).toEqual([ + "catalog.json (run `dbt docs generate`)", + "dbt project name not resolved — no readable dbt_project.yml next to the manifest, so compiled SQL cannot be located", + ]) + }) +}) + +async function writeDbtArtifacts(root: string, catalog = false) { + const target = path.join(root, "target") + await fs.mkdir(target, { recursive: true }) + await fs.writeFile( + path.join(target, "manifest.json"), + JSON.stringify({ + metadata: { adapter_type: "duckdb" }, + nodes: { + "model.analytics.existing": { + resource_type: "model", + name: "existing", + original_file_path: "models/existing.sql", + depends_on: { nodes: [] }, + }, + }, + sources: {}, + }), + ) + if (catalog) await fs.writeFile(path.join(target, "catalog.json"), USABLE_CATALOG) + await fs.writeFile(path.join(root, "dbt_project.yml"), "name: analytics\n") +} + +describe("review artifact hint scope", () => { + test("a README-only diff renders only the empty-scope message", async () => { + await using tmp = await tmpdir() + await writeDbtArtifacts(tmp.path) + + const env = await reviewPullRequest({ + cwd: tmp.path, + changedFiles: [{ path: "README.md", status: "modified", diff: "+docs\n" }], + getContent: async () => undefined, + noAi: true, + }) + const summary = renderSummary(env) + + expect(env.summary.emptyScopeReason).toBe("no_dbt_files") + expect(summary).toContain("Nothing to review") + expect(summary).not.toContain("Missing artifacts") + expect(summary).not.toContain("No issues found") + expect(summary).not.toContain("AI reviewer:") + }) + + test("committed build artifacts without dbt-classified paths report no dbt files", async () => { + await using tmp = await tmpdir() + await writeDbtArtifacts(tmp.path) + + const env = await reviewPullRequest({ + cwd: tmp.path, + changedFiles: [ + { path: "target/query.sql", status: "modified", diff: "+select 1\n" }, + { path: "compiled/output.yml", status: "modified", diff: "+version: 2\n" }, + { path: "dbt_packages/data.csv", status: "modified", diff: "+1\n" }, + ], + getContent: async () => undefined, + noAi: true, + }) + + expect(env.summary).toMatchObject({ + emptyScope: true, + emptyScopeReason: "no_dbt_files", + }) + expect(env.summary.emptyScopeFileCount).toBeUndefined() + }) + + test("an excluded extension-only file outside dbt directories reports no dbt files", async () => { + await using tmp = await tmpdir() + await writeDbtArtifacts(tmp.path) + await fs.mkdir(path.join(tmp.path, ".altimate"), { recursive: true }) + await fs.writeFile(path.join(tmp.path, ".altimate", "review.yml"), "exclude:\n - queries/report.sql\n") + + const env = await reviewPullRequest({ + cwd: tmp.path, + changedFiles: [{ path: "queries/report.sql", status: "modified", diff: "+select 1\n" }], + getContent: async () => undefined, + noAi: true, + }) + + expect(env.summary).toMatchObject({ + emptyScope: true, + emptyScopeReason: "no_dbt_files", + }) + expect(env.summary.emptyScopeFileCount).toBeUndefined() + }) + + test("excluded models and tracked compiled output do not produce hints", async () => { + await using tmp = await tmpdir() + await writeDbtArtifacts(tmp.path) + await fs.mkdir(path.join(tmp.path, ".altimate"), { recursive: true }) + await fs.writeFile(path.join(tmp.path, ".altimate", "review.yml"), "exclude:\n - models/excluded.sql\n") + + const env = await reviewPullRequest({ + cwd: tmp.path, + changedFiles: [ + { path: "models/excluded.sql", status: "modified", diff: "+select 1\n" }, + { path: "target/compiled/analytics/models/x.sql", status: "modified", diff: "+select 1\n" }, + ], + getContent: async () => undefined, + noAi: true, + }) + + expect(env.summary.artifactHints).toEqual([]) + expect(env.summary).toMatchObject({ + emptyScope: true, + emptyScopeReason: "all_excluded", + emptyScopeFileCount: 2, + }) + expect(renderSummary(env)).toContain( + "⚙️ Nothing to review — all 2 changed dbt files are excluded by the review configuration (`exclude` globs)", + ) + }) + + test("--no-ai excludes an unused configured model from the policy signature", async () => { + await using tmp = await tmpdir() + await writeDbtArtifacts(tmp.path) + + const review = (aiModel: string) => + reviewPullRequest({ + cwd: tmp.path, + changedFiles: [{ path: "models/new_model.sql", status: "added", diff: "+select 1\n" }], + getContent: async () => "select 1", + noAi: true, + aiModel, + }) + + const first = await review("altimate-gateway/altimate-base") + const second = await review("altimate-gateway/altimate-pro") + + expect(first.policySignature).toBe(second.policySignature) + }) + + test("a changed Python model is included in compiled artifact hints", async () => { + await using tmp = await tmpdir() + await writeDbtArtifacts(tmp.path, true) + + const env = await reviewPullRequest({ + cwd: tmp.path, + changedFiles: [{ path: "models/new_model.py", status: "added", diff: "+def model(): pass\n" }], + getContent: async () => undefined, + noAi: true, + }) + + expect(env.summary.artifactHints).toEqual([ + "target/compiled missing for 1 changed model(s) (run `dbt compile` for the head)", + ]) + }) +}) diff --git a/packages/opencode/test/altimate/review-runner.test.ts b/packages/opencode/test/altimate/review-runner.test.ts index 538b2d7631..928077080e 100644 --- a/packages/opencode/test/altimate/review-runner.test.ts +++ b/packages/opencode/test/altimate/review-runner.test.ts @@ -38,6 +38,15 @@ describe("review manifest loading", () => { expect(await runner.manifestAvailable?.()).toBe(true) expect(await runner.impact("orders")).toEqual({ hasManifest: true, + resolved: true, + severity: "SAFE", + directCount: 0, + transitiveCount: 0, + testCount: 0, + }) + expect(await runner.impact("missing_model")).toEqual({ + hasManifest: true, + resolved: false, severity: "SAFE", directCount: 0, transitiveCount: 0, diff --git a/packages/opencode/test/altimate/review-subdir-invocation.test.ts b/packages/opencode/test/altimate/review-subdir-invocation.test.ts index 14fe9543b8..e211196270 100644 --- a/packages/opencode/test/altimate/review-subdir-invocation.test.ts +++ b/packages/opencode/test/altimate/review-subdir-invocation.test.ts @@ -117,6 +117,25 @@ describe("R20: subdir-invocation content resolution (PR #1027 consensus MAJOR)", expect(content).toBe("-- compiled at root\n") }) + test("makeCompiledResolver can use a different BASE-side dbt project name", async () => { + const { root } = await mkTempRepo() + const headPath = path.join(root, "target", "compiled", "new_analytics", "models", "customers.sql") + const basePath = path.join(root, "target-base", "compiled", "old_analytics", "models", "customers.sql") + await fs.mkdir(path.dirname(headPath), { recursive: true }) + await fs.mkdir(path.dirname(basePath), { recursive: true }) + await fs.writeFile(headPath, "-- head project\n") + await fs.writeFile(basePath, "-- base project\n") + + const resolver = makeCompiledResolver({ + cwd: root, + projectName: "new_analytics", + baseProjectName: "old_analytics", + }) + + expect(await resolver("models/customers.sql", "new")).toBe("-- head project\n") + expect(await resolver("models/customers.sql", "old")).toBe("-- base project\n") + }) + test("makeCompiledResolver pathPrefix does NOT strip when path doesn't start with prefix", async () => { // Precision guard: a file outside the prefixed subtree must remain // untouched. Repo-relative `docs/foo.md` with prefix `packages/dbt` diff --git a/packages/opencode/test/altimate/review.test.ts b/packages/opencode/test/altimate/review.test.ts index 24bde20a7a..ba5def38ee 100644 --- a/packages/opencode/test/altimate/review.test.ts +++ b/packages/opencode/test/altimate/review.test.ts @@ -34,6 +34,7 @@ import { parseReviewConfig, resolveRubric, DEFAULT_REVIEW_CONFIG, + NO_MODEL_REASON, } from "../../src/altimate/review" // --------------------------------------------------------------------------- @@ -201,6 +202,20 @@ describe("verdict", () => { expect(computeIdealVerdict([mk("suggestion")], DEFAULT_RUBRIC)).toBe("COMMENT") }) + test("summary keeps lint-only separate from undecidable findings", () => { + const undecidable = makeFinding({ + severity: "warning", + category: "semantic_change", + title: "could not prove equivalence", + body: "compiled SQL is missing", + file: "models/m.sql", + degraded: true, + ruleKey: "undecidable", + }) + const env = buildEnvelope({ findings: [undecidable], tier: "full", mode: "comment", lintOnly: false }) + expect(env.summary).toMatchObject({ degraded: false, lintOnly: false, undecidableFindings: 1 }) + }) + test("the bot NEVER emits a formal APPROVE review event", () => { // A bot approval could satisfy branch protection and merge a PR without human // sign-off. An APPROVE verdict must post a COMMENT review event instead. @@ -848,7 +863,7 @@ describe("risk-tier", () => { const runner: ReviewRunner = { async check() { return { issues: [], ran: false } }, async detectPii() { return { columns: [] } }, - async impact() { return { hasManifest: false, severity: "SAFE", directCount: 0, transitiveCount: 0, testCount: 0 } }, + async impact() { return { hasManifest: false, resolved: false, severity: "SAFE", directCount: 0, transitiveCount: 0, testCount: 0 } }, async equivalence() { return { decided: true, equivalent: true } as EquivalenceResult }, async grade() { return { grade: "A", decided: true } }, } @@ -898,7 +913,7 @@ describe("risk-tier", () => { const runner: ReviewRunner = { async check() { return { issues: [], ran: false } }, async detectPii() { return { columns: [] } }, - async impact() { return { hasManifest: false, severity: "SAFE", directCount: 0, transitiveCount: 0, testCount: 0 } }, + async impact() { return { hasManifest: false, resolved: false, severity: "SAFE", directCount: 0, transitiveCount: 0, testCount: 0 } }, async equivalence() { return { decided: true, equivalent: true } as EquivalenceResult }, async grade() { return { grade: "A", decided: true } }, } @@ -1015,7 +1030,45 @@ describe("config", () => { }) test("empty config yields defaults", () => { - expect(parseReviewConfig("").mode).toBe("comment") + const config = parseReviewConfig("") + expect(config.mode).toBe("comment") + expect(config.aiTimeoutSeconds).toBeUndefined() + expect(config.aiMaxOutputTokens).toBe(8_192) + }) + + test("aiModel accepts only provider/model identifiers", () => { + expect(parseReviewConfig("aiModel: altimate-gateway/altimate-base\n").aiModel).toBe( + "altimate-gateway/altimate-base", + ) + expect(parseReviewConfig("aiModel: openrouter/openai/gpt-5\n").aiModel).toBe("openrouter/openai/gpt-5") + expect(() => parseReviewConfig("aiModel: altimate-base\n")).toThrow("provider/model") + expect(() => parseReviewConfig("aiModel: 'altimate-gateway/model with spaces'\n")).toThrow("provider/model") + }) + + test("aiReasoningEffort accepts only supported reasoning levels", () => { + for (const value of ["none", "minimal", "low", "medium", "high"] as const) { + expect(parseReviewConfig(`aiReasoningEffort: ${value}\n`).aiReasoningEffort).toBe(value) + } + for (const value of ["off", "NONE", "maximum"]) { + expect(() => parseReviewConfig(`aiReasoningEffort: ${value}\n`)).toThrow() + } + }) + + test("AI timeout and output budget accept only bounded integers", () => { + expect(parseReviewConfig("aiTimeoutSeconds: 10\naiMaxOutputTokens: 512\n")).toMatchObject({ + aiTimeoutSeconds: 10, + aiMaxOutputTokens: 512, + }) + expect(parseReviewConfig("aiTimeoutSeconds: 1800\naiMaxOutputTokens: 32768\n")).toMatchObject({ + aiTimeoutSeconds: 1800, + aiMaxOutputTokens: 32_768, + }) + for (const value of [9, 1801, 10.5]) { + expect(() => parseReviewConfig(`aiTimeoutSeconds: ${value}\n`)).toThrow() + } + for (const value of [511, 32_769, 1_024.5]) { + expect(() => parseReviewConfig(`aiMaxOutputTokens: ${value}\n`)).toThrow() + } }) test("resolveRubric folds exclude globs into rubric", () => { @@ -1062,6 +1115,7 @@ describe("orchestrate", () => { return ( opts.impact?.[model] ?? { hasManifest: true, + resolved: true, severity: "SAFE", directCount: 0, transitiveCount: 0, @@ -1092,7 +1146,14 @@ describe("orchestrate", () => { const files: ChangedFile[] = [{ path: "models/staging/stg_orders.sql", status: "deleted", diff: "" }] const runner = fakeRunner({ impact: { - stg_orders: { hasManifest: true, severity: "BREAKING", directCount: 3, transitiveCount: 8, testCount: 4 }, + stg_orders: { + hasManifest: true, + resolved: true, + severity: "BREAKING", + directCount: 3, + transitiveCount: 8, + testCount: 4, + }, }, }) const env = await runReview({ @@ -1585,6 +1646,12 @@ describe("orchestrate", () => { expect(topo!.severity).toBe("warning") expect(topo!.degraded).toBe(true) expect(env.verdict).not.toBe("REQUEST_CHANGES") + expect(env.summary).toMatchObject({ degraded: false, lintOnly: false, undecidableFindings: 1 }) + const summary = renderSummary(env) + expect(summary).not.toContain("Lint-only") + expect(summary).toContain( + "ℹ️ 1 finding could not be decided — compiled SQL missing for base or head, unsupported SQL for this dialect, or no schema — see each finding.", + ) }) test("topology lane: no compiled SQL available → skips (no crash)", async () => { @@ -1729,6 +1796,36 @@ describe("orchestrate", () => { expect(env.findings.some((x) => /no uniqueness\/grain test/.test(x.title))).toBe(false) }) + test("missing grain test: multiple new models remain atomic findings", async () => { + const runner: ReviewRunner = { + ...fakeRunner({}), + async declaredPrimaryKey() { + return undefined + }, + } + const env = await runReview({ + changedFiles: [ + { path: "models/intermediate/int_new_orders.sql", status: "added", diff: "+select 1\n" }, + { path: "models/marts/fct_new_orders.sql", status: "added", diff: "+select 1\n" }, + ], + config: { ...DEFAULT_REVIEW_CONFIG, reviewers: ["test_coverage"], ai: false }, + rubric: DEFAULT_RUBRIC, + mode: "comment", + runner, + getContent: content("select 1"), + generatedAt: "2026-05-29T00:00:00Z", + }) + const findings = env.findings.filter((x) => /no uniqueness\/grain test/.test(x.title)) + expect(findings).toHaveLength(2) + expect(findings.map((finding) => finding.file)).toEqual([ + "models/intermediate/int_new_orders.sql", + "models/marts/fct_new_orders.sql", + ]) + expect(findings.map((finding) => finding.model)).toEqual(["int_new_orders", "fct_new_orders"]) + expect(findings.every((finding) => finding.groupKey === "missing_grain_test")).toBe(true) + expect(new Set(findings.map((finding) => finding.id)).size).toBe(2) + }) + test("missing grain test: new mart WITH a declared PK → not flagged", async () => { const runner: ReviewRunner = { ...fakeRunner({}), async declaredPrimaryKey() { return ["id"] } } const env = await runReview({ @@ -1776,33 +1873,54 @@ describe("orchestrate", () => { runner: fakeRunner({}), getContent: content(sql), prTitle: "Add revenue mart", + aiModel: "altimate-gateway/altimate-base", + aiReasoningEffort: "none", + aiTimeoutMs: 180_000, + aiMaxOutputTokens: 12_288, + allowSessionModel: false, + sessionModel: "openrouter/openai/gpt-5", // Fake AI reviewer: returns a contextual comment + a (disallowed) critical // that must be downgraded — the AI must never block. aiReview: async (input) => { groundingSeen = input.grounding.length - return [ - makeFinding({ - severity: "warning", - category: "sql_correctness", - title: "m: `revenue` may double-count un-deduped orders", - body: "If orders has multiple rows per order_id, summing amount inflates revenue.", - file: "models/marts/m.sql", - model: "m", - confidence: "medium", - evidence: { tool: "ai-review", result: { confidence: "medium" } }, - ruleKey: "ai:sql_correctness:revenue-double-count", - }), - makeFinding({ - severity: "critical", // disallowed for AI — must be downgraded, must not block - category: "sql_correctness", - title: "m: bogus critical", - body: "AI should never block.", - file: "models/marts/m.sql", - model: "m", - evidence: { tool: "ai-review", result: {} }, - ruleKey: "ai:sql_correctness:bogus", - }), - ] + expect(input.model).toBe("altimate-gateway/altimate-base") + expect(input.reasoningEffort).toBe("none") + expect(input.allowSessionModel).toBe(false) + expect(input.sessionModel).toBe("openrouter/openai/gpt-5") + expect(input.timeoutMs).toBe(180_000) + expect(input.maxOutputTokens).toBe(12_288) + return { + status: "ok", + model: "altimate-gateway/altimate-base", + durationMs: 142_000, + promptChars: 24_000, + promptTokens: 6_000, + completionTokens: 4_000, + reasoningTokens: 3_000, + findings: [ + makeFinding({ + severity: "warning", + category: "sql_correctness", + title: "m: `revenue` may double-count un-deduped orders", + body: "If orders has multiple rows per order_id, summing amount inflates revenue.", + file: "models/marts/m.sql", + model: "m", + confidence: "medium", + evidence: { tool: "ai-review", result: { confidence: "medium" } }, + ruleKey: "ai:sql_correctness:revenue-double-count", + }), + makeFinding({ + severity: "critical", // disallowed for AI — must be downgraded, must not block + category: "sql_correctness", + title: "m: bogus critical", + body: "AI should never block.", + file: "models/marts/m.sql", + model: "m", + evidence: { tool: "spoofed-engine", result: {} }, + ruleKey: "ai:sql_correctness:bogus", + }), + ], + } }, generatedAt: "2026-05-29T00:00:00Z", }) @@ -1813,6 +1931,99 @@ describe("orchestrate", () => { // No AI finding survived as critical, and the AI did NOT cause a block. expect(env.findings.some((f) => f.evidence?.tool === "ai-review" && f.severity === "critical")).toBe(false) expect(env.verdict).not.toBe("REQUEST_CHANGES") + expect(env.summary.aiReview).toEqual({ + status: "ok", + findings: 2, + model: "altimate-gateway/altimate-base", + reasoningEffort: "none", + durationMs: 142_000, + promptChars: 24_000, + promptTokens: 6_000, + completionTokens: 4_000, + reasoningTokens: 3_000, + }) + }) + + test("AI reviewer lane: disabled configuration skips without invoking the reviewer", async () => { + let aiCalls = 0 + const env = await runReview({ + changedFiles: [{ path: "models/marts/m.sql", status: "modified", diff: "+select 1\n" }], + config: { ...DEFAULT_REVIEW_CONFIG, reviewers: ["ai_review"], ai: false }, + rubric: DEFAULT_RUBRIC, + mode: "comment", + runner: fakeRunner({}), + getContent: content("select 1"), + aiReview: async () => { + aiCalls++ + return { status: "ok", findings: [] } + }, + }) + + expect(aiCalls).toBe(0) + expect(env.summary.aiReview).toEqual({ + status: "skipped", + reason: "disabled by configuration", + findings: 0, + }) + }) + + test("AI reviewer status renders each outcome and never changes the verdict", () => { + const cases = [ + { + aiReview: { + status: "ok" as const, + findings: 2, + model: "altimate-gateway/altimate-base", + reasoningEffort: "none" as const, + durationMs: 142_000, + }, + expected: "🤖 AI reviewer (altimate-gateway/altimate-base, reasoning: none): 2 advisory findings · 142s", + }, + { + aiReview: { + status: "skipped" as const, + reason: NO_MODEL_REASON, + findings: 0, + }, + expected: `🤖 AI reviewer: skipped — ${NO_MODEL_REASON}`, + }, + { + aiReview: { status: "timeout" as const, reason: "timed out after 74s", findings: 0 }, + expected: "🤖 AI reviewer: timed out after 74s", + }, + { + aiReview: { status: "error" as const, reason: "ProviderError: unavailable", findings: 0 }, + expected: "🤖 AI reviewer: error — ProviderError: unavailable", + }, + ] + for (const { aiReview, expected } of cases) { + const env = buildEnvelope({ findings: [], tier: "lite", mode: "gate", aiReview }) + expect(env.verdict).toBe("APPROVE") + expect(renderSummary(env)).toContain(expected) + } + + const trivial = buildEnvelope({ + findings: [], + tier: "trivial", + mode: "comment", + aiReview: { status: "skipped", reason: "no changed model files", findings: 0 }, + }) + expect(renderSummary(trivial)).toContain("🤖 AI reviewer: skipped — no changed model files") + + const withoutAiReview = buildEnvelope({ findings: [], tier: "trivial", mode: "comment" }) + expect(renderSummary(withoutAiReview)).not.toContain("AI reviewer:") + }) + + test("format renders an AI status whenever the envelope contains one", () => { + const env = buildEnvelope({ + findings: [], + tier: "lite", + mode: "comment", + aiReview: { status: "ok", findings: 2, model: "altimate-gateway/altimate-base" }, + }) + const summary = renderSummary(env) + + expect(summary).toContain("🤖 AI reviewer (altimate-gateway/altimate-base): 2 advisory findings") }) test("FUSION: proven non-equivalent + downstream → critical → blocks (gate)", async () => { @@ -1820,7 +2031,7 @@ describe("orchestrate", () => { const runner: ReviewRunner = { ...fakeRunner({}), async impact() { - return { hasManifest: true, severity: "MEDIUM", directCount: 4, transitiveCount: 2, testCount: 1 } + return { hasManifest: true, resolved: true, severity: "MEDIUM", directCount: 4, transitiveCount: 2, testCount: 1 } }, async equivalence() { return { @@ -1852,7 +2063,7 @@ describe("orchestrate", () => { const runner: ReviewRunner = { ...fakeRunner({}), async impact() { - return { hasManifest: false, severity: "UNKNOWN", directCount: 0, transitiveCount: 0, testCount: 0 } + return { hasManifest: false, resolved: false, severity: "UNKNOWN", directCount: 0, transitiveCount: 0, testCount: 0 } }, async equivalence() { return { decided: true, equivalent: false, differences: ["filter changed"], confidence: "high" } @@ -1877,7 +2088,7 @@ describe("orchestrate", () => { const runner: ReviewRunner = { ...fakeRunner({}), async impact() { - return { hasManifest: true, severity: "HIGH", directCount: 12, transitiveCount: 30, testCount: 5 } + return { hasManifest: true, resolved: true, severity: "HIGH", directCount: 12, transitiveCount: 30, testCount: 5 } }, } const env = await runReview({ @@ -1891,6 +2102,7 @@ describe("orchestrate", () => { }) const lin = env.findings.find((f) => f.category === "lineage_breakage") expect(lin?.severity).toBe("warning") + expect(lin?.groupKey).toBe("lineage_fanout") expect(env.verdict).not.toBe("REQUEST_CHANGES") }) @@ -1958,6 +2170,7 @@ describe("orchestrate", () => { expect(sem).toBeDefined() expect(sem!.confidence).toBe("unknown") expect(sem!.severity).not.toBe("critical") + expect(sem!.groupKey).toBe("equivalence_undecided") }) test("PII exposure → critical pii_exposure finding", async () => { @@ -2414,7 +2627,7 @@ describe("orchestrate", () => { const runner: ReviewRunner = { ...fakeRunner({}), async impact() { - return { hasManifest: false, severity: "UNKNOWN", directCount: 0, transitiveCount: 0, testCount: 0 } + return { hasManifest: false, resolved: false, severity: "UNKNOWN", directCount: 0, transitiveCount: 0, testCount: 0 } }, } const env = await runReview({ @@ -2426,10 +2639,47 @@ describe("orchestrate", () => { getContent: content("select 1"), }) expect(env.summary.degraded).toBe(true) + expect(env.summary.lintOnly).toBe(true) + expect(env.summary.emptyScope).toBe(false) + expect(renderSummary(env)).toContain( + "⚙️ Lint-only run — no changed model resolved against a dbt manifest (missing manifest, or the changed models are not in it). Run `dbt compile` on this branch so lineage/equivalence can run.", + ) expect(["APPROVE", "COMMENT"]).toContain(env.verdict) }) - test("loaded manifest is not marked lint-only when a changed model is absent from it", async () => { + test("README-only diff with a manifest is empty scope, not lint-only", async () => { + let aiCalls = 0 + const runner: ReviewRunner = { + ...fakeRunner({}), + async manifestAvailable() { + return true + }, + } + const env = await runReview({ + changedFiles: [{ path: "README.md", status: "modified", diff: "+docs\n" }], + config: { ...DEFAULT_REVIEW_CONFIG, reviewers: ["ai_review"] }, + rubric: DEFAULT_RUBRIC, + mode: "comment", + runner, + aiReview: async () => { + aiCalls++ + return { status: "ok", findings: [] } + }, + }) + + expect(aiCalls).toBe(0) + expect(env.summary.aiReview).toBeUndefined() + expect(env.summary).toMatchObject({ degraded: true, lintOnly: false, emptyScope: true }) + const summary = renderSummary(env) + expect(summary).not.toContain("Lint-only") + expect(summary).toContain( + "⚙️ Nothing to review — no dbt model, schema, or macro files changed in this diff.", + ) + expect(summary).not.toContain("No issues found in the changed dbt models") + expect(summary).not.toContain("AI reviewer:") + }) + + test("loaded manifest is lint-only when no changed model resolves", async () => { const files: ChangedFile[] = [{ path: "models/staging/new_model.sql", status: "added", diff: "+select 1\n" }] const runner: ReviewRunner = { ...fakeRunner({}), @@ -2437,7 +2687,122 @@ describe("orchestrate", () => { return true }, async impact() { - return { hasManifest: false, severity: "UNKNOWN", directCount: 0, transitiveCount: 0, testCount: 0 } + return { hasManifest: true, resolved: false, severity: "SAFE", directCount: 0, transitiveCount: 0, testCount: 0 } + }, + } + const env = await runReview({ + changedFiles: files, + config: { ...DEFAULT_REVIEW_CONFIG, reviewers: ["sql_quality"] }, + rubric: DEFAULT_RUBRIC, + mode: "comment", + runner, + getContent: content("select 1 as value"), + }) + expect(env.summary.degraded).toBe(true) + expect(env.summary.lintOnly).toBe(true) + }) + + test("deletion-only diffs use manifest availability for lint-only status", async () => { + const review = async (manifestAvailable: boolean) => + runReview({ + changedFiles: [{ path: "models/staging/removed_model.sql", status: "deleted", diff: "" }], + config: { ...DEFAULT_REVIEW_CONFIG, reviewers: ["sql_quality"] }, + rubric: DEFAULT_RUBRIC, + mode: "comment", + runner: { + ...fakeRunner({}), + async manifestAvailable() { + return manifestAvailable + }, + async impact() { + return { + hasManifest: manifestAvailable, + resolved: false, + severity: "SAFE", + directCount: 0, + transitiveCount: 0, + testCount: 0, + } + }, + }, + }) + + expect((await review(true)).summary.lintOnly).toBe(false) + expect((await review(false)).summary.lintOnly).toBe(true) + }) + + test("the policy signature reflects configured policy, not the tier-derived lane set", async () => { + // A rerun that crosses trivial → lite gains the AI lane under default + // reviewers. That is an analysis-scope change (reported from the tier + // marker), not a settings change, so the signature must not move. + const input = { + changedFiles: [{ path: "models/staging/model.sql", status: "added" as const, diff: "+select 1\n" }], + config: { ...DEFAULT_REVIEW_CONFIG, reviewers: [] }, + rubric: DEFAULT_RUBRIC, + mode: "comment" as const, + runner: fakeRunner({}), + getContent: content("select 1"), + } + const lite = await runReview({ ...input, forceTier: "lite" }) + const full = await runReview({ ...input, forceTier: "full" }) + const trivial = await runReview({ ...input, forceTier: "trivial" }) + + expect(lite.policySignature).toBe(full.policySignature) + expect(trivial.policySignature).toBe(lite.policySignature) + + // Turning the AI lane off in configuration IS a settings change. + const disabled = await runReview({ ...input, config: { ...input.config, ai: false }, forceTier: "lite" }) + expect(disabled.policySignature).not.toBe(lite.policySignature) + }) + + test("an AI model does not affect policy when the selected reviewers omit the AI lane", async () => { + const input = { + changedFiles: [{ path: "models/staging/model.sql", status: "added" as const, diff: "+select 1\n" }], + config: { ...DEFAULT_REVIEW_CONFIG, reviewers: ["sql_quality"] }, + rubric: DEFAULT_RUBRIC, + mode: "comment" as const, + runner: fakeRunner({}), + getContent: content("select 1"), + } + const first = await runReview({ ...input, aiModel: "altimate-gateway/altimate-base" }) + const second = await runReview({ ...input, aiModel: "altimate-gateway/altimate-pro" }) + + expect(first.policySignature).toBe(second.policySignature) + }) + + test("AI reasoning effort does not affect the policy signature", async () => { + const input = { + changedFiles: [{ path: "models/staging/model.sql", status: "added" as const, diff: "+select 1\n" }], + config: { ...DEFAULT_REVIEW_CONFIG, reviewers: ["ai_review"] }, + rubric: DEFAULT_RUBRIC, + mode: "comment" as const, + runner: fakeRunner({}), + getContent: content("select 1"), + aiModel: "altimate-gateway/altimate-base", + aiReview: async () => ({ status: "ok" as const, findings: [] }), + } + const none = await runReview({ ...input, aiReasoningEffort: "none" }) + const high = await runReview({ ...input, aiReasoningEffort: "high" }) + + expect(none.policySignature).toBe(high.policySignature) + }) + + test("one resolved changed model keeps a mixed-model run out of lint-only", async () => { + const files: ChangedFile[] = [ + { path: "models/staging/known_model.sql", status: "added", diff: "+select 1\n" }, + { path: "models/staging/new_model.sql", status: "added", diff: "+select 1\n" }, + ] + const runner: ReviewRunner = { + ...fakeRunner({}), + async impact(model) { + return { + hasManifest: true, + resolved: model === "known_model", + severity: "SAFE", + directCount: 0, + transitiveCount: 0, + testCount: 0, + } }, } const env = await runReview({ @@ -2448,7 +2813,9 @@ describe("orchestrate", () => { runner, getContent: content("select 1 as value"), }) + expect(env.summary.degraded).toBe(false) + expect(env.summary.lintOnly).toBe(false) }) test("manifest availability errors degrade safely instead of aborting the review", async () => { @@ -2459,7 +2826,7 @@ describe("orchestrate", () => { throw new Error("manifest unreadable") }, async impact() { - return { hasManifest: false, severity: "UNKNOWN", directCount: 0, transitiveCount: 0, testCount: 0 } + return { hasManifest: false, resolved: false, severity: "UNKNOWN", directCount: 0, transitiveCount: 0, testCount: 0 } }, } const env = await runReview({ @@ -2533,4 +2900,83 @@ describe("orchestrate", () => { expect(inline.length).toBe(1) expect(inline[0]).toMatchObject({ path: "models/x.sql", line: 5, side: "RIGHT" }) }) + + test("renderSummary groups related model findings while a singleton renders normally", () => { + const groupedFindings = [ + makeFinding({ + severity: "suggestion", + category: "test_coverage", + title: "model_a: new model has no uniqueness/grain test", + body: "Add a uniqueness test for model_a.", + file: "models/model_a.sql", + model: "model_a", + startLine: 1, + groupKey: "missing_grain_test", + ruleKey: "test_coverage:missing-grain-test", + }), + makeFinding({ + severity: "suggestion", + category: "test_coverage", + title: "model_b: new model has no uniqueness/grain test", + body: "Add a uniqueness test for model_b.", + file: "models/model_b.sql", + model: "model_b", + startLine: 1, + groupKey: "missing_grain_test", + ruleKey: "test_coverage:missing-grain-test", + }), + ] + const groupedEnv = buildEnvelope({ findings: groupedFindings, tier: "lite", mode: "comment" }) + const groupedSummary = renderSummary(groupedEnv) + expect(groupedSummary).toContain( + "- **2 new models have no uniqueness/grain test** — `model_a`, `model_b` · test_coverage", + ) + expect(groupedSummary).not.toContain("Add a uniqueness test for model_a.") + expect(inlineComments(groupedEnv)).toHaveLength(2) + + const singletonEnv = buildEnvelope({ findings: [groupedFindings[0]], tier: "lite", mode: "comment" }) + const singletonSummary = renderSummary(singletonEnv) + expect(singletonSummary).toContain("- **model_a: new model has no uniqueness/grain test**") + expect(singletonSummary).toContain("Add a uniqueness test for model_a.") + }) + + test("renderSummary fences grouped model identifiers containing backticks", () => { + const findings = [ + makeFinding({ + severity: "suggestion", + category: "test_coverage", + title: "model`a: new model has no uniqueness/grain test", + body: "Add a uniqueness test.", + file: "models/model_a.sql", + model: "model`a", + groupKey: "missing_grain_test", + ruleKey: "test_coverage:missing-grain-test", + }), + makeFinding({ + severity: "suggestion", + category: "test_coverage", + title: "model_b: new model has no uniqueness/grain test", + body: "Add a uniqueness test.", + file: "models/model_b.sql", + model: "model_b", + groupKey: "missing_grain_test", + ruleKey: "test_coverage:missing-grain-test", + }), + ] + + const summary = renderSummary(buildEnvelope({ findings, tier: "lite", mode: "comment" })) + expect(summary).toContain("``model`a``, `model_b`") + }) + + test("renderSummary collapses missing artifact hints onto one line", () => { + const env = buildEnvelope({ + findings: [], + tier: "lite", + mode: "comment", + artifactHints: ["catalog.json (run `dbt docs generate`)", "target-base/compiled (compile the base ref)"], + }) + expect(renderSummary(env)).toContain( + "🧩 Missing artifacts: catalog.json (run `dbt docs generate`) · target-base/compiled (compile the base ref) — some analyses run at reduced fidelity; affected findings say which", + ) + }) }) diff --git a/packages/opencode/test/altimate/review/format.test.ts b/packages/opencode/test/altimate/review/format.test.ts new file mode 100644 index 0000000000..616a282cea --- /dev/null +++ b/packages/opencode/test/altimate/review/format.test.ts @@ -0,0 +1,431 @@ +import { describe, expect, test } from "bun:test" +import { makeFinding, type Finding } from "../../../src/altimate/review/finding" +import { renderSummary } from "../../../src/altimate/review/format" +import { computeFindingDelta, parseFindingIds } from "../../../src/altimate/review/post-github" +import { DEFAULT_RUBRIC } from "../../../src/altimate/review/rubric" +import { + buildEnvelope, + makeReviewPolicySignature, + type ReviewPolicySignatureInput, +} from "../../../src/altimate/review/verdict" + +function finding(id: string, overrides: Partial[0]> = {}): Finding { + return makeFinding({ + id, + severity: "warning", + category: "sql_quality", + title: `Finding ${id}`, + body: `Body ${id}.`, + file: `models/${id}.sql`, + ruleKey: id, + ...overrides, + }) +} + +function summary(findings: Finding[], options: { lintOnly?: boolean; artifactHints?: string[] } = {}): string { + return renderSummary(buildEnvelope({ findings, tier: "full", mode: "comment", ...options })) +} + +describe("review summary readability", () => { + test("describes every condition that makes a review lint-only", () => { + expect(summary([], { lintOnly: true })).toContain( + "⚙️ Lint-only run — no changed model resolved against a dbt manifest (missing manifest, or the changed models are not in it). Run `dbt compile` on this branch so lineage/equivalence can run.", + ) + }) + + test("escapes singleton locations that contain backticks", () => { + const rendered = summary([finding("singleton", { file: "models/odd`name.sql" })]) + + expect(rendered).toContain("``models/odd`name.sql`` · sql_quality") + }) + + test("grouped items retain distinct locations and an unverified marker", () => { + const rendered = summary([ + finding("group-a", { file: "models/a.sql", groupKey: "shared" }), + finding("group-b", { file: "models/b.sql", groupKey: "shared", degraded: true }), + finding("group-a-again", { file: "models/a.sql", groupKey: "shared" }), + finding("group-c", { file: "models/c.sql", groupKey: "shared" }), + finding("group-d", { file: "models/d.sql", groupKey: "shared" }), + ]) + + expect(rendered).toContain("`models/a.sql`, `models/b.sql`, `models/c.sql` · +1 more · _unverified_") + }) + + test("repetitive families render their member specifics in compact grouped items", () => { + const equivalenceBody = (model: string) => + `The logic of \`${model}\` changed and equivalence could not be decided (no schema, or unsupported SQL). ` + + "Treat as a potential behavior change and verify with a data-diff." + const grainBody = (column: string) => + `The \`unique_combination_of_columns\` test on \`orders\` names \`${column}\` as a grain key. ` + + `Add \`not_null\` to \`${column}\`'s \`data_tests:\` on \`orders\` (contract is not enforced, so use a data_test).` + const findings = [ + finding("fanout-orders", { + category: "lineage_breakage", + title: "orders: high downstream fan-out (7 models)", + body: "Fan-out body for orders.", + file: "models/orders.sql", + model: "orders", + groupKey: "lineage_fanout", + evidence: { + tool: "impact_analysis", + result: { directCount: 2, transitiveCount: 5, testCount: 3 }, + }, + }), + finding("fanout-customers", { + category: "lineage_breakage", + title: "customers: high downstream fan-out (4 models)", + body: "Fan-out body for customers.", + file: "models/customers.sql", + model: "customers", + groupKey: "lineage_fanout", + }), + finding("equivalence-orders", { + category: "semantic_change", + title: "orders: refactor could not be proven equivalent", + body: equivalenceBody("orders"), + file: "models/orders.sql", + model: "orders", + groupKey: "equivalence_undecided", + confidence: "unknown", + degraded: true, + }), + finding("equivalence-customers", { + category: "semantic_change", + title: "customers: refactor could not be proven equivalent", + body: equivalenceBody("customers"), + file: "models/customers.sql", + model: "customers", + groupKey: "equivalence_undecided", + confidence: "unknown", + degraded: true, + }), + finding("grain-id", { + category: "test_coverage", + title: "schema.yml: grain column `id` lacks `not_null`", + body: grainBody("id"), + file: "models/schema.yml", + model: "orders", + column: "id", + groupKey: "grain_not_null:orders", + }), + finding("grain-created", { + category: "test_coverage", + title: "schema.yml: grain column `created_at` lacks `not_null`", + body: grainBody("created_at"), + file: "models/schema.yml", + model: "orders", + column: "created_at", + groupKey: "grain_not_null:orders", + }), + ] + + const rendered = summary(findings, { + artifactHints: ["target/compiled missing for 2 changed model(s)"], + }) + + expect(rendered).toContain("### ⚠️ Warning (6 findings · 3 items)") + expect(rendered).toContain( + "**Downstream fan-out on 2 models** (informational) — `orders` (2 direct/5 transitive, +3 tests), `customers`", + ) + expect(rendered).toContain( + "**Equivalence could not be decided for 2 models** — no schema, or unsupported SQL. Fix once: compile base and head (see missing-artifact line). Models: `orders`, `customers`", + ) + expect(rendered).toContain("**`orders`: grain columns without `not_null`** — `id`, `created_at` · test_coverage") + expect(rendered).toContain("Add `not_null` to each listed column's `data_tests:` on `orders`") + expect(rendered.split("Add `not_null`")).toHaveLength(2) + + const withoutCompiledHint = summary(findings) + expect(withoutCompiledHint).toContain( + "**Equivalence could not be decided for 2 models** — no schema, or unsupported SQL. Undecidable with the available artifacts — unsupported SQL for this dialect or missing schema; verify with a data-diff. Models: `orders`, `customers`", + ) + expect(withoutCompiledHint).not.toContain("Fix once: compile base and head") + }) + + test("long non-critical sections fold after 12 rendered items, while critical never folds", () => { + const suggestions = Array.from({ length: 30 }, (_, index) => + finding(`suggestion-${index + 1}`, { + severity: "suggestion", + title: `Suggestion ${index + 1}`, + confidence: "medium", + }), + ) + const rendered = summary(suggestions) + const detailsAt = rendered.indexOf("
") + const beforeDetails = rendered.slice(0, detailsAt) + + expect(rendered).toContain("### 💡 Suggestion (30)") + expect(beforeDetails).toContain("Suggestion 12") + expect(beforeDetails).not.toContain("Suggestion 13") + expect(rendered.slice(detailsAt)).toContain("Suggestion 13") + expect(rendered).toContain("18 more …") + + const critical = Array.from({ length: 3 }, (_, index) => + finding(`critical-${index + 1}`, { + severity: "critical", + category: "contract_violation", + title: `Critical ${index + 1}`, + }), + ) + expect(summary(critical)).not.toContain("
") + }) + + test("Read first prioritizes critical, contextual AI, then high-confidence ungrouped warning", () => { + const findings = [ + finding("repeat-1", { title: "Repeated 1", groupKey: "repeated" }), + finding("repeat-2", { title: "Repeated 2", groupKey: "repeated" }), + finding("repeat-3", { title: "Repeated 3", groupKey: "repeated" }), + finding("low", { title: "Low warning", confidence: "low" }), + finding("critical", { + severity: "critical", + category: "contract_violation", + title: "Critical contract break", + file: "models/critical.sql", + }), + finding("ai", { + severity: "suggestion", + title: "Unused CTE bypasses the documented gate", + file: "models/context.sql", + evidence: { tool: "ai-review" }, + }), + finding("warning", { + title: "High-confidence warning", + file: "models/warning.sql", + confidence: "high", + }), + finding("extra", { severity: "suggestion", title: "Extra suggestion" }), + ] + const rendered = summary(findings, { lintOnly: true }) + const readFirstAt = rendered.indexOf("**Read first**") + const bannerAt = rendered.indexOf("> ⚙️ Lint-only run") + const block = rendered.slice(readFirstAt, bannerAt) + + expect(readFirstAt).toBeGreaterThan(rendered.indexOf("## ")) + expect(bannerAt).toBeGreaterThan(readFirstAt) + expect(block.indexOf("Critical contract break")).toBeLessThan(block.indexOf("Unused CTE")) + expect(block.indexOf("Unused CTE")).toBeLessThan(block.indexOf("High-confidence warning")) + expect(block).toContain("`models/critical.sql`") + expect(block).not.toContain("Repeated 1") + expect(block).not.toContain("Low warning") + expect(summary(findings.slice(0, 7))).not.toContain("**Read first**") + }) + + test("finding id blocks round-trip and drive the rerun delta line", () => { + const rubric = { + ...DEFAULT_RUBRIC, + exclusions: { ...DEFAULT_RUBRIC.exclusions, excludeGlobs: ["models/archive/**", "seeds/tmp/**"] }, + } + const policy: ReviewPolicySignatureInput = { + severityThreshold: "suggestion", + enabledReviewers: ["semantic_change", "sql_quality"], + dialect: "snowflake", + rubric, + aiEnabled: true, + aiModel: "altimate-gateway/altimate-base", + dataDiff: { enabled: false, warehouse: "" }, + } + const policySignature = makeReviewPolicySignature(policy) + expect(policySignature).toBe( + makeReviewPolicySignature({ + ...policy, + enabledReviewers: ["sql_quality", "semantic_change"], + rubric: { + ...rubric, + blockOn: [...rubric.blockOn].reverse(), + thresholds: Object.fromEntries(Object.entries(rubric.thresholds).reverse()) as typeof rubric.thresholds, + exclusions: { ...rubric.exclusions, excludeGlobs: ["seeds/tmp/**", "models/archive/**"] }, + }, + }), + ) + expect(policySignature).not.toBe( + makeReviewPolicySignature({ + ...policy, + rubric: { + ...rubric, + exclusions: { ...rubric.exclusions, excludeGlobs: ["models/other/**", "seeds/tmp/**"] }, + }, + }), + ) + const firstEnabledExclusion = makeReviewPolicySignature({ + ...policy, + rubric: { + ...rubric, + exclusions: { + ...rubric.exclusions, + allowSelectStarInStaging: false, + skipMissingContractWhenNotEnforced: true, + skipNonProdModels: false, + }, + }, + }) + const secondEnabledExclusion = makeReviewPolicySignature({ + ...policy, + rubric: { + ...rubric, + exclusions: { + ...rubric.exclusions, + allowSelectStarInStaging: true, + skipMissingContractWhenNotEnforced: false, + skipNonProdModels: false, + }, + }, + }) + expect(firstEnabledExclusion).not.toBe(secondEnabledExclusion) + expect(policySignature).not.toBe( + makeReviewPolicySignature({ + ...policy, + dataDiff: { enabled: true, warehouse: "" }, + }), + ) + expect(policySignature).not.toBe( + makeReviewPolicySignature({ + ...policy, + dataDiff: { enabled: false, warehouse: "production" }, + }), + ) + expect(policySignature).not.toBe(makeReviewPolicySignature({ ...policy, aiEnabled: false })) + expect(makeReviewPolicySignature({ ...policy, aiEnabled: false })).toBe( + makeReviewPolicySignature({ + ...policy, + aiEnabled: false, + aiModel: "altimate-gateway/altimate-pro", + }), + ) + expect(policySignature).not.toBe( + makeReviewPolicySignature({ ...policy, aiModel: "altimate-gateway/altimate-pro" }), + ) + expect(policySignature).not.toBe(makeReviewPolicySignature({ ...policy, aiModel: "session" })) + expect(policySignature).not.toBe(makeReviewPolicySignature({ ...policy, dialect: "bigquery" })) + expect(policySignature).not.toBe( + makeReviewPolicySignature({ + ...policy, + rubric: { + ...rubric, + thresholds: { + ...rubric.thresholds, + lineageWarnConsumers: rubric.thresholds.lineageWarnConsumers + 1, + }, + }, + }), + ) + expect(policySignature).not.toBe( + makeReviewPolicySignature({ + ...policy, + rubric: { ...rubric, warningPatternThreshold: rubric.warningPatternThreshold + 1 }, + }), + ) + expect(policySignature).not.toBe( + makeReviewPolicySignature({ + ...policy, + rubric: { ...rubric, blockOn: rubric.blockOn.slice(1) }, + }), + ) + const previous = buildEnvelope({ + findings: [finding("fixed"), finding("unchanged")], + tier: "lite", + mode: "comment", + policySignature, + }) + const current = buildEnvelope({ + findings: [finding("unchanged"), finding("new")], + tier: "lite", + mode: "comment", + policySignature, + }) + const previousBody = renderSummary(previous) + + expect([...parseFindingIds(previousBody)!]).toEqual(["fixed", "unchanged"]) + const delta = computeFindingDelta(previousBody, current) + expect(delta).toEqual({ noLongerSurfaced: 1, new: 1, unchanged: 1, reviewSettingsChanged: undefined }) + + const currentBody = renderSummary(current, delta) + expect(currentBody).toContain("**Since last review:** 1 no longer surfaced · 1 new · 1 unchanged") + expect(currentBody).toContain(``) + expect(currentBody).toContain("") + expect(currentBody.endsWith("")).toBe(true) + + const changedTier = buildEnvelope({ + findings: current.findings, + tier: "full", + mode: "comment", + policySignature, + }) + const changedTierDelta = computeFindingDelta(previousBody, changedTier) + expect(changedTierDelta?.reviewSettingsChanged).toBeUndefined() + expect(changedTierDelta?.analysisScopeChanged).toEqual({ from: "lite", to: "full" }) + expect(renderSummary(changedTier, changedTierDelta)).toContain( + "**Since last review:** 1 no longer surfaced · 1 new · 1 unchanged (analysis scope changed: lite → full)", + ) + + const changedPolicy = buildEnvelope({ + findings: current.findings, + tier: "lite", + mode: "comment", + policySignature: makeReviewPolicySignature({ + ...policy, + severityThreshold: "warning", + rubric: { + ...rubric, + exclusions: { ...rubric.exclusions, excludeGlobs: ["models/other/**", "seeds/tmp/**"] }, + }, + dataDiff: { enabled: true, warehouse: "" }, + }), + }) + const changedDelta = computeFindingDelta(previousBody, changedPolicy) + expect(changedDelta?.reviewSettingsChanged).toBe(true) + expect(renderSummary(changedPolicy, changedDelta)).toContain( + "**Since last review:** 1 no longer surfaced · 1 new · 1 unchanged (review settings changed)", + ) + }) + + test("omits the rerun delta when both finding sets are empty", () => { + const previous = buildEnvelope({ findings: [], tier: "lite", mode: "comment" }) + const current = buildEnvelope({ findings: [], tier: "lite", mode: "comment" }) + const delta = computeFindingDelta(renderSummary(previous), current) + + expect(delta).toBeUndefined() + expect(renderSummary(current, delta)).not.toContain("Since last review") + }) + + test("uses the final footer markers when finding titles contain marker-like lines", () => { + const policySignature = makeReviewPolicySignature({ + severityThreshold: "suggestion", + enabledReviewers: [], + dialect: "snowflake", + rubric: DEFAULT_RUBRIC, + aiEnabled: true, + dataDiff: { enabled: false, warehouse: "" }, + }) + const previous = buildEnvelope({ + findings: [ + finding("real", { + title: [ + "Marker-like title", + "", + "", + "", + "still part of the title", + ].join("\n"), + }), + ], + tier: "lite", + mode: "comment", + policySignature, + }) + const current = buildEnvelope({ + findings: [finding("real")], + tier: "lite", + mode: "comment", + policySignature, + }) + const body = renderSummary(previous) + + expect([...parseFindingIds(body)!]).toEqual(["real"]) + expect(computeFindingDelta(body, current)?.reviewSettingsChanged).toBeUndefined() + + const matchingFakePolicy = body.replace( + "", + ``, + ) + expect(computeFindingDelta(matchingFakePolicy, current)?.analysisScopeChanged).toBeUndefined() + }) +}) diff --git a/packages/opencode/test/altimate/review/post-github.test.ts b/packages/opencode/test/altimate/review/post-github.test.ts new file mode 100644 index 0000000000..5ca631d497 --- /dev/null +++ b/packages/opencode/test/altimate/review/post-github.test.ts @@ -0,0 +1,155 @@ +import { describe, expect, test } from "bun:test" +import { makeFinding } from "../../../src/altimate/review/finding" +import { REVIEW_MARKER, renderSummary } from "../../../src/altimate/review/format" +import { postGitHubReview } from "../../../src/altimate/review/post-github" +import { buildEnvelope } from "../../../src/altimate/review/verdict" + +function finding(id: string) { + return makeFinding({ + id, + severity: "warning", + category: "sql_quality", + title: `Finding ${id}`, + body: `Body ${id}.`, + file: `models/${id}.sql`, + ruleKey: id, + }) +} + +function fakeOctokit( + comments: Array<{ id: number; body: string; user: { login: string; type: string } }>, + getAuthenticated: () => Promise<{ data: { login: string } }>, + getAuthenticatedApp: () => Promise<{ data: { slug: string } }> = async () => { + throw new Error("not an app token") + }, +) { + const calls = { + authenticated: 0, + appAuthenticated: 0, + updated: [] as Array<{ comment_id: number; body: string }>, + created: [] as Array<{ body: string }>, + } + const octo = { + paginate: async () => comments, + rest: { + users: { + getAuthenticated: async () => { + calls.authenticated++ + return getAuthenticated() + }, + }, + apps: { + getAuthenticated: async () => { + calls.appAuthenticated++ + return getAuthenticatedApp() + }, + }, + issues: { + listComments: async () => ({ data: comments }), + updateComment: async (input: { comment_id: number; body: string }) => { + calls.updated.push(input) + return { data: { id: input.comment_id } } + }, + createComment: async (input: { body: string }) => { + calls.created.push(input) + return { data: { id: 30 } } + }, + }, + pulls: { + get: async () => ({ data: { head: { sha: "head-sha" } } }), + createReview: async () => ({ data: { id: 40 } }), + }, + }, + } + return { calls, octo } +} + +const target = { token: "token", owner: "owner", repo: "repo", prNumber: 7 } + +describe("GitHub sticky review ownership", () => { + test("updates the authenticated bot's marker comment and ignores a forged human marker", async () => { + const forged = renderSummary(buildEnvelope({ findings: [finding("new")], tier: "lite", mode: "comment" })) + const owned = renderSummary(buildEnvelope({ findings: [finding("old")], tier: "lite", mode: "comment" })) + const { calls, octo } = fakeOctokit( + [ + { id: 10, body: forged, user: { login: "human-reviewer", type: "User" } }, + { id: 20, body: owned, user: { login: "altimate-review[bot]", type: "Bot" } }, + ], + async () => ({ data: { login: "altimate-review[bot]" } }), + ) + + await postGitHubReview( + buildEnvelope({ findings: [finding("new")], tier: "lite", mode: "comment" }), + target, + octo as any, + ) + + expect(calls.authenticated).toBe(1) + expect(calls.appAuthenticated).toBe(0) + expect(calls.updated).toHaveLength(1) + expect(calls.updated[0].comment_id).toBe(20) + expect(calls.updated[0].body).toContain("**Since last review:** 1 no longer surfaced · 1 new · 0 unchanged") + expect(calls.created).toHaveLength(0) + }) + + test("falls back to the GitHub Actions bot when an installation token cannot resolve a user", async () => { + const { calls, octo } = fakeOctokit( + [ + { id: 10, body: REVIEW_MARKER, user: { login: "human-reviewer", type: "User" } }, + { id: 20, body: REVIEW_MARKER, user: { login: "github-actions[bot]", type: "Bot" } }, + ], + async () => { + throw Object.assign(new Error("Resource not accessible by integration"), { status: 403 }) + }, + ) + + await postGitHubReview(buildEnvelope({ findings: [], tier: "trivial", mode: "comment" }), target, octo as any) + + expect(calls.authenticated).toBe(1) + expect(calls.updated.map((call) => call.comment_id)).toEqual([20]) + expect(calls.created).toHaveLength(0) + }) + + test("adopts only the bot login named by ALTIMATE_REVIEW_BOT_LOGIN for a GitHub App token", async () => { + // An installation token resolves neither GET /user nor GET /app, so the + // App's own bot login must be configured explicitly. + const previous = process.env.ALTIMATE_REVIEW_BOT_LOGIN + process.env.ALTIMATE_REVIEW_BOT_LOGIN = "altimate-review[bot]" + try { + const { calls, octo } = fakeOctokit( + [ + { id: 10, body: REVIEW_MARKER, user: { login: "different-app[bot]", type: "Bot" } }, + { id: 20, body: REVIEW_MARKER, user: { login: "altimate-review[bot]", type: "Bot" } }, + { id: 30, body: REVIEW_MARKER, user: { login: "github-actions[bot]", type: "Bot" } }, + ], + async () => { + throw new Error("must not be called when the login is configured") + }, + ) + + await postGitHubReview(buildEnvelope({ findings: [], tier: "trivial", mode: "comment" }), target, octo as any) + + expect(calls.authenticated).toBe(0) + expect(calls.appAuthenticated).toBe(0) + expect(calls.updated.map((call) => call.comment_id)).toEqual([20]) + expect(calls.created).toHaveLength(0) + } finally { + if (previous === undefined) delete process.env.ALTIMATE_REVIEW_BOT_LOGIN + else process.env.ALTIMATE_REVIEW_BOT_LOGIN = previous + } + }) + + test("never adopts a marker comment owned by a different bot", async () => { + const { calls, octo } = fakeOctokit( + [{ id: 10, body: REVIEW_MARKER, user: { login: "different-app[bot]", type: "Bot" } }], + async () => { + throw Object.assign(new Error("Resource not accessible by integration"), { status: 403 }) + }, + ) + + await postGitHubReview(buildEnvelope({ findings: [], tier: "trivial", mode: "comment" }), target, octo as any) + + expect(calls.updated).toHaveLength(0) + expect(calls.created).toHaveLength(1) + }) +}) diff --git a/packages/opencode/test/altimate/review/telemetry.test.ts b/packages/opencode/test/altimate/review/telemetry.test.ts index a9ed6ea557..3e537a0b5a 100644 --- a/packages/opencode/test/altimate/review/telemetry.test.ts +++ b/packages/opencode/test/altimate/review/telemetry.test.ts @@ -32,7 +32,23 @@ function envelope(over: Record = {}) { idealVerdict: "REQUEST_CHANGES", mode: "comment", tier: "full", - summary: { critical: 1, warning: 2, suggestion: 0, degraded: false }, + summary: { + critical: 1, + warning: 2, + suggestion: 0, + degraded: false, + lintOnly: false, + undecidableFindings: 0, + artifactHints: [], + aiReview: { + status: "ok", + findings: 2, + model: "altimate-gateway/altimate-base", + durationMs: 142_000, + promptChars: 24_000, + reasoningTokens: 3_000, + }, + }, findings: [ { category: "join_risk", severity: "critical" }, { category: "join_risk", severity: "warning" }, @@ -58,6 +74,27 @@ describe("review_run", () => { expect(e.ideal_verdict).toBe("REQUEST_CHANGES") expect(e.critical).toBe(1) expect(e.duration_ms).toBe(1234) + expect(e.ai_status).toBe("ok") + expect(e.ai_model).toBe("altimate-gateway/altimate-base") + expect(e.ai_findings).toBe(2) + expect(e.ai_duration_ms).toBe(142_000) + expect(e.ai_prompt_chars).toBe(24_000) + expect(e.ai_reasoning_tokens).toBe(3_000) + expect(e.undecidable_findings).toBe(0) + expect(e.lint_only).toBe(false) + expect(e.empty_scope).toBe(false) + }) + + test("custom provider model ids are hashed before emission", () => { + const events = captureEvents() + const env = envelope() + env.summary.aiReview.model = "private-provider/secret-model" + + emitReviewRun({ invocation: "cli", durationMs: 1, sessionID: "", envelope: env }) + + expect((events[0] as any).ai_model).toBe("custom/cd6c5615") + expect(JSON.stringify(events[0])).not.toContain("private-provider") + expect(JSON.stringify(events[0])).not.toContain("secret-model") }) test("tier_forced normalises absent to false", () => { @@ -123,7 +160,7 @@ describe("review_run", () => { for (const v of Object.values(byCategory)) expect(typeof v).toBe("number") }) - test("stale_manifest and degraded are carried from the envelope", () => { + test("stale_manifest and run-level degraded states are carried from the envelope", () => { // Same `=== true` normalisation as tier_forced, which has its own test; these two had none, // and the shared envelope() helper omits staleManifest so every other test covers only the // undefined case. @@ -131,6 +168,8 @@ describe("review_run", () => { emitReviewRun({ invocation: "cli", durationMs: 1, sessionID: "", envelope: envelope() }) expect((events[0] as any).stale_manifest).toBe(false) expect((events[0] as any).degraded).toBe(false) + expect((events[0] as any).lint_only).toBe(false) + expect((events[0] as any).empty_scope).toBe(false) events.length = 0 emitReviewRun({ @@ -139,11 +178,122 @@ describe("review_run", () => { sessionID: "", envelope: envelope({ staleManifest: true, - summary: { critical: 0, warning: 0, suggestion: 0, degraded: true }, + summary: { + critical: 0, + warning: 0, + suggestion: 0, + degraded: true, + lintOnly: true, + undecidableFindings: 0, + artifactHints: [], + }, }), }) expect((events[0] as any).stale_manifest).toBe(true) expect((events[0] as any).degraded).toBe(true) + expect((events[0] as any).lint_only).toBe(true) + expect((events[0] as any).empty_scope).toBe(false) + + events.length = 0 + emitReviewRun({ + invocation: "cli", + durationMs: 1, + sessionID: "", + envelope: envelope({ + summary: { + critical: 0, + warning: 0, + suggestion: 0, + degraded: true, + lintOnly: false, + emptyScope: true, + undecidableFindings: 0, + artifactHints: [], + }, + }), + }) + expect((events[0] as any).degraded).toBe(true) + expect((events[0] as any).lint_only).toBe(false) + expect((events[0] as any).empty_scope).toBe(true) + + events.length = 0 + emitReviewRun({ + invocation: "cli", + durationMs: 1, + sessionID: "", + envelope: envelope({ + summary: { + critical: 0, + warning: 0, + suggestion: 0, + degraded: true, + emptyScope: true, + undecidableFindings: 0, + artifactHints: [], + }, + }), + }) + expect((events[0] as any).lint_only).toBe(false) + expect((events[0] as any).empty_scope).toBe(true) + + events.length = 0 + emitReviewRun({ + invocation: "cli", + durationMs: 1, + sessionID: "", + envelope: envelope({ + summary: { + critical: 0, + warning: 0, + suggestion: 0, + degraded: true, + undecidableFindings: 0, + artifactHints: [], + }, + }), + }) + expect((events[0] as any).lint_only).toBe(true) + expect((events[0] as any).empty_scope).toBe(false) + }) + + test("undecidable findings do not turn review_run degraded", () => { + const events = captureEvents() + emitReviewRun({ + invocation: "cli", + durationMs: 1, + sessionID: "", + envelope: envelope({ + summary: { + critical: 0, + warning: 1, + suggestion: 0, + degraded: false, + lintOnly: false, + undecidableFindings: 1, + artifactHints: [], + aiReview: { status: "timeout", reason: "timed out after 62s", findings: 0 }, + }, + }), + }) + + expect((events[0] as any).degraded).toBe(false) + expect((events[0] as any).undecidable_findings).toBe(1) + expect((events[0] as any).ai_status).toBe("timeout") + expect((events[0] as any).ai_findings).toBe(0) + }) + + test("undecidable findings fall back to degraded findings for compatibility envelopes", () => { + const events = captureEvents() + const env = envelope() + delete env.summary.undecidableFindings + env.findings = [ + { category: "semantic_change", severity: "warning", degraded: true }, + { category: "sql_quality", severity: "warning", degraded: false }, + ] + + emitReviewRun({ invocation: "cli", durationMs: 1, sessionID: "", envelope: env }) + + expect((events[0] as any).undecidable_findings).toBe(1) }) test("the tool path carries its session, the CLI path does not", () => { diff --git a/packages/opencode/test/skill/release-v0.8.5-adversarial.test.ts b/packages/opencode/test/skill/release-v0.8.5-adversarial.test.ts index 00b1ee1724..29b6ae4e49 100644 --- a/packages/opencode/test/skill/release-v0.8.5-adversarial.test.ts +++ b/packages/opencode/test/skill/release-v0.8.5-adversarial.test.ts @@ -238,9 +238,18 @@ describe("v0.8.5 adversarial - composite action", () => { const changelog = await fs.readFile(path.join(repoRoot, "CHANGELOG.md"), "utf8") expect(changelog).toContain(`## [${version}]`) + // The docs may move the pin forward as releases ship (they do — the review + // gained inputs after 0.8.5); the gate is that they never point at or + // below the broken tag. Assert every pin is a release >= 0.8.5. + const atLeast085 = (v: string) => { + const [a, b, c] = v.split(".").map(Number) + return a > 0 || b > 8 || (b === 8 && c >= 5) + } for (const relative of ["docs/docs/usage/dbt-pr-review.md", "github/review/examples/altimate-ingestion.yml"]) { const content = await fs.readFile(path.join(repoRoot, relative), "utf8") - expect(content).toContain(`AltimateAI/altimate-code/github/review@v${version}`) + const pins = [...content.matchAll(/AltimateAI\/altimate-code\/github\/review@v(\d+\.\d+\.\d+)/g)].map((m) => m[1]) + expect(pins.length).toBeGreaterThan(0) + for (const pin of pins) expect(atLeast085(pin)).toBe(true) expect(content).not.toContain("AltimateAI/altimate-code/github/review@v0.8.4") } }) diff --git a/packages/opencode/test/skill/release-v0.9.3-adversarial.test.ts b/packages/opencode/test/skill/release-v0.9.3-adversarial.test.ts index 94b1c07c9a..3ca1bbfdf5 100644 --- a/packages/opencode/test/skill/release-v0.9.3-adversarial.test.ts +++ b/packages/opencode/test/skill/release-v0.9.3-adversarial.test.ts @@ -50,7 +50,7 @@ const inertRunner: ReviewRunner = { return { columns: [] } }, async impact() { - return { hasManifest: false, severity: "SAFE", directCount: 0, transitiveCount: 0, testCount: 0 } + return { hasManifest: false, resolved: false, severity: "SAFE", directCount: 0, transitiveCount: 0, testCount: 0 } }, async equivalence() { return { decided: true, equivalent: true } as EquivalenceResult diff --git a/packages/opencode/test/skill/tracker-leak-check.test.ts b/packages/opencode/test/skill/tracker-leak-check.test.ts index 62114ed85a..ebd8c6318a 100644 --- a/packages/opencode/test/skill/tracker-leak-check.test.ts +++ b/packages/opencode/test/skill/tracker-leak-check.test.ts @@ -111,6 +111,23 @@ describe("Atlassian URL regex", () => { }) }) +describe("Internal hostname regex — public gateway allowlist", () => { + // Built from parts so this file never contains the apex literally. + const APEX = "oneal" + "timate.com" + const hostRule = RULES.find((r) => r.name.startsWith("Internal hostname"))! + + test("flags the bare apex and arbitrary subdomains", () => { + expect(matches(hostRule.pattern, `see https://${APEX}/x`)).toEqual([APEX]) + expect(matches(hostRule.pattern, `dashboard.${APEX}`)).toEqual([APEX]) + expect(matches(hostRule.pattern, `altimate-gateway-staging.${APEX}`)).toEqual([APEX]) + }) + + test("allows the public LLM gateway hostname (a documented action default)", () => { + expect(matches(hostRule.pattern, `https://altimate-gateway.${APEX}/v1`)).toEqual([]) + expect(matches(hostRule.pattern, `default: "https://altimate-gateway.${APEX}"`)).toEqual([]) + }) +}) + // altimate_change — bot-review fix: end-to-end coverage of the scanner's git // behaviour, not just its regexes. These run the real script against throwaway // repositories, because the failures the reviewers found were all in how the diff --git a/script/check-tracker-leaks.ts b/script/check-tracker-leaks.ts index b30aed7ee1..4ec6400b16 100755 --- a/script/check-tracker-leaks.ts +++ b/script/check-tracker-leaks.ts @@ -43,6 +43,12 @@ import { $ } from "bun" // (same technique used for the Jira-key/Atlassian-host fixtures in // packages/opencode/test/skill/tracker-leak-check.test.ts). const INTERNAL_HOST = "oneal" + "timate.com" +// Public product endpoints under the internal apex that ARE allowed in this repo. +// The LLM gateway's hostname is a documented default of the review action +// (`altimate_gateway_url`), so it is not a leak. Anything else under the apex +// (dashboards, staging boxes, tenant hosts) still is. Keep this list short and +// reviewed; adding a host here is a product decision, not a convenience. +const PUBLIC_HOSTS = ["altimate-gateway"] export const RULES = [ { @@ -58,7 +64,13 @@ export const RULES = [ }, { name: `Internal hostname (${INTERNAL_HOST})`, - pattern: new RegExp(`\\b${INTERNAL_HOST.replace(/\./g, "\\.")}\\b`, "g"), + // Negative lookbehind excludes the allowlisted public subdomains, e.g. + // `altimate-gateway.` passes while `dashboard.` or the bare + // apex is still flagged. + pattern: new RegExp( + `(? h.replace(/[.-]/g, "\\$&") + "\\.").join("|")})\\b${INTERNAL_HOST.replace(/\./g, "\\.")}\\b`, + "g", + ), remediation: "Replace with the corresponding GitHub issue link or drop the reference.", }, ]