From 337144a789d236fe2e22b9d975d2740a263c0ece Mon Sep 17 00:00:00 2001 From: anandgupta42 Date: Sat, 18 Jul 2026 22:31:30 -0700 Subject: [PATCH 1/4] test: [de-fork] trace-golden e2e harness (session trace as behavioral oracle) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A novel integration-test technique for de-fork stages S5 (native/plugin tool parity) and S7 (validator continuation): drive a real headless session with a scripted model, capture the trace it emits, normalize out nondeterminism, and partial-order-match it against a committed golden. Test infra only — no product-code change. Explicitly NOT a security oracle: tracing is best-effort/failure-suppressing and TraceSpan has no deny state, so an absent span is not proof of non-execution (S3's HardPolicy uses an independent audit probe instead). This harness corroborates observable behavior only. - `normalize.ts` — versioned allowlist-projection normalizer; every field classified stable-keep/volatile-drop/volatile-canonicalize; throws on an unknown span/trace field so schema drift surfaces. Structural, timing- independent sibling ranking (tool-call siblings under a generation collapse to one concurrent bucket; root-attached tools keep real ranks). callId uniqueness canonicalized; loop-summary hashes stripped structurally; realpath-variant path scrubbing (no macOS `/private` leak). - `match.ts` — partial-order structural diff; concurrent siblings compared as an unordered multiset via full canonical signature. - `driver.ts` — drives a real `opencode run` subprocess against the fork's own `TestLLMServer`, recovering the trace path from the CLI's `trace_saved` event. - `trace-golden.test.ts` — 34 tests; each of the reviewed correctness fixes has a describe block that fails on revert. Missing golden fails unless `TRACE_GOLDEN_UPDATE=1` (rejected under CI). Scenario auto-discovery. - `stability-check.ts` — drives the smoke scenario through N independent subprocess runs; verified 100/100 → 1 unique hash. - `fixtures/real-traces/` — 8 benign smoke-session traces (PONG/BANANA/etc.; no PII) that pin the normalizer against real schema shapes. Test Plan: - `bun test test/altimate/trace-golden/`: 34 pass, 0 fail. - `stability-check.ts`: 100/100 runs, 1 unique hash, matches golden. - `bun run typecheck`: clean. NOTE: the CI wiring that runs the live subprocess test lands in a combined "wire de-fork CI checks" follow-up (that ci.yml region is co-edited by a sibling stage). Co-Authored-By: Claude Fable 5 --- .../2026-07-18-trace-golden-e2e-technique.md | 71 ++ .../altimate/trace-golden/DRIVER-NOTES.md | 183 +++ .../test/altimate/trace-golden/driver.ts | 98 ++ .../ses_0c832d792ffeGW44jvnbD1ofca.json | 179 +++ .../ses_0c9a811f7ffeVHtkexaP3SKoMg.json | 246 ++++ .../ses_0dfeccc49ffetVwiuQmrDKxNxh.json | 43 + .../ses_0e3a95d6bffexTX5KtOaS4FOT9.json | 43 + .../ses_0e3f286e5ffeGgjTZeBHqhT14w.json | 126 ++ .../ses_0fb0d89c3ffemtOM9qTPAFNZyK.json | 77 ++ .../ses_2e819da4cffeMhWRAOCOslv3mp.json | 158 +++ .../ses_2e81c7baeffeaz1CvDUlNzeL4Y.json | 115 ++ .../test/altimate/trace-golden/match.ts | 303 +++++ .../test/altimate/trace-golden/normalize.ts | 801 ++++++++++++ .../trace-golden/scenarios/smoke/golden.json | 93 ++ .../scenarios/smoke/model-script.json | 11 + .../trace-golden/scenarios/smoke/prompt.json | 3 + .../trace-golden/scenarios/smoke/setup.json | 5 + .../altimate/trace-golden/stability-check.ts | 265 ++++ .../trace-golden/trace-golden.test.ts | 1090 +++++++++++++++++ 19 files changed, 3910 insertions(+) create mode 100644 docs/internal/2026-07-18-trace-golden-e2e-technique.md create mode 100644 packages/opencode/test/altimate/trace-golden/DRIVER-NOTES.md create mode 100644 packages/opencode/test/altimate/trace-golden/driver.ts create mode 100644 packages/opencode/test/altimate/trace-golden/fixtures/real-traces/ses_0c832d792ffeGW44jvnbD1ofca.json create mode 100644 packages/opencode/test/altimate/trace-golden/fixtures/real-traces/ses_0c9a811f7ffeVHtkexaP3SKoMg.json create mode 100644 packages/opencode/test/altimate/trace-golden/fixtures/real-traces/ses_0dfeccc49ffetVwiuQmrDKxNxh.json create mode 100644 packages/opencode/test/altimate/trace-golden/fixtures/real-traces/ses_0e3a95d6bffexTX5KtOaS4FOT9.json create mode 100644 packages/opencode/test/altimate/trace-golden/fixtures/real-traces/ses_0e3f286e5ffeGgjTZeBHqhT14w.json create mode 100644 packages/opencode/test/altimate/trace-golden/fixtures/real-traces/ses_0fb0d89c3ffemtOM9qTPAFNZyK.json create mode 100644 packages/opencode/test/altimate/trace-golden/fixtures/real-traces/ses_2e819da4cffeMhWRAOCOslv3mp.json create mode 100644 packages/opencode/test/altimate/trace-golden/fixtures/real-traces/ses_2e81c7baeffeaz1CvDUlNzeL4Y.json create mode 100644 packages/opencode/test/altimate/trace-golden/match.ts create mode 100644 packages/opencode/test/altimate/trace-golden/normalize.ts create mode 100644 packages/opencode/test/altimate/trace-golden/scenarios/smoke/golden.json create mode 100644 packages/opencode/test/altimate/trace-golden/scenarios/smoke/model-script.json create mode 100644 packages/opencode/test/altimate/trace-golden/scenarios/smoke/prompt.json create mode 100644 packages/opencode/test/altimate/trace-golden/scenarios/smoke/setup.json create mode 100644 packages/opencode/test/altimate/trace-golden/stability-check.ts create mode 100644 packages/opencode/test/altimate/trace-golden/trace-golden.test.ts diff --git a/docs/internal/2026-07-18-trace-golden-e2e-technique.md b/docs/internal/2026-07-18-trace-golden-e2e-technique.md new file mode 100644 index 000000000..ac2fa2928 --- /dev/null +++ b/docs/internal/2026-07-18-trace-golden-e2e-technique.md @@ -0,0 +1,71 @@ +# Trace-Golden E2E: using the fork's own trace subsystem as the integration-test oracle + +**Date:** 2026-07-18 · **Status:** technique spec (build target), **revised after Codex review** · **For:** de-fork spike verification **(S5 observable parity, S7 continuation invariants)** and general regression safety — **NOT an S3 security oracle** (see box). + +> **⚠️ Codex review correction (2026-07-18, verified in code) — scope + design changes:** +> 1. **Not a security oracle.** The trace has no deny status (`TraceSpan.status` is only `ok|error`), config denials throw `DeniedError` without emitting permission events, `tool.execute.before/after` are plugin calls not spans, and tracing is best-effort/failure-suppressing — so **absence of an execute span is not proof of non-execution**. S3's HardPolicy kill-gate uses an independent structured audit probe + dispatcher counters; the trace only *corroborates* user-visible surfacing. +> 2. **Partial-order matching required.** Spans append on completion (not start), Batch runs children with `Promise.all`, and 6 of 102 real local traces already show sibling order inversions. The matcher totally-orders only deliberately-serial scenarios; concurrent siblings compare as a keyed multiset/DAG. +> 3. **Normalizer = versioned allowlist projection, not a scrub blacklist.** Every field explicitly classified; unknown fields fail until classified. The trace is deliberately lossy (truncation, 10K output cap, elision) so "no diff" ≠ full behavioral equivalence. +> 4. **A zero-diff does not prove the route.** Native and plugin tools look identical in the trace by design — so S5 parity checks must pair the golden with a separate dispatcher sentinel proving the intended route actually ran. +> 5. Driver = real subprocess fixture (`test/lib/cli-process.ts`) + test LLM server (`test/lib/llm-server.ts`, deterministic call IDs); run each scenario 50–100× to prove hash stability under load before accepting a golden; `--update` disabled in CI. + +## The idea + +The fork already records every session as a **span tree** (`~/.local/share/altimate-code/traces/ses_*.json`): each span has `{spanId, parentSpanId, name, kind, input, output, status, startTime, endTime}`, plus a `summary {totalToolCalls, topTools, tokens, narrative}`. This is a complete, structured record of what a session *did* — the tool-call topology, the order, the inputs/outputs, and (critically for de-fork) permission/deny events. + +**So the trace IS the behavioral oracle.** Instead of asserting on a function's return value (unit-level, misses integration), we: +1. Drive a **real headless session** end-to-end with a deterministic prompt + a **scripted model** (fixed tool-call sequence, no network). +2. Capture the trace the session emits. +3. **Normalize** out nondeterminism (timestamps, spanIds, costs, abs paths, durations, host). +4. **Diff the normalized span topology** against a committed golden. + +A change in observable behavior = a diff in the golden. A tool moving from native→plugin (S5) that changes nothing observable produces an identical golden. A `DROP DATABASE` that should be blocked produces a `hard_policy_denied` span — the deny is **in the trace**, so the golden proves enforcement end-to-end, through whatever route the session took, not just at the unit that implements it. + +## Why this is the right oracle for de-fork specifically + +- **Route-agnostic:** the trace captures the actual execution path (native resolver, plugin tool, batch, subagent). If S3's HardPolicy has a hole on one route, a golden driving that route shows an un-denied span. This is the executable complement to S2's static route matrix. +- **Refactor-invariant:** de-fork is "move code, change nothing observable." Return-value tests are brittle to internal shape changes; span topology is the invariant the user actually experiences. +- **Parity by construction:** S5 migrates a tool to a plugin — record golden on native, switch to plugin, diff. Zero-diff = parity proven. Any diff is exactly the behavior change to explain. +- **Permission events are first-class:** the deny path emits a span/status; goldens assert denies happened AND that `execute`/`tool.execute.after` did NOT (absence of the execute span). + +## Architecture + +``` +packages/opencode/test/altimate/trace-golden/ + driver.ts # spins a headless session with a scripted model + captures its trace + normalize.ts # deterministic scrub (timestamps→0, spanId→ordinal, paths→, cost→0, sort stable) + match.ts # structural diff normalized-actual vs golden; pretty conflict output + scenarios/ # one dir per scenario: prompt.json + model-script.json + golden.json + drop-database-denied/ + sql-select-allowed/ + tool-native-vs-plugin-parity/ (S5) + validator-inject-continue/ (S7) + trace-golden.test.ts # runs every scenario dir, asserts match; --update regenerates goldens +``` + +### Scripted model (determinism source) +Reuse the fork's existing test-CLI / fake-provider path (ci.yml references an `OPENCODE_TEST_CLI` scriptable harness; `tracing-e2e.test.ts` shows the `Recap` API). The model script is a fixed list of assistant turns: text + tool calls with exact args. No LLM, no network → byte-stable except the nondeterminism `normalize.ts` strips. If no reusable fake provider exists, the driver registers a stub provider returning the scripted turns (smallest viable: a provider plugin that replays a JSON script). + +### Normalization contract (must be exhaustive or goldens flap) +Strip/canonicalize: `startTime`/`endTime`/`duration`→omitted; `spanId`/`parentSpanId`→ordinal by DFS order; `traceId`/`sessionId`→``; costs/tokens→bucketed or omitted; absolute paths→``; tmp dirs→``; env/host fields dropped; arrays that are set-like sorted by a stable key. Keep: span `name`, `kind`, `status`, tool `input`/`output` (with volatile sub-fields scrubbed), parent/child structure, sibling order where order is semantic. + +### Match output +On mismatch: print the minimal structural diff (added/removed/changed spans by path), not a 600KB blob. Exit nonzero. `--update` writes the normalized actual as the new golden (human reviews the diff in the PR — goldens are reviewed artifacts, like snapshots). + +## De-fork stage hooks + +- **S3 (HardPolicy):** scenarios `drop-database-denied` (every route: bash, sql tool, MCP, batch-wrapped, subagent), `drop-table-allowed` (near-miss control), `user-config-allow-all-still-denied`. Golden asserts the deny span present and execute span absent. These are the S3 kill-gate's executable proof, and they double as the bypass matrix's runtime arm. +- **S5 (tool migration):** record golden with native tool → flip the tool to plugin → same golden must pass. Repeat per migrated tool; the recipe. +- **S7 (validator continuation):** scenario where a failed dbt validation must produce an injected follow-up turn + continuation, exactly once — the trace shows the extra generation span; concurrency variants assert no double-inject. + +## Inspiration from the past upstream-merge traces + +The `~/.local/share/altimate-code/traces/` store (100+ sessions) and the downloaded merge-session bundles (`~/Downloads/handover-bundle/A3_altimate-code-traces/`, `~/Downloads/altimate-code-traces/`) are **real recorded sessions from actual work incl. the v1.17.9 merge**. Uses: +1. **Realistic scenario seeds:** mine them for the actual tool-call sequences real sessions produce (which tools, what order, how permissions surfaced) → scenarios grounded in real usage, not invented. +2. **Schema ground truth:** they define the exact span shape the normalizer must handle (kinds, status values, de-attributes) — build `normalize.ts` against real files, not a guess. +3. **Regression corpus:** replay-normalize a sample of them through the pipeline to confirm the normalizer is stable (same session → identical normalized output across runs) before trusting goldens. + +## Guardrails +- Goldens are committed, reviewed artifacts; `--update` diffs go in the PR body for human sign-off. +- Scenarios are hermetic: no network, no real warehouse, temp HOME, scripted model. A scenario that can't be made deterministic is not a trace-golden scenario (use a unit test). +- Keep the corpus small and load-bearing (each scenario must pin a behavior a de-fork stage could break), per the no-slop test rule. diff --git a/packages/opencode/test/altimate/trace-golden/DRIVER-NOTES.md b/packages/opencode/test/altimate/trace-golden/DRIVER-NOTES.md new file mode 100644 index 000000000..86e96f72e --- /dev/null +++ b/packages/opencode/test/altimate/trace-golden/DRIVER-NOTES.md @@ -0,0 +1,183 @@ +# driver.ts discovery trail + +Referenced from `driver.ts`'s header comment. This documents how the driver +was built, what existing infrastructure it reuses, and two real bugs the +normalizer had that only surfaced by actually running the driver multiple +times against a live subprocess. + +## Scope: what this technique is (and is not) for + +Trace-golden is scoped to **S5 behavioral parity** (does a session driven +end-to-end still produce the same shape of work after a de-fork/upstream +change?) and **S7 continuation** (regression coverage on later stages that +build on top of the same driver/normalizer plumbing). It is a snapshot-diff +tool over an observability trace, nothing more. + +**It is explicitly NOT a security or HardPolicy enforcement oracle**, and +must never be used or cited as one. Reasons, confirmed in code during this +harness's construction: + +- `TraceSpan.status` is only `ok | error` — there is no denial/blocked + status, so a HardPolicy deny has no distinct trace representation to + assert on. +- `logToolCall` and the tracer generally are best-effort and + failure-suppressing (see `match.ts`'s header comment on partial-order + matching for the concurrency half of this same fragility). A missing or + reshaped span can mean "correctly denied," "tracer swallowed an error," or + "unrelated normalizer/timing flake" — those are indistinguishable from the + trace alone, so **absence of an execute span is not proof of + non-execution**. +- Golden-diffing is inherently a UX/shape corroboration signal (does the + user-visible surface look right), not a proof that a dispatcher was + actually short-circuited before a side effect ran. + +For anything security-relevant (e.g. S3's HardPolicy kill gate), the +correct oracle is an independent structured audit probe emitted by the +policy check itself plus dispatcher execute-counters — see +`scratchpad/s3-build-brief.md`'s "Tests (the kill-gate proof)" section. A +trace-golden scenario may exist alongside that as a corroborating UX check +(e.g. confirming a denied tool call surfaces a sane error to the model/user) +but must always be paired with the audit-probe assertion, never relied on +by itself. + +## 1. Finding a scriptable session harness (no new fake provider needed) + +The technique spec calls for driving "a real headless session with a +deterministic prompt + scripted model." Before building anything new, we +searched the repo for an existing deterministic test harness and found one +already used by the CLI's own test suite: + +- `packages/opencode/test/lib/cli-process.ts` — `withCliFixture()` spins up an + isolated `CliFixture` (its own `HOME`/XDG dirs, own config, own git-free + workspace) and exposes `fixture.opencode.run(prompt, opts)` to invoke the + real `opencode` CLI binary as a subprocess, plus `fixture.llm` bound to a + `TestLLMServer`. +- `packages/opencode/test/lib/llm-server.ts` (via `test-provider.ts`) — a + scriptable fake LLM endpoint. `llm.text(str)` queues a plain assistant + reply; `llm.tool(name, input)` queues a tool-call turn. The CLI subprocess + talks to this local server instead of a real provider, giving us fully + deterministic model output without mocking anything inside the CLI itself. + +`driver.ts`'s `driveScenario()` is a thin composition of these two pieces: +push every `ScriptedTurn` from the scenario's `model-script.json` onto +`fixture.llm`, call `fixture.opencode.run(prompt, { format: "json", ... })`, +then recover the trace file path from the `trace_saved` JSON event the CLI +emits on exit (`src/cli/cmd/run.ts`) — no directory-convention guessing, no +polling `~/.local/share/altimate-code/traces/` for the newest file. + +## 2. `--dangerously-skip-permissions` is required for a non-interactive driver + +Without it, `opencode run` defaults every permission ask (e.g. for `read`) to +`"ask"`, which an in-process run with no TUI and no connected client to +answer can never resolve — the session just hangs. `--dangerously-skip-permissions` +auto-approves anything not explicitly denied (see the yolo-mode branch in +`src/cli/cmd/run.ts`). This is passed via `runOpts: { extraArgs: [...] }` in +the smoke scenario's test. + +## 3. `HOME` env vs `os.homedir()` — a normalization gotcha + +`isolatedEnv()` in `cli-process.ts` sets the child subprocess's `HOME` (and +related XDG vars) to the fixture's own tmp-created home directory, and also +sets the subprocess's `cwd` to that same directory. That means every +home-relative path inside the driven session's own trace is +`fixture.home`-rooted, NOT this test process's `os.homedir()`. `normalize()` +defaults `homeRoots` to `[os.homedir()]`, so calling it on a driven-session +trace without overriding `homeRoots` silently no-ops on every path in that +trace. The test passes `normalize(result.trace, { homeRoots: [fixture.home] })` +explicitly to fix this. + +## 4. Two real normalizer bugs found by running the driver more than once + +The first `golden.json` was generated via `TRACE_GOLDEN_UPDATE=1` and looked +correct on inspection. Re-running the exact same scenario immediately after +(no code changes, same script, same prompt) — as a sanity check before +trusting the golden — failed with a 7-diff mismatch. Two consecutive real +subprocess-driven sessions naturally produce different random session IDs, +different random tmp-dir names (`fixture.home` has a random per-run suffix), +and different random generation part-IDs, and the pre-fix normalizer didn't +fully scrub any of these: + +**Bug A — path-scrubbing order dependency.** The original `normalize.ts` ran +three *separate* passes over each string: a `tmpRoots` pass, then a +`repoRoots` pass, then a `homeRoots` pass. `fixture.home` is itself created +*inside* the OS tmp dir with a random per-run suffix, e.g. +`/private/var/folders/.../T/oc-cli-Wp3D9s`. The default `tmpRoots` list +contains the generic, shorter prefix (`os.tmpdir()`, `/private/tmp`, etc.). +Because the tmpRoots pass ran first, it consumed just the generic prefix and +replaced it with ``, leaving the random `oc-cli-Wp3D9s` suffix exposed +as literal, nondeterministic text in every path in the trace — by the time +the homeRoots pass ran, the string had already been mutated so the full +`fixture.home` path no longer matched as a contiguous substring. + +Fix: replaced the three separate passes with a single merged, **longest-root- +first** pass (`buildRootEntries` / `RootEntry` in `normalize.ts`). A root +from one category can be a strict prefix of a root from another category +(home-inside-tmp is exactly this case), so the more specific (longer) root +must always be tried — and win — before the more generic (shorter) one it +happens to live inside. + +**Bug B — non-path random IDs leaking through span names.** `kind: "session"` +spans are named `metadata.instance_id || sessionId` (`src/altimate/ +observability/tracing.ts` ~L600) — the trace's own random `ses_...` id. +`kind: "generation"` spans are named `` `generation-${part.id}` `` (~L798), +where `part.id` is a random-per-run `prt_...` id. Neither is a filesystem +path, so no path-scrubbing pass ever touched them, and they flapped on every +run independent of Bug A. + +Fix: added `scrubSpanName()` with `SESSION_NAME_PATTERN` / +`GENERATION_NAME_PATTERN` to canonicalize these two span-name shapes +(`""` and `"generation"` respectively) before they reach the diff. + +**Verification (original, 3 runs, pre-rework normalizer).** After both fixes: +deleted the stale golden, regenerated it once, then ran the full suite 3 +additional independent times (each a fresh real `opencode run` subprocess +with new random IDs). All 3 passed with zero diffs, confirming the golden is +stable across genuinely different process runs — not just stable because it +was compared against itself. + +This is exactly the failure mode the technique spec's normalization-contract +section warned about ("must be exhaustive or goldens flap"). It was only +caught by actually driving the scenario multiple times rather than trusting +a single golden-generation run — a single run has no way to distinguish +"correctly normalized" from "coincidentally matched its own random IDs." + +## 5. Stability under load — 100 independent runs (post-rework: partial-order matcher + allowlist normalizer) + +3 runs is not enough to trust a golden — it's enough to catch a bug that +flaps most of the time, not one that flaps rarely. After `match.ts` was +rewritten to partial-order/rank-aware diffing and `normalize.ts` was rewritten +to a versioned allowlist projection (see the top-level technique doc), the +smoke scenario's golden was regenerated under the new normalizer and then +re-verified against **100 fully independent runs**: + +``` +bun run test/altimate/trace-golden/stability-check.ts 100 6 +``` + +`stability-check.ts` (new standalone script, sibling to this file — deliberately +NOT a `*.test.ts` file so `bun test` never picks it up automatically) drives +the `smoke` scenario 100 times, each through its own independent `CliFixture` +(own random-suffixed tmp home dir, own `TestLLMServer`, own `opencode run` +subprocess — never reusing one fixture across drives, since Bug A above only +manifested because of that exact per-run randomness), and compares the +`stableStringify(normalize(trace))` output across all 100 runs plus the +committed golden. + +Result: + +``` +runs requested: 100 +runs succeeded: 100 +runs failed: 0 +unique hashes: 1 (1 = fully stable) +matches golden: true +total wall time: 46.6s +avg run time: 2763ms +``` + +100/100 runs produced a byte-identical normalized hash, and that hash matches +the committed `scenarios/smoke/golden.json`. This meets the bar of proving +stability under load rather than trusting a single (or triple) golden- +generation run. `stability-check.ts` is intentionally opt-in only (not wired +into `bun test` or CI) — see its header comment for why, and for how to +re-run it (e.g. after normalizer changes, before accepting a new golden). diff --git a/packages/opencode/test/altimate/trace-golden/driver.ts b/packages/opencode/test/altimate/trace-golden/driver.ts new file mode 100644 index 000000000..15eede6a7 --- /dev/null +++ b/packages/opencode/test/altimate/trace-golden/driver.ts @@ -0,0 +1,98 @@ +// Real headless-session driver for trace-golden scenarios. Spins up the +// actual `opencode` CLI as a subprocess (via the fork's own `withCliFixture` +// test harness) against a scripted `TestLLMServer`, drives one prompt to +// completion, and returns the exact TraceFile the session wrote to disk. +// +// This is NOT a stub/fake provider built for this task — it reuses the fork's +// existing deterministic test-CLI harness (packages/opencode/test/lib/ +// cli-process.ts + llm-server.ts), which the technique spec explicitly asked +// us to prefer over inventing a new one. See DRIVER-NOTES.md for the +// discovery trail. +import fs from "node:fs/promises" +import { Effect, Scope } from "effect" +import type { HttpClient } from "effect/unstable/http" +import type { CliFixture, RunOpts } from "../../lib/cli-process" +import { withCliFixture } from "../../lib/cli-process" +import type { TraceFile } from "@/altimate/observability/tracing" + +export { withCliFixture } +export type { CliFixture } + +/** One scripted assistant turn: either plain text (session ends) or a tool call (session continues). */ +export interface ScriptedTurn { + readonly kind: "text" | "tool" + readonly text?: string + readonly toolName?: string + readonly toolInput?: unknown +} + +export interface DriveScenarioOptions { + readonly prompt: string + readonly script: ScriptedTurn[] + readonly runOpts?: Omit +} + +export interface DriveScenarioResult { + readonly trace: TraceFile + readonly tracePath: string + readonly exitCode: number + readonly stderr: string +} + +function pushTurn(llm: CliFixture["llm"], turn: ScriptedTurn): Effect.Effect { + if (turn.kind === "tool") { + if (!turn.toolName) throw new Error("driveScenario: tool turn missing toolName") + return llm.tool(turn.toolName, turn.toolInput ?? {}) + } + return llm.text(turn.text ?? "") +} + +/** + * Runs ONE real headless `opencode run` session inside a CliFixture, scripted + * turn-by-turn via TestLLMServer, and returns the TraceFile it wrote to disk. + * + * Must be called from inside `withCliFixture` (pass its `fixture` argument + * straight through) — this function doesn't manage the fixture's lifecycle + * itself, so callers can compose it with other fixture-scoped assertions in + * the same test. + * + * The trace file location is read directly off the `trace_saved` JSON event + * that `opencode run --format json` emits on exit (see src/cli/cmd/run.ts) — + * no directory-convention guessing required. + */ +export function driveScenario( + fixture: CliFixture, + options: DriveScenarioOptions, +): Effect.Effect { + return Effect.gen(function* () { + for (const turn of options.script) { + yield* pushTurn(fixture.llm, turn) + } + + const result = yield* fixture.opencode.run(options.prompt, { + ...options.runOpts, + format: "json", + }) + + const events = fixture.opencode.parseJsonEvents(result.stdout) + const traceSaved = events.find((e) => e.type === "trace_saved") + if (!traceSaved || typeof traceSaved.path !== "string") { + return yield* Effect.fail( + new Error( + `driveScenario: no trace_saved event in CLI output (exit ${result.exitCode}).\n` + + `stderr (last 1000): ${result.stderr.slice(-1000)}\n` + + `stdout (last 1000): ${result.stdout.slice(-1000)}`, + ), + ) + } + const tracePath = traceSaved.path + + const raw = yield* Effect.tryPromise({ + try: () => fs.readFile(tracePath, "utf-8"), + catch: (cause) => new Error(`driveScenario: failed to read trace file ${tracePath}: ${String(cause)}`), + }) + const trace = JSON.parse(raw) as TraceFile + + return { trace, tracePath, exitCode: result.exitCode, stderr: result.stderr } + }) +} diff --git a/packages/opencode/test/altimate/trace-golden/fixtures/real-traces/ses_0c832d792ffeGW44jvnbD1ofca.json b/packages/opencode/test/altimate/trace-golden/fixtures/real-traces/ses_0c832d792ffeGW44jvnbD1ofca.json new file mode 100644 index 000000000..666037d46 --- /dev/null +++ b/packages/opencode/test/altimate/trace-golden/fixtures/real-traces/ses_0c832d792ffeGW44jvnbD1ofca.json @@ -0,0 +1,179 @@ +{ + "version": 2, + "traceId": "019f37cd-290c-7000-8f4e-ca8ab9856cf7", + "sessionId": "ses_0c832d792ffeGW44jvnbD1ofca", + "startedAt": "2026-07-06T14:20:20.108Z", + "endedAt": "2026-07-06T15:48:11.664Z", + "metadata": { + "title": "SQL query equivalence and formatting check", + "model": "google-vertex-anthropic/claude-haiku-4-5@20251001", + "providerId": "google-vertex-anthropic", + "agent": "builder", + "prompt": "Do exactly this, then stop: 1) call altimate_core_equivalence comparing `SELECT a, b FROM t WHERE a > 1` vs `SELECT b, a FROM t WHERE a > 1` with inline schema `CREATE TABLE t (a INT, b INT)`; 2) call sql_format on `select 1 as x,2 as y from information_schema.tables where 1=1`; 3) call finops_query_history (report its error if any). Use no other tools. One-paragraph summary." + }, + "spans": [ + { + "spanId": "019f37cd-2939-7000-b287-1a604781e457", + "parentSpanId": null, + "name": "ses_0c832d792ffeGW44jvnbD1ofca", + "kind": "session", + "startTime": 1783347620108, + "status": "ok", + "endTime": 1783352891664, + "output": "2 generations, 3 tool calls, 125537 tokens, $0.1583" + }, + { + "spanId": "019f37cd-2939-7001-91ea-aee4d8a5ac96", + "parentSpanId": "019f37cd-2939-7000-b287-1a604781e457", + "name": "user-message", + "kind": "user-message", + "startTime": 1783347620153, + "endTime": 1783347620153, + "status": "ok", + "input": "Do exactly this, then stop: 1) call altimate_core_equivalence comparing `SELECT a, b FROM t WHERE a > 1` vs `SELECT b, a FROM t WHERE a > 1` with inline schema `CREATE TABLE t (a INT, b INT)`; 2) call sql_format on `select 1 as x,2 as y from information_schema.tables where 1=1`; 3) call finops_query_history (report its error if any). Use no other tools. One-paragraph summary." + }, + { + "spanId": "019f37cd-2f8e-7000-9693-c2b5fe0441f5", + "parentSpanId": "019f37cd-2939-7000-b287-1a604781e457", + "name": "generation-prt_f37cd2f8d001p7zb2ArHQI7LyO", + "kind": "generation", + "startTime": 1783347621774, + "status": "ok", + "model": { + "modelId": "google-vertex-anthropic/claude-haiku-4-5@20251001", + "providerId": "google-vertex-anthropic" + }, + "endTime": 1783347623171, + "output": "I'll execute exactly those three calls.", + "finishReason": "tool-calls", + "cost": 0.0793645, + "tokens": { + "input": 2, + "output": 287, + "reasoning": 0, + "cacheRead": 0, + "cacheWrite": 62342, + "total": 62631 + } + }, + { + "spanId": "019f37cd-333a-7000-ad4f-3fdd8065bada", + "parentSpanId": "019f37cd-2f8e-7000-9693-c2b5fe0441f5", + "name": "altimate_core_equivalence", + "kind": "tool", + "startTime": 1783347622659, + "endTime": 1783347622713, + "status": "ok", + "tool": { + "callId": "toolu_vrtx_01AskndC57FHRDRTvPY56e4q", + "durationMs": 54 + }, + "input": { + "sql1": "SELECT a, b FROM t WHERE a > 1", + "sql2": "SELECT b, a FROM t WHERE a > 1", + "schema_context": { + "t": { + "a": "INT", + "b": "INT" + } + } + }, + "output": "Queries are semantically equivalent.\n\nDifferences:\n - Different output column names: 'a' vs 'b', 'b' vs 'a'\n\nConfidence: 0.95" + }, + { + "spanId": "019f37cd-33f4-7000-93a9-0debd0527d7e", + "parentSpanId": "019f37cd-2f8e-7000-9693-c2b5fe0441f5", + "name": "sql_format", + "kind": "tool", + "startTime": 1783347622899, + "endTime": 1783347622900, + "status": "ok", + "tool": { + "callId": "toolu_vrtx_016Dixgi8sXBFgNSK6Y9YRwH", + "durationMs": 1 + }, + "input": { + "sql": "select 1 as x,2 as y from information_schema.tables where 1=1", + "dialect": "snowflake", + "indent": 2 + }, + "output": "SELECT 1 AS x, 2 AS y FROM information_schema.tables WHERE 1 = 1" + }, + { + "spanId": "019f37cd-34e9-7000-86c1-f4023d0459cf", + "parentSpanId": "019f37cd-2f8e-7000-9693-c2b5fe0441f5", + "name": "finops_query_history", + "kind": "tool", + "startTime": 1783347623052, + "endTime": 1783347623145, + "status": "ok", + "tool": { + "callId": "toolu_vrtx_0162YSchk1eTmSkJJmeSaACx", + "durationMs": 93 + }, + "input": { + "days": 7, + "limit": 100 + }, + "output": "Failed to fetch query history: Snowflake driver not installed. Run: npm install snowflake-sdk" + }, + { + "spanId": "019f37cd-3c37-7000-a026-76e4fed8415b", + "parentSpanId": "019f37cd-2939-7000-b287-1a604781e457", + "name": "generation-prt_f37cd3c36001WrVvGvyRfmoWNV", + "kind": "generation", + "startTime": 1783347625015, + "status": "ok", + "model": { + "modelId": "google-vertex-anthropic/claude-haiku-4-5@20251001", + "providerId": "google-vertex-anthropic" + }, + "input": "[altimate_core_equivalence] Queries are semantically equivalent.\n\nDifferences:\n - Different output column names: 'a' vs 'b', 'b' vs 'a'\n\nConfidence: 0.95\n[sql_format] SELECT 1 AS x, 2 AS y FROM information_schema.tables WHERE 1 = 1\n[finops_query_history] Failed to fetch query history: Snowflake driver not installed. Run: npm install snowflake-sdk", + "endTime": 1783347625891, + "output": "**Summary:** The equivalence check found the two SELECT queries semantically equivalent despite swapped column order (both filter `a > 1` and return the same data, just reordered columns). The SQL formatter normalized the second query to proper casing and spacing. The query history call failed because the Snowflake driver is not installed; the error message recommends installing `snowflake-sdk`.", + "finishReason": "stop", + "cost": 0.07895725, + "tokens": { + "input": 6, + "output": 87, + "reasoning": 0, + "cacheRead": 0, + "cacheWrite": 62813, + "total": 62906 + } + } + ], + "summary": { + "totalTokens": 125537, + "totalCost": 0.15832175, + "totalToolCalls": 3, + "totalGenerations": 2, + "duration": 5271556, + "status": "completed", + "tokens": { + "input": 8, + "output": 374, + "reasoning": 0, + "cacheRead": 0, + "cacheWrite": 125155 + }, + "topTools": [ + { + "name": "altimate_core_equivalence", + "count": 1, + "totalDuration": 54 + }, + { + "name": "sql_format", + "count": 1, + "totalDuration": 1 + }, + { + "name": "finops_query_history", + "count": 1, + "totalDuration": 93 + } + ], + "narrative": "Completed in 87m51s. Made 2 LLM calls using 3 tools (altimate_core_equivalence, sql_format, finops_query_history). Total cost: $0.1583." + } +} \ No newline at end of file diff --git a/packages/opencode/test/altimate/trace-golden/fixtures/real-traces/ses_0c9a811f7ffeVHtkexaP3SKoMg.json b/packages/opencode/test/altimate/trace-golden/fixtures/real-traces/ses_0c9a811f7ffeVHtkexaP3SKoMg.json new file mode 100644 index 000000000..d58b211ad --- /dev/null +++ b/packages/opencode/test/altimate/trace-golden/fixtures/real-traces/ses_0c9a811f7ffeVHtkexaP3SKoMg.json @@ -0,0 +1,246 @@ +{ + "version": 2, + "traceId": "019f3658-27e9-7000-85f7-f7ef46ed6f50", + "sessionId": "ses_0c9a811f7ffeVHtkexaP3SKoMg", + "startedAt": "2026-07-06T07:32:54.889Z", + "endedAt": "2026-07-06T07:34:08.785Z", + "metadata": { + "title": "studio smoke: altimate tools", + "model": "google-vertex-anthropic/claude-haiku-4-5@20251001", + "providerId": "google-vertex-anthropic", + "agent": "general", + "prompt": "Call altimate_core_equivalence again for the same two queries (`SELECT a, b FROM t WHERE a > 1` vs `SELECT b, a FROM t WHERE a > 1`), this time passing schema context inline: `CREATE TABLE t (a INT, b INT)`. Also run finops_query_history if it works without a warehouse (if it errors, that is fine, just report the error). Then stop and summarize in 2 sentences." + }, + "spans": [ + { + "spanId": "019f3658-27fa-7000-b1b0-8cc53672e3b0", + "parentSpanId": null, + "name": "ses_0c9a811f7ffeVHtkexaP3SKoMg", + "kind": "session", + "startTime": 1783323174889, + "status": "ok" + }, + { + "spanId": "019f3658-27fb-7000-a5c2-bb7880a3969b", + "parentSpanId": "019f3658-27fa-7000-b1b0-8cc53672e3b0", + "name": "user-message", + "kind": "user-message", + "startTime": 1783323174907, + "endTime": 1783323174907, + "status": "ok", + "input": "Please do exactly these two things using tools, then stop:\n1. Use the altimate_core_equivalence tool to check whether these two queries are equivalent: `SELECT a, b FROM t WHERE a > 1` and `SELECT b, a FROM t WHERE a > 1`.\n2. Use the sql_format tool to format: `select 1 as x,2 as y from information_schema.tables where 1=1`.\nDo not use any other tools. Summarize results in one short paragraph." + }, + { + "spanId": "019f3658-a277-7000-8797-1dfb0a0f7fd3", + "parentSpanId": "019f3658-27fa-7000-b1b0-8cc53672e3b0", + "name": "user-message", + "kind": "user-message", + "startTime": 1783323206263, + "endTime": 1783323206263, + "status": "ok", + "input": "Please do exactly these two things using tools, then stop:\n1. Use the altimate_core_equivalence tool to check whether these two queries are equivalent: `SELECT a, b FROM t WHERE a > 1` and `SELECT b, a FROM t WHERE a > 1`.\n2. Use the sql_format tool to format: `select 1 as x,2 as y from information_schema.tables where 1=1`.\nDo not use any other tools. Summarize results in one short paragraph." + }, + { + "spanId": "019f3658-ac50-7000-8512-8d200c6f1ec2", + "parentSpanId": "019f3658-27fa-7000-b1b0-8cc53672e3b0", + "name": "generation-prt_f3658ac4e001ZTv60yfkf1a2G2", + "kind": "generation", + "startTime": 1783323208784, + "status": "ok", + "model": { + "modelId": "google-vertex-anthropic/claude-haiku-4-5@20251001", + "providerId": "google-vertex-anthropic" + }, + "endTime": 1783323209087, + "output": "[tool calls: altimate_core_equivalence, sql_format]", + "finishReason": "tool-calls", + "cost": 0.07350325, + "tokens": { + "input": 2, + "output": 190, + "reasoning": 0, + "cacheRead": 0, + "cacheWrite": 58041, + "total": 58233 + } + }, + { + "spanId": "019f3658-ac87-7000-acec-0e709e993af6", + "parentSpanId": "019f3658-ac50-7000-8512-8d200c6f1ec2", + "name": "altimate_core_equivalence", + "kind": "tool", + "startTime": 1783323208838, + "endTime": 1783323208839, + "status": "ok", + "tool": { + "callId": "toolu_vrtx_01DvqydYULcxxv7GA5Xtj6EU", + "durationMs": 1 + }, + "input": { + "sql1": "SELECT a, b FROM t WHERE a > 1", + "sql2": "SELECT b, a FROM t WHERE a > 1" + }, + "output": "Error: No schema provided. Provide schema_context or schema_path so table/column references can be resolved." + }, + { + "spanId": "019f3658-ad63-7000-bc7e-63945c93225b", + "parentSpanId": "019f3658-ac50-7000-8512-8d200c6f1ec2", + "name": "sql_format", + "kind": "tool", + "startTime": 1783323209050, + "endTime": 1783323209058, + "status": "ok", + "tool": { + "callId": "toolu_vrtx_013aH9kVGNmDx7R54M8M9MKe", + "durationMs": 8 + }, + "input": { + "sql": "select 1 as x,2 as y from information_schema.tables where 1=1", + "dialect": "snowflake", + "indent": 2 + }, + "output": "SELECT 1 AS x, 2 AS y FROM information_schema.tables WHERE 1 = 1" + }, + { + "spanId": "019f3658-b5a9-7000-9946-0494f73677d8", + "parentSpanId": "019f3658-27fa-7000-b1b0-8cc53672e3b0", + "name": "generation-prt_f3658b5a8001p41cf0lk82egbi", + "kind": "generation", + "startTime": 1783323211177, + "status": "ok", + "model": { + "modelId": "google-vertex-anthropic/claude-haiku-4-5@20251001", + "providerId": "google-vertex-anthropic" + }, + "input": "[altimate_core_equivalence] Error: No schema provided. Provide schema_context or schema_path so table/column references can be resolved.\n[sql_format] SELECT 1 AS x, 2 AS y FROM information_schema.tables WHERE 1 = 1", + "endTime": 1783323212304, + "output": "The equivalence check requires schema context and couldn't complete, but semantically these queries are **not equivalent** — they return columns in different orders (a, b vs. b, a), which matters for positional access and result set contracts. The formatted query output is: `SELECT 1 AS x, 2 AS y FROM information_schema.tables WHERE 1 = 1`, with consistent spacing and keyword capitalization applied.", + "finishReason": "stop", + "cost": 0.0734085, + "tokens": { + "input": 6, + "output": 95, + "reasoning": 0, + "cacheRead": 0, + "cacheWrite": 58342, + "total": 58443 + } + }, + { + "spanId": "019f3659-3944-7000-97b9-e8f3e24af1f1", + "parentSpanId": "019f3658-27fa-7000-b1b0-8cc53672e3b0", + "name": "user-message", + "kind": "user-message", + "startTime": 1783323244868, + "endTime": 1783323244868, + "status": "ok", + "input": "Call altimate_core_equivalence again for the same two queries (`SELECT a, b FROM t WHERE a > 1` vs `SELECT b, a FROM t WHERE a > 1`), this time passing schema context inline: `CREATE TABLE t (a INT, b INT)`. Also run finops_query_history if it works without a warehouse (if it errors, that is fine, just report the error). Then stop and summarize in 2 sentences." + }, + { + "spanId": "019f3659-3e76-7000-a61c-9d46c8b87a2d", + "parentSpanId": "019f3658-27fa-7000-b1b0-8cc53672e3b0", + "name": "generation-prt_f36593e74001wlzTiGGPapD4wD", + "kind": "generation", + "startTime": 1783323246198, + "status": "ok", + "model": { + "modelId": "google-vertex-anthropic/claude-haiku-4-5@20251001", + "providerId": "google-vertex-anthropic" + }, + "endTime": 1783323246671, + "output": "[tool calls: altimate_core_equivalence, finops_query_history]", + "finishReason": "tool-calls", + "cost": 0.00801, + "tokens": { + "input": 2, + "output": 186, + "reasoning": 0, + "cacheRead": 57480, + "cacheWrite": 1064, + "total": 58732 + } + }, + { + "spanId": "019f3659-3f15-7000-b6dd-cc3fdb46c46e", + "parentSpanId": "019f3659-3e76-7000-a61c-9d46c8b87a2d", + "name": "altimate_core_equivalence", + "kind": "tool", + "startTime": 1783323246347, + "endTime": 1783323246356, + "status": "ok", + "tool": { + "callId": "toolu_vrtx_01Lder6j1wBmLaf5kqWayDqh", + "durationMs": 9 + }, + "input": { + "sql1": "SELECT a, b FROM t WHERE a > 1", + "sql2": "SELECT b, a FROM t WHERE a > 1", + "schema_context": { + "t": { + "a": "INT", + "b": "INT" + } + } + }, + "output": "Queries are semantically equivalent.\n\nDifferences:\n - Different output column names: 'a' vs 'b', 'b' vs 'a'\n\nConfidence: 0.95" + }, + { + "spanId": "019f3659-403a-7000-9abc-b0eebda43f4e", + "parentSpanId": "019f3659-3e76-7000-a61c-9d46c8b87a2d", + "name": "finops_query_history", + "kind": "tool", + "startTime": 1783323246534, + "endTime": 1783323246650, + "status": "ok", + "tool": { + "callId": "toolu_vrtx_01MqWHUHszfZThjoihEr9YGR", + "durationMs": 116 + }, + "input": { + "days": 7, + "limit": 10 + }, + "output": "Failed to fetch query history: Snowflake driver not installed. Run: npm install snowflake-sdk" + }, + { + "spanId": "019f3659-4671-7000-83b8-5e4db89599e9", + "parentSpanId": "019f3658-27fa-7000-b1b0-8cc53672e3b0", + "name": "generation-prt_f365946700012g3Px8COYUrG0K", + "kind": "generation", + "startTime": 1783323248241, + "status": "ok", + "model": { + "modelId": "google-vertex-anthropic/claude-haiku-4-5@20251001", + "providerId": "google-vertex-anthropic" + }, + "input": "[altimate_core_equivalence] Queries are semantically equivalent.\n\nDifferences:\n - Different output column names: 'a' vs 'b', 'b' vs 'a'\n\nConfidence: 0.95\n[finops_query_history] Failed to fetch query history: Snowflake driver not installed. Run: npm install snowflake-sdk", + "endTime": 1783323248785, + "output": "The two queries are semantically equivalent (they filter and fetch the same rows) despite returning columns in different order—the engine treats them as equivalent because column order doesn't affect the logical result set. The `finops_query_history` call failed because no Snowflake warehouse is configured.", + "finishReason": "stop", + "cost": 0.073906, + "tokens": { + "input": 6, + "output": 64, + "reasoning": 0, + "cacheRead": 0, + "cacheWrite": 58864, + "total": 58934 + } + } + ], + "summary": { + "totalTokens": 234342, + "totalCost": 0.22882775, + "totalToolCalls": 4, + "totalGenerations": 4, + "duration": 73896, + "status": "completed", + "tokens": { + "input": 16, + "output": 535, + "reasoning": 0, + "cacheRead": 57480, + "cacheWrite": 176311 + } + } +} \ No newline at end of file diff --git a/packages/opencode/test/altimate/trace-golden/fixtures/real-traces/ses_0dfeccc49ffetVwiuQmrDKxNxh.json b/packages/opencode/test/altimate/trace-golden/fixtures/real-traces/ses_0dfeccc49ffetVwiuQmrDKxNxh.json new file mode 100644 index 000000000..081008732 --- /dev/null +++ b/packages/opencode/test/altimate/trace-golden/fixtures/real-traces/ses_0dfeccc49ffetVwiuQmrDKxNxh.json @@ -0,0 +1,43 @@ +{ + "version": 2, + "traceId": "019f2013-33b6-7000-a830-5b15e10fa1d9", + "sessionId": "ses_0dfeccc49ffetVwiuQmrDKxNxh", + "startedAt": "2026-07-01T23:45:57.174Z", + "endedAt": "2026-07-01T23:47:29.963Z", + "metadata": { + "title": "\"Reply with exactly one word: PONG\"\n", + "model": "anthropic/claude-haiku-4-5-20251001", + "providerId": "anthropic", + "agent": "builder", + "prompt": "\"Reply with exactly one word: PONG\"\n" + }, + "spans": [ + { + "spanId": "019f2013-33ba-7000-9c73-319b78aa7686", + "parentSpanId": null, + "name": "ses_0dfeccc49ffetVwiuQmrDKxNxh", + "kind": "session", + "startTime": 1782949557174, + "status": "error", + "input": "\"Reply with exactly one word: PONG\"\n", + "endTime": 1782949649963, + "statusMessage": "Process interrupted" + } + ], + "summary": { + "totalTokens": 0, + "totalCost": 0, + "totalToolCalls": 0, + "totalGenerations": 0, + "duration": 92789, + "status": "crashed", + "error": "Process interrupted", + "tokens": { + "input": 0, + "output": 0, + "reasoning": 0, + "cacheRead": 0, + "cacheWrite": 0 + } + } +} \ No newline at end of file diff --git a/packages/opencode/test/altimate/trace-golden/fixtures/real-traces/ses_0e3a95d6bffexTX5KtOaS4FOT9.json b/packages/opencode/test/altimate/trace-golden/fixtures/real-traces/ses_0e3a95d6bffexTX5KtOaS4FOT9.json new file mode 100644 index 000000000..2c92aa0e3 --- /dev/null +++ b/packages/opencode/test/altimate/trace-golden/fixtures/real-traces/ses_0e3a95d6bffexTX5KtOaS4FOT9.json @@ -0,0 +1,43 @@ +{ + "version": 2, + "traceId": "019f1c56-a293-7000-8065-579ad0ff4de1", + "sessionId": "ses_0e3a95d6bffexTX5KtOaS4FOT9", + "startedAt": "2026-07-01T06:21:07.603Z", + "endedAt": "2026-07-01T06:22:06.692Z", + "metadata": { + "title": "\"say PONG\"\n", + "model": "google-vertex/claude-sonnet-4-6@default", + "providerId": "google-vertex", + "agent": "builder", + "prompt": "\"say PONG\"\n" + }, + "spans": [ + { + "spanId": "019f1c56-a296-7000-8ee0-1acd5c438ed6", + "parentSpanId": null, + "name": "ses_0e3a95d6bffexTX5KtOaS4FOT9", + "kind": "session", + "startTime": 1782886867603, + "status": "error", + "input": "\"say PONG\"\n", + "endTime": 1782886926692, + "statusMessage": "Process interrupted" + } + ], + "summary": { + "totalTokens": 0, + "totalCost": 0, + "totalToolCalls": 0, + "totalGenerations": 0, + "duration": 59089, + "status": "crashed", + "error": "Process interrupted", + "tokens": { + "input": 0, + "output": 0, + "reasoning": 0, + "cacheRead": 0, + "cacheWrite": 0 + } + } +} \ No newline at end of file diff --git a/packages/opencode/test/altimate/trace-golden/fixtures/real-traces/ses_0e3f286e5ffeGgjTZeBHqhT14w.json b/packages/opencode/test/altimate/trace-golden/fixtures/real-traces/ses_0e3f286e5ffeGgjTZeBHqhT14w.json new file mode 100644 index 000000000..a2444a18c --- /dev/null +++ b/packages/opencode/test/altimate/trace-golden/fixtures/real-traces/ses_0e3f286e5ffeGgjTZeBHqhT14w.json @@ -0,0 +1,126 @@ +{ + "version": 2, + "traceId": "019f1c0d-7c2a-7000-97af-2d0a7ec3afdf", + "sessionId": "ses_0e3f286e5ffeGgjTZeBHqhT14w", + "startedAt": "2026-07-01T05:01:13.642Z", + "endedAt": "2026-07-01T05:01:21.982Z", + "metadata": { + "title": "Creating newfile.txt with BANANA", + "model": "opencode/big-pickle", + "providerId": "opencode", + "agent": "builder", + "prompt": "Create a file called newfile.txt containing the text BANANA. Use the write tool." + }, + "spans": [ + { + "spanId": "019f1c0d-7c3d-7000-abc2-227506bab39b", + "parentSpanId": null, + "name": "ses_0e3f286e5ffeGgjTZeBHqhT14w", + "kind": "session", + "startTime": 1782882073642, + "status": "ok", + "endTime": 1782882081982, + "output": "2 generations, 1 tool calls, 89098 tokens, $0.0000" + }, + { + "spanId": "019f1c0d-7c3d-7001-a9ef-c0979f45c85a", + "parentSpanId": "019f1c0d-7c3d-7000-abc2-227506bab39b", + "name": "user-message", + "kind": "user-message", + "startTime": 1782882073661, + "endTime": 1782882073661, + "status": "ok", + "input": "Create a file called newfile.txt containing the text BANANA. Use the write tool." + }, + { + "spanId": "019f1c0d-81d8-7000-a0d5-b3601a9ecdcf", + "parentSpanId": "019f1c0d-7c3d-7000-abc2-227506bab39b", + "name": "generation-prt_f1c0d81d60013oB3cJ99Cnd63m", + "kind": "generation", + "startTime": 1782882075096, + "status": "ok", + "model": { + "modelId": "opencode/big-pickle", + "providerId": "opencode" + }, + "endTime": 1782882076960, + "output": "[tool calls: write]", + "finishReason": "tool-calls", + "cost": 0, + "tokens": { + "input": 44414, + "output": 100, + "reasoning": 25, + "cacheRead": 0, + "cacheWrite": 0, + "total": 44539 + } + }, + { + "spanId": "019f1c0d-88a9-7000-9c07-bccdd11e28d5", + "parentSpanId": "019f1c0d-81d8-7000-a0d5-b3601a9ecdcf", + "name": "write", + "kind": "tool", + "startTime": 1782882076837, + "endTime": 1782882076841, + "status": "ok", + "tool": { + "callId": "call_00_WgILI9flpTEheMYQxLGn9176", + "durationMs": 4 + }, + "input": { + "filePath": "/private/tmp/diffverify-CGXm/newfile.txt", + "content": "BANANA" + }, + "output": "Wrote file successfully." + }, + { + "spanId": "019f1c0d-8cd0-7000-afa7-6f0e32cb2c21", + "parentSpanId": "019f1c0d-7c3d-7000-abc2-227506bab39b", + "name": "generation-prt_f1c0d8ccf001UQED5QRVtMT7By", + "kind": "generation", + "startTime": 1782882077904, + "status": "ok", + "model": { + "modelId": "opencode/big-pickle", + "providerId": "opencode" + }, + "input": "[write] Wrote file successfully.", + "endTime": 1782882079681, + "output": "Done! Created `newfile.txt` with the text `BANANA`.", + "finishReason": "stop", + "cost": 0, + "tokens": { + "input": 40434, + "output": 23, + "reasoning": 6, + "cacheRead": 4096, + "cacheWrite": 0, + "total": 44559 + } + } + ], + "summary": { + "totalTokens": 89098, + "totalCost": 0, + "totalToolCalls": 1, + "totalGenerations": 2, + "duration": 8340, + "status": "completed", + "tokens": { + "input": 84848, + "output": 123, + "reasoning": 31, + "cacheRead": 4096, + "cacheWrite": 0 + }, + "topTools": [ + { + "name": "write", + "count": 1, + "totalDuration": 4 + } + ], + "narrative": "Completed in 8.3s. Made 2 LLM calls using 1 tools (write). Total cost: $0.0000." + } +} \ No newline at end of file diff --git a/packages/opencode/test/altimate/trace-golden/fixtures/real-traces/ses_0fb0d89c3ffemtOM9qTPAFNZyK.json b/packages/opencode/test/altimate/trace-golden/fixtures/real-traces/ses_0fb0d89c3ffemtOM9qTPAFNZyK.json new file mode 100644 index 000000000..7c1acffcb --- /dev/null +++ b/packages/opencode/test/altimate/trace-golden/fixtures/real-traces/ses_0fb0d89c3ffemtOM9qTPAFNZyK.json @@ -0,0 +1,77 @@ +{ + "version": 2, + "traceId": "019f04f2-7649-7000-b924-561e89551e38", + "sessionId": "ses_0fb0d89c3ffemtOM9qTPAFNZyK", + "startedAt": "2026-06-26T17:20:26.697Z", + "endedAt": "2026-06-26T17:20:50.782Z", + "metadata": { + "title": "New session - 2026-06-26T17:20:26.684Z", + "model": "azure/gpt-5.5", + "providerId": "azure", + "agent": "builder", + "prompt": "what is 6 times 7? number only" + }, + "spans": [ + { + "spanId": "019f04f2-766d-7000-b828-e9820b0dbf80", + "parentSpanId": null, + "name": "ses_0fb0d89c3ffemtOM9qTPAFNZyK", + "kind": "session", + "startTime": 1782494426697, + "status": "ok", + "endTime": 1782494450782, + "output": "1 generations, 0 tool calls, 29113 tokens, $0.1462" + }, + { + "spanId": "019f04f2-766d-7001-ae86-f55aded28564", + "parentSpanId": "019f04f2-766d-7000-b828-e9820b0dbf80", + "name": "user-message", + "kind": "user-message", + "startTime": 1782494426733, + "endTime": 1782494426733, + "status": "ok", + "input": "what is 6 times 7? number only" + }, + { + "spanId": "019f04f2-7d45-7000-91a2-c81a9ee63800", + "parentSpanId": "019f04f2-766d-7000-b828-e9820b0dbf80", + "name": "generation-prt_f04f27d43001415BAYpKBBs1wu", + "kind": "generation", + "startTime": 1782494428485, + "status": "ok", + "model": { + "modelId": "azure/gpt-5.5", + "providerId": "azure" + }, + "endTime": 1782494430222, + "output": "42", + "finishReason": "stop", + "cost": 0.14624, + "tokens": { + "input": 29086, + "output": 17, + "reasoning": 10, + "cacheRead": 0, + "cacheWrite": 0, + "total": 29113 + } + } + ], + "summary": { + "totalTokens": 29113, + "totalCost": 0.14624, + "totalToolCalls": 0, + "totalGenerations": 1, + "duration": 24085, + "status": "completed", + "tokens": { + "input": 29086, + "output": 17, + "reasoning": 10, + "cacheRead": 0, + "cacheWrite": 0 + }, + "topTools": [], + "narrative": "Completed in 24.1s. Made 1 LLM call. Total cost: $0.1462." + } +} \ No newline at end of file diff --git a/packages/opencode/test/altimate/trace-golden/fixtures/real-traces/ses_2e819da4cffeMhWRAOCOslv3mp.json b/packages/opencode/test/altimate/trace-golden/fixtures/real-traces/ses_2e819da4cffeMhWRAOCOslv3mp.json new file mode 100644 index 000000000..0ee5d9d78 --- /dev/null +++ b/packages/opencode/test/altimate/trace-golden/fixtures/real-traces/ses_2e819da4cffeMhWRAOCOslv3mp.json @@ -0,0 +1,158 @@ +{ + "version": 2, + "traceId": "019d17e6-25a7-7000-9a31-6f456a479292", + "sessionId": "ses_2e819da4cffeMhWRAOCOslv3mp", + "startedAt": "2026-03-22T23:34:12.135Z", + "endedAt": "2026-03-22T23:34:18.400Z", + "metadata": { + "title": "\"read the Dockerfile and explain the build process\"\n", + "prompt": "\"read the Dockerfile and explain the build process\"\n" + }, + "spans": [ + { + "spanId": "019d17e6-25b5-7000-a793-fde7d0922d93", + "parentSpanId": null, + "name": "ses_2e819da4cffeMhWRAOCOslv3mp", + "kind": "session", + "startTime": 1774222452135, + "status": "error", + "input": "\"read the Dockerfile and explain the build process\"\n", + "endTime": 1774222458400, + "statusMessage": "Budget exceeded: reached 2 assistant turns limit", + "output": "Error: Budget exceeded: reached 2 assistant turns limit" + }, + { + "spanId": "019d17e6-261f-7000-b03c-6cfef6802390", + "parentSpanId": "019d17e6-25b5-7000-a793-fde7d0922d93", + "name": "system-prompt", + "kind": "span", + "startTime": 1774222452255, + "endTime": 1774222452255, + "status": "ok", + "input": { + "agent": "builder", + "step": 1 + }, + "output": { + "parts": 4, + "content": "You are powered by the model named claude-sonnet-4-6. The exact model ID is anthropic/claude-sonnet-4-6\nHere is some useful information about the environment you are running in:\n\n Working directory: /Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/packages/opencode\n Workspace root folder: /Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui\n Is directory a git repo: yes\n Platform: darwin\n Today's date: Sun Mar 22 2026\n\n\n \n\n\nSkills provide specialized instructions and workflows for specific tasks.\nUse the skill tool to load a skill when a task matches its description.\n\n \n dbt-analyze\n Analyze downstream impact of dbt model changes using column-level lineage and the dependency graph. Use when evaluating the blast radius of a change before shipping. Powered by altimate-dbt.\n file:///Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/dbt-analyze/SKILL.md\n \n \n altimate-setup\n Configure Altimate platform credentials for datamate and API access\n file:///Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/altimate-setup/SKILL.md\n \n \n sql-translate\n Translate SQL queries between database dialects (Snowflake, BigQuery, PostgreSQL, MySQL, etc.)\n file:///Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/sql-translate/SKILL.md\n \n \n dbt-docs\n Document dbt models and columns in schema.yml with business context — model descriptions, column definitions, and doc blocks. Use when adding or improving documentation for discoverability. Powered by altimate-dbt.\n file:///Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/dbt-docs/SKILL.md\n \n \n lineage-diff\n Compare column-level lineage between two versions of a SQL query to show added, removed, and changed data flow edges.\n file:///Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/lineage-diff/SKILL.md\n \n \n cost-report\n Analyze Snowflake query costs and identify optimization opportunities\n file:///Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/cost-report/SKILL.md\n \n \n query-optimize\n Analyze and optimize SQL queries for better performance\n file:///Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/query-optimize/SKILL.md\n \n \n training-status\n Show what your AI teammate has learned — patterns, rules, glossary, and standards\n file:///Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/training-status/SKILL.md\n \n \n sql-review\n Pre-merge SQL quality gate — lint 26 anti-patterns, grade readability/performance A-F, validate syntax, and scan for injection threats. Use before committing or reviewing SQL changes.\n file:///Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/sql-review/SKILL.md\n \n \n data-viz\n Build modern, interactive data visualizations and dashboards using code-based component libraries (shadcn/ui, Recharts, Tremor, Nivo, D3, Victory, visx). Use this skill whenever the user asks to visualize data, build dashboards, create analytics views, chart metrics, tell a data story, build a reporting interface, create KPI cards, plot graphs, or explore a dataset — even if they mention PowerBI, Tableau, Streamlit, Metabase, Looker, Grafana, or similar tools. Also trigger when the user says \"make a dashboard\", \"show me the data\", \"chart this\", \"visualize trends\", \"build an analytics page\", \"data story\", or anything involving turning raw data into interactive visual interfaces. If the task involves presenting data visually — this is the skill. Always prefer building a real, interactive, code-based UI over exporting to or recommending a BI platform.\n\n file:///Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/data-viz/SKILL.md\n \n \n dbt-develop\n Create and modify dbt models — staging, intermediate, marts, incremental, medallion architecture. Use when building new SQL models, extending existing ones, scaffolding YAML configs, or reorganizing project structure. Powered by altimate-dbt.\n file:///Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/dbt-develop/SKILL.md\n \n \n pii-audit\n Classify schema columns for PII (SSN, email, phone, name, address, credit card) and check whether queries expose them. Use for GDPR/CCPA/HIPAA compliance audits.\n file:///Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/pii-audit/SKILL.md\n \n \n dbt-troubleshoot\n Debug dbt errors — compilation failures, runtime database errors, test failures, wrong data, and performance issues. Use when something is broken, producing wrong results, or failing to build. Powered by altimate-dbt.\n file:///Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/dbt-troubleshoot/SKILL.md\n \n \n dbt-test\n Add schema tests, unit tests, and data quality checks to dbt models. Use when validating data integrity, adding test definitions to schema.yml, writing unit tests, or practicing test-driven development in dbt. Powered by altimate-dbt.\n file:///Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/dbt-test/SKILL.md\n \n \n train\n Train your AI teammate on team standards from a document or style guide\n file:///Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/train/SKILL.md\n \n \n teach\n Teach your AI teammate a pattern by showing it an example file from your codebase\n file:///Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/teach/SKILL.md\n \n \n schema-migration\n Analyze DDL migrations for data loss risks — type narrowing, missing defaults, dropped constraints, breaking column changes. Use before applying schema changes to production.\n file:///Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/schema-migration/SKILL.md\n \n\n\n## Altimate Knowledge\n\nKnowledge from previous sessions and team training. Apply it consistently.\n\n### Learned Rules\n_Always follow these rules. They were taught by the user through corrections and explicit instruction._\n\n#### no-positional-group-by (applied 36x)\n## Never use positional GROUP BY\n\n**Rule:** Always use explicit column names in GROUP BY. Never use positional references (`GROUP BY 1, 2, 3`).\n\n**Why:** Positional GROUP BY is fragile. Adding, removing, or reordering a SELECT column silently shifts which columns are grouped — the query still runs but produces wrong aggregations with no error.\n\n**Wrong:**\n```sql\nselect\n order_date,\n order_channel,\n sum(revenue) as revenue\nfrom orders\ngroup by 1, 2\n```\n\n**Right:**\n```sql\nselect\n order_date,\n order_channel,\n sum(revenue) as revenue\nfrom orders\ngroup by order_date, order_channel\n```\n\n**Edge case — computed expressions:** When grouping by an expression (e.g. `date_trunc`, `date_part`), repeat the full expression in GROUP BY rather than using a positional alias:\n\n```sql\n-- Wrong\nselect date_trunc('month', order_date) as month, sum(revenue)\nfrom orders\ngroup by 1\n\n-- Right\nselect date_trunc('month', order_date) as month, sum(revenue)\nfrom orders\ngroup by date_trunc('month', order_date)\n```\n\n**Applies to:** All SQL written in dbt models, ad-hoc queries, CTEs, subqueries, and window function PARTITION BY clauses.\n\n\nInstructions from: /Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/packages/opencode/AGENTS.md\n# opencode database guide\n\n## Database\n\n- **Schema**: Drizzle schema lives in `src/**/*.sql.ts`.\n- **Naming**: tables and columns use snake*case; join columns are `_id`; indexes are `*\\_idx`.\n- **Migrations**: generated by Drizzle Kit using `drizzle.config.ts` (schema: `./src/**/*.sql.ts`, output: `./migration`).\n- **Command**: `bun run db generate --name `.\n- **Output**: creates `migration/_/migration.sql` and `snapshot.json`.\n- **Tests**: migration tests should read the per-folder layout (no `_journal.json`).\n\n# opencode Effect guide\n\nInstructions to follow when writing Effect.\n\n## Schemas\n\n- Use `Schema.Class` for data types with multiple fields.\n- Use branded schemas (`Schema.brand`) for single-value types.\n\n## Services\n\n- Services use `ServiceMap.Service()(\"@console/\")`.\n- In `Layer.effect`, always return service implementations with `ServiceName.of({ ... })`, never a plain object.\n\n## Errors\n\n- Use `Schema.TaggedErrorClass` for typed errors.\n- For defect-like causes, use `Schema.Defect` instead of `unknown`.\n- In `Effect.gen`, prefer `yield* new MyError(...)` over `yield* Effect.fail(new MyError(...))` for direct early-failure branches.\n\n## Effects\n\n- Use `Effect.gen(function* () { ... })` for composition.\n- Use `Effect.fn(\"ServiceName.method\")` for named/traced effects and `Effect.fnUntraced` for internal helpers.\n- `Effect.fn` / `Effect.fnUntraced` accept pipeable operators as extra arguments, so avoid unnecessary `flow` or outer `.pipe()` wrappers.\n\n## Time\n\n- Prefer `DateTime.nowAsDate` over `new Date(yield* Clock.currentTimeMillis)` when you need a `Date`.\n\n## Errors\n\n- In `Effect.gen/fn`, prefer `yield* new MyError(...)` over `yield* Effect.fail(new MyError(...))` for direct early-failure branches.\n" + } + }, + { + "spanId": "019d17e6-2f0c-7000-937e-45acda31c8e4", + "parentSpanId": "019d17e6-25b5-7000-a793-fde7d0922d93", + "name": "generation-prt_d17e62f0a001t5UHReTiHQUzNy", + "kind": "generation", + "startTime": 1774222454540, + "status": "ok", + "model": {}, + "endTime": 1774222454647, + "output": "[tool calls: glob]", + "finishReason": "tool-calls", + "cost": 0, + "tokens": { + "input": 2, + "output": 59, + "reasoning": 0, + "cacheRead": 0, + "cacheWrite": 35787, + "total": 35848 + } + }, + { + "spanId": "019d17e6-2f4e-7000-8486-181f2224577a", + "parentSpanId": "019d17e6-2f0c-7000-937e-45acda31c8e4", + "name": "glob", + "kind": "tool", + "startTime": 1774222454597, + "endTime": 1774222454605, + "status": "ok", + "tool": { + "callId": "toolu_011gbQhAZ9Rhq1Vo8DxPhyaK", + "durationMs": 8 + }, + "input": { + "pattern": "**/Dockerfile*" + }, + "output": "/Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/packages/opencode/Dockerfile" + }, + { + "spanId": "019d17e6-35d0-7000-abb4-fb7e7c232341", + "parentSpanId": "019d17e6-25b5-7000-a793-fde7d0922d93", + "name": "generation-prt_d17e635cf001RmtjqkbjBfsKGP", + "kind": "generation", + "startTime": 1774222456272, + "status": "ok", + "model": {}, + "input": "[glob] /Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/packages/opencode/Dockerfile", + "endTime": 1774222456460, + "output": "[tool calls: read]", + "finishReason": "tool-calls", + "cost": 0, + "tokens": { + "input": 1, + "output": 87, + "reasoning": 0, + "cacheRead": 29100, + "cacheWrite": 6791, + "total": 35979 + } + }, + { + "spanId": "019d17e6-365f-7000-807e-924debb53734", + "parentSpanId": "019d17e6-35d0-7000-abb4-fb7e7c232341", + "name": "read", + "kind": "tool", + "startTime": 1774222456411, + "endTime": 1774222456415, + "status": "ok", + "tool": { + "callId": "toolu_01G2ywBM8oKKwbQYi6e3PwFN", + "durationMs": 4 + }, + "input": { + "filePath": "/Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/packages/opencode/Dockerfile" + }, + "output": "/Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/packages/opencode/Dockerfile\nfile\n1: FROM alpine AS base\n2: \n3: # Disable the runtime transpiler cache by default inside Docker containers.\n4: # On ephemeral containers, the cache is not useful\n5: ARG BUN_RUNTIME_TRANSPILER_CACHE_PATH=0\n6: ENV BUN_RUNTIME_TRANSPILER_CACHE_PATH=${BUN_RUNTIME_TRANSPILER_CACHE_PATH}\n7: RUN apk add libgcc libstdc++ ripgrep\n8: \n9: FROM base AS build-amd64\n10: COPY dist/@altimateai/altimate-code-linux-x64-baseline-musl/bin/altimate /usr/local/bin/altimate\n11: \n12: FROM base AS build-arm64\n13: COPY dist/@altimateai/altimate-code-linux-arm64-musl/bin/altimate /usr/local/bin/altimate\n14: \n15: ARG TARGETARCH\n16: FROM build-${TARGETARCH}\n17: RUN ln -sf /usr/local/bin/altimate /usr/local/bin/altimate-code\n18: RUN altimate --version\n19: ENTRYPOINT [\"altimate\"]\n\n(End of file - total 19 lines)\n" + }, + { + "spanId": "019d17e6-3c2e-7000-be11-69c546adcd20", + "parentSpanId": "019d17e6-25b5-7000-a793-fde7d0922d93", + "name": "generation-prt_d17e63c2d001xFX8S9atYg1N13", + "kind": "generation", + "startTime": 1774222457902, + "status": "ok", + "model": {}, + "input": "[read] /Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/packages/opencode/Dockerfile\nfile\n1: FROM alpine AS base\n2: \n3: # Disable the runtime transpiler cache by default inside Docker containers.\n4: # On ephemeral containers, the cache is not useful\n5: ARG BUN_RUNTIME_TRANSPILER_CACHE_PATH=0\n6: ENV BUN_RUNTIME_TRANSPILER_CACHE_PATH=${BUN_RUNTIME_TRANSPILER_CACHE_PATH}\n7: RUN apk add libgcc libstdc++ ripgrep\n8: \n9: FROM base AS build-amd64\n10: COPY dis" + } + ], + "summary": { + "totalTokens": 71827, + "totalCost": 0, + "totalToolCalls": 2, + "totalGenerations": 3, + "duration": 6265, + "status": "error", + "error": "Budget exceeded: reached 2 assistant turns limit", + "tokens": { + "input": 3, + "output": 146, + "reasoning": 0, + "cacheRead": 29100, + "cacheWrite": 42578 + }, + "topTools": [ + { + "name": "glob", + "count": 1, + "totalDuration": 8 + }, + { + "name": "read", + "count": 1, + "totalDuration": 4 + } + ], + "narrative": "Failed after 6.3s. Made 3 LLM calls using 2 tools (glob, read). Total cost: $0.0000." + } +} \ No newline at end of file diff --git a/packages/opencode/test/altimate/trace-golden/fixtures/real-traces/ses_2e81c7baeffeaz1CvDUlNzeL4Y.json b/packages/opencode/test/altimate/trace-golden/fixtures/real-traces/ses_2e81c7baeffeaz1CvDUlNzeL4Y.json new file mode 100644 index 000000000..ddba7c9e6 --- /dev/null +++ b/packages/opencode/test/altimate/trace-golden/fixtures/real-traces/ses_2e81c7baeffeaz1CvDUlNzeL4Y.json @@ -0,0 +1,115 @@ +{ + "version": 2, + "traceId": "019d17e3-83bb-7000-8242-e9ed595a696e", + "sessionId": "ses_2e81c7baeffeaz1CvDUlNzeL4Y", + "startedAt": "2026-03-22T23:31:19.611Z", + "endedAt": "2026-03-22T23:31:27.387Z", + "metadata": { + "title": "\"search for TODO in tracing.ts\"\n", + "prompt": "\"search for TODO in tracing.ts\"\n" + }, + "spans": [ + { + "spanId": "019d17e3-8465-7000-88f1-3e5bb1c8764f", + "parentSpanId": null, + "name": "ses_2e81c7baeffeaz1CvDUlNzeL4Y", + "kind": "session", + "startTime": 1774222279611, + "status": "error", + "input": "\"search for TODO in tracing.ts\"\n", + "endTime": 1774222287387, + "statusMessage": "Budget exceeded: reached 1 assistant turn limit", + "output": "Error: Budget exceeded: reached 1 assistant turn limit" + }, + { + "spanId": "019d17e3-8625-7000-9b54-a0a1ec98fb55", + "parentSpanId": "019d17e3-8465-7000-88f1-3e5bb1c8764f", + "name": "system-prompt", + "kind": "span", + "startTime": 1774222280229, + "endTime": 1774222280229, + "status": "ok", + "input": { + "agent": "builder", + "step": 1 + }, + "output": { + "parts": 4, + "content": "You are powered by the model named claude-sonnet-4-6. The exact model ID is anthropic/claude-sonnet-4-6\nHere is some useful information about the environment you are running in:\n\n Working directory: /Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/packages/opencode\n Workspace root folder: /Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui\n Is directory a git repo: yes\n Platform: darwin\n Today's date: Sun Mar 22 2026\n\n\n \n\n\nSkills provide specialized instructions and workflows for specific tasks.\nUse the skill tool to load a skill when a task matches its description.\n\n \n teach\n Teach your AI teammate a pattern by showing it an example file from your codebase\n file:///Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/teach/SKILL.md\n \n \n training-status\n Show what your AI teammate has learned — patterns, rules, glossary, and standards\n file:///Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/training-status/SKILL.md\n \n \n pii-audit\n Classify schema columns for PII (SSN, email, phone, name, address, credit card) and check whether queries expose them. Use for GDPR/CCPA/HIPAA compliance audits.\n file:///Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/pii-audit/SKILL.md\n \n \n cost-report\n Analyze Snowflake query costs and identify optimization opportunities\n file:///Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/cost-report/SKILL.md\n \n \n data-viz\n Build modern, interactive data visualizations and dashboards using code-based component libraries (shadcn/ui, Recharts, Tremor, Nivo, D3, Victory, visx). Use this skill whenever the user asks to visualize data, build dashboards, create analytics views, chart metrics, tell a data story, build a reporting interface, create KPI cards, plot graphs, or explore a dataset — even if they mention PowerBI, Tableau, Streamlit, Metabase, Looker, Grafana, or similar tools. Also trigger when the user says \"make a dashboard\", \"show me the data\", \"chart this\", \"visualize trends\", \"build an analytics page\", \"data story\", or anything involving turning raw data into interactive visual interfaces. If the task involves presenting data visually — this is the skill. Always prefer building a real, interactive, code-based UI over exporting to or recommending a BI platform.\n\n file:///Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/data-viz/SKILL.md\n \n \n sql-review\n Pre-merge SQL quality gate — lint 26 anti-patterns, grade readability/performance A-F, validate syntax, and scan for injection threats. Use before committing or reviewing SQL changes.\n file:///Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/sql-review/SKILL.md\n \n \n dbt-troubleshoot\n Debug dbt errors — compilation failures, runtime database errors, test failures, wrong data, and performance issues. Use when something is broken, producing wrong results, or failing to build. Powered by altimate-dbt.\n file:///Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/dbt-troubleshoot/SKILL.md\n \n \n altimate-setup\n Configure Altimate platform credentials for datamate and API access\n file:///Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/altimate-setup/SKILL.md\n \n \n dbt-docs\n Document dbt models and columns in schema.yml with business context — model descriptions, column definitions, and doc blocks. Use when adding or improving documentation for discoverability. Powered by altimate-dbt.\n file:///Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/dbt-docs/SKILL.md\n \n \n dbt-analyze\n Analyze downstream impact of dbt model changes using column-level lineage and the dependency graph. Use when evaluating the blast radius of a change before shipping. Powered by altimate-dbt.\n file:///Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/dbt-analyze/SKILL.md\n \n \n lineage-diff\n Compare column-level lineage between two versions of a SQL query to show added, removed, and changed data flow edges.\n file:///Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/lineage-diff/SKILL.md\n \n \n dbt-develop\n Create and modify dbt models — staging, intermediate, marts, incremental, medallion architecture. Use when building new SQL models, extending existing ones, scaffolding YAML configs, or reorganizing project structure. Powered by altimate-dbt.\n file:///Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/dbt-develop/SKILL.md\n \n \n train\n Train your AI teammate on team standards from a document or style guide\n file:///Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/train/SKILL.md\n \n \n sql-translate\n Translate SQL queries between database dialects (Snowflake, BigQuery, PostgreSQL, MySQL, etc.)\n file:///Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/sql-translate/SKILL.md\n \n \n schema-migration\n Analyze DDL migrations for data loss risks — type narrowing, missing defaults, dropped constraints, breaking column changes. Use before applying schema changes to production.\n file:///Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/schema-migration/SKILL.md\n \n \n query-optimize\n Analyze and optimize SQL queries for better performance\n file:///Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/query-optimize/SKILL.md\n \n \n dbt-test\n Add schema tests, unit tests, and data quality checks to dbt models. Use when validating data integrity, adding test definitions to schema.yml, writing unit tests, or practicing test-driven development in dbt. Powered by altimate-dbt.\n file:///Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/dbt-test/SKILL.md\n \n\n\n## Altimate Knowledge\n\nKnowledge from previous sessions and team training. Apply it consistently.\n\n### Learned Rules\n_Always follow these rules. They were taught by the user through corrections and explicit instruction._\n\n#### no-positional-group-by (applied 24x)\n## Never use positional GROUP BY\n\n**Rule:** Always use explicit column names in GROUP BY. Never use positional references (`GROUP BY 1, 2, 3`).\n\n**Why:** Positional GROUP BY is fragile. Adding, removing, or reordering a SELECT column silently shifts which columns are grouped — the query still runs but produces wrong aggregations with no error.\n\n**Wrong:**\n```sql\nselect\n order_date,\n order_channel,\n sum(revenue) as revenue\nfrom orders\ngroup by 1, 2\n```\n\n**Right:**\n```sql\nselect\n order_date,\n order_channel,\n sum(revenue) as revenue\nfrom orders\ngroup by order_date, order_channel\n```\n\n**Edge case — computed expressions:** When grouping by an expression (e.g. `date_trunc`, `date_part`), repeat the full expression in GROUP BY rather than using a positional alias:\n\n```sql\n-- Wrong\nselect date_trunc('month', order_date) as month, sum(revenue)\nfrom orders\ngroup by 1\n\n-- Right\nselect date_trunc('month', order_date) as month, sum(revenue)\nfrom orders\ngroup by date_trunc('month', order_date)\n```\n\n**Applies to:** All SQL written in dbt models, ad-hoc queries, CTEs, subqueries, and window function PARTITION BY clauses.\n\n\nInstructions from: /Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/packages/opencode/AGENTS.md\n# opencode database guide\n\n## Database\n\n- **Schema**: Drizzle schema lives in `src/**/*.sql.ts`.\n- **Naming**: tables and columns use snake*case; join columns are `_id`; indexes are `
*\\_idx`.\n- **Migrations**: generated by Drizzle Kit using `drizzle.config.ts` (schema: `./src/**/*.sql.ts`, output: `./migration`).\n- **Command**: `bun run db generate --name `.\n- **Output**: creates `migration/_/migration.sql` and `snapshot.json`.\n- **Tests**: migration tests should read the per-folder layout (no `_journal.json`).\n\n# opencode Effect guide\n\nInstructions to follow when writing Effect.\n\n## Schemas\n\n- Use `Schema.Class` for data types with multiple fields.\n- Use branded schemas (`Schema.brand`) for single-value types.\n\n## Services\n\n- Services use `ServiceMap.Service()(\"@console/\")`.\n- In `Layer.effect`, always return service implementations with `ServiceName.of({ ... })`, never a plain object.\n\n## Errors\n\n- Use `Schema.TaggedErrorClass` for typed errors.\n- For defect-like causes, use `Schema.Defect` instead of `unknown`.\n- In `Effect.gen`, prefer `yield* new MyError(...)` over `yield* Effect.fail(new MyError(...))` for direct early-failure branches.\n\n## Effects\n\n- Use `Effect.gen(function* () { ... })` for composition.\n- Use `Effect.fn(\"ServiceName.method\")` for named/traced effects and `Effect.fnUntraced` for internal helpers.\n- `Effect.fn` / `Effect.fnUntraced` accept pipeable operators as extra arguments, so avoid unnecessary `flow` or outer `.pipe()` wrappers.\n\n## Time\n\n- Prefer `DateTime.nowAsDate` over `new Date(yield* Clock.currentTimeMillis)` when you need a `Date`.\n\n## Errors\n\n- In `Effect.gen/fn`, prefer `yield* new MyError(...)` over `yield* Effect.fail(new MyError(...))` for direct early-failure branches.\n" + } + }, + { + "spanId": "019d17e3-9641-7000-9262-42114a2e7b1b", + "parentSpanId": "019d17e3-8465-7000-88f1-3e5bb1c8764f", + "name": "generation-prt_d17e3963f0019phZk4S6WbDjPO", + "kind": "generation", + "startTime": 1774222284353, + "status": "ok", + "model": {}, + "endTime": 1774222284660, + "output": "[tool calls: grep]", + "finishReason": "tool-calls", + "cost": 0, + "tokens": { + "input": 2, + "output": 75, + "reasoning": 0, + "cacheRead": 0, + "cacheWrite": 35783, + "total": 35860 + } + }, + { + "spanId": "019d17e3-974f-7000-a2ee-96f70b3afab5", + "parentSpanId": "019d17e3-9641-7000-9262-42114a2e7b1b", + "name": "grep", + "kind": "tool", + "startTime": 1774222284614, + "endTime": 1774222284622, + "status": "ok", + "tool": { + "callId": "toolu_01WHbaFUKGbeYje5zF6KjR2x", + "durationMs": 8 + }, + "input": { + "pattern": "TODO", + "include": "tracing.ts" + }, + "output": "No files found" + }, + { + "spanId": "019d17e3-a13d-7000-8b9b-cdf289571ac6", + "parentSpanId": "019d17e3-8465-7000-88f1-3e5bb1c8764f", + "name": "generation-prt_d17e3a13b0014RBZod3U6600JE", + "kind": "generation", + "startTime": 1774222287165, + "status": "ok", + "model": {}, + "input": "[grep] No files found" + } + ], + "summary": { + "totalTokens": 35860, + "totalCost": 0, + "totalToolCalls": 1, + "totalGenerations": 2, + "duration": 7776, + "status": "error", + "error": "Budget exceeded: reached 1 assistant turn limit", + "tokens": { + "input": 2, + "output": 75, + "reasoning": 0, + "cacheRead": 0, + "cacheWrite": 35783 + }, + "topTools": [ + { + "name": "grep", + "count": 1, + "totalDuration": 8 + } + ], + "narrative": "Failed after 7.8s. Made 2 LLM calls using 1 tools (grep). Total cost: $0.0000." + } +} \ No newline at end of file diff --git a/packages/opencode/test/altimate/trace-golden/match.ts b/packages/opencode/test/altimate/trace-golden/match.ts new file mode 100644 index 000000000..bd5f2f608 --- /dev/null +++ b/packages/opencode/test/altimate/trace-golden/match.ts @@ -0,0 +1,303 @@ +// Structural diff between a normalized "actual" trace and a committed golden. +// Produces a minimal, path-based diff — not a blob dump — so a broken golden +// tells a reviewer exactly which span(s) changed and how. +// +// Partial-order matching: `logToolCall` appends spans on completion (not +// start) and both Batch (`Promise.all`, batch.ts:158) and the parallel +// session/prompt.ts resolver dispatch tool calls concurrently, so two +// structurally-identical runs can legitimately resolve a race between +// concurrent siblings to different array positions / different ordinal ids. +// normalize.ts's `rank` field is derived structurally from each span's own +// `kind` (all `"tool"`-kind siblings under a parent always share one rank, +// since their completion order is never trustworthy; every other kind gets +// a real, unique rank from its genuinely synchronous array order) — not +// from timing. `rank` tells us which sibling groups are safe to diff 1:1 by +// id (rank-unique — a real, reproducible order) vs. which are genuinely +// concurrent (same rank, size > 1 on either side) and must be compared as +// an unordered multiset of subtrees instead. +import { stableStringify, type NormalizedSpan, type NormalizedTrace } from "./normalize" + +export type DiffKind = "added" | "removed" | "changed" + +export interface Diff { + kind: DiffKind + /** Dotted/bracketed path, e.g. `spans[2].status` or `spans[5]` (whole span added/removed). */ + path: string + /** Present for "removed" and "changed". */ + expected?: unknown + /** Present for "added" and "changed". */ + actual?: unknown +} + +export interface MatchResult { + readonly pass: boolean + readonly diffs: Diff[] + /** Human-readable report; empty string when pass is true. */ + format(): string +} + +/** Compares two normalized traces and returns a pass/fail result plus a minimal diff list. */ +export function match(golden: NormalizedTrace, actual: NormalizedTrace): MatchResult { + const diffs: Diff[] = [] + + diffScalar(diffs, "version", golden.version, actual.version) + diffScalar(diffs, "sessionId", golden.sessionId, actual.sessionId) + diffObject(diffs, "metadata", golden.metadata as Record, actual.metadata as Record) + diffObject(diffs, "summary", golden.summary as Record, actual.summary as Record) + diffSpans(diffs, golden.spans, actual.spans) + + const pass = diffs.length === 0 + return { + pass, + diffs, + format: () => (pass ? "" : formatDiffs(diffs)), + } +} + +/** parentId ("" for top-level spans) → rank → sibling spans, in the order they appear in the (already DFS-sorted) input. */ +function groupByParentAndRank(spans: NormalizedSpan[]): Map> { + const byParent = new Map>() + for (const span of spans) { + const parentKey = span.parentId ?? "" + let byRank = byParent.get(parentKey) + if (!byRank) { + byRank = new Map() + byParent.set(parentKey, byRank) + } + const bucket = byRank.get(span.rank) + if (bucket) bucket.push(span) + else byRank.set(span.rank, [span]) + } + return byParent +} + +/** + * Canonical signature of a span's subtree, ignoring the run-local ordinal + * (`id`/`parentId`) and the grouping key (`rank`) that got it here — only + * the span's own content and its children's own canonical signatures + * contribute. Two spans with the same signature are behaviorally + * indistinguishable regardless of which ordinal each run happened to + * assign them. + */ +function canonicalSignature(span: NormalizedSpan, childrenByParentId: Map): string { + const { id, parentId, rank, ...own } = span + const children = childrenByParentId.get(id) ?? [] + const childSigs = canonicalChildSignatures(children, childrenByParentId) + return stableStringify({ own, children: childSigs }) +} + +/** + * Signs a span's children rank-aware: children are grouped by their own + * `rank`, groups are emitted in ascending-rank order (preserving the real, + * reproducible order normalize.ts assigned to non-concurrent siblings), and + * only the signatures WITHIN a tied-rank (size > 1) group are sorted — + * because that's the only case where two runs may have assigned physically + * different ordinals to behaviorally-interchangeable concurrent spans. + * + * An indiscriminate sort of ALL descendant signatures (the prior behavior) + * erased a real order whenever an ordered child sequence was reached through + * a concurrent ancestor: two accepted raw traces where ordered child + * sequences changed ownership between concurrent parents normalized to + * different bytes in normalize.ts, yet this function's old `.sort()` still + * signed them identically, so `match()` returned `pass: true` on a real + * topology change. See codex-tracegolden-code-review.md finding #4 (half B). + */ +function canonicalChildSignatures( + children: NormalizedSpan[], + childrenByParentId: Map, +): string[] { + const byRank = new Map() + for (const child of children) { + const bucket = byRank.get(child.rank) + if (bucket) bucket.push(child) + else byRank.set(child.rank, [child]) + } + const out: string[] = [] + for (const rank of [...byRank.keys()].sort((a, b) => a - b)) { + const group = byRank.get(rank)! + const sigs = group.map((c) => canonicalSignature(c, childrenByParentId)) + out.push(...(group.length > 1 ? sigs.sort() : sigs)) + } + return out +} + + +function buildChildrenByParentId(spans: NormalizedSpan[]): Map { + const byParentId = new Map() + for (const span of spans) { + if (span.parentId === null) continue + const bucket = byParentId.get(span.parentId) + if (bucket) bucket.push(span) + else byParentId.set(span.parentId, [span]) + } + return byParentId +} + +/** + * Compares a rank-tied sibling group (genuinely concurrent — size > 1 on + * either side) as an unordered multiset of subtree signatures, rather than + * pairing by id. Matches are removed from both pools; whatever's left is a + * real diff, reported against the group rather than a specific (arbitrary) + * id pairing. + */ +function diffConcurrentGroup( + diffs: Diff[], + parentKey: string, + rank: number, + golden: NormalizedSpan[], + actual: NormalizedSpan[], + goldenChildren: Map, + actualChildren: Map, +) { + const goldenSigs = golden.map((span) => ({ span, sig: canonicalSignature(span, goldenChildren) })) + const actualPool = actual.map((span) => ({ span, sig: canonicalSignature(span, actualChildren) })) + + const unmatchedGolden: typeof goldenSigs = [] + for (const g of goldenSigs) { + const idx = actualPool.findIndex((a) => a.sig === g.sig) + if (idx >= 0) actualPool.splice(idx, 1) + else unmatchedGolden.push(g) + } + + const groupPath = `spans[parent=${parentKey}][concurrent-rank=${rank}]` + for (const g of unmatchedGolden) { + diffs.push({ kind: "removed", path: `${groupPath}.${g.span.id}`, expected: g.span }) + } + for (const a of actualPool) { + diffs.push({ kind: "added", path: `${groupPath}.${a.span.id}`, actual: a.span }) + } +} + +function diffSpans(diffs: Diff[], goldenSpans: NormalizedSpan[], actualSpans: NormalizedSpan[]) { + const goldenChildren = buildChildrenByParentId(goldenSpans) + const actualChildren = buildChildrenByParentId(actualSpans) + + const goldenGroups = groupByParentAndRank(goldenSpans) + const actualGroups = groupByParentAndRank(actualSpans) + const parentKeys = new Set([...goldenGroups.keys(), ...actualGroups.keys()]) + + for (const parentKey of [...parentKeys].sort()) { + const goldenByRank = goldenGroups.get(parentKey) ?? new Map() + const actualByRank = actualGroups.get(parentKey) ?? new Map() + const ranks = new Set([...goldenByRank.keys(), ...actualByRank.keys()]) + + for (const rank of [...ranks].sort((a, b) => a - b)) { + const goldenGroup = goldenByRank.get(rank) ?? [] + const actualGroup = actualByRank.get(rank) ?? [] + + if (goldenGroup.length <= 1 && actualGroup.length <= 1) { + // Rank-unique on both sides — a real, reproducible order. Diff 1:1 by id for a precise, + // field-level report (and to catch e.g. a singleton span silently changing identity). + const g = goldenGroup[0] + const a = actualGroup[0] + if (g && !a) { + diffs.push({ kind: "removed", path: `spans[${g.id}]`, expected: g }) + } else if (!g && a) { + diffs.push({ kind: "added", path: `spans[${a.id}]`, actual: a }) + } else if (g && a) { + diffObject(diffs, `spans[${g.id}]`, g as unknown as Record, a as unknown as Record) + } + continue + } + + // Genuinely concurrent on at least one side — never trust id pairing here. + diffConcurrentGroup(diffs, parentKey, rank, goldenGroup, actualGroup, goldenChildren, actualChildren) + } + } + + // Sanity check: every span id must be unique within its own trace. A duplicate means + // normalize.ts's DFS-ordinal invariant broke — a bug in this file's assumptions, not a + // real golden mismatch — so fail loudly rather than silently under/over-report diffs. + if (new Set(goldenSpans.map((s) => s.id)).size !== goldenSpans.length) { + throw new Error("match: duplicate span id in golden trace — normalize.ts invariant violated") + } + if (new Set(actualSpans.map((s) => s.id)).size !== actualSpans.length) { + throw new Error("match: duplicate span id in actual trace — normalize.ts invariant violated") + } +} + +function diffObject(diffs: Diff[], basePath: string, golden: Record, actual: Record) { + const keys = new Set([...Object.keys(golden), ...Object.keys(actual)]) + for (const key of [...keys].sort()) { + diffValue(diffs, `${basePath}.${key}`, golden[key], actual[key]) + } +} + +function diffValue(diffs: Diff[], path: string, expected: unknown, actual: unknown) { + if (deepEqual(expected, actual)) return + + if (isPlainObject(expected) && isPlainObject(actual)) { + diffObject(diffs, path, expected as Record, actual as Record) + return + } + if (Array.isArray(expected) && Array.isArray(actual)) { + diffArray(diffs, path, expected, actual) + return + } + diffs.push({ kind: "changed", path, expected, actual }) +} + +function diffArray(diffs: Diff[], basePath: string, expected: unknown[], actual: unknown[]) { + const max = Math.max(expected.length, actual.length) + if (expected.length !== actual.length) { + diffs.push({ kind: "changed", path: `${basePath}.length`, expected: expected.length, actual: actual.length }) + } + for (let i = 0; i < max; i++) { + if (i >= expected.length) { + diffs.push({ kind: "added", path: `${basePath}[${i}]`, actual: actual[i] }) + continue + } + if (i >= actual.length) { + diffs.push({ kind: "removed", path: `${basePath}[${i}]`, expected: expected[i] }) + continue + } + diffValue(diffs, `${basePath}[${i}]`, expected[i], actual[i]) + } +} + +function diffScalar(diffs: Diff[], path: string, expected: unknown, actual: unknown) { + if (!deepEqual(expected, actual)) diffs.push({ kind: "changed", path, expected, actual }) +} + +function isPlainObject(value: unknown): value is Record { + return !!value && typeof value === "object" && !Array.isArray(value) +} + +function deepEqual(a: unknown, b: unknown): boolean { + if (a === b) return true + if (typeof a !== typeof b) return false + if (isPlainObject(a) && isPlainObject(b)) { + const aKeys = Object.keys(a) + const bKeys = Object.keys(b) + if (aKeys.length !== bKeys.length) return false + return aKeys.every((k) => deepEqual(a[k], b[k])) + } + if (Array.isArray(a) && Array.isArray(b)) { + if (a.length !== b.length) return false + return a.every((v, i) => deepEqual(v, b[i])) + } + return false +} + +function formatValue(value: unknown): string { + if (value === undefined) return "" + const json = JSON.stringify(value) + if (json === undefined) return String(value) + return json.length > 120 ? `${json.slice(0, 117)}...` : json +} + +/** Renders diffs as a compact, reviewable report — one line per conflict, sorted by path. */ +export function formatDiffs(diffs: Diff[]): string { + const sorted = [...diffs].sort((a, b) => a.path.localeCompare(b.path)) + const lines = sorted.map((d) => { + switch (d.kind) { + case "added": + return ` + ${d.path} = ${formatValue(d.actual)}` + case "removed": + return ` - ${d.path} = ${formatValue(d.expected)}` + case "changed": + return ` ~ ${d.path}: ${formatValue(d.expected)} → ${formatValue(d.actual)}` + } + }) + return [`trace-golden mismatch: ${diffs.length} diff(s)`, ...lines].join("\n") +} diff --git a/packages/opencode/test/altimate/trace-golden/normalize.ts b/packages/opencode/test/altimate/trace-golden/normalize.ts new file mode 100644 index 000000000..ea72f4b81 --- /dev/null +++ b/packages/opencode/test/altimate/trace-golden/normalize.ts @@ -0,0 +1,801 @@ +// Deterministic normalizer for TraceFile (see tracing.ts). +// +// Design: a VERSIONED ALLOWLIST PROJECTION, not a scrub blacklist. Every field +// at every fixed-schema level (TraceFile, metadata, TraceSpan, span.model, +// span.tool, span.tokens, summary, summary.tokens, summary.topTools[] +// elements, summary.loops[] elements) is explicitly classified below as +// stable-keep / volatile-drop / volatile-canonicalize. `assertKnownKeys` fails +// LOUDLY on any field tracing.ts adds that hasn't been classified here, so the +// normalizer can never silently start hiding — or silently start leaking — a +// new field. Freeform payloads (span.input/output/attributes, and the two +// derived English-text fields, summary.narrative and the root span's closing +// `output`) cannot be schema-validated field-by-field; those get pattern-based +// volatile-content scrubbing instead (`scrubDynamicTokens`). +// +// This does NOT prove full behavioral equivalence: the trace is deliberately +// lossy (user messages truncated at USER_MESSAGE_INPUT_MAX_CHARS, tool output +// capped at 10,000 chars, no copied attachments, long traces elide their +// middle past ALTIMATE_TRACE_MAX_SPANS). "No diff" means "no diff in what the +// trace records," not "no behavioral difference at all." See +// docs/internal/2026-07-18-trace-golden-e2e-technique.md for the scope this +// is limited to (S5 parity / S7 continuation corroboration — NOT an S3 +// security oracle). +// +// Ordering: `logToolCall` (tracing.ts:926) appends a tool span at COMPLETION +// time, not start time, and tool calls are dispatched concurrently by BOTH +// Batch (`Promise.all`, batch.ts:158) and the parallel session/prompt.ts +// resolver — confirmed structurally: `parentSpanId: this.currentGenerationSpanId +// ?? this.rootSpanId` (tracing.ts:928) means every tool call issued within one +// step is a direct sibling under that step's generation span, with no +// sub-span distinguishing "this one was batched" from "this one wasn't." So +// raw array position for `kind: "tool"` siblings reflects completion/event- +// loop race order, not real dispatch order, full stop — there is no +// trustworthy sequential sub-case to recover among them. Every OTHER kind +// (`session`, `generation`, `user-message`, the elision-marker `span`) is +// pushed synchronously at the real moment it occurs — `logStepStart` opens a +// generation span at step START, not finish — so raw array order for those +// IS a genuine, reproducible order. +// +// `computeRanks` encodes exactly this: siblings of kind `"tool"` under a +// given parent all share ONE rank (a "concurrent bucket" — no claim of +// order among them); every other-kind sibling gets a unique, strictly +// increasing rank in its real array order. (An earlier version of this +// derived rank from startTime/endTime interval overlap instead — dropped +// because wall-clock overlap is itself a noisy heuristic: a fast sibling can +// finish before a slower one starts even when both came from the same +// `Promise.all`, which would nondeterministically reclassify a genuinely +// concurrent group as "rank-unique" on one run and "same-rank" on another.) +// `buildDfsOrdinals` then assigns ordinals by (rank, then a stable content +// key — kind + name + normalized-input digest, NOT array/original-index) +// instead of raw array order, so two independent runs of the same +// concurrent scenario assign the SAME ordinal to the SAME logical tool call +// regardless of which one happened to complete first. The `rank` also +// survives into the normalized output so match.ts can tell which sibling +// groups are single spans (safe to diff 1:1 by id) vs. genuinely concurrent +// (must be diffed as an unordered multiset) — belt-and-suspenders with the +// content-key ordinal assignment above, not a substitute for it. +import fs from "node:fs" +import os from "node:os" +import path from "node:path" +import type { TraceFile, TraceSpan } from "@/altimate/observability/tracing" + +export interface NormalizeOptions { + /** Absolute path(s) to replace with ``. Defaults to `process.cwd()`. */ + repoRoots?: string[] + /** Absolute path(s) to replace with ``. Defaults to `os.homedir()`. */ + homeRoots?: string[] + /** + * Absolute path(s) to replace with ``. Defaults to `os.tmpdir()` plus + * common macOS/Linux temp-dir prefixes, so goldens built on a Mac still + * normalize traces recorded on Linux CI (and vice versa). + */ + tmpRoots?: string[] +} + +export interface NormalizedSpan { + id: string + parentId: string | null + /** + * Sibling rank under the same parent, derived from each raw span's `kind` + * (see computeRanks) — NOT from array position or startTime/endTime + * overlap. `kind: "tool"` siblings always share one rank (their completion + * order is never trustworthy, by construction — see computeRanks); every + * other kind gets a strictly increasing rank in its real, synchronous + * chronological order. Spans sharing a rank are a concurrent group that + * match.ts must compare as a multiset, not pair by id. + */ + rank: number + name: string + kind: TraceSpan["kind"] + status: TraceSpan["status"] + statusMessage?: string + interrupted?: boolean + model?: { modelId?: string; providerId?: string; variant?: string } + finishReason?: string + /** Bucketed, not exact — exact token counts/costs vary by provider pricing and are not "behavior". */ + hasTokens?: boolean + hasCost?: boolean + /** + * Canonicalized to `call:` — the + * raw tool-call id itself is provider-random, but which spans share (or + * don't share) a raw id is real, reproducible information, so it's + * preserved as an equality/uniqueness pattern rather than dropped. + */ + toolCallId?: string + input?: unknown + output?: unknown + attributes?: Record +} + +export interface NormalizedTrace { + version: number + sessionId: "" + metadata: { + title?: string + model?: string + providerId?: string + agent?: string + variant?: string + prompt?: string + /** Arbitrary tags are scenario config, not runtime nondeterminism — kept, sorted for determinism. */ + tags?: string[] + } + spans: NormalizedSpan[] + summary: { + status: TraceFile["summary"]["status"] + totalToolCalls: number + totalGenerations: number + error?: string + /** count kept (real behavior signal); totalDuration dropped; re-sorted (count desc, name asc — see below). */ + topTools?: Array<{ name: string; count: number }> + /** Loop detection is a real behavior signal; inputHash (a derived digest) is dropped, description is scrubbed. */ + loops?: Array<{ tool: string; count: number; description: string }> + /** Derived English summary text; kept with cost/duration/token-count phrases scrubbed (see scrubDynamicTokens). */ + narrative?: string + } +} + +// --------------------------------------------------------------------------- +// Allowlist: every key TraceFile/TraceSpan/summary can carry, per schema +// level, and what happens to it. Any key NOT in the relevant set below is +// unclassified — assertKnownKeys throws rather than silently drop or leak it. +// --------------------------------------------------------------------------- + +/** TraceFile top level. `spans`/`summary`/`metadata` handled structurally below; the rest here. */ +const TRACE_FILE_KEYS = new Set([ + "version", // stable-keep + "traceId", // volatile-drop (UUIDv7, tracing.ts:528) + "sessionId", // volatile-drop (replaced with "") + "startedAt", // volatile-drop (wall-clock timestamp) + "endedAt", // volatile-drop (wall-clock timestamp) + "metadata", // structural — see METADATA_KEYS + "spans", // structural — see SPAN_KEYS + "summary", // structural — see SUMMARY_KEYS +]) + +const METADATA_KEYS = new Set([ + "title", // stable-keep + "model", // stable-keep + "providerId", // stable-keep + "agent", // stable-keep + "variant", // stable-keep + "prompt", // stable-keep (scrubbed for paths/dynamic tokens) + "userId", // volatile-drop (per-user identity, not scenario behavior) + "environment", // volatile-drop (deploy environment, not scenario behavior) + "version", // volatile-drop (app release version — bumping it must not flap goldens) + "tags", // stable-keep, sorted (arbitrary but scenario-configured, not runtime-random) +]) + +const SPAN_KEYS = new Set([ + "spanId", // volatile-drop (replaced with DFS ordinal) + "parentSpanId", // volatile-drop (replaced with DFS ordinal) + "name", // stable-keep (scrubbed for session/generation dynamic parts) + "kind", // stable-keep + "startTime", // volatile-drop (timing; not used for ordering — see computeRanks's kind-based partial order) + "endTime", // volatile-drop (timing; not used for ordering — see computeRanks's kind-based partial order) + "status", // stable-keep + "statusMessage", // stable-keep (scrubbed) + "interrupted", // stable-keep + "model", // structural — see SPAN_MODEL_KEYS + "finishReason", // stable-keep + "tokens", // volatile-canonicalize — collapsed to hasTokens (exact counts vary by provider pricing) + "cost", // volatile-canonicalize — collapsed to hasCost + "tool", // structural — see SPAN_TOOL_KEYS + "input", // stable-keep (scrubbed for paths/dynamic tokens) + "output", // stable-keep (scrubbed for paths/dynamic tokens) + "attributes", // structural — see VOLATILE_ATTRIBUTE_KEYS filtering +]) + +/** + * Compile-time pin, sitting right next to `SPAN_KEYS` on purpose: if + * `TraceSpan` (tracing.ts) gains, loses, or renames a field without + * `SpanKeyLiteral` below being updated to match, `_SpanKeysPinnedToTraceSpan` + * stops typechecking and `bun run typecheck` fails — unconditionally, before + * any test even runs. `SpanKeyLiteral` is a hand-maintained literal-string + * mirror of `SPAN_KEYS` (TypeScript has no way to turn a runtime `Set` into a + * type), so the two must always be edited together. This is the compile-time + * half of "reject unknown until classified"; `assertKnownKeys` below is the + * runtime half, firing on live data (a real or synthetic trace) that a type + * check alone can't see. + */ +type SpanKeyLiteral = + | "spanId" + | "parentSpanId" + | "name" + | "kind" + | "startTime" + | "endTime" + | "status" + | "statusMessage" + | "interrupted" + | "model" + | "finishReason" + | "tokens" + | "cost" + | "tool" + | "input" + | "output" + | "attributes" +type Equals = (() => T extends X ? 1 : 2) extends () => T extends Y ? 1 : 2 ? true : false +type AssertTrue = T +// Unused by design — this type's only job is to exist and typecheck. +type _SpanKeysPinnedToTraceSpan = AssertTrue> + +const SPAN_MODEL_KEYS = new Set(["modelId", "providerId", "variant"]) // all stable-keep + +const SPAN_TOOL_KEYS = new Set([ + "callId", // volatile-canonicalize (provider-random; toolCallId is rewritten to a first-occurrence-ordinal label instead — see computeToolCallIdLabels) + "durationMs", // volatile-drop (timing) +]) + +/** TraceSpan.tokens (TokenUsage) — every field volatile-drop; collapsed to hasTokens. */ +const SPAN_TOKENS_KEYS = new Set(["input", "output", "reasoning", "cacheRead", "cacheWrite", "total"]) + +const SUMMARY_KEYS = new Set([ + "totalTokens", // volatile-drop (exact magnitude, not behavior) + "totalCost", // volatile-drop + "totalToolCalls", // stable-keep + "totalGenerations", // stable-keep + "duration", // volatile-drop (timing) + "status", // stable-keep + "error", // stable-keep (scrubbed) + "tokens", // structural — see SUMMARY_TOKENS_KEYS (all volatile-drop) + "loops", // structural — see LOOP_ELEMENT_KEYS + "narrative", // volatile-canonicalize — kept with embedded cost/duration/token-count scrubbed + "topTools", // structural — see TOP_TOOL_ELEMENT_KEYS +]) + +/** TraceFile.summary.tokens — note: unlike TokenUsage this object has no "total" field. */ +const SUMMARY_TOKENS_KEYS = new Set(["input", "output", "reasoning", "cacheRead", "cacheWrite"]) + +const LOOP_ELEMENT_KEYS = new Set([ + "tool", // stable-keep + "inputHash", // volatile-drop (derived digest, not independently meaningful once dropped from description) + "count", // stable-keep + "description", // stable-keep, scrubbed (embeds the inputHash as literal text — must go through scrubDynamicTokens) +]) + +const TOP_TOOL_ELEMENT_KEYS = new Set([ + "name", // stable-keep + "count", // stable-keep + "totalDuration", // volatile-drop (timing) +]) + +/** + * The allowlist projection this enforces (only fields listed in `allowed` + * survive; everything else throws) is an intentional design choice, not an + * oversight — see the module-level comment at the top of this file. A + * silent-drop of an unrecognized field would let a real new TraceSpan/ + * TraceFile field (added in tracing.ts) quietly vanish from every golden + * forever with no signal; throwing instead forces a human to classify it + * (stable-keep / volatile-drop / volatile-canonicalize) before it can be + * normalized at all. + */ +function assertKnownKeys(obj: Record, allowed: Set, label: string) { + const unknown = Object.keys(obj).filter((k) => !allowed.has(k)) + if (unknown.length > 0) { + throw new Error( + `normalize: unknown field(s) at "${label}": ${unknown.join(", ")}. ` + + `Classify each as stable-keep / volatile-drop / volatile-canonicalize in normalize.ts before accepting this golden.`, + ) + } +} + +const VOLATILE_ATTRIBUTE_KEYS = new Set([ + "pid", + "hostname", + "host", + "cwd", + "env", + "platform", + "arch", + "nodeVersion", + "user", + "uid", + "gid", + "timestamp", + "time", + "date", + "durationMs", + "duration", +]) + +// --------------------------------------------------------------------------- +// Kind-based rank derivation (the partial-order fix) +// --------------------------------------------------------------------------- + +interface RankedChild { + span: TraceSpan + originalIndex: number +} + +/** Groups spans by parentSpanId, treating an unknown/dangling parentSpanId as root — same rule buildDfsOrdinals uses. */ +function groupByParent(spans: TraceSpan[]): Map { + const knownIds = new Set(spans.map((s) => s.spanId)) + const byParent = new Map() + spans.forEach((span, originalIndex) => { + const parentKey = span.parentSpanId && knownIds.has(span.parentSpanId) ? span.parentSpanId : null + const bucket = byParent.get(parentKey) + if (bucket) bucket.push({ span, originalIndex }) + else byParent.set(parentKey, [{ span, originalIndex }]) + }) + return byParent +} + +/** + * Derives a per-sibling-group `rank` purely from each span's own `kind`, not + * timing. `kind: "tool"` siblings under a given parent always share ONE + * rank — a "concurrent bucket" — because their array order reflects + * completion time (`logToolCall` appends on finish, tracing.ts:926), and + * both `Promise.all` dispatchers (Batch, batch.ts:158; the parallel + * session/prompt.ts resolver) mean that order is a race outcome, not a + * dispatch order, with no trustworthy sub-case to recover. An earlier + * version of this function tried to recover a partial order from + * startTime/endTime interval overlap instead: siblings whose intervals + * don't overlap got distinct, increasing ranks (claiming a real order), + * while overlapping ones shared a rank. That's unsound across runs — a fast + * sibling can finish before a slower one starts even when both came from the + * same `Promise.all` call, so whether two siblings' intervals "overlap" can + * itself flip between two runs of the identical scenario under ordinary + * scheduling jitter, silently reclassifying a genuinely concurrent pair as + * rank-unique on one run and rank-tied on another — which is exactly the + * false-positive diff this harness exists to prevent. + * + * Every OTHER kind (`session`, `generation`, `user-message`, the + * elision-marker `span`) is pushed synchronously at the real moment it + * occurs — `logStepStart` opens a generation span at step START, not finish + * (tracing.ts:786); `logUserMessage` appends in real time (tracing.ts:761). + * So raw array order for those kinds IS a genuine, reproducible order, and + * each such sibling gets its own strictly increasing rank in that order. + * + * The concurrent-bucket treatment ONLY applies to a `kind: "tool"` span + * whose parent is itself a `kind: "generation"` span — because that's the + * one topology `logToolCall` actually races on (`parentSpanId: + * this.currentGenerationSpanId ?? this.rootSpanId`, tracing.ts:928). When no + * generation is active, `logToolCall` attaches the tool directly to the + * session root instead, alongside ordered siblings like `user-message` and + * `generation` — and THAT ordering is real and reproducible, not a race. + * Blindly bucketing every `kind: "tool"` span regardless of its parent + * erased this real order and let a reordering of a root-attached tool + * around a user-message/generation sibling normalize byte-identically and + * false-pass — see codex-tracegolden-code-review.md finding #4 (half A). + */ +function computeRanks(spans: TraceSpan[]): Map { + const byId = new Map(spans.map((s) => [s.spanId, s])) + const byParent = groupByParent(spans) + const ranks = new Map() + // Sentinel rank for the shared tool-kind bucket. Ordered ranks below start + // at 0 and only increase, so -1 can never collide with a real ordered + // rank — the two kinds of rank are never compared against each other + // within one parent's sibling group. + const TOOL_BUCKET_RANK = -1 + for (const [parentKey, siblings] of byParent) { + const parentIsGeneration = parentKey !== null && byId.get(parentKey)?.kind === "generation" + let nextOrderedRank = 0 + for (const { span } of siblings) { + if (span.kind === "tool" && parentIsGeneration) { + ranks.set(span.spanId, TOOL_BUCKET_RANK) + } else { + ranks.set(span.spanId, nextOrderedRank) + nextOrderedRank += 1 + } + } + } + return ranks +} + +/** + * Deterministic content key for ordering same-rank (concurrent) siblings: + * `kind`, then `name`, then the full stably-stringified `input`, then the + * full stably-stringified `output` — not a hash, to avoid any collision + * risk, and `stableStringify` already produces a canonical, + * key-order-independent serialization, so two structurally identical + * inputs/outputs always produce the same key. This is what lets two + * independent runs of the same concurrent scenario assign the SAME ordinal + * to the SAME logical tool call, regardless of which one happened to + * complete first. + * + * `output` is included (not just `input`) because two same-name/same-input + * concurrent siblings that resolve to DIFFERENT outputs are, in fact, + * distinguishable — omitting output collapsed them onto the raw-array-order + * fallback below, which is a race outcome, not a stable key. See + * codex-tracegolden-code-review.md finding #3. + */ +function toolContentKey(span: TraceSpan): string { + return `${span.kind} ${span.name} ${stableStringify(span.input ?? null)} ${stableStringify(span.output ?? null)}` +} + +/** + * Builds spanId → ordinal ("s0", "s1", ...) via DFS pre-order, root-first, + * siblings ordered by (rank, then a stable content key) rather than raw + * array position. Rank alone already fully orders non-tool siblings (each + * gets a unique rank from computeRanks); the content key only matters + * WITHIN the shared tool-bucket rank, where it sorts by each span's own + * `kind`/`name`/normalized-`input` — never by array position or completion + * order — so two independent runs of the same concurrent scenario produce + * the SAME ordinal assignment for the SAME logical tool call. This is what + * makes normalize()'s OWN output order-invariant across runs, not just a + * property match.ts has to work around downstream. + * + * The one case this can't fully resolve is two tool spans with + * byte-identical (kind, name, input) — genuinely indistinguishable + * concurrent duplicates — where the final tiebreak falls back to + * original-index. That residual is harmless: match.ts's rank-aware multiset + * comparison treats same-signature spans as interchangeable regardless of + * which physical ordinal either run happened to assign them. + */ +function buildDfsOrdinals(spans: TraceSpan[], ranks: Map): Map { + const byParent = groupByParent(spans) + const ordinals = new Map() + let counter = 0 + const visit = (parentId: string | null) => { + const children = byParent.get(parentId) ?? [] + const sorted = [...children].sort((a, b) => { + const ra = ranks.get(a.span.spanId) ?? 0 + const rb = ranks.get(b.span.spanId) ?? 0 + if (ra !== rb) return ra - rb + const ka = toolContentKey(a.span) + const kb = toolContentKey(b.span) + if (ka !== kb) return ka < kb ? -1 : 1 + return a.originalIndex - b.originalIndex + }) + for (const { span } of sorted) { + ordinals.set(span.spanId, `s${counter}`) + counter += 1 + visit(span.spanId) + } + } + visit(null) + + if (ordinals.size !== spans.length) { + throw new Error( + `normalize: DFS ordinal assignment covered ${ordinals.size}/${spans.length} spans — ` + + `likely a cycle in parentSpanId links`, + ) + } + return ordinals +} + +/** + * Canonicalizes raw provider tool-call ids to a label based on the ordinal + * of their FIRST DFS occurrence, so distinct raw ids map to distinct + * labels, and two calls that (incorrectly) share one raw id map to the SAME + * label — preserving the raw id's equality/uniqueness PATTERN without + * leaking the literal (provider-random, non-reproducible) value. + * + * The prior behavior rewrote `toolCallId` to the span's OWN ordinal + * (`call:${id}`), independent of the raw callId value entirely. That threw + * away the one thing worth keeping: whether two spans share a raw callId at + * all. Two sibling calls with distinct raw ids, and two calls incorrectly + * sharing one duplicate raw id, both normalized byte-identically and + * matched successfully — hiding an S7-relevant continuation-correctness + * regression. See codex-tracegolden-code-review.md finding #5. + */ +function computeToolCallIdLabels(spans: TraceSpan[], ordinals: Map): Map { + const withOrdinal = spans + .filter((s) => s.tool?.callId !== undefined) + .map((s) => ({ span: s, ordinal: ordinals.get(s.spanId) ?? "s0" })) + .sort((a, b) => Number(a.ordinal.slice(1)) - Number(b.ordinal.slice(1))) + + const firstOrdinalByRawId = new Map() + for (const { span, ordinal } of withOrdinal) { + const rawId = span.tool!.callId! + if (!firstOrdinalByRawId.has(rawId)) { + firstOrdinalByRawId.set(rawId, ordinal) + } + } + + const labels = new Map() + for (const { span } of withOrdinal) { + const rawId = span.tool!.callId! + labels.set(span.spanId, `call:${firstOrdinalByRawId.get(rawId)}`) + } + return labels +} + +// --------------------------------------------------------------------------- +// Path scrubbing (unchanged from the prior version — still correct) +// --------------------------------------------------------------------------- + +/** + * Expands each root with its `fs.realpathSync()` resolution, so a golden + * built with one root form (e.g. macOS's `/var/folders/...` symlink) still + * matches a trace recorded through the other form (`/private/var/folders/...`) + * — and vice versa. This is the fix for the platform-specific `/private` + * leak: `replaceAllPrefixed` (below) is literal substring replacement, so a + * root and its realpath must BOTH be registered as scrub prefixes, longest + * first, or whichever form isn't the literal prefix of the recorded string + * survives as leaked text. Non-existent paths (synthetic test roots, or a + * root for a fixture already torn down) are left as-is; realpath resolution + * is best-effort, not required. + */ +function withRealpathVariants(roots: string[]): string[] { + const out = new Set() + for (const root of roots) { + if (!root) continue + out.add(root) + try { + const real = fs.realpathSync(root) + if (real) out.add(real) + } catch { + // Path doesn't exist (synthetic test root, torn-down fixture) — skip silently. + } + } + return [...out] +} + +function resolvedRoots(input: string[] | undefined, fallback: string[]): string[] { + const roots = input && input.length > 0 ? input : fallback + const expanded = withRealpathVariants(roots) + // Longest-first so a nested tmp dir under a repo root (or vice versa) matches its more specific prefix. + return [...new Set(expanded)].sort((a, b) => b.length - a.length) +} + +function defaultTmpRoots(): string[] { + return [os.tmpdir(), "/tmp", "/private/tmp", "/var/folders"] +} + +interface RootEntry { + root: string + placeholder: string +} + +/** + * Merges repo/home/tmp roots into a single longest-root-first list. A single + * merged pass (rather than three separate category passes) is required + * because a root from one category can be a strict prefix of a root from + * another — e.g. a CliFixture's home dir is itself created inside the OS tmp + * dir with a per-run-random suffix (`$TMPDIR/oc-cli-`). If the + * generic tmp root ran first it would consume just its own prefix and leave + * the random suffix exposed as literal, nondeterministic text — the specific + * (longer) home root must always win over the generic (shorter) tmp root it + * happens to live inside. + */ +function buildRootEntries(repoRoots: string[], homeRoots: string[], tmpRoots: string[]): RootEntry[] { + const entries: RootEntry[] = [ + ...repoRoots.map((root) => ({ root, placeholder: "" })), + ...homeRoots.map((root) => ({ root, placeholder: "" })), + ...tmpRoots.map((root) => ({ root, placeholder: "" })), + ] + return entries.filter((e) => e.root).sort((a, b) => b.root.length - a.root.length) +} + +// --------------------------------------------------------------------------- +// Dynamic-token scrubbing — pattern-based, for freeform text that can't be +// schema-validated field-by-field (input/output/attributes/narrative/error). +// Patterns are ordered so more-specific matches (human dates) run before +// more-general ones that could otherwise partially consume them. +// --------------------------------------------------------------------------- + +// `simpleHash().toString(36)` embedded verbatim in loop-summary descriptions (tracing.ts); a signed +// 32-bit hash can toString(36) with a leading "-", matched here so the sign doesn't leak either. +const LOOP_HASH_PATTERN = /\(hash: -?[0-9a-z]+\)/g +const UUID_PATTERN = /[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}/g +// Fork-minted prefixed ids: ses_/prt_/msg_/call_ (see PartID/session id generators). +const PREFIXED_ID_PATTERN = /\b(?:ses|prt|msg|call)_[A-Za-z0-9]+\b/g +const ISO_TIMESTAMP_PATTERN = /\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z?/g +// `new Date().toDateString()` shape, e.g. "Sat Jul 18 2026" — embedded verbatim in the system prompt (system.ts:87). +const HUMAN_DATE_PATTERN = /\b(?:Mon|Tue|Wed|Thu|Fri|Sat|Sun) (?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) \d{1,2} \d{4}\b/g +// `Platform: ${process.platform}` (system.ts:81). +const PLATFORM_LINE_PATTERN = /Platform: \S+/g +// formatDurationShort's exact output shapes (tracing.ts:396-402): "150ms", "2.3s", "1m5s". +const DURATION_PHRASE_PATTERN = /\b\d+ms\b|\b\d+\.\d+s\b|\b\d+m\d+s\b/g +// `$${cost.toFixed(4)}`-style cost strings embedded in narrative/root-span output. +const COST_PHRASE_PATTERN = /\$\d+(?:\.\d+)?/g + +/** Scrubs volatile dynamic content (ids, timestamps, dates, platform, durations, costs) out of freeform text. */ +function scrubDynamicTokens(text: string): string { + return text + .replace(LOOP_HASH_PATTERN, "(hash: )") + .replace(UUID_PATTERN, "") + .replace(PREFIXED_ID_PATTERN, "") + .replace(ISO_TIMESTAMP_PATTERN, "") + .replace(HUMAN_DATE_PATTERN, "") + .replace(PLATFORM_LINE_PATTERN, "Platform: ") + .replace(DURATION_PHRASE_PATTERN, "") + .replace(COST_PHRASE_PATTERN, "") +} + +/** Recursively replaces absolute-path prefixes anywhere inside a JSON-cloneable value with placeholders. */ +function scrubPaths(value: unknown, entries: RootEntry[]): unknown { + if (typeof value === "string") return scrubText(value, entries) + if (Array.isArray(value)) return value.map((v) => scrubPaths(v, entries)) + if (value && typeof value === "object") { + const out: Record = {} + for (const [k, v] of Object.entries(value as Record)) { + out[k] = scrubPaths(v, entries) + } + return out + } + return value +} + +/** Path scrubbing followed by dynamic-token scrubbing — every string that flows through input/output/attributes gets both passes. */ +function scrubText(text: string, entries: RootEntry[]): string { + let out = text + for (const { root, placeholder } of entries) out = replaceAllPrefixed(out, root, placeholder) + return scrubDynamicTokens(out) +} + +function replaceAllPrefixed(text: string, root: string, placeholder: string): string { + if (!root) return text + const normalizedRoot = root.endsWith(path.sep) ? root.slice(0, -1) : root + if (!normalizedRoot) return text + return text.split(normalizedRoot).join(placeholder) +} + +function scrubAttributes( + attributes: Record | undefined, + entries: RootEntry[], +): Record | undefined { + if (!attributes) return undefined + const out: Record = {} + let any = false + for (const [key, value] of Object.entries(attributes)) { + if (VOLATILE_ATTRIBUTE_KEYS.has(key)) continue + out[key] = scrubPaths(value, entries) + any = true + } + return any ? out : undefined +} + +// `kind: "session"` spans are named after the trace's own session id (a +// random-per-run `ses_...` identifier) and `kind: "generation"` spans are +// named `generation-`, where the part id is also random-per-run +// (see tracing.ts). Neither is a filesystem path, so scrubPathString never +// touches them — they need their own targeted normalization or every trace +// flaps on its own span names. +const SESSION_NAME_PATTERN = /^ses_[A-Za-z0-9]+$/ +const GENERATION_NAME_PATTERN = /^generation-.+$/ + +function scrubSpanName(span: TraceSpan, sessionId: string): string { + if (span.kind === "session" && (span.name === sessionId || SESSION_NAME_PATTERN.test(span.name))) return "" + if (span.kind === "generation" && GENERATION_NAME_PATTERN.test(span.name)) return "generation" + return span.name +} + +function normalizeSpan( + span: TraceSpan, + ordinals: Map, + ranks: Map, + sessionId: string, + entries: RootEntry[], + toolCallIdLabels: Map, +): NormalizedSpan { + assertKnownKeys(span as unknown as Record, SPAN_KEYS, `spans[spanId=${span.spanId}]`) + if (span.model) assertKnownKeys(span.model as unknown as Record, SPAN_MODEL_KEYS, "span.model") + if (span.tool) assertKnownKeys(span.tool as unknown as Record, SPAN_TOOL_KEYS, "span.tool") + if (span.tokens) assertKnownKeys(span.tokens as unknown as Record, SPAN_TOKENS_KEYS, "span.tokens") + + const id = ordinals.get(span.spanId) + if (!id) throw new Error(`normalize: span ${span.spanId} missing from ordinal map`) + const parentId = span.parentSpanId ? (ordinals.get(span.parentSpanId) ?? null) : null + + const out: NormalizedSpan = { + id, + parentId, + rank: ranks.get(span.spanId) ?? 0, + name: scrubSpanName(span, sessionId), + kind: span.kind, + status: span.status, + } + if (span.statusMessage !== undefined) out.statusMessage = scrubText(span.statusMessage, entries) + if (span.interrupted !== undefined) out.interrupted = span.interrupted + if (span.model) { + out.model = { + modelId: span.model.modelId, + providerId: span.model.providerId, + variant: span.model.variant, + } + } + if (span.finishReason !== undefined) out.finishReason = span.finishReason + if (span.tokens !== undefined) out.hasTokens = span.tokens.total > 0 + if (span.cost !== undefined) out.hasCost = span.cost > 0 + if (span.tool?.callId !== undefined) out.toolCallId = toolCallIdLabels.get(span.spanId) + if (span.input !== undefined) out.input = scrubPaths(span.input, entries) + if (span.output !== undefined) out.output = scrubPaths(span.output, entries) + const attrs = scrubAttributes(span.attributes, entries) + if (attrs) out.attributes = attrs + return out +} + +/** + * Normalizes a TraceFile into a diffable, deterministic shape. Idempotent and + * stable: calling this twice on the same TraceFile with the same options + * always produces deep-equal (and JSON.stringify-identical) output — but see + * the module-level comment on rank ties: two DIFFERENT recordings of the same + * scenario are only guaranteed identical ordinal assignment for + * non-overlapping (rank-unique) siblings. Concurrent siblings must be + * compared via match.ts's rank-aware multiset logic, not raw equality. + * + * Fails loudly (throws) rather than silently drop or leak an unclassified + * field — see assertKnownKeys and the allowlist sets above. + */ +export function normalize(trace: TraceFile, options: NormalizeOptions = {}): NormalizedTrace { + assertKnownKeys(trace as unknown as Record, TRACE_FILE_KEYS, "TraceFile") + assertKnownKeys(trace.metadata as unknown as Record, METADATA_KEYS, "metadata") + assertKnownKeys(trace.summary as unknown as Record, SUMMARY_KEYS, "summary") + if (trace.summary.tokens) { + assertKnownKeys(trace.summary.tokens as unknown as Record, SUMMARY_TOKENS_KEYS, "summary.tokens") + } + for (const t of trace.summary.topTools ?? []) { + assertKnownKeys(t as unknown as Record, TOP_TOOL_ELEMENT_KEYS, "summary.topTools[]") + } + for (const l of trace.summary.loops ?? []) { + assertKnownKeys(l as unknown as Record, LOOP_ELEMENT_KEYS, "summary.loops[]") + } + + const repoRoots = resolvedRoots(options.repoRoots, [process.cwd()]) + const homeRoots = resolvedRoots(options.homeRoots, [os.homedir()]) + const tmpRoots = resolvedRoots(options.tmpRoots, defaultTmpRoots()) + const entries = buildRootEntries(repoRoots, homeRoots, tmpRoots) + + const ranks = computeRanks(trace.spans) + const ordinals = buildDfsOrdinals(trace.spans, ranks) + const toolCallIdLabels = computeToolCallIdLabels(trace.spans, ordinals) + const spans = trace.spans + .map((span) => normalizeSpan(span, ordinals, ranks, trace.sessionId, entries, toolCallIdLabels)) + // Canonical order: DFS order (== ordinal order), not raw storage order. This makes the diff + // robust to append-order jitter between two structurally-identical runs (e.g. two spans opened + // in the same tick finishing in a different wall-clock order) while still preserving the + // parent/child hierarchy and each parent's own rank-derived child ordering. + .sort((a, b) => Number(a.id.slice(1)) - Number(b.id.slice(1))) + + const topTools = trace.summary.topTools + ?.map((t) => ({ name: t.name, count: t.count })) + // count desc (real signal) then name asc (deterministic tiebreak) — the raw topTools array ties + // on Map-insertion order, which is completion order and therefore not reproducible (tracing.ts:1332). + .sort((a, b) => b.count - a.count || a.name.localeCompare(b.name)) + + const loops = trace.summary.loops + ?.map((l) => ({ + tool: l.tool, + count: l.count, + description: scrubDynamicTokens(l.description), + })) + // Multiple loops retain first-detection/completion order in the raw summary, which is not + // reproducible across runs. Sort by a stable semantic key: tool name, then count, then the + // (already-scrubbed) description as a final tiebreak. + .sort((a, b) => a.tool.localeCompare(b.tool) || a.count - b.count || a.description.localeCompare(b.description)) + + return { + version: trace.version, + sessionId: "", + metadata: { + title: trace.metadata.title, + model: trace.metadata.model, + providerId: trace.metadata.providerId, + agent: trace.metadata.agent, + variant: trace.metadata.variant, + prompt: trace.metadata.prompt ? scrubText(trace.metadata.prompt, entries) : trace.metadata.prompt, + tags: trace.metadata.tags ? [...trace.metadata.tags].sort() : undefined, + }, + spans, + summary: { + status: trace.summary.status, + totalToolCalls: trace.summary.totalToolCalls, + totalGenerations: trace.summary.totalGenerations, + error: trace.summary.error ? scrubText(trace.summary.error, entries) : trace.summary.error, + topTools, + loops, + narrative: trace.summary.narrative ? scrubDynamicTokens(trace.summary.narrative) : trace.summary.narrative, + }, + } +} + +/** Deterministic key order for stable JSON.stringify comparisons across two normalize() calls. */ +export function stableStringify(value: unknown): string { + return JSON.stringify(sortKeysDeep(value), null, 2) +} + +function sortKeysDeep(value: unknown): unknown { + if (Array.isArray(value)) return value.map(sortKeysDeep) + if (value && typeof value === "object") { + const out: Record = {} + for (const key of Object.keys(value as Record).sort()) { + out[key] = sortKeysDeep((value as Record)[key]) + } + return out + } + return value +} diff --git a/packages/opencode/test/altimate/trace-golden/scenarios/smoke/golden.json b/packages/opencode/test/altimate/trace-golden/scenarios/smoke/golden.json new file mode 100644 index 000000000..7ec3d2d61 --- /dev/null +++ b/packages/opencode/test/altimate/trace-golden/scenarios/smoke/golden.json @@ -0,0 +1,93 @@ +{ + "metadata": { + "model": "test/test-model", + "prompt": "\"Read smoke.txt in the working directory and summarize its contents in one sentence.\"\n", + "title": "\"Read smoke.txt in the working directory and summarize its contents in one sente" + }, + "sessionId": "", + "spans": [ + { + "id": "s0", + "input": "\"Read smoke.txt in the working directory and summarize its contents in one sentence.\"\n", + "kind": "session", + "name": "", + "output": "2 generations, 1 tool calls, 0 tokens, ", + "parentId": null, + "rank": 0, + "status": "ok" + }, + { + "id": "s1", + "input": { + "agent": "builder", + "step": 1 + }, + "kind": "span", + "name": "system-prompt", + "output": { + "content": "You are powered by the model named test-model. The exact model ID is test/test-model\nHere is some useful information about the environment you are running in:\n\n Working directory: \n Workspace root folder: /\n Is directory a git repo: no\n Platform: \n Today's date: \n\n\n \n\n\nSkills provide specialized instructions and workflows for specific tasks.\nUse the skill tool to load a skill when a task matches its description.\n\n \n customize-opencode\n Use ONLY when the user is editing or creating Altimate Code's own configuration: altimate-code.json, opencode.json, opencode.jsonc, files under .altimate-code/, files under .opencode/, or files under ~/.config/altimate-code/. Also use when creating or fixing Altimate Code agents, subagents, skills, plugins, MCP servers, or permission rules. Do not use for the user's own application code, or for any project that is not configuring Altimate Code itself.\n file:///%3Cbuilt-in%3E\n \n", + "parts": 2 + }, + "parentId": "s0", + "rank": 0, + "status": "ok" + }, + { + "finishReason": "tool-calls", + "hasCost": false, + "hasTokens": false, + "id": "s2", + "kind": "generation", + "model": { + "modelId": "test/test-model" + }, + "name": "generation", + "output": "[tool calls: read]", + "parentId": "s0", + "rank": 1, + "status": "ok" + }, + { + "id": "s3", + "input": { + "filePath": "/smoke.txt" + }, + "kind": "tool", + "name": "read", + "output": "/smoke.txt\nfile\n\n1: hello from the trace-golden smoke scenario\n\n(End of file - total 1 lines)\n", + "parentId": "s2", + "rank": -1, + "status": "ok", + "toolCallId": "call:s3" + }, + { + "finishReason": "stop", + "hasCost": false, + "hasTokens": false, + "id": "s4", + "input": "[read] /smoke.txt\nfile\n\n1: hello from the trace-golden smoke scenario\n\n(End of file - total 1 lines)\n", + "kind": "generation", + "model": { + "modelId": "test/test-model" + }, + "name": "generation", + "output": "The file contains a short greeting from the trace-golden smoke scenario.", + "parentId": "s0", + "rank": 2, + "status": "ok" + } + ], + "summary": { + "narrative": "Completed in . Made 2 LLM calls using 1 tools (read). Total cost: .", + "status": "completed", + "topTools": [ + { + "count": 1, + "name": "read" + } + ], + "totalGenerations": 2, + "totalToolCalls": 1 + }, + "version": 2 +} diff --git a/packages/opencode/test/altimate/trace-golden/scenarios/smoke/model-script.json b/packages/opencode/test/altimate/trace-golden/scenarios/smoke/model-script.json new file mode 100644 index 000000000..9f281e01d --- /dev/null +++ b/packages/opencode/test/altimate/trace-golden/scenarios/smoke/model-script.json @@ -0,0 +1,11 @@ +[ + { + "kind": "tool", + "toolName": "read", + "toolInput": { "filePath": "/smoke.txt" } + }, + { + "kind": "text", + "text": "The file contains a short greeting from the trace-golden smoke scenario." + } +] diff --git a/packages/opencode/test/altimate/trace-golden/scenarios/smoke/prompt.json b/packages/opencode/test/altimate/trace-golden/scenarios/smoke/prompt.json new file mode 100644 index 000000000..192226ab9 --- /dev/null +++ b/packages/opencode/test/altimate/trace-golden/scenarios/smoke/prompt.json @@ -0,0 +1,3 @@ +{ + "prompt": "Read smoke.txt in the working directory and summarize its contents in one sentence." +} diff --git a/packages/opencode/test/altimate/trace-golden/scenarios/smoke/setup.json b/packages/opencode/test/altimate/trace-golden/scenarios/smoke/setup.json new file mode 100644 index 000000000..be17d52a0 --- /dev/null +++ b/packages/opencode/test/altimate/trace-golden/scenarios/smoke/setup.json @@ -0,0 +1,5 @@ +{ + "files": { + "smoke.txt": "hello from the trace-golden smoke scenario\n" + } +} diff --git a/packages/opencode/test/altimate/trace-golden/stability-check.ts b/packages/opencode/test/altimate/trace-golden/stability-check.ts new file mode 100644 index 000000000..6d9f9d8aa --- /dev/null +++ b/packages/opencode/test/altimate/trace-golden/stability-check.ts @@ -0,0 +1,265 @@ +// stability-check.ts — proves the trace-golden normalizer produces a +// byte-identical hash across many INDEPENDENT real subprocess-driven runs of +// the same scenario. This is the corrective-redirect requirement (#4) that +// was still open after the partial-order matcher and allowlist-projection +// normalizer rework: "run each scenario 50-100x to prove the normalized hash +// is stable under load before accepting any golden." +// +// Deliberately a STANDALONE SCRIPT, not a bun:test file — it spawns 50-100 +// real CLI subprocesses (several minutes of wall-clock time) and must never +// run automatically as part of `bun test` / CI. Invoke explicitly: +// +// bun run packages/opencode/test/altimate/trace-golden/stability-check.ts [runs] [concurrency] +// TRACE_GOLDEN_STABILITY_RUNS=100 bun run .../stability-check.ts +// +// Each run provisions its OWN independent CliFixture (own random-suffixed tmp +// home dir, own TestLLMServer instance, own `opencode run` subprocess) rather +// than reusing one fixture across N drives — that independence is the point. +// DRIVER-NOTES.md documents a real bug (Bug A: path-scrubbing order +// dependency) that only manifested because `fixture.home` gets a random +// per-run suffix; reusing a single fixture would silently stop exercising +// exactly the class of nondeterminism this check exists to catch. +import fsSync from "node:fs" +import fs from "node:fs/promises" +import path from "node:path" +import { Effect } from "effect" +import { withCliFixture } from "../../lib/cli-process" +import { driveScenario, type ScriptedTurn } from "./driver" +import { formatDiffs, match } from "./match" +import { normalize, stableStringify, type NormalizedTrace } from "./normalize" + +const SCENARIOS_DIR = path.join(import.meta.dir, "scenarios") + +/** + * Discovers scenario directories under `scenarios/` instead of hard-coding + * `smoke`, so a newly added S5/S7 scenario is automatically exercised by the + * 50-100x stability check the moment its directory exists — closing the + * other half of codex-tracegolden-code-review.md finding #1 ("both the test + * and stability runner hard-code smoke rather than discovering scenario + * directories"). Kept as a standalone copy (not imported from + * trace-golden.test.ts) because this file is a standalone script, not part + * of the bun:test module graph. + * + * Skips directories missing one of the three REQUIRED driver inputs — see + * the matching filter in trace-golden.test.ts's discoverScenarios for why + * (an incomplete scaffold directory must not break every other scenario's + * stability run). + */ +function discoverScenarios(dir: string): string[] { + const REQUIRED_FILES = ["prompt.json", "setup.json", "model-script.json"] + return fsSync + .readdirSync(dir, { withFileTypes: true }) + .filter((e) => e.isDirectory()) + .filter((e) => REQUIRED_FILES.every((f) => fsSync.existsSync(path.join(dir, e.name, f)))) + .map((e) => e.name) + .sort() +} + +const RUNS = Number(process.argv[2] ?? process.env.TRACE_GOLDEN_STABILITY_RUNS ?? 60) +const CONCURRENCY = Number(process.argv[3] ?? process.env.TRACE_GOLDEN_STABILITY_CONCURRENCY ?? 4) + +function resolvePlaceholders(value: T, home: string): T { + if (typeof value === "string") return value.replaceAll("", home) as unknown as T + if (Array.isArray(value)) return value.map((v) => resolvePlaceholders(v, home)) as unknown as T + if (value && typeof value === "object") { + const out: Record = {} + for (const [k, v] of Object.entries(value as Record)) out[k] = resolvePlaceholders(v, home) + return out as T + } + return value +} + +type RunOutcome = + | { readonly index: number; readonly ok: true; readonly hash: string; readonly durationMs: number } + | { readonly index: number; readonly ok: false; readonly error: string; readonly durationMs: number } + +async function runOnce(scenarioDir: string, index: number): Promise { + const start = Date.now() + try { + const hash = await Effect.runPromise( + Effect.scoped( + withCliFixture((fixture) => + Effect.gen(function* () { + const [promptRaw, setupRaw, scriptRaw] = yield* Effect.all([ + Effect.tryPromise(() => fs.readFile(path.join(scenarioDir, "prompt.json"), "utf-8")), + Effect.tryPromise(() => fs.readFile(path.join(scenarioDir, "setup.json"), "utf-8")), + Effect.tryPromise(() => fs.readFile(path.join(scenarioDir, "model-script.json"), "utf-8")), + ]) + const { prompt } = JSON.parse(promptRaw) as { prompt: string } + const setup = JSON.parse(setupRaw) as { files?: Record } + const script = resolvePlaceholders(JSON.parse(scriptRaw) as ScriptedTurn[], fixture.home) + + for (const [name, content] of Object.entries(setup.files ?? {})) { + yield* Effect.tryPromise(() => fs.writeFile(path.join(fixture.home, name), content, "utf-8")) + } + + const result = yield* driveScenario(fixture, { + prompt, + script, + runOpts: { extraArgs: ["--dangerously-skip-permissions"] }, + }) + + if (result.exitCode !== 0) { + return yield* Effect.fail( + new Error( + `run ${index}: opencode run exited ${result.exitCode}\nstderr (last 1500):\n${result.stderr.slice(-1500)}`, + ), + ) + } + + const actual = normalize(result.trace, { homeRoots: [fixture.home] }) + return stableStringify(actual) + }), + ), + ), + ) + return { index, ok: true, hash, durationMs: Date.now() - start } + } catch (cause) { + return { index, ok: false, error: String(cause), durationMs: Date.now() - start } + } +} + +async function runPool(n: number, concurrency: number, fn: (i: number) => Promise): Promise { + const results: T[] = new Array(n) + let next = 0 + async function worker() { + for (;;) { + const i = next++ + if (i >= n) return + results[i] = await fn(i) + // eslint-disable-next-line no-console + console.log(`[stability-check] run ${i + 1}/${n} done`) + } + } + await Promise.all(Array.from({ length: Math.min(concurrency, n) }, () => worker())) + return results +} + +/** + * Drives a single scenario RUNS times and reports whether its normalized + * hash is stable and matches the committed golden. Returns `true` iff this + * scenario is fully stable, so `main()` can aggregate pass/fail across every + * discovered scenario rather than only ever checking `smoke`. + */ +async function checkScenario(name: string, dir: string): Promise { + // eslint-disable-next-line no-console + console.log(`[stability-check] driving '${name}' scenario ${RUNS}x real subprocess runs (concurrency ${CONCURRENCY})`) + const overallStart = Date.now() + + const goldenPath = path.join(dir, "golden.json") + const golden = JSON.parse(await fs.readFile(goldenPath, "utf-8")) as NormalizedTrace + const goldenHash = stableStringify(golden) + + const outcomes = await runPool(RUNS, CONCURRENCY, (i) => runOnce(dir, i)) + + const failures = outcomes.filter((o): o is Extract => !o.ok) + const successes = outcomes.filter((o): o is Extract => o.ok) + const uniqueHashes = new Set(successes.map((o) => o.hash)) + const totalMs = Date.now() - overallStart + const avgMs = outcomes.reduce((sum, o) => sum + o.durationMs, 0) / outcomes.length + + // eslint-disable-next-line no-console + console.log("") + // eslint-disable-next-line no-console + console.log(`=== stability-check summary: ${name} ===`) + // eslint-disable-next-line no-console + console.log(`runs requested: ${RUNS}`) + // eslint-disable-next-line no-console + console.log(`runs succeeded: ${successes.length}`) + // eslint-disable-next-line no-console + console.log(`runs failed: ${failures.length}`) + // eslint-disable-next-line no-console + console.log(`unique hashes: ${uniqueHashes.size} (1 = fully stable)`) + // eslint-disable-next-line no-console + console.log(`matches golden: ${uniqueHashes.size === 1 && uniqueHashes.has(goldenHash)}`) + // eslint-disable-next-line no-console + console.log(`total wall time: ${(totalMs / 1000).toFixed(1)}s`) + // eslint-disable-next-line no-console + console.log(`avg run time: ${avgMs.toFixed(0)}ms`) + + if (failures.length > 0) { + // eslint-disable-next-line no-console + console.log("") + // eslint-disable-next-line no-console + console.log("--- failures ---") + for (const f of failures.slice(0, 5)) { + // eslint-disable-next-line no-console + console.log(`run ${f.index}: ${f.error.slice(0, 500)}`) + } + } + + if (uniqueHashes.size > 1) { + // eslint-disable-next-line no-console + console.log("") + // eslint-disable-next-line no-console + console.log("--- INSTABILITY DETECTED: diff between first two distinct normalized outputs ---") + const [firstHash, secondHash] = [...uniqueHashes] + const a = successes.find((o) => o.hash === firstHash)! + const b = successes.find((o) => o.hash === secondHash)! + const diffs = match(JSON.parse(a.hash) as NormalizedTrace, JSON.parse(b.hash) as NormalizedTrace).diffs + // eslint-disable-next-line no-console + console.log(formatDiffs(diffs)) + } else if (successes.length > 0 && !uniqueHashes.has(goldenHash)) { + // eslint-disable-next-line no-console + console.log("") + // eslint-disable-next-line no-console + console.log("--- runs are mutually stable but diverge from the COMMITTED golden ---") + const actual = JSON.parse(successes[0].hash) as NormalizedTrace + const diffs = match(golden, actual).diffs + // eslint-disable-next-line no-console + console.log(formatDiffs(diffs)) + } + + const stable = failures.length === 0 && uniqueHashes.size === 1 && uniqueHashes.has(goldenHash) + // eslint-disable-next-line no-console + console.log("") + // eslint-disable-next-line no-console + console.log( + stable + ? `[stability-check] PASS: '${name}' normalized hash is stable and matches golden.` + : `[stability-check] FAIL: '${name}' — see above.`, + ) + return stable +} + +async function main() { + if (!Number.isFinite(RUNS) || RUNS < 1) throw new Error(`stability-check: invalid RUNS=${RUNS}`) + + const scenarioNames = discoverScenarios(SCENARIOS_DIR) + if (scenarioNames.length === 0) throw new Error(`stability-check: no scenario directories found under ${SCENARIOS_DIR}`) + + // eslint-disable-next-line no-console + console.log(`[stability-check] discovered ${scenarioNames.length} scenario(s): ${scenarioNames.join(", ")}`) + + const results: Array<{ name: string; stable: boolean }> = [] + for (const name of scenarioNames) { + // eslint-disable-next-line no-console + console.log("") + const stable = await checkScenario(name, path.join(SCENARIOS_DIR, name)) + results.push({ name, stable }) + } + + // eslint-disable-next-line no-console + console.log("") + // eslint-disable-next-line no-console + console.log("=== overall stability-check summary ===") + for (const r of results) { + // eslint-disable-next-line no-console + console.log(` ${r.stable ? "PASS" : "FAIL"} ${r.name}`) + } + + const allStable = results.every((r) => r.stable) + // eslint-disable-next-line no-console + console.log( + allStable + ? "[stability-check] PASS: all scenarios are stable and match their goldens." + : "[stability-check] FAIL: at least one scenario is unstable or diverges from its golden — see above.", + ) + process.exit(allStable ? 0 : 1) +} + +main().catch((err) => { + // eslint-disable-next-line no-console + console.error("[stability-check] fatal error:", err) + process.exit(1) +}) diff --git a/packages/opencode/test/altimate/trace-golden/trace-golden.test.ts b/packages/opencode/test/altimate/trace-golden/trace-golden.test.ts new file mode 100644 index 000000000..519535c50 --- /dev/null +++ b/packages/opencode/test/altimate/trace-golden/trace-golden.test.ts @@ -0,0 +1,1090 @@ +// trace-golden.test.ts — proves the technique described in +// docs/internal/2026-07-18-trace-golden-e2e-technique.md end-to-end: +// 1. normalize() is idempotent and stable on real recorded traces. +// 2. A normalized real trace round-trips through match() as its own golden. +// 3. match()/formatDiffs() actually DETECT a planted behavioral change and +// produce a readable diff — this is the load-bearing guarantee: a golden +// that can't catch a real regression is worthless. (Most important test +// in this file.) +// 4. The full pipeline — real `opencode run` subprocess, scripted +// TestLLMServer, real Recap tracer, real trace file on disk — is driven +// end-to-end via driver.ts's driveScenario() against the `smoke` +// scenario, and the result matches a committed golden. +// +// Run `TRACE_GOLDEN_UPDATE=1 bun test packages/opencode/test/altimate/trace-golden/` +// to (re)generate scenario goldens after an intentional behavior change — +// review the diff in the PR body, same convention as UI snapshot updates. +import { describe, expect, test } from "bun:test" +import fsSync from "node:fs" +import fs from "node:fs/promises" +import os from "node:os" +import path from "node:path" +import { Effect } from "effect" +import type { TraceFile, TraceSpan } from "@/altimate/observability/tracing" +import { cliIt } from "../../lib/cli-process" +import { driveScenario, type ScriptedTurn } from "./driver" +import { formatDiffs, match } from "./match" +import { normalize, stableStringify, type NormalizedTrace } from "./normalize" + +const FIXTURES_DIR = path.join(import.meta.dir, "fixtures/real-traces") +const SCENARIOS_DIR = path.join(import.meta.dir, "scenarios") +const TRACE_GOLDEN_UPDATE = process.env.TRACE_GOLDEN_UPDATE === "1" + +async function loadRealTraces(): Promise> { + const files = (await fs.readdir(FIXTURES_DIR)).filter((f) => f.endsWith(".json")).sort() + return Promise.all( + files.map(async (file) => ({ + file, + trace: JSON.parse(await fs.readFile(path.join(FIXTURES_DIR, file), "utf-8")) as TraceFile, + })), + ) +} + +async function fileExists(file: string): Promise { + return fs + .access(file) + .then(() => true) + .catch(() => false) +} + +/** Recursively replaces the `` placeholder token in scenario fixtures with the CliFixture's actual home dir. */ +function resolvePlaceholders(value: T, home: string): T { + if (typeof value === "string") return value.replaceAll("", home) as unknown as T + if (Array.isArray(value)) return value.map((v) => resolvePlaceholders(v, home)) as unknown as T + if (value && typeof value === "object") { + const out: Record = {} + for (const [k, v] of Object.entries(value as Record)) out[k] = resolvePlaceholders(v, home) + return out as T + } + return value +} + +const readJson = (file: string): Effect.Effect => + Effect.tryPromise({ + try: () => fs.readFile(file, "utf-8").then((raw) => JSON.parse(raw) as T), + catch: (cause) => new Error(`trace-golden: failed to read ${file}: ${String(cause)}`), + }) + +/** Minimal, schema-valid synthetic TraceFile wrapping the given spans — for tests that need exact, hand-built topology rather than a real recording. */ +function buildSyntheticTrace(spans: TraceSpan[]): TraceFile { + return { + version: 2, + traceId: "trace_synthetic", + sessionId: "ses_synthetic", + startedAt: "2026-01-01T00:00:00.000Z", + endedAt: "2026-01-01T00:00:01.000Z", + metadata: {}, + spans, + summary: { + totalTokens: 0, + totalCost: 0, + totalToolCalls: spans.filter((s) => s.kind === "tool").length, + totalGenerations: spans.filter((s) => s.kind === "generation").length, + duration: 1000, + status: "completed", + tokens: { input: 0, output: 0, reasoning: 0, cacheRead: 0, cacheWrite: 0 }, + }, + } +} + +/** + * Discovers scenario directories under `scenarios/` (each with its own + * `prompt.json`/`setup.json`/`model-script.json`/`golden.json`) so new S5/S7 + * scenarios are picked up automatically instead of requiring a hand-edit of + * this file every time one is added — fixing the "only `smoke` is wired" + * half of codex-tracegolden-code-review.md finding #1. + * + * Skips directories missing one of the three REQUIRED driver inputs + * (prompt.json/setup.json/model-script.json — golden.json is intentionally + * excluded here since fix #2 handles a missing golden as its own, explicit + * failure mode, not as "not a scenario"). Without this filter, an in-progress + * or abandoned scenario scaffold directory (created but never populated) + * silently breaks every OTHER discovered scenario's test run too, since + * discovery is a single flat list consumed by a `for` loop that registers + * one test per name — exactly the kind of coverage gap finding #1 exists to + * close, just one level down. + */ +function discoverScenarios(dir: string): string[] { + const REQUIRED_FILES = ["prompt.json", "setup.json", "model-script.json"] + return fsSync + .readdirSync(dir, { withFileTypes: true }) + .filter((e) => e.isDirectory()) + .filter((e) => REQUIRED_FILES.every((f) => fsSync.existsSync(path.join(dir, e.name, f)))) + .map((e) => e.name) + .sort() +} + +type GoldenAction = + | { readonly kind: "write" } + | { readonly kind: "compare" } + | { readonly kind: "fail"; readonly reason: string } + +/** + * Pure decision function for what to do with a scenario's golden — kept + * separate from the Effect-based live test body so all four branches are + * directly unit-testable without spinning up a real subprocess. + * + * Fixes codex-tracegolden-code-review.md finding #2: a missing golden used + * to self-approve (write-then-pass) even in a plain, non-update run, which + * silently turns "compare actual to golden" into "compare actual to itself" + * the moment a golden.json is deleted from a writable CI checkout. A missing + * golden must FAIL unless the run explicitly opts into regenerating it via + * `TRACE_GOLDEN_UPDATE=1` — and that opt-in is itself rejected under CI, so + * a CI run can never silently rewrite (and thus self-approve) a golden. + */ +function resolveGoldenAction(opts: { + goldenExists: boolean + updateRequested: boolean + isCI: boolean + goldenPath: string +}): GoldenAction { + if (opts.updateRequested) { + if (opts.isCI) { + return { + kind: "fail", + reason: + `trace-golden: TRACE_GOLDEN_UPDATE=1 is not allowed under CI (would silently rewrite ${opts.goldenPath}). ` + + `Regenerate goldens locally and commit the reviewed diff.`, + } + } + return { kind: "write" } + } + if (!opts.goldenExists) { + return { + kind: "fail", + reason: + `trace-golden: golden missing at ${opts.goldenPath}. A missing golden must FAIL, not self-approve — ` + + `run with TRACE_GOLDEN_UPDATE=1 locally to (re)generate it, then review and commit the diff.`, + } + } + return { kind: "compare" } +} + +describe("discoverScenarios(): scenario-directory discovery (fix #1)", () => { + function makeCompleteScenario(dir: string) { + fsSync.mkdirSync(dir) + fsSync.writeFileSync(path.join(dir, "prompt.json"), "{}") + fsSync.writeFileSync(path.join(dir, "setup.json"), "{}") + fsSync.writeFileSync(path.join(dir, "model-script.json"), "[]") + } + + test("finds all COMPLETE scenario subdirectories under a temp dir, sorted, excluding files", () => { + const tmp = fsSync.mkdtempSync(path.join(os.tmpdir(), "trace-golden-discover-")) + try { + makeCompleteScenario(path.join(tmp, "zeta")) + makeCompleteScenario(path.join(tmp, "alpha")) + makeCompleteScenario(path.join(tmp, "mid")) + fsSync.writeFileSync(path.join(tmp, "not-a-scenario.json"), "{}") + + expect(discoverScenarios(tmp)).toEqual(["alpha", "mid", "zeta"]) + } finally { + fsSync.rmSync(tmp, { recursive: true, force: true }) + } + }) + + test("excludes an INCOMPLETE scenario directory (missing driver inputs) instead of breaking discovery for its siblings", () => { + // Regression test for a real failure this exact filter caught: an + // abandoned/in-progress scenario scaffold directory (created, never + // populated with prompt.json/setup.json/model-script.json) existed + // alongside `smoke` in the real SCENARIOS_DIR and, before this filter + // existed, caused driver.ts's live test to crash with ENOENT reading its + // missing prompt.json — taking down the whole discovered-scenario test + // loop, not just that one entry. + const tmp = fsSync.mkdtempSync(path.join(os.tmpdir(), "trace-golden-discover-incomplete-")) + try { + makeCompleteScenario(path.join(tmp, "complete-one")) + fsSync.mkdirSync(path.join(tmp, "incomplete-scaffold")) // no files inside — simulates an abandoned scenario dir + + expect(discoverScenarios(tmp)).toEqual(["complete-one"]) + } finally { + fsSync.rmSync(tmp, { recursive: true, force: true }) + } + }) + + test("the real SCENARIOS_DIR contains at least the smoke scenario", () => { + const found = discoverScenarios(SCENARIOS_DIR) + expect(found).toContain("smoke") + expect(found.length).toBeGreaterThanOrEqual(1) + }) +}) + +describe("resolveGoldenAction(): golden write/compare/fail decision (fix #2)", () => { + test("golden exists, no update requested → compare", () => { + const action = resolveGoldenAction({ goldenExists: true, updateRequested: false, isCI: false, goldenPath: "g.json" }) + expect(action.kind).toBe("compare") + }) + + test("golden missing, no update requested → fail (this is the fix: no self-approve)", () => { + const action = resolveGoldenAction({ goldenExists: false, updateRequested: false, isCI: false, goldenPath: "g.json" }) + expect(action.kind).toBe("fail") + if (action.kind === "fail") expect(action.reason).toContain("golden missing") + }) + + test("golden missing, update requested, not CI → write", () => { + const action = resolveGoldenAction({ goldenExists: false, updateRequested: true, isCI: false, goldenPath: "g.json" }) + expect(action.kind).toBe("write") + }) + + test("golden exists, update requested, not CI → write (explicit regeneration)", () => { + const action = resolveGoldenAction({ goldenExists: true, updateRequested: true, isCI: false, goldenPath: "g.json" }) + expect(action.kind).toBe("write") + }) + + test("update requested under CI → fail regardless of golden existence", () => { + const withGolden = resolveGoldenAction({ goldenExists: true, updateRequested: true, isCI: true, goldenPath: "g.json" }) + const withoutGolden = resolveGoldenAction({ goldenExists: false, updateRequested: true, isCI: true, goldenPath: "g.json" }) + expect(withGolden.kind).toBe("fail") + expect(withoutGolden.kind).toBe("fail") + if (withGolden.kind === "fail") expect(withGolden.reason).toContain("not allowed under CI") + }) +}) + +describe("normalize(): idempotency on real traces", () => { + test("fixtures directory has real trace files to test against", async () => { + const traces = await loadRealTraces() + expect(traces.length).toBeGreaterThanOrEqual(5) + }) + + test("normalizing the same real trace twice yields byte-identical output", async () => { + const traces = await loadRealTraces() + expect(traces.length).toBeGreaterThan(0) + for (const { file, trace } of traces) { + const first = stableStringify(normalize(trace)) + const second = stableStringify(normalize(trace)) + expect(second).toBe(first) + // Also confirm re-parsing round-trips (normalize output is valid, stable JSON). + expect(() => JSON.parse(first)).not.toThrow() + void file + } + }) +}) + +describe("normalize() + match(): golden round-trip on a real trace", () => { + test("a real trace normalized twice matches itself via match()", async () => { + const traces = await loadRealTraces() + const { trace } = traces[traces.length - 1] // pick the largest fixture (most spans, most coverage) + const golden = normalize(trace) + const actual = normalize(trace) + const result = match(golden, actual) + expect(result.pass).toBe(true) + expect(result.diffs).toEqual([]) + }) +}) + +describe("match(): detects a planted behavioral regression (most important test in this file)", () => { + test("flips a span status and injects an extra span — match() must catch both", async () => { + const traces = await loadRealTraces() + const withMultipleSpans = traces.find(({ trace }) => trace.spans.length >= 3) + if (!withMultipleSpans) throw new Error("expected at least one fixture with >= 3 spans") + + const golden = normalize(withMultipleSpans.trace) + // Deep clone so mutating "actual" cannot affect "golden". + const actual: NormalizedTrace = JSON.parse(JSON.stringify(golden)) + + // Plant 1: flip an existing span's status — simulates a route that silently + // started failing (or a deny that silently started succeeding). + const targetIndex = Math.min(1, actual.spans.length - 1) + const originalStatus = actual.spans[targetIndex].status + actual.spans[targetIndex].status = originalStatus === "ok" ? "error" : "ok" + + // Plant 2: inject a phantom span — simulates an extra, unexpected tool + // execution (e.g. a HardPolicy gap letting an execute span through after a deny). + // rank is a value no existing sibling under this parent uses, so match()'s + // rank-aware grouping treats it as a singleton (real, reproducible order) + // rather than folding it into a concurrent-group multiset diff — this test + // asserts the precise 1:1 "spans[s999]" path, not a "concurrent-rank" path. + actual.spans.push({ + id: "s999", + parentId: actual.spans[0].id, + name: "phantom-tool", + kind: "tool", + status: "ok", + rank: 999999, + }) + + const result = match(golden, actual) + + expect(result.pass).toBe(false) + expect(result.diffs.length).toBeGreaterThanOrEqual(2) + + const statusDiff = result.diffs.find((d) => d.path === `spans[${golden.spans[targetIndex].id}].status`) + expect(statusDiff).toBeDefined() + expect(statusDiff?.kind).toBe("changed") + expect(statusDiff?.expected).toBe(originalStatus) + + const addedDiff = result.diffs.find((d) => d.kind === "added" && d.path === "spans[s999]") + expect(addedDiff).toBeDefined() + + // The formatted report must be readable, not a blob dump. + const report = result.format() + expect(report).toContain("trace-golden mismatch") + expect(report).toContain(`spans[${golden.spans[targetIndex].id}].status`) + expect(report).toContain("+ spans[s999]") + expect(report.length).toBeLessThan(2000) // minimal diff, not a 600KB blob + + // formatDiffs() is the same function match().format() delegates to — confirm parity. + expect(formatDiffs(result.diffs)).toBe(report) + }) +}) + +describe("computeRanks/buildDfsOrdinals: concurrent-sibling ordinal stability (fix #1)", () => { + // Two sibling `kind: "tool"` spans under one `kind: "generation"` parent, + // dispatched the way Batch (`Promise.all`, batch.ts:158) and the parallel + // session/prompt.ts resolver actually dispatch tool calls: SAME name (so + // an old (kind, name)-only tiebreak ties and falls through to array + // position) but DIFFERENT input (so the two spans are genuinely distinct + // logical tool calls, not indistinguishable duplicates), with identical + // start/end times (so even the old startTime/endTime-overlap rank scheme + // would have clustered them into the same "concurrent" group). The only + // thing that differs between the two traces below is which of the two + // spans appears FIRST in the raw `spans` array — a difference that is + // exactly what a real race between two `Promise.all`-dispatched tool + // calls produces on two runs of the identical scenario. + // + // This test must FAIL if buildDfsOrdinals is reverted to numbering + // same-rank siblings by raw array/original-index order instead of by + // toolContentKey: with array-order numbering, the span appearing first in + // the array always gets the lower ordinal, so the two traces below would + // normalize to different span identities/order (`s1` denoting `a.txt` in + // one and `b.txt` in the other) and this test's deep-equal assertion + // would fail. + const parent: TraceSpan = { + spanId: "gen1", + parentSpanId: null, + name: "generation-abc123", + kind: "generation", + startTime: 1000, + endTime: 3000, + status: "ok", + } + const toolA: TraceSpan = { + spanId: "toolA", + parentSpanId: "gen1", + name: "read_file", + kind: "tool", + startTime: 1000, + endTime: 2000, + status: "ok", + input: { path: "/repo/a.txt" }, + } + const toolB: TraceSpan = { + spanId: "toolB", + parentSpanId: "gen1", + name: "read_file", + kind: "tool", + startTime: 1000, + endTime: 2000, + status: "ok", + input: { path: "/repo/b.txt" }, + } + + test("normalize() produces byte-identical output regardless of which concurrent sibling was recorded first", () => { + const traceOrderAB = buildSyntheticTrace([parent, toolA, toolB]) + const traceOrderBA = buildSyntheticTrace([parent, toolB, toolA]) + + const normalizedAB = stableStringify(normalize(traceOrderAB)) + const normalizedBA = stableStringify(normalize(traceOrderBA)) + + expect(normalizedAB).toBe(normalizedBA) + }) + + test("match() reports zero diffs between the two array orderings", () => { + const traceOrderAB = buildSyntheticTrace([parent, toolA, toolB]) + const traceOrderBA = buildSyntheticTrace([parent, toolB, toolA]) + + const result = match(normalize(traceOrderAB), normalize(traceOrderBA)) + + expect(result.pass).toBe(true) + expect(result.diffs).toEqual([]) + }) +}) + +describe("computeRanks: structural rank is timing-independent (fix #1b — cross-order AND cross-timing)", () => { + // Closes a gap in the fix #1 block above: that test kept startTime/endTime + // IDENTICAL across both compared traces, so it could not distinguish a + // genuinely structural (kind-only) rank scheme from a wall-clock-overlap + // scheme that happens to see the same overlap in both variants. This test + // varies the WALL-CLOCK RELATIONSHIP itself: one variant's two tool + // siblings overlap in time (as a real Promise.all race often does), the + // other variant's two tool siblings are jittered so they do NOT overlap + // (one fully completes before the other starts) — and their array order is + // also swapped, so the ONLY invariant across both variants is "these are + // two `kind: tool` siblings under the same parent, with the same names and + // inputs." Under a wall-clock overlap-based rank scheme these two variants + // would land in different rank shapes (one clustered as concurrent, the + // other split into two distinct ordered ranks); under the current kind-only + // structural scheme (see normalize.ts's computeRanks) they must normalize + // identically regardless of timing. + const parent: TraceSpan = { + spanId: "gen1", + parentSpanId: null, + name: "generation-abc123", + kind: "generation", + startTime: 1000, + endTime: 5000, + status: "ok", + } + // Different name AND different input — distinguishable regardless of the + // toolContentKey tiebreak, so a mismatch can't hide behind ambiguous input. + const readTool: TraceSpan = { + spanId: "toolRead", + parentSpanId: "gen1", + name: "read_file", + kind: "tool", + startTime: 1000, + endTime: 2000, + status: "ok", + input: { path: "/repo/a.txt" }, + } + const writeTool: TraceSpan = { + spanId: "toolWrite", + parentSpanId: "gen1", + name: "write_file", + kind: "tool", + startTime: 1500, + endTime: 2500, + status: "ok", + input: { path: "/repo/b.txt" }, + } + + // Variant 1: array order [read, write]; times OVERLAP (1000-2000 vs 1500-2500). + const overlapping = buildSyntheticTrace([parent, readTool, writeTool]) + + // Variant 2: array order SWAPPED [write, read]; times jittered so they do + // NOT overlap — write fully completes (3000-3500) before read starts + // (4000-4500). Only each span's own kind/name/input survives from variant + // 1; every timing value and the array position differ. + const nonOverlappingSwapped = buildSyntheticTrace([ + parent, + { ...writeTool, startTime: 3000, endTime: 3500 }, + { ...readTool, startTime: 4000, endTime: 4500 }, + ]) + + test("normalize() is byte-identical whether the two tool siblings overlap in time or not (swapped order + non-overlapping times)", () => { + const normalizedOverlap = stableStringify(normalize(overlapping)) + const normalizedNonOverlap = stableStringify(normalize(nonOverlappingSwapped)) + + expect(normalizedOverlap).toBe(normalizedNonOverlap) + }) + + test("match() reports zero diffs between the overlapping and non-overlapping variants", () => { + const result = match(normalize(overlapping), normalize(nonOverlappingSwapped)) + + expect(result.pass).toBe(true) + expect(result.diffs).toEqual([]) + }) + + test("NEGATIVE control: changing one sibling's input produces exactly one add/remove pair scoped to the concurrent group, not spurious churn on the unchanged sibling", () => { + const mutated = buildSyntheticTrace([parent, readTool, { ...writeTool, input: { path: "/repo/CHANGED.txt" } }]) + + const result = match(normalize(overlapping), normalize(mutated)) + + expect(result.pass).toBe(false) + // Exactly one logical change manifests as one "removed" (old writeTool + // signature no longer present) + one "added" (new writeTool signature) — + // both scoped to the concurrent-rank group. If ordinal renumbering ever + // leaked timing or array-position into the rank or tiebreak, the + // unchanged readTool sibling would ALSO show up as a spurious diff, and + // this exact-count assertion (2, not 3+) would catch it. + expect(result.diffs.length).toBe(2) + + const removed = result.diffs.find((d) => d.kind === "removed") + const added = result.diffs.find((d) => d.kind === "added") + expect(removed).toBeDefined() + expect(added).toBeDefined() + expect(removed?.path).toContain("concurrent-rank") + expect(added?.path).toContain("concurrent-rank") + + // The unchanged readTool sibling must not appear in either diff's payload. + for (const d of result.diffs) { + const payload = JSON.stringify(d.expected ?? d.actual ?? {}) + expect(payload).not.toContain("/repo/a.txt") + } + }) +}) + +describe("toolContentKey: output-based tiebreak for identical-input concurrent siblings (fix #3)", () => { + // Codex review finding #3: toolContentKey only used kind/name/input, so two + // concurrent tool siblings with the SAME name/input but DIFFERENT output — + // a real, distinguishable pair (e.g. two identical reads against a file + // that changed between calls) — fell through to the raw-array-position + // (originalIndex) tiebreak. Recorded in opposite completion order across + // two runs of the identical scenario, that produced spurious add/remove + // diffs even though nothing behaviorally different happened. Adding output + // to toolContentKey (normalize.ts) fixes this: these tests must FAIL if + // output is ever dropped back out of toolContentKey. + const parent: TraceSpan = { + spanId: "gen3", + parentSpanId: null, + name: "generation-samekey", + kind: "generation", + startTime: 1000, + endTime: 3000, + status: "ok", + } + const toolSameKeyA: TraceSpan = { + spanId: "sameA", + parentSpanId: "gen3", + name: "read_file", + kind: "tool", + startTime: 1000, + endTime: 2000, + status: "ok", + input: { path: "/repo/x.txt" }, + output: { content: "AAA" }, + } + const toolSameKeyB: TraceSpan = { + spanId: "sameB", + parentSpanId: "gen3", + name: "read_file", + kind: "tool", + startTime: 1000, + endTime: 2000, + status: "ok", + input: { path: "/repo/x.txt" }, + output: { content: "BBB" }, + } + + test("normalize() is byte-identical regardless of which same-name/same-input/different-output sibling completed first", () => { + const traceOrderAB = buildSyntheticTrace([parent, toolSameKeyA, toolSameKeyB]) + const traceOrderBA = buildSyntheticTrace([parent, toolSameKeyB, toolSameKeyA]) + + expect(stableStringify(normalize(traceOrderAB))).toBe(stableStringify(normalize(traceOrderBA))) + }) + + test("match() reports zero diffs between the two completion orderings", () => { + const traceOrderAB = buildSyntheticTrace([parent, toolSameKeyA, toolSameKeyB]) + const traceOrderBA = buildSyntheticTrace([parent, toolSameKeyB, toolSameKeyA]) + + const result = match(normalize(traceOrderAB), normalize(traceOrderBA)) + + expect(result.pass).toBe(true) + expect(result.diffs).toEqual([]) + }) +}) + +describe("computeRanks: root-attached tool preserves real sibling order (fix #4a)", () => { + // Codex review finding #4 (half A): every kind:"tool" sibling used to get + // rank -1 (the concurrent bucket) regardless of its parent's kind. But + // logToolCall only races when a generation is active — with none active it + // attaches the tool directly to the session root (tracing.ts:928), right + // alongside genuinely ORDERED siblings like user-message/generation. Under + // the old rule, reordering such a root-attached tool around its ordered + // siblings normalized byte-identically and matchPass: true, silently + // erasing a real, reproducible order. The fix scopes the concurrent bucket + // to tools whose parent is itself a generation span; every other tool + // parent (including the session root) gets a real, increasing rank. These + // tests must FAIL if that scoping is reverted. + const root: TraceSpan = { + spanId: "root1", + parentSpanId: null, + name: "ses_synthetic", + kind: "session", + startTime: 0, + endTime: 10000, + status: "ok", + } + const userMsg: TraceSpan = { + spanId: "um1", + parentSpanId: "root1", + name: "user-message", + kind: "user-message", + startTime: 100, + endTime: 100, + status: "ok", + } + const rootTool: TraceSpan = { + spanId: "rt1", + parentSpanId: "root1", + name: "bash", + kind: "tool", + startTime: 200, + endTime: 300, + status: "ok", + input: { cmd: "ls" }, + } + const gen: TraceSpan = { + spanId: "gen1", + parentSpanId: "root1", + name: "generation-root-attached", + kind: "generation", + startTime: 400, + endTime: 500, + status: "ok", + } + + // Real chronological order: user-message, then the root-attached tool, then the generation. + const realOrder = buildSyntheticTrace([root, userMsg, rootTool, gen]) + // Reordered: the root-attached tool now precedes the user-message — a genuine topology change, + // not a race outcome (its parent, root1, is kind "session", not "generation"). + const reordered = buildSyntheticTrace([root, rootTool, userMsg, gen]) + + test("normalize() produces DIFFERENT output when a root-attached tool is reordered around ordered siblings", () => { + expect(stableStringify(normalize(realOrder))).not.toBe(stableStringify(normalize(reordered))) + }) + + test("match() detects the reordering as a real diff, not a false pass", () => { + const result = match(normalize(realOrder), normalize(reordered)) + + expect(result.pass).toBe(false) + expect(result.diffs.length).toBeGreaterThan(0) + }) +}) + +describe("canonicalSignature: ordered children within a concurrent parent are not erased by sorting (fix #4b)", () => { + // Codex review finding #4 (half B): canonicalChildSignatures used to sort + // ALL descendant signatures indiscriminately, discarding the real, + // reproducible order of a concurrent parent's OWN ordered children. Two + // accepted raw traces where an ordered child sequence's order changed + // (without changing the multiset of signatures) normalized to different + // bytes in normalize.ts, yet the old, fully-sorted canonicalSignature still + // signed them identically, so match() returned pass: true on a real + // topology change. The fix groups children by their own rank and only + // sorts WITHIN a tied-rank group. concA below is byte-identical on both + // sides except for which of its two ordered children came first; concB is + // an untouched peer included only to force the outer group into the + // concurrent (multiset) comparison path (diffConcurrentGroup requires size + // > 1 on at least one side). This test must FAIL (report pass: true) if + // canonicalChildSignatures is reverted to an indiscriminate full sort. + const genX: TraceSpan = { + spanId: "genX", + parentSpanId: null, + name: "generation-nested", + kind: "generation", + startTime: 0, + endTime: 10000, + status: "ok", + } + // concA/concB: two concurrent tool siblings under genX (both kind "tool", parent is a + // generation) — same rank bucket, forming the outer concurrent group. + const concA: TraceSpan = { + spanId: "concA", + parentSpanId: "genX", + name: "batch", + kind: "tool", + startTime: 100, + endTime: 200, + status: "ok", + input: { branch: "A" }, + } + const concB: TraceSpan = { + spanId: "concB", + parentSpanId: "genX", + name: "batch", + kind: "tool", + startTime: 100, + endTime: 200, + status: "ok", + input: { branch: "B" }, + } + // concA's own children — concA's kind is "tool", not "generation", so these get real, + // increasing ranks in their array/original-index order, not the concurrent bucket. + const gchild1: TraceSpan = { + spanId: "gchild1", + parentSpanId: "concA", + name: "sub_one", + kind: "tool", + startTime: 110, + endTime: 120, + status: "ok", + input: { tag: "one" }, + } + const gchild2: TraceSpan = { + spanId: "gchild2", + parentSpanId: "concA", + name: "sub_two", + kind: "tool", + startTime: 130, + endTime: 140, + status: "ok", + input: { tag: "two" }, + } + + test("swapping which of concA's two ordered children came first is detected as a real diff, not a false pass", () => { + const golden = normalize(buildSyntheticTrace([genX, concA, concB, gchild1, gchild2])) + const actual = normalize(buildSyntheticTrace([genX, concA, concB, gchild2, gchild1])) + + const result = match(golden, actual) + + // Real observed shape: reordering concA's children changes concA's own + // canonical signature, so the outer concurrent-group comparison reports + // concA as removed+added as a whole subtree (path carries + // "concurrent-rank"); the positional per-span walk ALSO independently + // reports the two children as "changed" at their now-mismatched DFS + // ordinals (s2/s3). Both signals firing together is stronger evidence the + // fix works than either alone — the exact count is an implementation + // detail, not the load-bearing assertion. + expect(result.pass).toBe(false) + expect(result.diffs.length).toBeGreaterThan(0) + expect(result.diffs.some((d) => d.path.includes("concurrent-rank"))).toBe(true) + expect(result.diffs.some((d) => d.kind === "removed")).toBe(true) + expect(result.diffs.some((d) => d.kind === "added")).toBe(true) + }) + + test("NEGATIVE control: unchanged child order matches with zero diffs (proves this isn't just always failing on nested-concurrent topology)", () => { + const golden = normalize(buildSyntheticTrace([genX, concA, concB, gchild1, gchild2])) + const actual = normalize(buildSyntheticTrace([genX, concA, concB, gchild1, gchild2])) + + const result = match(golden, actual) + + expect(result.pass).toBe(true) + expect(result.diffs).toEqual([]) + }) +}) + +describe("computeToolCallIdLabels: raw callId equality/uniqueness pattern is preserved (fix #5)", () => { + // Codex review finding #5: toolCallId used to be rewritten to the span's + // OWN ordinal, independent of the raw provider callId value entirely. Two + // sibling calls with genuinely DISTINCT raw ids, and two calls that + // (incorrectly) share ONE duplicate raw id, both normalized + // byte-identically and matched successfully — hiding an S7-relevant + // continuation-correctness regression. The fix maps each raw callId to the + // ordinal of its FIRST DFS occurrence, so distinct raw ids get distinct + // labels and a shared raw id gets a shared label. + const genY: TraceSpan = { + spanId: "genY", + parentSpanId: null, + name: "generation-callid", + kind: "generation", + startTime: 0, + endTime: 1000, + status: "ok", + } + const toolBase1: TraceSpan = { + spanId: "td1", + parentSpanId: "genY", + name: "read_file", + kind: "tool", + startTime: 10, + endTime: 20, + status: "ok", + input: { path: "/a" }, + } + const toolBase2: TraceSpan = { + spanId: "td2", + parentSpanId: "genY", + name: "read_file", + kind: "tool", + startTime: 30, + endTime: 40, + status: "ok", + input: { path: "/b" }, + } + + test("two spans with distinct raw callIds normalize to DIFFERENT toolCallId labels", () => { + const distinctTrace = buildSyntheticTrace([ + genY, + { ...toolBase1, tool: { callId: "call_AAA", durationMs: 5 } }, + { ...toolBase2, tool: { callId: "call_BBB", durationMs: 5 } }, + ]) + + const normalized = normalize(distinctTrace) + const toolSpans = normalized.spans.filter((s) => s.kind === "tool") + expect(toolSpans.length).toBe(2) + expect(toolSpans[0].toolCallId).toBeDefined() + expect(toolSpans[1].toolCallId).toBeDefined() + expect(toolSpans[0].toolCallId).not.toBe(toolSpans[1].toolCallId) + }) + + test("two spans that (incorrectly) share ONE duplicate raw callId normalize to the SAME toolCallId label", () => { + const duplicateTrace = buildSyntheticTrace([ + genY, + { ...toolBase1, tool: { callId: "call_SHARED", durationMs: 5 } }, + { ...toolBase2, tool: { callId: "call_SHARED", durationMs: 5 } }, + ]) + + const normalized = normalize(duplicateTrace) + const toolSpans = normalized.spans.filter((s) => s.kind === "tool") + expect(toolSpans.length).toBe(2) + expect(toolSpans[0].toolCallId).toBe(toolSpans[1].toolCallId) + }) + + test("match() detects a distinct-ids-vs-duplicate-ids regression (the S7-relevant case the review reproduced)", () => { + const distinctTrace = buildSyntheticTrace([ + genY, + { ...toolBase1, tool: { callId: "call_AAA", durationMs: 5 } }, + { ...toolBase2, tool: { callId: "call_BBB", durationMs: 5 } }, + ]) + const duplicateTrace = buildSyntheticTrace([ + genY, + { ...toolBase1, tool: { callId: "call_SHARED", durationMs: 5 } }, + { ...toolBase2, tool: { callId: "call_SHARED", durationMs: 5 } }, + ]) + + const result = match(normalize(distinctTrace), normalize(duplicateTrace)) + + // The two tool spans sit in the same concurrent-rank bucket, so match() + // reports the mismatched one as a whole-object removed/added pair (path + // is the span id, not a "toolCallId" field path) — assert on the + // embedded object payload rather than the diff path string. + expect(result.pass).toBe(false) + expect(result.diffs.length).toBeGreaterThan(0) + expect(JSON.stringify(result.diffs)).toContain("toolCallId") + const toolCallIds = result.diffs.flatMap((d) => + [d.expected, d.actual].filter((v): v is { toolCallId?: string } => typeof v === "object" && v !== null && "toolCallId" in v), + ) + expect(toolCallIds.length).toBeGreaterThan(0) + }) +}) + +describe("loop summary determinism: hash stripped, loops order-independent (fix #6)", () => { + // Codex review finding #6: loop descriptions embed a base-36 + // `simpleHash().toString(36)` value as literal text — "(hash: xyz)" — that + // scrubDynamicTokens didn't strip, so two runs whose loop-triggering inputs + // differed only in something irrelevant to behavior (e.g. a temp path) + // still changed the emitted hash and flapped the golden. Multiple loops + // also retained raw first-detection/completion order, which is not + // reproducible. The fix adds a LOOP_HASH_PATTERN scrub and sorts loops by + // (tool, count, description). + function buildTraceWithLoops( + loops: Array<{ tool: string; inputHash: string; count: number; description: string }>, + ): TraceFile { + const base = buildSyntheticTrace([ + { + spanId: "loopRoot", + parentSpanId: null, + name: "ses_synthetic", + kind: "session", + startTime: 0, + endTime: 1000, + status: "ok", + }, + ]) + return { ...base, summary: { ...base.summary, loops } } + } + + test("differing (hash: ...) values in an otherwise-identical loop description normalize identically", () => { + const traceA = buildTraceWithLoops([ + { tool: "read_file", inputHash: "a1b2c3", count: 3, description: "read_file called 3x (hash: a1b2c3)" }, + ]) + const traceB = buildTraceWithLoops([ + { tool: "read_file", inputHash: "z9y8x7", count: 3, description: "read_file called 3x (hash: z9y8x7)" }, + ]) + + expect(stableStringify(normalize(traceA))).toBe(stableStringify(normalize(traceB))) + }) + + test("multiple loops normalize identically regardless of their original array order", () => { + const loop1 = { tool: "read_file", inputHash: "h1", count: 3, description: "read_file called 3x (hash: h1)" } + const loop2 = { tool: "write_file", inputHash: "h2", count: 2, description: "write_file called 2x (hash: h2)" } + + const traceOrder12 = buildTraceWithLoops([loop1, loop2]) + const traceOrder21 = buildTraceWithLoops([loop2, loop1]) + + expect(stableStringify(normalize(traceOrder12))).toBe(stableStringify(normalize(traceOrder21))) + }) + + test("NEGATIVE control: match() still detects a genuine loop count regression", () => { + const loop1 = { tool: "read_file", inputHash: "h1", count: 3, description: "read_file called 3x (hash: h1)" } + const loop2 = { tool: "write_file", inputHash: "h2", count: 2, description: "write_file called 2x (hash: h2)" } + + const golden = normalize(buildTraceWithLoops([loop1, loop2])) + const mutated = normalize(buildTraceWithLoops([{ ...loop1, count: 99 }, loop2])) + + const result = match(golden, mutated) + + expect(result.pass).toBe(false) + expect(result.diffs.some((d) => d.path.includes("loops"))).toBe(true) + }) +}) + +describe("withRealpathVariants: symlink-vs-realpath home root leak (fix #7)", () => { + // Codex review finding #7: root replacement is literal substring + // replacement. On macOS, os.homedir()/os.tmpdir() report the SYMLINK form + // (e.g. /var/folders/...), but some recorded strings resolve through the + // symlink to its REALPATH form (/private/var/folders/...) — a single + // literal-prefix root only ever catches ONE of the two forms, leaking + // `/private` (or the reverse) whenever the trace happens to contain + // the other form. The fix expands every root to include its own + // fs.realpathSync() resolution before scrubbing. This test builds a REAL + // symlink (so it reproduces the bug's mechanism on every platform, not + // just macOS) rather than relying on the test runner's own OS to happen to + // have symlinked temp dirs. + test("both the symlink form and its realpath form scrub to the identical output", () => { + const tmpBase = fsSync.mkdtempSync(path.join(os.tmpdir(), "trace-golden-realpath-")) + try { + const realTarget = path.join(tmpBase, "real-target") + fsSync.mkdirSync(realTarget) + const symlinkHome = path.join(tmpBase, "home-symlink") + fsSync.symlinkSync(realTarget, symlinkHome, "dir") + const resolvedHome = fsSync.realpathSync(symlinkHome) + + // Sanity precondition: the symlink must actually resolve to a DIFFERENT + // literal string, or this test would pass trivially without exercising + // the fix at all. + expect(resolvedHome).not.toBe(symlinkHome) + + const spanSymlinkForm: TraceSpan = { + spanId: "s1", + parentSpanId: null, + name: "ses_synthetic", + kind: "session", + startTime: 0, + endTime: 1000, + status: "ok", + input: { path: path.join(symlinkHome, "file.txt") }, + } + const spanRealpathForm: TraceSpan = { + ...spanSymlinkForm, + input: { path: path.join(resolvedHome, "file.txt") }, + } + + // homeRoots mirrors how this is actually invoked (driver's `fixture.home`, or + // os.homedir()) — always the symlink-style form the OS reports, never the realpath. + const normalizedSymlink = stableStringify( + normalize(buildSyntheticTrace([spanSymlinkForm]), { homeRoots: [symlinkHome] }), + ) + const normalizedRealpath = stableStringify( + normalize(buildSyntheticTrace([spanRealpathForm]), { homeRoots: [symlinkHome] }), + ) + + expect(normalizedSymlink).toBe(normalizedRealpath) + expect(normalizedSymlink).not.toContain(resolvedHome) + expect(normalizedSymlink).not.toContain(symlinkHome) + expect(normalizedSymlink).toContain("") + } finally { + fsSync.rmSync(tmpBase, { recursive: true, force: true }) + } + }) +}) + +describe("normalize(): schema-drift guard on unclassified span fields (fix #2)", () => { + const baseSpan: TraceSpan = { + spanId: "root1", + parentSpanId: null, + name: "ses_synthetic", + kind: "session", + startTime: 1000, + endTime: 2000, + status: "ok", + } + + test("throws when a span carries a field TraceSpan doesn't declare (simulated schema drift)", () => { + // Deliberately bypasses the type system — this simulates tracing.ts's + // TraceSpan gaining a real new field before normalize.ts's SPAN_KEYS/ + // SpanKeyLiteral have been updated to classify it. A cast is required + // precisely because the compile-time pin next to SPAN_KEYS (see + // normalize.ts) would otherwise catch a *declared* drift at typecheck + // time; this test proves the runtime backstop for undeclared/dynamic + // data (e.g. a real trace file written by an older or newer binary). + const drifted = { ...baseSpan, plannedNewField: "simulates a future TraceSpan field" } as unknown as TraceSpan + const trace = buildSyntheticTrace([drifted]) + + expect(() => normalize(trace)).toThrow(/unknown field/) + }) + + test("does NOT throw on a span using only today's known field set (negative control, pins the other boundary)", () => { + const fullyPopulated: TraceSpan = { + ...baseSpan, + statusMessage: "done", + interrupted: false, + model: { modelId: "m1", providerId: "p1", variant: "high" }, + finishReason: "stop", + tokens: { input: 1, output: 1, reasoning: 0, cacheRead: 0, cacheWrite: 0, total: 2 }, + cost: 0.01, + tool: { callId: "call_1", durationMs: 10 }, + input: { a: 1 }, + output: { b: 2 }, + attributes: { foo: "bar" }, + } + const trace = buildSyntheticTrace([fullyPopulated]) + + expect(() => normalize(trace)).not.toThrow() + }) +}) + +describe("CI wiring: trace-golden's live subprocess test actually runs in CI (fix #1)", () => { + // cliIt.live (cli-process.ts) is a no-op when OPENCODE_SKIP_SUBPROCESS=1, which the main CI test + // pass sets. CI only runs subprocess-dependent suites that are explicitly named in its dedicated + // SUBPROCESS_PATHS list — a suite not named there silently never executes its live scenarios at + // all (not "skipped and visible", just absent), which is exactly how "only smoke is wired and + // even that never truly runs in CI" went unnoticed. This test fails loudly the moment that list + // stops including this directory, instead of relying on someone noticing a coverage gap. + test("ci.yml's SUBPROCESS_PATHS includes test/altimate/trace-golden/", () => { + const ciYmlPath = path.join(import.meta.dir, "../../../../../.github/workflows/ci.yml") + const ciYml = fsSync.readFileSync(ciYmlPath, "utf-8") + const subprocessPathsLine = ciYml.split("\n").find((line) => line.includes("SUBPROCESS_PATHS=")) + expect(subprocessPathsLine).toBeDefined() + expect(subprocessPathsLine).toContain("test/altimate/trace-golden/") + }) +}) + +describe("driver.ts: real headless session driven end-to-end (all discovered scenarios)", () => { + const isCI = process.env.CI === "true" || process.env.CI === "1" + + // Discovers scenario directories rather than hard-coding `smoke`, so a new S5/S7 scenario is + // automatically wired into both this test file AND stability-check.ts (which uses the same + // discoverScenarios shape) the moment its directory is added — closing the other half of + // codex-tracegolden-code-review.md finding #1 ("both the test and stability runner hard-code + // smoke rather than discovering scenario directories"). + for (const scenarioName of discoverScenarios(SCENARIOS_DIR)) { + const scenarioDir = path.join(SCENARIOS_DIR, scenarioName) + + cliIt.live(`${scenarioName} scenario trace matches its golden`, (fixture) => + Effect.gen(function* () { + const { prompt } = yield* readJson<{ prompt: string }>(path.join(scenarioDir, "prompt.json")) + const setup = yield* readJson<{ files?: Record }>(path.join(scenarioDir, "setup.json")) + const scriptRaw = yield* readJson(path.join(scenarioDir, "model-script.json")) + const script = resolvePlaceholders(scriptRaw, fixture.home) + + for (const [name, content] of Object.entries(setup.files ?? {})) { + yield* Effect.tryPromise({ + try: () => fs.writeFile(path.join(fixture.home, name), content, "utf-8"), + catch: (cause) => new Error(`trace-golden: failed to write scenario fixture ${name}: ${String(cause)}`), + }) + } + + const result = yield* driveScenario(fixture, { + prompt, + script, + // Non-interactive `run` defaults every permission ask to "ask", which + // an in-process run (no TUI, no connected client to answer) can never + // resolve. --dangerously-skip-permissions auto-approves anything not + // explicitly denied — see src/cli/cmd/run.ts's yolo-mode branch. + runOpts: { extraArgs: ["--dangerously-skip-permissions"] }, + }) + + if (result.exitCode !== 0) { + return yield* Effect.fail( + new Error( + `${scenarioName} scenario: opencode run exited ${result.exitCode}\nstderr:\n${result.stderr.slice(-2000)}`, + ), + ) + } + + // The child subprocess's HOME (and therefore every home-relative path in + // its trace) is fixture.home, NOT this test process's os.homedir() — see + // isolatedEnv() in cli-process.ts. Must be passed explicitly or path + // scrubbing silently no-ops on the driven session's own paths. + const actual = normalize(result.trace, { homeRoots: [fixture.home] }) + const goldenPath = path.join(scenarioDir, "golden.json") + const goldenExists = yield* Effect.tryPromise({ + try: () => fileExists(goldenPath), + catch: (cause) => new Error(String(cause)), + }) + + const action = resolveGoldenAction({ goldenExists, updateRequested: TRACE_GOLDEN_UPDATE, isCI, goldenPath }) + + if (action.kind === "fail") { + return yield* Effect.fail(new Error(action.reason)) + } + + if (action.kind === "write") { + yield* Effect.tryPromise({ + try: () => fs.writeFile(goldenPath, stableStringify(actual) + "\n", "utf-8"), + catch: (cause) => new Error(`trace-golden: failed to write golden ${goldenPath}: ${String(cause)}`), + }) + // eslint-disable-next-line no-console + console.warn(`[trace-golden] wrote golden: ${goldenPath} (review this diff before committing)`) + return + } + + const golden = yield* readJson(goldenPath) + const matchResult = match(golden, actual) + if (!matchResult.pass) { + return yield* Effect.fail(new Error(matchResult.format())) + } + }), + ) + } +}) From 516eb684c5ab5a47552e6e2c98ad3fadcd500c67 Mon Sep 17 00:00:00 2001 From: anandgupta42 Date: Sun, 19 Jul 2026 07:35:25 -0700 Subject: [PATCH 2/4] =?UTF-8?q?test:=20[de-fork]=20address=20review=20?= =?UTF-8?q?=E2=80=94=20sanitize=20fixture=20home=20paths,=20guard=20CONCUR?= =?UTF-8?q?RENCY?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Two `real-traces/` fixtures contained a developer's `/Users/anandgupta/...` home path (PII + CI-portability risk). Replaced with `/home/developer/...` throughout — the fixtures are used only for normalizer idempotency (normalize twice, compare), so the content swap doesn't affect any assertion (CodeRabbit). - `stability-check.ts` now validates `CONCURRENCY` the same way as `RUNS` (`Number.isFinite && >= 1`), so a bad env/arg fails with a clear message instead of `runPool` spawning zero workers and crashing downstream (CodeRabbit). Co-Authored-By: Claude Fable 5 --- .../real-traces/ses_2e819da4cffeMhWRAOCOslv3mp.json | 12 ++++++------ .../real-traces/ses_2e81c7baeffeaz1CvDUlNzeL4Y.json | 2 +- .../test/altimate/trace-golden/stability-check.ts | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/packages/opencode/test/altimate/trace-golden/fixtures/real-traces/ses_2e819da4cffeMhWRAOCOslv3mp.json b/packages/opencode/test/altimate/trace-golden/fixtures/real-traces/ses_2e819da4cffeMhWRAOCOslv3mp.json index 0ee5d9d78..0f5473362 100644 --- a/packages/opencode/test/altimate/trace-golden/fixtures/real-traces/ses_2e819da4cffeMhWRAOCOslv3mp.json +++ b/packages/opencode/test/altimate/trace-golden/fixtures/real-traces/ses_2e819da4cffeMhWRAOCOslv3mp.json @@ -35,7 +35,7 @@ }, "output": { "parts": 4, - "content": "You are powered by the model named claude-sonnet-4-6. The exact model ID is anthropic/claude-sonnet-4-6\nHere is some useful information about the environment you are running in:\n\n Working directory: /Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/packages/opencode\n Workspace root folder: /Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui\n Is directory a git repo: yes\n Platform: darwin\n Today's date: Sun Mar 22 2026\n\n\n \n\n\nSkills provide specialized instructions and workflows for specific tasks.\nUse the skill tool to load a skill when a task matches its description.\n\n \n dbt-analyze\n Analyze downstream impact of dbt model changes using column-level lineage and the dependency graph. Use when evaluating the blast radius of a change before shipping. Powered by altimate-dbt.\n file:///Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/dbt-analyze/SKILL.md\n \n \n altimate-setup\n Configure Altimate platform credentials for datamate and API access\n file:///Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/altimate-setup/SKILL.md\n \n \n sql-translate\n Translate SQL queries between database dialects (Snowflake, BigQuery, PostgreSQL, MySQL, etc.)\n file:///Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/sql-translate/SKILL.md\n \n \n dbt-docs\n Document dbt models and columns in schema.yml with business context — model descriptions, column definitions, and doc blocks. Use when adding or improving documentation for discoverability. Powered by altimate-dbt.\n file:///Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/dbt-docs/SKILL.md\n \n \n lineage-diff\n Compare column-level lineage between two versions of a SQL query to show added, removed, and changed data flow edges.\n file:///Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/lineage-diff/SKILL.md\n \n \n cost-report\n Analyze Snowflake query costs and identify optimization opportunities\n file:///Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/cost-report/SKILL.md\n \n \n query-optimize\n Analyze and optimize SQL queries for better performance\n file:///Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/query-optimize/SKILL.md\n \n \n training-status\n Show what your AI teammate has learned — patterns, rules, glossary, and standards\n file:///Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/training-status/SKILL.md\n \n \n sql-review\n Pre-merge SQL quality gate — lint 26 anti-patterns, grade readability/performance A-F, validate syntax, and scan for injection threats. Use before committing or reviewing SQL changes.\n file:///Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/sql-review/SKILL.md\n \n \n data-viz\n Build modern, interactive data visualizations and dashboards using code-based component libraries (shadcn/ui, Recharts, Tremor, Nivo, D3, Victory, visx). Use this skill whenever the user asks to visualize data, build dashboards, create analytics views, chart metrics, tell a data story, build a reporting interface, create KPI cards, plot graphs, or explore a dataset — even if they mention PowerBI, Tableau, Streamlit, Metabase, Looker, Grafana, or similar tools. Also trigger when the user says \"make a dashboard\", \"show me the data\", \"chart this\", \"visualize trends\", \"build an analytics page\", \"data story\", or anything involving turning raw data into interactive visual interfaces. If the task involves presenting data visually — this is the skill. Always prefer building a real, interactive, code-based UI over exporting to or recommending a BI platform.\n\n file:///Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/data-viz/SKILL.md\n \n \n dbt-develop\n Create and modify dbt models — staging, intermediate, marts, incremental, medallion architecture. Use when building new SQL models, extending existing ones, scaffolding YAML configs, or reorganizing project structure. Powered by altimate-dbt.\n file:///Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/dbt-develop/SKILL.md\n \n \n pii-audit\n Classify schema columns for PII (SSN, email, phone, name, address, credit card) and check whether queries expose them. Use for GDPR/CCPA/HIPAA compliance audits.\n file:///Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/pii-audit/SKILL.md\n \n \n dbt-troubleshoot\n Debug dbt errors — compilation failures, runtime database errors, test failures, wrong data, and performance issues. Use when something is broken, producing wrong results, or failing to build. Powered by altimate-dbt.\n file:///Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/dbt-troubleshoot/SKILL.md\n \n \n dbt-test\n Add schema tests, unit tests, and data quality checks to dbt models. Use when validating data integrity, adding test definitions to schema.yml, writing unit tests, or practicing test-driven development in dbt. Powered by altimate-dbt.\n file:///Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/dbt-test/SKILL.md\n \n \n train\n Train your AI teammate on team standards from a document or style guide\n file:///Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/train/SKILL.md\n \n \n teach\n Teach your AI teammate a pattern by showing it an example file from your codebase\n file:///Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/teach/SKILL.md\n \n \n schema-migration\n Analyze DDL migrations for data loss risks — type narrowing, missing defaults, dropped constraints, breaking column changes. Use before applying schema changes to production.\n file:///Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/schema-migration/SKILL.md\n \n\n\n## Altimate Knowledge\n\nKnowledge from previous sessions and team training. Apply it consistently.\n\n### Learned Rules\n_Always follow these rules. They were taught by the user through corrections and explicit instruction._\n\n#### no-positional-group-by (applied 36x)\n## Never use positional GROUP BY\n\n**Rule:** Always use explicit column names in GROUP BY. Never use positional references (`GROUP BY 1, 2, 3`).\n\n**Why:** Positional GROUP BY is fragile. Adding, removing, or reordering a SELECT column silently shifts which columns are grouped — the query still runs but produces wrong aggregations with no error.\n\n**Wrong:**\n```sql\nselect\n order_date,\n order_channel,\n sum(revenue) as revenue\nfrom orders\ngroup by 1, 2\n```\n\n**Right:**\n```sql\nselect\n order_date,\n order_channel,\n sum(revenue) as revenue\nfrom orders\ngroup by order_date, order_channel\n```\n\n**Edge case — computed expressions:** When grouping by an expression (e.g. `date_trunc`, `date_part`), repeat the full expression in GROUP BY rather than using a positional alias:\n\n```sql\n-- Wrong\nselect date_trunc('month', order_date) as month, sum(revenue)\nfrom orders\ngroup by 1\n\n-- Right\nselect date_trunc('month', order_date) as month, sum(revenue)\nfrom orders\ngroup by date_trunc('month', order_date)\n```\n\n**Applies to:** All SQL written in dbt models, ad-hoc queries, CTEs, subqueries, and window function PARTITION BY clauses.\n\n\nInstructions from: /Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/packages/opencode/AGENTS.md\n# opencode database guide\n\n## Database\n\n- **Schema**: Drizzle schema lives in `src/**/*.sql.ts`.\n- **Naming**: tables and columns use snake*case; join columns are `_id`; indexes are `
*\\_idx`.\n- **Migrations**: generated by Drizzle Kit using `drizzle.config.ts` (schema: `./src/**/*.sql.ts`, output: `./migration`).\n- **Command**: `bun run db generate --name `.\n- **Output**: creates `migration/_/migration.sql` and `snapshot.json`.\n- **Tests**: migration tests should read the per-folder layout (no `_journal.json`).\n\n# opencode Effect guide\n\nInstructions to follow when writing Effect.\n\n## Schemas\n\n- Use `Schema.Class` for data types with multiple fields.\n- Use branded schemas (`Schema.brand`) for single-value types.\n\n## Services\n\n- Services use `ServiceMap.Service()(\"@console/\")`.\n- In `Layer.effect`, always return service implementations with `ServiceName.of({ ... })`, never a plain object.\n\n## Errors\n\n- Use `Schema.TaggedErrorClass` for typed errors.\n- For defect-like causes, use `Schema.Defect` instead of `unknown`.\n- In `Effect.gen`, prefer `yield* new MyError(...)` over `yield* Effect.fail(new MyError(...))` for direct early-failure branches.\n\n## Effects\n\n- Use `Effect.gen(function* () { ... })` for composition.\n- Use `Effect.fn(\"ServiceName.method\")` for named/traced effects and `Effect.fnUntraced` for internal helpers.\n- `Effect.fn` / `Effect.fnUntraced` accept pipeable operators as extra arguments, so avoid unnecessary `flow` or outer `.pipe()` wrappers.\n\n## Time\n\n- Prefer `DateTime.nowAsDate` over `new Date(yield* Clock.currentTimeMillis)` when you need a `Date`.\n\n## Errors\n\n- In `Effect.gen/fn`, prefer `yield* new MyError(...)` over `yield* Effect.fail(new MyError(...))` for direct early-failure branches.\n" + "content": "You are powered by the model named claude-sonnet-4-6. The exact model ID is anthropic/claude-sonnet-4-6\nHere is some useful information about the environment you are running in:\n\n Working directory: /home/developer/codebase/altimate-code/.claude/worktrees/openui/packages/opencode\n Workspace root folder: /home/developer/codebase/altimate-code/.claude/worktrees/openui\n Is directory a git repo: yes\n Platform: darwin\n Today's date: Sun Mar 22 2026\n\n\n \n\n\nSkills provide specialized instructions and workflows for specific tasks.\nUse the skill tool to load a skill when a task matches its description.\n\n \n dbt-analyze\n Analyze downstream impact of dbt model changes using column-level lineage and the dependency graph. Use when evaluating the blast radius of a change before shipping. Powered by altimate-dbt.\n file:///home/developer/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/dbt-analyze/SKILL.md\n \n \n altimate-setup\n Configure Altimate platform credentials for datamate and API access\n file:///home/developer/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/altimate-setup/SKILL.md\n \n \n sql-translate\n Translate SQL queries between database dialects (Snowflake, BigQuery, PostgreSQL, MySQL, etc.)\n file:///home/developer/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/sql-translate/SKILL.md\n \n \n dbt-docs\n Document dbt models and columns in schema.yml with business context — model descriptions, column definitions, and doc blocks. Use when adding or improving documentation for discoverability. Powered by altimate-dbt.\n file:///home/developer/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/dbt-docs/SKILL.md\n \n \n lineage-diff\n Compare column-level lineage between two versions of a SQL query to show added, removed, and changed data flow edges.\n file:///home/developer/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/lineage-diff/SKILL.md\n \n \n cost-report\n Analyze Snowflake query costs and identify optimization opportunities\n file:///home/developer/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/cost-report/SKILL.md\n \n \n query-optimize\n Analyze and optimize SQL queries for better performance\n file:///home/developer/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/query-optimize/SKILL.md\n \n \n training-status\n Show what your AI teammate has learned — patterns, rules, glossary, and standards\n file:///home/developer/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/training-status/SKILL.md\n \n \n sql-review\n Pre-merge SQL quality gate — lint 26 anti-patterns, grade readability/performance A-F, validate syntax, and scan for injection threats. Use before committing or reviewing SQL changes.\n file:///home/developer/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/sql-review/SKILL.md\n \n \n data-viz\n Build modern, interactive data visualizations and dashboards using code-based component libraries (shadcn/ui, Recharts, Tremor, Nivo, D3, Victory, visx). Use this skill whenever the user asks to visualize data, build dashboards, create analytics views, chart metrics, tell a data story, build a reporting interface, create KPI cards, plot graphs, or explore a dataset — even if they mention PowerBI, Tableau, Streamlit, Metabase, Looker, Grafana, or similar tools. Also trigger when the user says \"make a dashboard\", \"show me the data\", \"chart this\", \"visualize trends\", \"build an analytics page\", \"data story\", or anything involving turning raw data into interactive visual interfaces. If the task involves presenting data visually — this is the skill. Always prefer building a real, interactive, code-based UI over exporting to or recommending a BI platform.\n\n file:///home/developer/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/data-viz/SKILL.md\n \n \n dbt-develop\n Create and modify dbt models — staging, intermediate, marts, incremental, medallion architecture. Use when building new SQL models, extending existing ones, scaffolding YAML configs, or reorganizing project structure. Powered by altimate-dbt.\n file:///home/developer/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/dbt-develop/SKILL.md\n \n \n pii-audit\n Classify schema columns for PII (SSN, email, phone, name, address, credit card) and check whether queries expose them. Use for GDPR/CCPA/HIPAA compliance audits.\n file:///home/developer/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/pii-audit/SKILL.md\n \n \n dbt-troubleshoot\n Debug dbt errors — compilation failures, runtime database errors, test failures, wrong data, and performance issues. Use when something is broken, producing wrong results, or failing to build. Powered by altimate-dbt.\n file:///home/developer/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/dbt-troubleshoot/SKILL.md\n \n \n dbt-test\n Add schema tests, unit tests, and data quality checks to dbt models. Use when validating data integrity, adding test definitions to schema.yml, writing unit tests, or practicing test-driven development in dbt. Powered by altimate-dbt.\n file:///home/developer/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/dbt-test/SKILL.md\n \n \n train\n Train your AI teammate on team standards from a document or style guide\n file:///home/developer/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/train/SKILL.md\n \n \n teach\n Teach your AI teammate a pattern by showing it an example file from your codebase\n file:///home/developer/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/teach/SKILL.md\n \n \n schema-migration\n Analyze DDL migrations for data loss risks — type narrowing, missing defaults, dropped constraints, breaking column changes. Use before applying schema changes to production.\n file:///home/developer/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/schema-migration/SKILL.md\n \n\n\n## Altimate Knowledge\n\nKnowledge from previous sessions and team training. Apply it consistently.\n\n### Learned Rules\n_Always follow these rules. They were taught by the user through corrections and explicit instruction._\n\n#### no-positional-group-by (applied 36x)\n## Never use positional GROUP BY\n\n**Rule:** Always use explicit column names in GROUP BY. Never use positional references (`GROUP BY 1, 2, 3`).\n\n**Why:** Positional GROUP BY is fragile. Adding, removing, or reordering a SELECT column silently shifts which columns are grouped — the query still runs but produces wrong aggregations with no error.\n\n**Wrong:**\n```sql\nselect\n order_date,\n order_channel,\n sum(revenue) as revenue\nfrom orders\ngroup by 1, 2\n```\n\n**Right:**\n```sql\nselect\n order_date,\n order_channel,\n sum(revenue) as revenue\nfrom orders\ngroup by order_date, order_channel\n```\n\n**Edge case — computed expressions:** When grouping by an expression (e.g. `date_trunc`, `date_part`), repeat the full expression in GROUP BY rather than using a positional alias:\n\n```sql\n-- Wrong\nselect date_trunc('month', order_date) as month, sum(revenue)\nfrom orders\ngroup by 1\n\n-- Right\nselect date_trunc('month', order_date) as month, sum(revenue)\nfrom orders\ngroup by date_trunc('month', order_date)\n```\n\n**Applies to:** All SQL written in dbt models, ad-hoc queries, CTEs, subqueries, and window function PARTITION BY clauses.\n\n\nInstructions from: /home/developer/codebase/altimate-code/.claude/worktrees/openui/packages/opencode/AGENTS.md\n# opencode database guide\n\n## Database\n\n- **Schema**: Drizzle schema lives in `src/**/*.sql.ts`.\n- **Naming**: tables and columns use snake*case; join columns are `_id`; indexes are `
*\\_idx`.\n- **Migrations**: generated by Drizzle Kit using `drizzle.config.ts` (schema: `./src/**/*.sql.ts`, output: `./migration`).\n- **Command**: `bun run db generate --name `.\n- **Output**: creates `migration/_/migration.sql` and `snapshot.json`.\n- **Tests**: migration tests should read the per-folder layout (no `_journal.json`).\n\n# opencode Effect guide\n\nInstructions to follow when writing Effect.\n\n## Schemas\n\n- Use `Schema.Class` for data types with multiple fields.\n- Use branded schemas (`Schema.brand`) for single-value types.\n\n## Services\n\n- Services use `ServiceMap.Service()(\"@console/\")`.\n- In `Layer.effect`, always return service implementations with `ServiceName.of({ ... })`, never a plain object.\n\n## Errors\n\n- Use `Schema.TaggedErrorClass` for typed errors.\n- For defect-like causes, use `Schema.Defect` instead of `unknown`.\n- In `Effect.gen`, prefer `yield* new MyError(...)` over `yield* Effect.fail(new MyError(...))` for direct early-failure branches.\n\n## Effects\n\n- Use `Effect.gen(function* () { ... })` for composition.\n- Use `Effect.fn(\"ServiceName.method\")` for named/traced effects and `Effect.fnUntraced` for internal helpers.\n- `Effect.fn` / `Effect.fnUntraced` accept pipeable operators as extra arguments, so avoid unnecessary `flow` or outer `.pipe()` wrappers.\n\n## Time\n\n- Prefer `DateTime.nowAsDate` over `new Date(yield* Clock.currentTimeMillis)` when you need a `Date`.\n\n## Errors\n\n- In `Effect.gen/fn`, prefer `yield* new MyError(...)` over `yield* Effect.fail(new MyError(...))` for direct early-failure branches.\n" } }, { @@ -74,7 +74,7 @@ "input": { "pattern": "**/Dockerfile*" }, - "output": "/Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/packages/opencode/Dockerfile" + "output": "/home/developer/codebase/altimate-code/.claude/worktrees/openui/packages/opencode/Dockerfile" }, { "spanId": "019d17e6-35d0-7000-abb4-fb7e7c232341", @@ -84,7 +84,7 @@ "startTime": 1774222456272, "status": "ok", "model": {}, - "input": "[glob] /Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/packages/opencode/Dockerfile", + "input": "[glob] /home/developer/codebase/altimate-code/.claude/worktrees/openui/packages/opencode/Dockerfile", "endTime": 1774222456460, "output": "[tool calls: read]", "finishReason": "tool-calls", @@ -111,9 +111,9 @@ "durationMs": 4 }, "input": { - "filePath": "/Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/packages/opencode/Dockerfile" + "filePath": "/home/developer/codebase/altimate-code/.claude/worktrees/openui/packages/opencode/Dockerfile" }, - "output": "/Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/packages/opencode/Dockerfile\nfile\n1: FROM alpine AS base\n2: \n3: # Disable the runtime transpiler cache by default inside Docker containers.\n4: # On ephemeral containers, the cache is not useful\n5: ARG BUN_RUNTIME_TRANSPILER_CACHE_PATH=0\n6: ENV BUN_RUNTIME_TRANSPILER_CACHE_PATH=${BUN_RUNTIME_TRANSPILER_CACHE_PATH}\n7: RUN apk add libgcc libstdc++ ripgrep\n8: \n9: FROM base AS build-amd64\n10: COPY dist/@altimateai/altimate-code-linux-x64-baseline-musl/bin/altimate /usr/local/bin/altimate\n11: \n12: FROM base AS build-arm64\n13: COPY dist/@altimateai/altimate-code-linux-arm64-musl/bin/altimate /usr/local/bin/altimate\n14: \n15: ARG TARGETARCH\n16: FROM build-${TARGETARCH}\n17: RUN ln -sf /usr/local/bin/altimate /usr/local/bin/altimate-code\n18: RUN altimate --version\n19: ENTRYPOINT [\"altimate\"]\n\n(End of file - total 19 lines)\n" + "output": "/home/developer/codebase/altimate-code/.claude/worktrees/openui/packages/opencode/Dockerfile\nfile\n1: FROM alpine AS base\n2: \n3: # Disable the runtime transpiler cache by default inside Docker containers.\n4: # On ephemeral containers, the cache is not useful\n5: ARG BUN_RUNTIME_TRANSPILER_CACHE_PATH=0\n6: ENV BUN_RUNTIME_TRANSPILER_CACHE_PATH=${BUN_RUNTIME_TRANSPILER_CACHE_PATH}\n7: RUN apk add libgcc libstdc++ ripgrep\n8: \n9: FROM base AS build-amd64\n10: COPY dist/@altimateai/altimate-code-linux-x64-baseline-musl/bin/altimate /usr/local/bin/altimate\n11: \n12: FROM base AS build-arm64\n13: COPY dist/@altimateai/altimate-code-linux-arm64-musl/bin/altimate /usr/local/bin/altimate\n14: \n15: ARG TARGETARCH\n16: FROM build-${TARGETARCH}\n17: RUN ln -sf /usr/local/bin/altimate /usr/local/bin/altimate-code\n18: RUN altimate --version\n19: ENTRYPOINT [\"altimate\"]\n\n(End of file - total 19 lines)\n" }, { "spanId": "019d17e6-3c2e-7000-be11-69c546adcd20", @@ -123,7 +123,7 @@ "startTime": 1774222457902, "status": "ok", "model": {}, - "input": "[read] /Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/packages/opencode/Dockerfile\nfile\n1: FROM alpine AS base\n2: \n3: # Disable the runtime transpiler cache by default inside Docker containers.\n4: # On ephemeral containers, the cache is not useful\n5: ARG BUN_RUNTIME_TRANSPILER_CACHE_PATH=0\n6: ENV BUN_RUNTIME_TRANSPILER_CACHE_PATH=${BUN_RUNTIME_TRANSPILER_CACHE_PATH}\n7: RUN apk add libgcc libstdc++ ripgrep\n8: \n9: FROM base AS build-amd64\n10: COPY dis" + "input": "[read] /home/developer/codebase/altimate-code/.claude/worktrees/openui/packages/opencode/Dockerfile\nfile\n1: FROM alpine AS base\n2: \n3: # Disable the runtime transpiler cache by default inside Docker containers.\n4: # On ephemeral containers, the cache is not useful\n5: ARG BUN_RUNTIME_TRANSPILER_CACHE_PATH=0\n6: ENV BUN_RUNTIME_TRANSPILER_CACHE_PATH=${BUN_RUNTIME_TRANSPILER_CACHE_PATH}\n7: RUN apk add libgcc libstdc++ ripgrep\n8: \n9: FROM base AS build-amd64\n10: COPY dis" } ], "summary": { diff --git a/packages/opencode/test/altimate/trace-golden/fixtures/real-traces/ses_2e81c7baeffeaz1CvDUlNzeL4Y.json b/packages/opencode/test/altimate/trace-golden/fixtures/real-traces/ses_2e81c7baeffeaz1CvDUlNzeL4Y.json index ddba7c9e6..5bc137a36 100644 --- a/packages/opencode/test/altimate/trace-golden/fixtures/real-traces/ses_2e81c7baeffeaz1CvDUlNzeL4Y.json +++ b/packages/opencode/test/altimate/trace-golden/fixtures/real-traces/ses_2e81c7baeffeaz1CvDUlNzeL4Y.json @@ -35,7 +35,7 @@ }, "output": { "parts": 4, - "content": "You are powered by the model named claude-sonnet-4-6. The exact model ID is anthropic/claude-sonnet-4-6\nHere is some useful information about the environment you are running in:\n\n Working directory: /Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/packages/opencode\n Workspace root folder: /Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui\n Is directory a git repo: yes\n Platform: darwin\n Today's date: Sun Mar 22 2026\n\n\n \n\n\nSkills provide specialized instructions and workflows for specific tasks.\nUse the skill tool to load a skill when a task matches its description.\n\n \n teach\n Teach your AI teammate a pattern by showing it an example file from your codebase\n file:///Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/teach/SKILL.md\n \n \n training-status\n Show what your AI teammate has learned — patterns, rules, glossary, and standards\n file:///Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/training-status/SKILL.md\n \n \n pii-audit\n Classify schema columns for PII (SSN, email, phone, name, address, credit card) and check whether queries expose them. Use for GDPR/CCPA/HIPAA compliance audits.\n file:///Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/pii-audit/SKILL.md\n \n \n cost-report\n Analyze Snowflake query costs and identify optimization opportunities\n file:///Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/cost-report/SKILL.md\n \n \n data-viz\n Build modern, interactive data visualizations and dashboards using code-based component libraries (shadcn/ui, Recharts, Tremor, Nivo, D3, Victory, visx). Use this skill whenever the user asks to visualize data, build dashboards, create analytics views, chart metrics, tell a data story, build a reporting interface, create KPI cards, plot graphs, or explore a dataset — even if they mention PowerBI, Tableau, Streamlit, Metabase, Looker, Grafana, or similar tools. Also trigger when the user says \"make a dashboard\", \"show me the data\", \"chart this\", \"visualize trends\", \"build an analytics page\", \"data story\", or anything involving turning raw data into interactive visual interfaces. If the task involves presenting data visually — this is the skill. Always prefer building a real, interactive, code-based UI over exporting to or recommending a BI platform.\n\n file:///Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/data-viz/SKILL.md\n \n \n sql-review\n Pre-merge SQL quality gate — lint 26 anti-patterns, grade readability/performance A-F, validate syntax, and scan for injection threats. Use before committing or reviewing SQL changes.\n file:///Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/sql-review/SKILL.md\n \n \n dbt-troubleshoot\n Debug dbt errors — compilation failures, runtime database errors, test failures, wrong data, and performance issues. Use when something is broken, producing wrong results, or failing to build. Powered by altimate-dbt.\n file:///Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/dbt-troubleshoot/SKILL.md\n \n \n altimate-setup\n Configure Altimate platform credentials for datamate and API access\n file:///Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/altimate-setup/SKILL.md\n \n \n dbt-docs\n Document dbt models and columns in schema.yml with business context — model descriptions, column definitions, and doc blocks. Use when adding or improving documentation for discoverability. Powered by altimate-dbt.\n file:///Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/dbt-docs/SKILL.md\n \n \n dbt-analyze\n Analyze downstream impact of dbt model changes using column-level lineage and the dependency graph. Use when evaluating the blast radius of a change before shipping. Powered by altimate-dbt.\n file:///Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/dbt-analyze/SKILL.md\n \n \n lineage-diff\n Compare column-level lineage between two versions of a SQL query to show added, removed, and changed data flow edges.\n file:///Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/lineage-diff/SKILL.md\n \n \n dbt-develop\n Create and modify dbt models — staging, intermediate, marts, incremental, medallion architecture. Use when building new SQL models, extending existing ones, scaffolding YAML configs, or reorganizing project structure. Powered by altimate-dbt.\n file:///Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/dbt-develop/SKILL.md\n \n \n train\n Train your AI teammate on team standards from a document or style guide\n file:///Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/train/SKILL.md\n \n \n sql-translate\n Translate SQL queries between database dialects (Snowflake, BigQuery, PostgreSQL, MySQL, etc.)\n file:///Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/sql-translate/SKILL.md\n \n \n schema-migration\n Analyze DDL migrations for data loss risks — type narrowing, missing defaults, dropped constraints, breaking column changes. Use before applying schema changes to production.\n file:///Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/schema-migration/SKILL.md\n \n \n query-optimize\n Analyze and optimize SQL queries for better performance\n file:///Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/query-optimize/SKILL.md\n \n \n dbt-test\n Add schema tests, unit tests, and data quality checks to dbt models. Use when validating data integrity, adding test definitions to schema.yml, writing unit tests, or practicing test-driven development in dbt. Powered by altimate-dbt.\n file:///Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/dbt-test/SKILL.md\n \n\n\n## Altimate Knowledge\n\nKnowledge from previous sessions and team training. Apply it consistently.\n\n### Learned Rules\n_Always follow these rules. They were taught by the user through corrections and explicit instruction._\n\n#### no-positional-group-by (applied 24x)\n## Never use positional GROUP BY\n\n**Rule:** Always use explicit column names in GROUP BY. Never use positional references (`GROUP BY 1, 2, 3`).\n\n**Why:** Positional GROUP BY is fragile. Adding, removing, or reordering a SELECT column silently shifts which columns are grouped — the query still runs but produces wrong aggregations with no error.\n\n**Wrong:**\n```sql\nselect\n order_date,\n order_channel,\n sum(revenue) as revenue\nfrom orders\ngroup by 1, 2\n```\n\n**Right:**\n```sql\nselect\n order_date,\n order_channel,\n sum(revenue) as revenue\nfrom orders\ngroup by order_date, order_channel\n```\n\n**Edge case — computed expressions:** When grouping by an expression (e.g. `date_trunc`, `date_part`), repeat the full expression in GROUP BY rather than using a positional alias:\n\n```sql\n-- Wrong\nselect date_trunc('month', order_date) as month, sum(revenue)\nfrom orders\ngroup by 1\n\n-- Right\nselect date_trunc('month', order_date) as month, sum(revenue)\nfrom orders\ngroup by date_trunc('month', order_date)\n```\n\n**Applies to:** All SQL written in dbt models, ad-hoc queries, CTEs, subqueries, and window function PARTITION BY clauses.\n\n\nInstructions from: /Users/anandgupta/codebase/altimate-code/.claude/worktrees/openui/packages/opencode/AGENTS.md\n# opencode database guide\n\n## Database\n\n- **Schema**: Drizzle schema lives in `src/**/*.sql.ts`.\n- **Naming**: tables and columns use snake*case; join columns are `_id`; indexes are `
*\\_idx`.\n- **Migrations**: generated by Drizzle Kit using `drizzle.config.ts` (schema: `./src/**/*.sql.ts`, output: `./migration`).\n- **Command**: `bun run db generate --name `.\n- **Output**: creates `migration/_/migration.sql` and `snapshot.json`.\n- **Tests**: migration tests should read the per-folder layout (no `_journal.json`).\n\n# opencode Effect guide\n\nInstructions to follow when writing Effect.\n\n## Schemas\n\n- Use `Schema.Class` for data types with multiple fields.\n- Use branded schemas (`Schema.brand`) for single-value types.\n\n## Services\n\n- Services use `ServiceMap.Service()(\"@console/\")`.\n- In `Layer.effect`, always return service implementations with `ServiceName.of({ ... })`, never a plain object.\n\n## Errors\n\n- Use `Schema.TaggedErrorClass` for typed errors.\n- For defect-like causes, use `Schema.Defect` instead of `unknown`.\n- In `Effect.gen`, prefer `yield* new MyError(...)` over `yield* Effect.fail(new MyError(...))` for direct early-failure branches.\n\n## Effects\n\n- Use `Effect.gen(function* () { ... })` for composition.\n- Use `Effect.fn(\"ServiceName.method\")` for named/traced effects and `Effect.fnUntraced` for internal helpers.\n- `Effect.fn` / `Effect.fnUntraced` accept pipeable operators as extra arguments, so avoid unnecessary `flow` or outer `.pipe()` wrappers.\n\n## Time\n\n- Prefer `DateTime.nowAsDate` over `new Date(yield* Clock.currentTimeMillis)` when you need a `Date`.\n\n## Errors\n\n- In `Effect.gen/fn`, prefer `yield* new MyError(...)` over `yield* Effect.fail(new MyError(...))` for direct early-failure branches.\n" + "content": "You are powered by the model named claude-sonnet-4-6. The exact model ID is anthropic/claude-sonnet-4-6\nHere is some useful information about the environment you are running in:\n\n Working directory: /home/developer/codebase/altimate-code/.claude/worktrees/openui/packages/opencode\n Workspace root folder: /home/developer/codebase/altimate-code/.claude/worktrees/openui\n Is directory a git repo: yes\n Platform: darwin\n Today's date: Sun Mar 22 2026\n\n\n \n\n\nSkills provide specialized instructions and workflows for specific tasks.\nUse the skill tool to load a skill when a task matches its description.\n\n \n teach\n Teach your AI teammate a pattern by showing it an example file from your codebase\n file:///home/developer/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/teach/SKILL.md\n \n \n training-status\n Show what your AI teammate has learned — patterns, rules, glossary, and standards\n file:///home/developer/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/training-status/SKILL.md\n \n \n pii-audit\n Classify schema columns for PII (SSN, email, phone, name, address, credit card) and check whether queries expose them. Use for GDPR/CCPA/HIPAA compliance audits.\n file:///home/developer/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/pii-audit/SKILL.md\n \n \n cost-report\n Analyze Snowflake query costs and identify optimization opportunities\n file:///home/developer/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/cost-report/SKILL.md\n \n \n data-viz\n Build modern, interactive data visualizations and dashboards using code-based component libraries (shadcn/ui, Recharts, Tremor, Nivo, D3, Victory, visx). Use this skill whenever the user asks to visualize data, build dashboards, create analytics views, chart metrics, tell a data story, build a reporting interface, create KPI cards, plot graphs, or explore a dataset — even if they mention PowerBI, Tableau, Streamlit, Metabase, Looker, Grafana, or similar tools. Also trigger when the user says \"make a dashboard\", \"show me the data\", \"chart this\", \"visualize trends\", \"build an analytics page\", \"data story\", or anything involving turning raw data into interactive visual interfaces. If the task involves presenting data visually — this is the skill. Always prefer building a real, interactive, code-based UI over exporting to or recommending a BI platform.\n\n file:///home/developer/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/data-viz/SKILL.md\n \n \n sql-review\n Pre-merge SQL quality gate — lint 26 anti-patterns, grade readability/performance A-F, validate syntax, and scan for injection threats. Use before committing or reviewing SQL changes.\n file:///home/developer/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/sql-review/SKILL.md\n \n \n dbt-troubleshoot\n Debug dbt errors — compilation failures, runtime database errors, test failures, wrong data, and performance issues. Use when something is broken, producing wrong results, or failing to build. Powered by altimate-dbt.\n file:///home/developer/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/dbt-troubleshoot/SKILL.md\n \n \n altimate-setup\n Configure Altimate platform credentials for datamate and API access\n file:///home/developer/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/altimate-setup/SKILL.md\n \n \n dbt-docs\n Document dbt models and columns in schema.yml with business context — model descriptions, column definitions, and doc blocks. Use when adding or improving documentation for discoverability. Powered by altimate-dbt.\n file:///home/developer/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/dbt-docs/SKILL.md\n \n \n dbt-analyze\n Analyze downstream impact of dbt model changes using column-level lineage and the dependency graph. Use when evaluating the blast radius of a change before shipping. Powered by altimate-dbt.\n file:///home/developer/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/dbt-analyze/SKILL.md\n \n \n lineage-diff\n Compare column-level lineage between two versions of a SQL query to show added, removed, and changed data flow edges.\n file:///home/developer/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/lineage-diff/SKILL.md\n \n \n dbt-develop\n Create and modify dbt models — staging, intermediate, marts, incremental, medallion architecture. Use when building new SQL models, extending existing ones, scaffolding YAML configs, or reorganizing project structure. Powered by altimate-dbt.\n file:///home/developer/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/dbt-develop/SKILL.md\n \n \n train\n Train your AI teammate on team standards from a document or style guide\n file:///home/developer/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/train/SKILL.md\n \n \n sql-translate\n Translate SQL queries between database dialects (Snowflake, BigQuery, PostgreSQL, MySQL, etc.)\n file:///home/developer/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/sql-translate/SKILL.md\n \n \n schema-migration\n Analyze DDL migrations for data loss risks — type narrowing, missing defaults, dropped constraints, breaking column changes. Use before applying schema changes to production.\n file:///home/developer/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/schema-migration/SKILL.md\n \n \n query-optimize\n Analyze and optimize SQL queries for better performance\n file:///home/developer/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/query-optimize/SKILL.md\n \n \n dbt-test\n Add schema tests, unit tests, and data quality checks to dbt models. Use when validating data integrity, adding test definitions to schema.yml, writing unit tests, or practicing test-driven development in dbt. Powered by altimate-dbt.\n file:///home/developer/codebase/altimate-code/.claude/worktrees/openui/.opencode/skills/dbt-test/SKILL.md\n \n\n\n## Altimate Knowledge\n\nKnowledge from previous sessions and team training. Apply it consistently.\n\n### Learned Rules\n_Always follow these rules. They were taught by the user through corrections and explicit instruction._\n\n#### no-positional-group-by (applied 24x)\n## Never use positional GROUP BY\n\n**Rule:** Always use explicit column names in GROUP BY. Never use positional references (`GROUP BY 1, 2, 3`).\n\n**Why:** Positional GROUP BY is fragile. Adding, removing, or reordering a SELECT column silently shifts which columns are grouped — the query still runs but produces wrong aggregations with no error.\n\n**Wrong:**\n```sql\nselect\n order_date,\n order_channel,\n sum(revenue) as revenue\nfrom orders\ngroup by 1, 2\n```\n\n**Right:**\n```sql\nselect\n order_date,\n order_channel,\n sum(revenue) as revenue\nfrom orders\ngroup by order_date, order_channel\n```\n\n**Edge case — computed expressions:** When grouping by an expression (e.g. `date_trunc`, `date_part`), repeat the full expression in GROUP BY rather than using a positional alias:\n\n```sql\n-- Wrong\nselect date_trunc('month', order_date) as month, sum(revenue)\nfrom orders\ngroup by 1\n\n-- Right\nselect date_trunc('month', order_date) as month, sum(revenue)\nfrom orders\ngroup by date_trunc('month', order_date)\n```\n\n**Applies to:** All SQL written in dbt models, ad-hoc queries, CTEs, subqueries, and window function PARTITION BY clauses.\n\n\nInstructions from: /home/developer/codebase/altimate-code/.claude/worktrees/openui/packages/opencode/AGENTS.md\n# opencode database guide\n\n## Database\n\n- **Schema**: Drizzle schema lives in `src/**/*.sql.ts`.\n- **Naming**: tables and columns use snake*case; join columns are `_id`; indexes are `
*\\_idx`.\n- **Migrations**: generated by Drizzle Kit using `drizzle.config.ts` (schema: `./src/**/*.sql.ts`, output: `./migration`).\n- **Command**: `bun run db generate --name `.\n- **Output**: creates `migration/_/migration.sql` and `snapshot.json`.\n- **Tests**: migration tests should read the per-folder layout (no `_journal.json`).\n\n# opencode Effect guide\n\nInstructions to follow when writing Effect.\n\n## Schemas\n\n- Use `Schema.Class` for data types with multiple fields.\n- Use branded schemas (`Schema.brand`) for single-value types.\n\n## Services\n\n- Services use `ServiceMap.Service()(\"@console/\")`.\n- In `Layer.effect`, always return service implementations with `ServiceName.of({ ... })`, never a plain object.\n\n## Errors\n\n- Use `Schema.TaggedErrorClass` for typed errors.\n- For defect-like causes, use `Schema.Defect` instead of `unknown`.\n- In `Effect.gen`, prefer `yield* new MyError(...)` over `yield* Effect.fail(new MyError(...))` for direct early-failure branches.\n\n## Effects\n\n- Use `Effect.gen(function* () { ... })` for composition.\n- Use `Effect.fn(\"ServiceName.method\")` for named/traced effects and `Effect.fnUntraced` for internal helpers.\n- `Effect.fn` / `Effect.fnUntraced` accept pipeable operators as extra arguments, so avoid unnecessary `flow` or outer `.pipe()` wrappers.\n\n## Time\n\n- Prefer `DateTime.nowAsDate` over `new Date(yield* Clock.currentTimeMillis)` when you need a `Date`.\n\n## Errors\n\n- In `Effect.gen/fn`, prefer `yield* new MyError(...)` over `yield* Effect.fail(new MyError(...))` for direct early-failure branches.\n" } }, { diff --git a/packages/opencode/test/altimate/trace-golden/stability-check.ts b/packages/opencode/test/altimate/trace-golden/stability-check.ts index 6d9f9d8aa..88875c539 100644 --- a/packages/opencode/test/altimate/trace-golden/stability-check.ts +++ b/packages/opencode/test/altimate/trace-golden/stability-check.ts @@ -224,6 +224,7 @@ async function checkScenario(name: string, dir: string): Promise { async function main() { if (!Number.isFinite(RUNS) || RUNS < 1) throw new Error(`stability-check: invalid RUNS=${RUNS}`) + if (!Number.isFinite(CONCURRENCY) || CONCURRENCY < 1) throw new Error(`stability-check: invalid CONCURRENCY=${CONCURRENCY}`) const scenarioNames = discoverScenarios(SCENARIOS_DIR) if (scenarioNames.length === 0) throw new Error(`stability-check: no scenario directories found under ${SCENARIOS_DIR}`) From e40d0d15b3376ed22898fe715e4d9813c60c5711 Mon Sep 17 00:00:00 2001 From: anandgupta42 Date: Sun, 19 Jul 2026 08:08:35 -0700 Subject: [PATCH 3/4] =?UTF-8?q?test:=20[de-fork]=20address=20trace-golden?= =?UTF-8?q?=20review=20=E2=80=94=20CI=20merge-blocker,=20temp-dir=20determ?= =?UTF-8?q?inism,=20tmpdir=20convention?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - CRITICAL (fails on merge): the plain `test(...)` asserting ci.yml's SUBPROCESS_PATHS includes `test/altimate/trace-golden/` runs in the MAIN CI pass but the branch tip didn't wire that path, so it would fail the moment this PR merged. Add `test/altimate/trace-golden/` to SUBPROCESS_PATHS in ci.yml — the assertion now passes AND the live golden scenario actually runs in the bounded subprocess pass where it belongs (it spawns a real CLI + mock LLM). Makes the PR self-contained instead of deferring to a sibling stage. - Temp-dir determinism (normalize.ts): a tool that creates its own scratch dir directly under the OS temp root (`mkdtemp` -> `$TMPDIR/diffverify-/`) left a per-run-random segment behind after `` prefix scrubbing, so any future live scenario touching such a tool would flap on the random suffix. Add TMP_SUBDIR_PATTERN to canonicalize the first path segment under `` to `` (any dir a tool makes there is inherently per-run) — keeps nested paths stable. +1 unit test (two different suffixes normalize equal). - tmpdir() convention (Major): the 3 manual `mkdtempSync` + `try/finally rmSync` scaffolds now use `await using tmp = await tmpdir()` (test/fixture/fixture.ts) with automatic per-test cleanup, per test/AGENTS.md. - Documented two known limitations tied to scenario types not shipped in this PR (per YAGNI — the guardrails land WITH the first such scenario, not before one exists): driver.ts's single-tool-per-turn schema can't yet generate S7 concurrent-sibling topology end-to-end; and the generic scenario runner needs a per-route dispatcher sentinel for S5 native->plugin parity (trace-match alone can't distinguish identical native/plugin routes). PII: fixture home paths already sanitized on the branch tip (verified no `/Users/...` remains). 34 pass / 1 skip / 0 fail, typecheck clean. Co-Authored-By: Claude Fable 5 --- .github/workflows/ci.yml | 2 +- .../test/altimate/trace-golden/driver.ts | 14 +- .../test/altimate/trace-golden/normalize.ts | 14 +- .../trace-golden/trace-golden.test.ts | 157 ++++++++++-------- 4 files changed, 117 insertions(+), 70 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4c2313656..0fa95cacc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -147,7 +147,7 @@ jobs: # pass is isolated from the 11k main-suite tests so 2-per-worker is plenty bounded. # (Bun 1.3.x can segfault during cleanup after all tests pass — we parse the summary, not the # exit code. --timeout 90000 gives headroom for slow fs/spawn tests.) - SUBPROCESS_PATHS="test/cli/acp/ test/cli/smokes/ test/cli/serve/ test/cli/run/run-process.test.ts test/cli/mcp-add.test.ts test/cli/help/help-snapshots.test.ts" + SUBPROCESS_PATHS="test/cli/acp/ test/cli/smokes/ test/cli/serve/ test/cli/run/run-process.test.ts test/cli/mcp-add.test.ts test/cli/help/help-snapshots.test.ts test/altimate/trace-golden/" OPENCODE_SKIP_SUBPROCESS=1 bun test --timeout 90000 > /tmp/test-main.txt 2>&1 || true bun test --timeout 90000 --max-concurrency=2 $SUBPROCESS_PATHS > /tmp/test-sub.txt 2>&1 || true diff --git a/packages/opencode/test/altimate/trace-golden/driver.ts b/packages/opencode/test/altimate/trace-golden/driver.ts index 15eede6a7..f8fb35a21 100644 --- a/packages/opencode/test/altimate/trace-golden/driver.ts +++ b/packages/opencode/test/altimate/trace-golden/driver.ts @@ -18,7 +18,19 @@ import type { TraceFile } from "@/altimate/observability/tracing" export { withCliFixture } export type { CliFixture } -/** One scripted assistant turn: either plain text (session ends) or a tool call (session continues). */ +/** + * One scripted assistant turn: either plain text (session ends) or a tool call (session continues). + * + * KNOWN LIMITATION (S7 concurrency, not yet exercised): this schema represents at most ONE tool + * call per assistant turn, and pushTurn() queues a separate `reply().tool(...)` generation for each + * turn. Two adjacent `tool` turns therefore become two sequential single-tool generations, NOT one + * generation that dispatches multiple sibling calls — so the live harness cannot yet produce the + * concurrent-sibling topology that the partial-order matcher (match.ts) and planned S7 concurrency + * scenarios are built to protect. The matcher and normalizer are already order-invariant (they were + * unit-tested against hand-built concurrent traces), but END-TO-END concurrent generation needs a + * grouped multi-tool turn (e.g. `kind: "tools"` with a `calls[]` array, all queued in one Reply). + * Add that alongside the first S7 concurrency scenario, not speculatively before one exists. + */ export interface ScriptedTurn { readonly kind: "text" | "tool" readonly text?: string diff --git a/packages/opencode/test/altimate/trace-golden/normalize.ts b/packages/opencode/test/altimate/trace-golden/normalize.ts index ea72f4b81..128ea146b 100644 --- a/packages/opencode/test/altimate/trace-golden/normalize.ts +++ b/packages/opencode/test/altimate/trace-golden/normalize.ts @@ -582,8 +582,17 @@ const PLATFORM_LINE_PATTERN = /Platform: \S+/g const DURATION_PHRASE_PATTERN = /\b\d+ms\b|\b\d+\.\d+s\b|\b\d+m\d+s\b/g // `$${cost.toFixed(4)}`-style cost strings embedded in narrative/root-span output. const COST_PHRASE_PATTERN = /\$\d+(?:\.\d+)?/g - -/** Scrubs volatile dynamic content (ids, timestamps, dates, platform, durations, costs) out of freeform text. */ +// A tool that creates its own scratch dir directly under the OS temp root (e.g. +// `mkdtemp` -> `$TMPDIR/diffverify-CGXm/`) leaves a per-run-random segment behind AFTER +// the `` root prefix has been scrubbed. The driver only knows `fixture.home` and +// cannot enumerate temp dirs a tool creates, so canonicalize the entire first path +// segment under `` to a stable placeholder — any directory a tool makes directly +// there is inherently per-run, so collapsing it keeps nested paths (`.../newfile.txt`) +// deterministic instead of flapping on the random suffix. Runs AFTER path scrubbing has +// produced the literal ``, so it is a no-op on text that never carried a temp path. +const TMP_SUBDIR_PATTERN = /()\/[^\/"'\s]+/g + +/** Scrubs volatile dynamic content (ids, timestamps, dates, platform, durations, costs, temp subdirs) out of freeform text. */ function scrubDynamicTokens(text: string): string { return text .replace(LOOP_HASH_PATTERN, "(hash: )") @@ -594,6 +603,7 @@ function scrubDynamicTokens(text: string): string { .replace(PLATFORM_LINE_PATTERN, "Platform: ") .replace(DURATION_PHRASE_PATTERN, "") .replace(COST_PHRASE_PATTERN, "") + .replace(TMP_SUBDIR_PATTERN, "$1/") } /** Recursively replaces absolute-path prefixes anywhere inside a JSON-cloneable value with placeholders. */ diff --git a/packages/opencode/test/altimate/trace-golden/trace-golden.test.ts b/packages/opencode/test/altimate/trace-golden/trace-golden.test.ts index 519535c50..942791614 100644 --- a/packages/opencode/test/altimate/trace-golden/trace-golden.test.ts +++ b/packages/opencode/test/altimate/trace-golden/trace-golden.test.ts @@ -22,6 +22,7 @@ import path from "node:path" import { Effect } from "effect" import type { TraceFile, TraceSpan } from "@/altimate/observability/tracing" import { cliIt } from "../../lib/cli-process" +import { tmpdir } from "../../fixture/fixture" import { driveScenario, type ScriptedTurn } from "./driver" import { formatDiffs, match } from "./match" import { normalize, stableStringify, type NormalizedTrace } from "./normalize" @@ -168,21 +169,17 @@ describe("discoverScenarios(): scenario-directory discovery (fix #1)", () => { fsSync.writeFileSync(path.join(dir, "model-script.json"), "[]") } - test("finds all COMPLETE scenario subdirectories under a temp dir, sorted, excluding files", () => { - const tmp = fsSync.mkdtempSync(path.join(os.tmpdir(), "trace-golden-discover-")) - try { - makeCompleteScenario(path.join(tmp, "zeta")) - makeCompleteScenario(path.join(tmp, "alpha")) - makeCompleteScenario(path.join(tmp, "mid")) - fsSync.writeFileSync(path.join(tmp, "not-a-scenario.json"), "{}") - - expect(discoverScenarios(tmp)).toEqual(["alpha", "mid", "zeta"]) - } finally { - fsSync.rmSync(tmp, { recursive: true, force: true }) - } + test("finds all COMPLETE scenario subdirectories under a temp dir, sorted, excluding files", async () => { + await using tmp = await tmpdir() + makeCompleteScenario(path.join(tmp.path, "zeta")) + makeCompleteScenario(path.join(tmp.path, "alpha")) + makeCompleteScenario(path.join(tmp.path, "mid")) + fsSync.writeFileSync(path.join(tmp.path, "not-a-scenario.json"), "{}") + + expect(discoverScenarios(tmp.path)).toEqual(["alpha", "mid", "zeta"]) }) - test("excludes an INCOMPLETE scenario directory (missing driver inputs) instead of breaking discovery for its siblings", () => { + test("excludes an INCOMPLETE scenario directory (missing driver inputs) instead of breaking discovery for its siblings", async () => { // Regression test for a real failure this exact filter caught: an // abandoned/in-progress scenario scaffold directory (created, never // populated with prompt.json/setup.json/model-script.json) existed @@ -190,15 +187,11 @@ describe("discoverScenarios(): scenario-directory discovery (fix #1)", () => { // existed, caused driver.ts's live test to crash with ENOENT reading its // missing prompt.json — taking down the whole discovered-scenario test // loop, not just that one entry. - const tmp = fsSync.mkdtempSync(path.join(os.tmpdir(), "trace-golden-discover-incomplete-")) - try { - makeCompleteScenario(path.join(tmp, "complete-one")) - fsSync.mkdirSync(path.join(tmp, "incomplete-scaffold")) // no files inside — simulates an abandoned scenario dir - - expect(discoverScenarios(tmp)).toEqual(["complete-one"]) - } finally { - fsSync.rmSync(tmp, { recursive: true, force: true }) - } + await using tmp = await tmpdir() + makeCompleteScenario(path.join(tmp.path, "complete-one")) + fsSync.mkdirSync(path.join(tmp.path, "incomplete-scaffold")) // no files inside — simulates an abandoned scenario dir + + expect(discoverScenarios(tmp.path)).toEqual(["complete-one"]) }) test("the real SCENARIOS_DIR contains at least the smoke scenario", () => { @@ -900,51 +893,72 @@ describe("withRealpathVariants: symlink-vs-realpath home root leak (fix #7)", () // symlink (so it reproduces the bug's mechanism on every platform, not // just macOS) rather than relying on the test runner's own OS to happen to // have symlinked temp dirs. - test("both the symlink form and its realpath form scrub to the identical output", () => { - const tmpBase = fsSync.mkdtempSync(path.join(os.tmpdir(), "trace-golden-realpath-")) - try { - const realTarget = path.join(tmpBase, "real-target") - fsSync.mkdirSync(realTarget) - const symlinkHome = path.join(tmpBase, "home-symlink") - fsSync.symlinkSync(realTarget, symlinkHome, "dir") - const resolvedHome = fsSync.realpathSync(symlinkHome) - - // Sanity precondition: the symlink must actually resolve to a DIFFERENT - // literal string, or this test would pass trivially without exercising - // the fix at all. - expect(resolvedHome).not.toBe(symlinkHome) - - const spanSymlinkForm: TraceSpan = { - spanId: "s1", - parentSpanId: null, - name: "ses_synthetic", - kind: "session", - startTime: 0, - endTime: 1000, - status: "ok", - input: { path: path.join(symlinkHome, "file.txt") }, - } - const spanRealpathForm: TraceSpan = { - ...spanSymlinkForm, - input: { path: path.join(resolvedHome, "file.txt") }, - } - - // homeRoots mirrors how this is actually invoked (driver's `fixture.home`, or - // os.homedir()) — always the symlink-style form the OS reports, never the realpath. - const normalizedSymlink = stableStringify( - normalize(buildSyntheticTrace([spanSymlinkForm]), { homeRoots: [symlinkHome] }), - ) - const normalizedRealpath = stableStringify( - normalize(buildSyntheticTrace([spanRealpathForm]), { homeRoots: [symlinkHome] }), - ) - - expect(normalizedSymlink).toBe(normalizedRealpath) - expect(normalizedSymlink).not.toContain(resolvedHome) - expect(normalizedSymlink).not.toContain(symlinkHome) - expect(normalizedSymlink).toContain("") - } finally { - fsSync.rmSync(tmpBase, { recursive: true, force: true }) + test("both the symlink form and its realpath form scrub to the identical output", async () => { + await using tmp = await tmpdir() + const realTarget = path.join(tmp.path, "real-target") + fsSync.mkdirSync(realTarget) + const symlinkHome = path.join(tmp.path, "home-symlink") + fsSync.symlinkSync(realTarget, symlinkHome, "dir") + const resolvedHome = fsSync.realpathSync(symlinkHome) + + // Sanity precondition: the symlink must actually resolve to a DIFFERENT + // literal string, or this test would pass trivially without exercising + // the fix at all. + expect(resolvedHome).not.toBe(symlinkHome) + + const spanSymlinkForm: TraceSpan = { + spanId: "s1", + parentSpanId: null, + name: "ses_synthetic", + kind: "session", + startTime: 0, + endTime: 1000, + status: "ok", + input: { path: path.join(symlinkHome, "file.txt") }, + } + const spanRealpathForm: TraceSpan = { + ...spanSymlinkForm, + input: { path: path.join(resolvedHome, "file.txt") }, } + + // homeRoots mirrors how this is actually invoked (driver's `fixture.home`, or + // os.homedir()) — always the symlink-style form the OS reports, never the realpath. + const normalizedSymlink = stableStringify( + normalize(buildSyntheticTrace([spanSymlinkForm]), { homeRoots: [symlinkHome] }), + ) + const normalizedRealpath = stableStringify( + normalize(buildSyntheticTrace([spanRealpathForm]), { homeRoots: [symlinkHome] }), + ) + + expect(normalizedSymlink).toBe(normalizedRealpath) + expect(normalizedSymlink).not.toContain(resolvedHome) + expect(normalizedSymlink).not.toContain(symlinkHome) + expect(normalizedSymlink).toContain("") + }) + + // A tool that creates its OWN scratch dir directly under the OS temp root (mkdtemp) emits a + // per-run-random segment that survives `` prefix scrubbing (`/diffverify-/…`). + // The driver can't enumerate tool-created temp dirs, so the normalizer must canonicalize that + // first segment or any scenario touching such a tool would flap on the random suffix. + test("a tool-created mkdtemp subdir under the OS temp root canonicalizes — different random suffixes normalize identically", () => { + const spanForm = (suffix: string): TraceSpan => ({ + spanId: "s1", + parentSpanId: null, + name: "ses_synthetic", + kind: "session", + startTime: 0, + endTime: 1000, + status: "ok", + input: { path: path.join(os.tmpdir(), `diffverify-${suffix}`, "newfile.txt") }, + }) + + const normalizedA = stableStringify(normalize(buildSyntheticTrace([spanForm("CGXm4a")]))) + const normalizedB = stableStringify(normalize(buildSyntheticTrace([spanForm("Zq9T1p")]))) + + expect(normalizedA).toBe(normalizedB) + expect(normalizedA).toContain("//newfile.txt") + expect(normalizedA).not.toContain("diffverify-CGXm4a") + expect(normalizedA).not.toContain("diffverify-Zq9T1p") }) }) @@ -1017,6 +1031,17 @@ describe("driver.ts: real headless session driven end-to-end (all discovered sce // discoverScenarios shape) the moment its directory is added — closing the other half of // codex-tracegolden-code-review.md finding #1 ("both the test and stability runner hard-code // smoke rather than discovering scenario directories"). + // + // KNOWN LIMITATION (S5 dispatcher parity): this generic runner accepts a scenario solely when its + // normalized trace matches the golden. For an S5 native->plugin migration that is INSUFFICIENT on + // its own — native and plugin routes are designed to produce identical user-visible traces (see + // DRIVER-NOTES.md: "not a proof that a dispatcher was executed"), so a migration that accidentally + // keeps invoking the OLD native implementation would still match the golden and pass its parity + // gate. An S5 scenario therefore MUST carry an independent route sentinel (e.g. a dispatcher + // execute-counter or a source-tagged span attribute the golden pins) and this loop must assert it + // before accepting the match. That sentinel is defined per-route, so it lands WITH the first S5 + // scenario rather than as a speculative schema field here (there are no S5 scenarios yet — the + // only shipped scenario is `smoke`, a plain continuation, which has no native/plugin ambiguity). for (const scenarioName of discoverScenarios(SCENARIOS_DIR)) { const scenarioDir = path.join(SCENARIOS_DIR, scenarioName) From b89b4ad38b3b1eda369956705446b287860a789a Mon Sep 17 00:00:00 2001 From: anandgupta42 Date: Sun, 19 Jul 2026 09:50:57 -0700 Subject: [PATCH 4/4] =?UTF-8?q?test:=20[de-fork]=20address=20trace-golden?= =?UTF-8?q?=20review=20round=202=20=E2=80=94=20targeted=20temp=20canon=20+?= =?UTF-8?q?=20full=20placeholder=20resolution?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - normalize.ts temp-dir canonicalization was too broad: it collapsed the ENTIRE first segment under `` to ``, so distinct STABLE names (`/input-a` vs `/input-b`) mapped to the same placeholder — a regression that swapped which fixed file a tool touched would still match the golden (codex). Narrow TMP_SUBDIR_PATTERN to canonicalize ONLY a recognizable mkdtemp random tail (`<>=6 base62>`), preserving the stable prefix, and accept `\` as well as `/` so it also fires on the Windows temp root (codex). +1 test that two stable names stay distinct; existing random-suffix test updated. - Fixture placeholder resolution: only `model-script.json` was passed through resolvePlaceholders, so a scenario with `` in its prompt or a setup file sent the literal placeholder to the real CLI — and both the live test and stability-check.ts reproduced the same broken setup (codex). Resolve prompt + setup as well, in both runners. - Documented the script-consumption gap (codex): TestLLMServer exposes `pending`/`misses`, but a blanket exact-consumption assertion would flake on a real CLI's auxiliary (title/summary) generations, so that guard belongs per-scenario with the first S7 continuation scenario, not as a blanket gate. 35 pass / 1 skip / 0 fail, typecheck clean. Co-Authored-By: Claude Fable 5 --- .../test/altimate/trace-golden/driver.ts | 11 ++++ .../test/altimate/trace-golden/normalize.ts | 19 +++---- .../altimate/trace-golden/stability-check.ts | 7 ++- .../trace-golden/trace-golden.test.ts | 51 ++++++++++++++++--- 4 files changed, 70 insertions(+), 18 deletions(-) diff --git a/packages/opencode/test/altimate/trace-golden/driver.ts b/packages/opencode/test/altimate/trace-golden/driver.ts index f8fb35a21..b75760108 100644 --- a/packages/opencode/test/altimate/trace-golden/driver.ts +++ b/packages/opencode/test/altimate/trace-golden/driver.ts @@ -86,6 +86,17 @@ export function driveScenario( format: "json", }) + // KNOWN LIMITATION (script-consumption, codex review): TestLLMServer exposes + // `fixture.llm.pending` (queued turns never served — session exited early) and + // `fixture.llm.misses` (requests served by the automatic "ok" fallback — an unscripted + // generation). During golden CREATION/UPDATE, neither is checked, so a scenario whose + // model-script.json doesn't match the session's actual generations can self-approve a + // truncated/over-run trace. A blanket `pending === 0 && misses === []` assertion is NOT + // added here because a real CLI run makes auxiliary generations (title/summary) that a + // scenario script legitimately does not cover, so those checks belong per-scenario with an + // explicit expected-request profile — added WITH the first S7 continuation scenario that + // needs the guarantee, not as a blanket gate that would flake the existing smoke live test. + const events = fixture.opencode.parseJsonEvents(result.stdout) const traceSaved = events.find((e) => e.type === "trace_saved") if (!traceSaved || typeof traceSaved.path !== "string") { diff --git a/packages/opencode/test/altimate/trace-golden/normalize.ts b/packages/opencode/test/altimate/trace-golden/normalize.ts index 128ea146b..67eafcd0e 100644 --- a/packages/opencode/test/altimate/trace-golden/normalize.ts +++ b/packages/opencode/test/altimate/trace-golden/normalize.ts @@ -583,14 +583,15 @@ const DURATION_PHRASE_PATTERN = /\b\d+ms\b|\b\d+\.\d+s\b|\b\d+m\d+s\b/g // `$${cost.toFixed(4)}`-style cost strings embedded in narrative/root-span output. const COST_PHRASE_PATTERN = /\$\d+(?:\.\d+)?/g // A tool that creates its own scratch dir directly under the OS temp root (e.g. -// `mkdtemp` -> `$TMPDIR/diffverify-CGXm/`) leaves a per-run-random segment behind AFTER -// the `` root prefix has been scrubbed. The driver only knows `fixture.home` and -// cannot enumerate temp dirs a tool creates, so canonicalize the entire first path -// segment under `` to a stable placeholder — any directory a tool makes directly -// there is inherently per-run, so collapsing it keeps nested paths (`.../newfile.txt`) -// deterministic instead of flapping on the random suffix. Runs AFTER path scrubbing has -// produced the literal ``, so it is a no-op on text that never carried a temp path. -const TMP_SUBDIR_PATTERN = /()\/[^\/"'\s]+/g +// `mkdtemp` -> `$TMPDIR/diffverify-CGXm4a/`) leaves a per-run-random suffix behind AFTER the +// `` root prefix has been scrubbed. Canonicalize ONLY that random suffix, not the whole +// segment: `mkdtemp` appends random chars to a caller prefix that conventionally ends in a +// delimiter (`-`/`_`/`.`), so we require `<>=6 base62 chars>`. This preserves STABLE +// names directly under the temp root — `/input-a` stays distinct from `/input-b`, so a +// regression that swaps which fixed file a tool touches still diffs — while killing the flap on a +// genuinely random `mkdtemp` tail. Both `/` and `\` (Windows temp root) separators are accepted. +// Runs AFTER path scrubbing produced the literal ``, so it is a no-op on non-temp text. +const TMP_SUBDIR_PATTERN = /([/\\][^/\\"'\s]*?[-_.])[A-Za-z0-9]{6,}(?=[/\\"'\s]|$)/g /** Scrubs volatile dynamic content (ids, timestamps, dates, platform, durations, costs, temp subdirs) out of freeform text. */ function scrubDynamicTokens(text: string): string { @@ -603,7 +604,7 @@ function scrubDynamicTokens(text: string): string { .replace(PLATFORM_LINE_PATTERN, "Platform: ") .replace(DURATION_PHRASE_PATTERN, "") .replace(COST_PHRASE_PATTERN, "") - .replace(TMP_SUBDIR_PATTERN, "$1/") + .replace(TMP_SUBDIR_PATTERN, "$1") } /** Recursively replaces absolute-path prefixes anywhere inside a JSON-cloneable value with placeholders. */ diff --git a/packages/opencode/test/altimate/trace-golden/stability-check.ts b/packages/opencode/test/altimate/trace-golden/stability-check.ts index 88875c539..aea185b35 100644 --- a/packages/opencode/test/altimate/trace-golden/stability-check.ts +++ b/packages/opencode/test/altimate/trace-golden/stability-check.ts @@ -85,8 +85,11 @@ async function runOnce(scenarioDir: string, index: number): Promise Effect.tryPromise(() => fs.readFile(path.join(scenarioDir, "setup.json"), "utf-8")), Effect.tryPromise(() => fs.readFile(path.join(scenarioDir, "model-script.json"), "utf-8")), ]) - const { prompt } = JSON.parse(promptRaw) as { prompt: string } - const setup = JSON.parse(setupRaw) as { files?: Record } + // Resolve `` across every fixture input (prompt + setup + script), matching + // trace-golden.test.ts — a `` in the prompt or a setup file must reach the CLI + // as the real fixture home, not the literal placeholder. + const { prompt } = resolvePlaceholders(JSON.parse(promptRaw) as { prompt: string }, fixture.home) + const setup = resolvePlaceholders(JSON.parse(setupRaw) as { files?: Record }, fixture.home) const script = resolvePlaceholders(JSON.parse(scriptRaw) as ScriptedTurn[], fixture.home) for (const [name, content] of Object.entries(setup.files ?? {})) { diff --git a/packages/opencode/test/altimate/trace-golden/trace-golden.test.ts b/packages/opencode/test/altimate/trace-golden/trace-golden.test.ts index 942791614..9c8eb29a0 100644 --- a/packages/opencode/test/altimate/trace-golden/trace-golden.test.ts +++ b/packages/opencode/test/altimate/trace-golden/trace-golden.test.ts @@ -956,9 +956,33 @@ describe("withRealpathVariants: symlink-vs-realpath home root leak (fix #7)", () const normalizedB = stableStringify(normalize(buildSyntheticTrace([spanForm("Zq9T1p")]))) expect(normalizedA).toBe(normalizedB) - expect(normalizedA).toContain("//newfile.txt") - expect(normalizedA).not.toContain("diffverify-CGXm4a") - expect(normalizedA).not.toContain("diffverify-Zq9T1p") + // Only the random tail is canonicalized; the stable `diffverify-` prefix is preserved. + expect(normalizedA).toContain("/diffverify-/newfile.txt") + expect(normalizedA).not.toContain("CGXm4a") + expect(normalizedA).not.toContain("Zq9T1p") + }) + + // Counterpart to the above: the canonicalization must NOT collapse STABLE directory names + // directly under the temp root, or a regression that swaps which fixed file a tool reads + // would silently keep matching the golden. + test("stable (non-random) directory names directly under the OS temp root are preserved and stay distinct", () => { + const spanForm = (dir: string): TraceSpan => ({ + spanId: "s1", + parentSpanId: null, + name: "ses_synthetic", + kind: "session", + startTime: 0, + endTime: 1000, + status: "ok", + input: { path: path.join(os.tmpdir(), dir, "data.txt") }, + }) + + const normalizedA = stableStringify(normalize(buildSyntheticTrace([spanForm("input-a")]))) + const normalizedB = stableStringify(normalize(buildSyntheticTrace([spanForm("input-b")]))) + + expect(normalizedA).not.toBe(normalizedB) + expect(normalizedA).toContain("/input-a/data.txt") + expect(normalizedB).toContain("/input-b/data.txt") }) }) @@ -1047,10 +1071,23 @@ describe("driver.ts: real headless session driven end-to-end (all discovered sce cliIt.live(`${scenarioName} scenario trace matches its golden`, (fixture) => Effect.gen(function* () { - const { prompt } = yield* readJson<{ prompt: string }>(path.join(scenarioDir, "prompt.json")) - const setup = yield* readJson<{ files?: Record }>(path.join(scenarioDir, "setup.json")) - const scriptRaw = yield* readJson(path.join(scenarioDir, "model-script.json")) - const script = resolvePlaceholders(scriptRaw, fixture.home) + // Resolve `` placeholders across EVERY fixture input (prompt, setup files, and + // model script), not just the script — otherwise a scenario that puts `` in its + // prompt or a setup file (e.g. a plugin path) would send the literal placeholder to the + // real CLI, and both this live test and stability-check.ts would reproduce the same + // broken setup rather than exercise the intended fixture. + const { prompt } = resolvePlaceholders( + yield* readJson<{ prompt: string }>(path.join(scenarioDir, "prompt.json")), + fixture.home, + ) + const setup = resolvePlaceholders( + yield* readJson<{ files?: Record }>(path.join(scenarioDir, "setup.json")), + fixture.home, + ) + const script = resolvePlaceholders( + yield* readJson(path.join(scenarioDir, "model-script.json")), + fixture.home, + ) for (const [name, content] of Object.entries(setup.files ?? {})) { yield* Effect.tryPromise({