diff --git a/README.md b/README.md index eb368dc..4bd141e 100644 --- a/README.md +++ b/README.md @@ -289,15 +289,40 @@ console.log(response.usage); // { input_tokens, output_tokens } The package re-exports the official SDK client, builders, and error types, plus the offline decision layer: ```typescript -import { evaluatePolicy, extractResponse, loadPack, buildRecord } from "@fiale-plus/jev-cli"; +import { TypeSafeClient, choice, noul } from "@fiale-plus/jev-cli"; + +const client = new TypeSafeClient({ apiKey: process.env.TYPESAFE_API_KEY! }); +const response = await client.systemOne({ + state: "Payouts failing 3 days, help!", + questions: { + is_urgent: noul("Does this convey urgency?"), + dept: choice("Which team handles this?", { billing: "Payments", technical: "Bugs" }), + }, +}); +console.log(response.answers.is_urgent); // { type: "noul", noul: 0.98 } +console.log(response.usage); // { input_tokens, output_tokens } +``` + +Reuse one client across calls, own your state and elapsed time, and never apply a policy to a failed request: + +```typescript +import { evaluatePolicy, loadPack, readRecord, buildRecord } from "@fiale-plus/jev-cli"; -const { pack, hash } = loadPack("verify"); -const response = await jev.systemOne({ state: claim, questions: pack.questions }); -const result = evaluatePolicy(pack.policy, response); +const { pack } = loadPack("verify"); +const started = Date.now(); +const response = await client.systemOne({ state: claim, questions: pack.questions }); +const record = buildRecord({ pack: null, modelRequested: undefined, state: claim, questions: pack.questions, latencyMs: Date.now() - started }, response); +``` + +`readRecord` is the validated boundary for stored records — it returns a checked `DecisionRecord` or throws — while `isRecord` only detects the envelope. `extractResponse` accepts either a bare response or a decision record, so gating code does not care which one it was handed: + +```typescript +const stored = readRecord(JSON.parse(savedText)); +const result = evaluatePolicy(pack.policy, stored.response); if (result.decision !== "accept") process.exit(result.exit_code); ``` -`GATE_EXIT` maps a decision to its exit code; `extractResponse` accepts either a bare response or a decision record, so gating code does not care which one it was handed. +A complete runnable version lives at `examples/library/record-and-gate.mts`. `GATE_EXIT` maps a decision to its exit code. ## Development diff --git a/examples/README.md b/examples/README.md index 963b1ba..7e2161f 100644 --- a/examples/README.md +++ b/examples/README.md @@ -104,3 +104,22 @@ npx tsx src/cli.ts packs verify | jq '.policy' > my-policy.json Keep the questions and the policy in step: change a question ID and the rules that name it abstain (exit 4), which is loud, not silent. + +## library — caller-owned recorded workflow + +`library/record-and-gate.mts` runs the full workflow with public imports only: +one shared SDK client, abort on SIGINT/SIGTERM, elapsed-time measurement, error +handling that never applies policy to a failed request, then offline policy over +the saved record. + +```bash +node tools/stub-server.mjs & +export TYPESAFE_BASE_URL=http://127.0.0.1:8787 TYPESAFE_API_KEY=stub +mkdir -p out +npx tsx examples/library/record-and-gate.mts out/library-record.json; echo "exit $?" +npx tsx src/cli.ts replay --record out/library-record.json +``` + +The record stores hashes of the supplied state and questions, model identity and +latency — never the supplied text. Keep the original evidence and its stable ID +outside version control if it is private. diff --git a/examples/library/record-and-gate.mts b/examples/library/record-and-gate.mts new file mode 100644 index 0000000..cc28578 --- /dev/null +++ b/examples/library/record-and-gate.mts @@ -0,0 +1,66 @@ +// Complete recorded library workflow: one shared client, caller-owned state, +// elapsed time and error handling, offline policy over a saved record. +// +// Run with: npx tsx examples/library/record-and-gate.mjs [out-path] +import { readFileSync, writeFileSync } from "node:fs"; +import { + TypeSafeClient, + buildRecord, + estimateCostUsd, + evaluatePolicy, + loadPack, + readRecord, +} from "../../src/index.ts"; + +const apiKey = process.env.TYPESAFE_API_KEY; +if (!apiKey) { + throw new Error("TYPESAFE_API_KEY is required (with `npm run stub`: TYPESAFE_BASE_URL=http://127.0.0.1:8787 TYPESAFE_API_KEY=stub)."); +} +const outPath = process.argv[2] ?? "out/library-record.json"; + +// Example input: caller-owned evidence and its stable ID. The record stores a +// hash of this state, never the text — keep the original outside version +// control if it is private. +const claim = { + id: "library-intro-1", + claim: "Paid plans add seats in billing settings.", + evidence: "On the billing page, choose Add seats to invite more members.", +}; + +const { pack } = loadPack("verify"); + +// One shared client: reuse it across calls instead of constructing one per row. +const client = new TypeSafeClient({ apiKey }); +const controller = new AbortController(); +process.on("SIGINT", () => controller.abort()); +process.on("SIGTERM", () => controller.abort()); + +let response; +const started = Date.now(); +try { + response = await client.systemOne( + { state: claim, questions: pack.questions }, + { signal: controller.signal }, + ); +} catch (err) { + // Client errors stay in the caller: no policy is applied to a failed request. + process.stderr.write(`request failed: ${err instanceof Error ? err.message : String(err)}\n`); + process.exit(1); +} +const latencyMs = Date.now() - started; + +const relation = response.answers.relation; +console.log(`relation: ${relation?.choice ?? "unknown"} (confidence ${(relation?.confidence ?? 0).toFixed(3)})`); +console.log(`model: ${response.model} latency: ${Math.round(latencyMs)}ms cost: $${estimateCostUsd(response.usage.input_tokens).toFixed(6)}`); + +const record = buildRecord({ pack: null, modelRequested: undefined, state: claim, questions: pack.questions, latencyMs }, response); +writeFileSync(outPath, JSON.stringify(record, null, 2) + "\n", "utf8"); +process.stderr.write(`record written: ${outPath}\n`); + +// Offline decision over the stored record, no inference: readRecord validates +// the envelope, evaluatePolicy returns a decision with per-rule reasons. +const stored = readRecord(JSON.parse(readFileSync(outPath, "utf8"))); +const result = evaluatePolicy(pack.policy, stored.response); +console.log(`decision: ${result.decision} (exit ${result.exit_code})`); +for (const rule of result.rules) console.log(` - ${rule.answer}: ${rule.outcome}${rule.reason ? ` — ${rule.reason}` : ""}`); +process.exitCode = result.exit_code; diff --git a/package-lock.json b/package-lock.json index 51d37a2..a72b2e0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@fiale-plus/jev-cli", - "version": "0.1.2", + "version": "0.1.3", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@fiale-plus/jev-cli", - "version": "0.1.2", + "version": "0.1.3", "license": "MIT", "dependencies": { "@typesafe-ai/sdk": "^0.6.0" diff --git a/package.json b/package.json index 8180424..d632c04 100644 --- a/package.json +++ b/package.json @@ -52,7 +52,7 @@ }, "type": "module", "types": "dist/index.d.ts", - "version": "0.1.2", + "version": "0.1.3", "dependencies": { "@typesafe-ai/sdk": "^0.6.0" } diff --git a/src/cli/records.ts b/src/cli/records.ts index 4971d7e..4c9b3b6 100644 --- a/src/cli/records.ts +++ b/src/cli/records.ts @@ -69,10 +69,10 @@ export function recordCost(record: DecisionRecord): { input_tokens: number; outp }; } -// Envelope detector. Anything carrying record fields is treated as a record and -// must validate as one: falling back to a bare response would let a malformed or -// newer record be reinterpreted as a different judgment. -export function isRecord(input: unknown): input is DecisionRecord { +// Envelope detector. Anything carrying record fields is treated as a candidate +// record and must validate as one: falling back to a bare response would let a +// malformed or newer record be reinterpreted as a different judgment. +export function isRecord(input: unknown): boolean { return ( typeof input === "object" && input !== null && @@ -87,7 +87,7 @@ function isObject(value: unknown): value is Record { // Validates the fields that gate and replay dereference, so a hand-edited or // truncated record fails with a message instead of a TypeError. -export function coerceRecord(input: unknown): DecisionRecord { +function coerceRecord(input: unknown): DecisionRecord { if (!isObject(input)) throw new Error("Invalid record: expected a JSON object."); if (input.record_version !== RECORD_VERSION) { throw new Error(`Unsupported record_version ${JSON.stringify(input.record_version ?? null)}: this CLI writes version ${RECORD_VERSION}.`); @@ -105,10 +105,19 @@ export function coerceRecord(input: unknown): DecisionRecord { return input as unknown as DecisionRecord; } +// Public boundary for stored records: returns a validated record or throws. Unlike +// isRecord, this is a claim the caller can rely on — a malformed envelope fails +// here instead of reaching response handling unvalidated. +export function readRecord(input: unknown): DecisionRecord { + if (!isRecord(input)) throw new Error('Invalid record: expected a decision record with a "record_version" or "response" field.'); + return coerceRecord(input); +} + + // Accepts either a record or a bare response object, so `gate` can read the output // of `ask` directly as well as a saved record. export function extractResponse(input: unknown): SystemOneResult { - if (isRecord(input)) return coerceRecord(input).response; + if (isRecord(input)) return readRecord(input).response; if (isObject(input) && "answers" in input) return input as unknown as SystemOneResult; throw new Error('Invalid input: expected a response object with an "answers" map, or a decision record with a "response" field.'); } diff --git a/src/commands/gate.ts b/src/commands/gate.ts index 0eedb4f..b830eb9 100644 --- a/src/commands/gate.ts +++ b/src/commands/gate.ts @@ -3,7 +3,7 @@ import type { OutputFormat } from "../cli/formatters.js"; import { formatGate } from "../cli/formatters.js"; import type { GatePolicy } from "../cli/policy.js"; import { GATE_EXIT, coercePolicy, evaluatePolicy } from "../cli/policy.js"; -import { coerceRecord, extractResponse, isRecord } from "../cli/records.js"; +import { extractResponse, isRecord, readRecord } from "../cli/records.js"; import { loadPack } from "./packs.js"; import { hashValue } from "../utils/hash.js"; import { readJsonFile } from "../utils/io.js"; @@ -54,7 +54,7 @@ export async function handleGate(global: GlobalOptions, format: OutputFormat): P const { policy, pack, questionsHash, source } = resolvePolicy(global); const input = readJsonFile(global.input); - const record = isRecord(input) ? coerceRecord(input) : null; + const record = isRecord(input) ? readRecord(input) : null; const response = record !== null ? record.response : extractResponse(input); // Gating a record against the pack that produced it: if the questions changed, diff --git a/src/commands/replay.ts b/src/commands/replay.ts index 1933415..be2c492 100644 --- a/src/commands/replay.ts +++ b/src/commands/replay.ts @@ -1,7 +1,7 @@ import type { GlobalOptions } from "../cli/parseArgs.js"; import type { OutputFormat } from "../cli/formatters.js"; import { formatReplay } from "../cli/formatters.js"; -import { RECORD_VERSION, coerceRecord } from "../cli/records.js"; +import { RECORD_VERSION, readRecord } from "../cli/records.js"; import { readJsonFile } from "../utils/io.js"; // Replay never calls the API: it re-emits answers that were already paid for, and @@ -13,7 +13,7 @@ export async function handleReplay(global: GlobalOptions, format: OutputFormat): const raw = readJsonFile(global.record); let record; try { - record = coerceRecord(raw); + record = readRecord(raw); } catch (err) { throw new Error(`Invalid record ${global.record}: ${err instanceof Error ? err.message : String(err)} (expected a decision record written by --record, record_version ${RECORD_VERSION}).`); } diff --git a/src/index.ts b/src/index.ts index bf0555d..37f6e58 100644 --- a/src/index.ts +++ b/src/index.ts @@ -6,7 +6,7 @@ export type { LintIssue, LintResult } from "./cli/lint.js"; // thresholds on their own records needs the same decision function the CLI uses. export { lintPolicy, coercePolicy, evaluatePolicy, policyHash, GATE_EXIT } from "./cli/policy.js"; export type { GateDecision, GatePolicy, GateResult, GateRule, RuleOutcome } from "./cli/policy.js"; -export { buildRecord, extractResponse, isRecord, recordCost, RECORD_VERSION } from "./cli/records.js"; +export { buildRecord, extractResponse, isRecord, readRecord, recordCost, RECORD_VERSION } from "./cli/records.js"; export type { DecisionRecord, RecordPackRef } from "./cli/records.js"; export { listPacks, loadPack } from "./commands/packs.js"; export type { Pack } from "./commands/packs.js"; diff --git a/src/tests/gate.test.ts b/src/tests/gate.test.ts index ff30f7a..b093a9d 100644 --- a/src/tests/gate.test.ts +++ b/src/tests/gate.test.ts @@ -3,7 +3,7 @@ import assert from "node:assert/strict"; import type { Questions, SystemOneResult } from "@typesafe-ai/sdk"; import { GATE_EXIT, coercePolicy, evaluatePolicy, lintPolicy, policyHash } from "../cli/policy.js"; import type { GatePolicy } from "../cli/policy.js"; -import { buildRecord, extractResponse, isRecord } from "../cli/records.js"; +import { buildRecord, extractResponse, isRecord, readRecord } from "../cli/records.js"; import { canonicalJson, hashValue } from "../utils/hash.js"; import { cliVersion } from "../utils/package.js"; import { listPacks, loadPack } from "../commands/packs.js"; @@ -296,6 +296,20 @@ describe("records", () => { assert.equal(new Set(hashes).size, hashes.length, `state hashes must be distinct per input: ${JSON.stringify(hashes)}`); }); + it("detects record envelopes without claiming them valid", () => { + assert.equal(isRecord({ response: null }), true); + assert.throws(() => readRecord({ response: null }), /Unsupported record_version/); + assert.throws(() => readRecord({ nope: true }), /Invalid record/); + }); + + it("validates stored records through the advertised boundary", () => { + const record = buildRecord( + { pack: null, modelRequested: undefined, state: "claim text", questions: {} as Questions, latencyMs: 1 }, + MIXED_RESPONSE, + ); + assert.deepEqual(readRecord(record), record); + }); + it("rejects a record envelope that is malformed or from another version", () => { assert.throws(() => extractResponse({ record_version: 2, response: { answers: {} }, answers: { a: {} } }), /Unsupported record_version 2/); assert.throws(() => extractResponse({ record_version: 1, response: null }), /"response" must be an object/);