Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 13 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -36,14 +36,25 @@ Requires Node.js >= 22.0.0.

## The decision pipeline

Inference and policy are separate steps, so a judgment is reproducible and auditable:
Inference and policy remain separate primitives. `decide` composes them when a caller wants one request, one inspectable envelope, and the existing gate exit codes:

```bash
jev decide --pack verify --state-file claim.json --state-format json --record decisions/claim-1.json
jev decide --pack-file .jev/claim-pack.json --state-file claim.json --format decision
```

JSON output distinguishes `response`, `gate`, `provenance`, and correlation IDs. `--format decision` prints only `accept`, `review`, `deny`, or `abstain`; diagnostics remain on stderr. `decide` never executes a follow-up command. `ask` and `gate` remain independently usable.

`--pack-file` loads one project-owned pack at the exact path supplied. It uses the same question/policy validation and content hash as bundled packs; there is no registry or implicit search.

For the lower-level workflow:

```bash
# 1. Judge — questions from a bundled pack, state from you, record for the audit trail
jev ask --pack verify --state-file claim.json --state-format json --record decisions/claim-1.json

# 2. Decide — offline, no API call, exit code is the decision
jev gate --input decisions/claim-1.json --pack verify; echo "exit $?" # 0 2 3 4
jev gate --input decisions/claim-1.json --pack verify; echo "exit $?"

# 3. Show your work — re-emit the stored answers without paying again
jev replay --record decisions/claim-1.json
Expand Down
4 changes: 2 additions & 2 deletions package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 2 additions & 2 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
"dist/",
"packs/",
"examples/",
"README.md",
"tools/",
"LICENSE"
],
"keywords": [
Expand Down Expand Up @@ -52,7 +52,7 @@
},
"type": "module",
"types": "dist/index.d.ts",
"version": "0.1.3",
"version": "0.2.0",
"dependencies": {
"@typesafe-ai/sdk": "^0.6.0"
}
Expand Down
14 changes: 9 additions & 5 deletions src/cli.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,27 +4,28 @@ import { APIError, TypeSafeError } from "@typesafe-ai/sdk";
import { parseGlobal, extractGlobalOpts } from "./cli/parseArgs.js";
import type { OutputFormat } from "./cli/formatters.js";
import { formatJsonError } from "./cli/formatters.js";
import { ASK_HELP, BATCH_HELP, DOCTOR_HELP, GATE_HELP, MAIN_HELP, MODELS_HELP, PACKS_HELP, REPLAY_HELP } from "./cli/help.js";
import { ASK_HELP, BATCH_HELP, DECIDE_HELP, DOCTOR_HELP, GATE_HELP, MAIN_HELP, MODELS_HELP, PACKS_HELP, REPLAY_HELP } from "./cli/help.js";
import { resolveApiKey } from "./utils/validation.js";
import { cliVersion } from "./utils/package.js";
import { handleBatch, handleLint, handleModels } from "./commands/ops.js";
import type { ClientOpts } from "./api/client.js";
import { clientOpts, handleAsk, handleChoice, handleNoul, handleScore } from "./commands/requests.js";
import { handleGate } from "./commands/gate.js";
import { handleReplay } from "./commands/replay.js";
import { handlePacks } from "./commands/packs.js";
import { handleDoctor } from "./commands/doctor.js";
import { handlePacks } from "./commands/packs.js";
import { handleDecide } from "./commands/decide.js";

const HELP_BY_COMMAND: Record<string, string> = {
ask: ASK_HELP,
batch: BATCH_HELP,
decide: DECIDE_HELP,
models: MODELS_HELP,
gate: GATE_HELP,
replay: REPLAY_HELP,
packs: PACKS_HELP,
doctor: DOCTOR_HELP,
};

async function main(): Promise<void> {
const argv = process.argv.slice(2);

Expand Down Expand Up @@ -59,8 +60,11 @@ async function main(): Promise<void> {
await handleLint(global);
return;

// Offline commands: no API key required, so policies and records can be
// evaluated in CI and in environments that must not hold a key.
case "decide": {
const opts = clientOpts(global, resolveApiKey(global.apiKey));
await handleDecide(global, opts, format);
return;
}
case "gate":
await handleGate(global, format);
return;
Expand Down
3 changes: 2 additions & 1 deletion src/cli/formatters.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import { estimateCostUsd } from "../api/client.js";
import type { DecisionRecord } from "./records.js";
import type { GateResult } from "./policy.js";

export type OutputFormat = "json" | "table";
export type OutputFormat = "json" | "table" | "decision";

export interface DoctorCheck {
name: string;
Expand Down Expand Up @@ -99,6 +99,7 @@ function formatTable(response: SystemOneResult<Questions>): string {
}

export function formatGate(result: GateResult, provenance: GateProvenance, format: OutputFormat): string {
if (format === "decision") return result.decision;
if (format === "json") return JSON.stringify({ gate: result, provenance }, null, 2);
const lines = [
`decision: ${result.decision} (exit ${result.exit_code})`,
Expand Down
25 changes: 18 additions & 7 deletions src/cli/help.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ COMMANDS
choice <instructions> Pick one option (--option name="desc" ×2+)
score <instructions> Rate along levels (--level "desc" ×2+, lowest first)
ask Batch questions over one state in a single call
batch Many requests from a JSONL file, bounded concurrency
decide Judge and apply a pack policy in one call
gate Apply a policy to a saved judgment — no API call
replay Re-emit a stored record — no API call
packs List the bundled question packs
Expand All @@ -26,9 +26,12 @@ STATE (noul/choice/score/ask)
REQUEST FILE (ask)
--request <path> Full request {"state","questions","model?"}
--questions <file> Questions map (with --state/--state-file/--stdin)
--pack <name> Bundled questions (jev packs) — "verify", "screen", "route"
--pack <name> Bundled questions/policy pack
--pack-file <path> Project-owned questions/policy pack
--record <path> Write a decision record (state hashed, not stored)

--run-id <id> Correlate related operations
--decision-id <id> Set the decision correlation ID
--parent-id <id> Link this decision to a parent operation
EXIT CODES (execution status)
0 success — inference completed, answers on stdout
1 usage, transport, or API error
Expand Down Expand Up @@ -75,6 +78,16 @@ DISCLAIMER
Unofficial, community-maintained. Not affiliated with TypeSafe.
Typed output guarantees the interface, not the truth — validate on your data.
`;
export const DECIDE_HELP = `jev decide — Run inference and policy together

USAGE
jev decide --pack <name> --state-file <path> [--record <path>]
jev decide --pack-file <path> --state-file <path> [--format decision]

The policy is explicit in the selected pack. JSON output separates response,
gate result, provenance, and correlation IDs. Exit codes are accept 0,
review 2, deny 3, abstain 4; execution failures remain 1.
`;

export const ASK_HELP = `jev ask — Questions over one state in a single call

Expand Down Expand Up @@ -124,11 +137,9 @@ USAGE
jev models

NOTE
The list carries aliases. Versioned IDs (e.g. jev-1.13.0) are accepted by
the model field whether or not they appear in the list. The response "model"
field reports the versioned ID that answered — log it.
The list carries aliases. Versioned IDs are accepted by the model field; the
response model field reports the versioned ID that answered — log it.
`;

export const GATE_HELP = `jev gate — Apply a policy to a saved judgment (no API call)

USAGE
Expand Down
24 changes: 20 additions & 4 deletions src/cli/parseArgs.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ export interface GlobalOptions {
model?: string;
baseUrl?: string;
logLevel?: string;
format: "json" | "table";
format: "json" | "table" | "decision";
request?: string;
state?: string;
stateFile?: string;
Expand All @@ -20,9 +20,14 @@ export interface GlobalOptions {
retries?: string;
concurrency?: string;
pack?: string;
packFile?: string;
policy?: string;
input?: string;
record?: string;
recordJsonl?: string;
runId?: string;
decisionId?: string;
parentId?: string;
live: boolean;
help: boolean;
version: boolean;
Expand All @@ -48,22 +53,28 @@ const OPTIONS = {
retries: { type: "string" as const },
concurrency: { type: "string" as const },
pack: { type: "string" as const },
"pack-file": { type: "string" as const },
policy: { type: "string" as const },
input: { type: "string" as const },
record: { type: "string" as const },
"record-jsonl": { type: "string" as const },
"run-id": { type: "string" as const },
"decision-id": { type: "string" as const },
"parent-id": { type: "string" as const },
live: { type: "boolean" as const, default: false },
help: { type: "boolean" as const, default: false },
version: { type: "boolean" as const, default: false },
};


export function parseGlobal(argv: string[]) {
return parseArgs({ args: argv, options: OPTIONS, strict: true, allowPositionals: true });
}

export function extractGlobalOpts(values: Record<string, unknown>): GlobalOptions {
const format = (values.format as string) || "json";
if (format !== "json" && format !== "table") {
throw new Error(`Invalid --format: "${format}". Expected json|table.`);
if (format !== "json" && format !== "table" && format !== "decision") {
throw new Error(`Invalid --format: "${format}". Expected json|table|decision.`);
}
const stateFormat = (values["state-format"] as string) || "text";
if (stateFormat !== "text" && stateFormat !== "json") {
Expand All @@ -79,7 +90,7 @@ export function extractGlobalOpts(values: Record<string, unknown>): GlobalOption
model: values.model as string | undefined,
baseUrl: values["base-url"] as string | undefined,
logLevel,
format: format as "json" | "table",
format: format as "json" | "table" | "decision",
request: values.request as string | undefined,
state: values.state as string | undefined,
stateFile: values["state-file"] as string | undefined,
Expand All @@ -94,9 +105,14 @@ export function extractGlobalOpts(values: Record<string, unknown>): GlobalOption
retries: values.retries as string | undefined,
concurrency: values.concurrency as string | undefined,
pack: values.pack as string | undefined,
packFile: values["pack-file"] as string | undefined,
policy: values.policy as string | undefined,
input: values.input as string | undefined,
record: values.record as string | undefined,
recordJsonl: values["record-jsonl"] as string | undefined,
runId: values["run-id"] as string | undefined,
decisionId: values["decision-id"] as string | undefined,
parentId: values["parent-id"] as string | undefined,
live: (values.live as boolean) || false,
help: (values.help as boolean) || false,
version: (values.version as boolean) || false,
Expand Down
15 changes: 14 additions & 1 deletion src/cli/records.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { writeFileSync } from "node:fs";
import { randomUUID } from "node:crypto";
import { appendFileSync, writeFileSync } from "node:fs";
import type { Questions, SystemOneResult } from "@typesafe-ai/sdk";
import { estimateCostUsd } from "../api/client.js";
import { hashValue, sha256Hex } from "../utils/hash.js";
Expand All @@ -16,6 +17,9 @@ export interface RecordPackRef {
// confidential text, and the hash is enough to commit to which input was judged.
export interface DecisionRecord {
record_version: number;
run_id?: string;
decision_id?: string;
parent_id?: string | null;
created_at: string;
cli_version: string;
pack: RecordPackRef | null;
Expand All @@ -28,6 +32,9 @@ export interface DecisionRecord {
}

export interface RecordInputs {
runId?: string;
decisionId?: string;
parentId?: string;
pack: RecordPackRef | null;
modelRequested: string | undefined;
state: unknown;
Expand All @@ -45,6 +52,9 @@ function stateHash(state: unknown): string | null {
export function buildRecord(inputs: RecordInputs, response: SystemOneResult<Questions>): DecisionRecord {
return {
record_version: RECORD_VERSION,
run_id: inputs.runId ?? randomUUID(),
decision_id: inputs.decisionId ?? randomUUID(),
parent_id: inputs.parentId ?? null,
created_at: new Date().toISOString(),
cli_version: cliVersion(),
pack: inputs.pack,
Expand All @@ -60,6 +70,9 @@ export function buildRecord(inputs: RecordInputs, response: SystemOneResult<Ques
export function writeRecord(path: string, record: DecisionRecord): void {
writeFileSync(path, JSON.stringify(record, null, 2) + "\n", "utf8");
}
export function appendRecordJsonl(path: string, record: DecisionRecord): void {
appendFileSync(path, JSON.stringify(record) + "\n", "utf8");
}

export function recordCost(record: DecisionRecord): { input_tokens: number; output_tokens: number; estimated_usd: number } {
return {
Expand Down
38 changes: 38 additions & 0 deletions src/commands/decide.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
import { mkdirSync } from "node:fs";
import { dirname, resolve } from "node:path";
import type { GlobalOptions } from "../cli/parseArgs.js";
import type { OutputFormat } from "../cli/formatters.js";
import { formatGate } from "../cli/formatters.js";
import { evaluatePolicy } from "../cli/policy.js";
import { buildRecord, writeRecord, appendRecordJsonl } from "../cli/records.js";
import { createClient, type ClientOpts } from "../api/client.js";
import { readJsonFile, readState } from "../utils/io.js";
import { abortSignal } from "./requests.js";
import { loadPack, loadPackFile } from "./packs.js";
import { resolvePolicy } from "./gate.js";
import { hashValue } from "../utils/hash.js";

export async function handleDecide(global: GlobalOptions, opts: ClientOpts, format: OutputFormat): Promise<void> {
if (global.pack === undefined && global.packFile === undefined) throw new Error("Missing policy pack: decide requires --pack <name> or --pack-file <path>.");
if (global.policy !== undefined) throw new Error("Conflicting inputs: decide uses the policy in --pack or --pack-file.");
const loaded = global.pack !== undefined ? loadPack(global.pack) : loadPackFile(global.packFile as string);
const state = await readState({ state: global.state, stateFile: global.stateFile, stateFormat: global.stateFormat, stdin: global.stdin, extra: [] });
const started = Date.now();
const response = await createClient(opts).systemOne({ state: state as never, questions: loaded.pack.questions, ...(global.model !== undefined ? { model: global.model } : {}) }, { signal: abortSignal() });
const record = buildRecord({ runId: global.runId, decisionId: global.decisionId, parentId: global.parentId, pack: { name: loaded.pack.name, pack_version: loaded.pack.pack_version, hash: loaded.hash }, modelRequested: global.model, state, questions: loaded.pack.questions, latencyMs: Date.now() - started }, response);
if (global.recordJsonl !== undefined) {
mkdirSync(dirname(resolve(global.recordJsonl)), { recursive: true });
appendRecordJsonl(global.recordJsonl, record);
}
if (global.record !== undefined) {
writeRecord(global.record, record);
process.stderr.write(`record written: ${global.record} (${record.decision_id})\n`);
}
const { policy, pack, questionsHash, source } = resolvePolicy(global);
const gate = evaluatePolicy(policy, response);
const provenance = { policy_source: source, pack, model: response.model ?? null, record_version: record.record_version, pack_hash_matches_record: true, questions_match_record: true, response_answers_hash: `sha256:${hashValue(response.answers ?? {})}` };
if (format === "decision") process.stdout.write(`${gate.decision}\n`);
else if (format === "json") process.stdout.write(JSON.stringify({ response, gate, provenance, record: { run_id: record.run_id, decision_id: record.decision_id } }, null, 2) + "\n");
else process.stdout.write(formatGate(gate, provenance, format) + "\n");
process.exitCode = gate.exit_code;
}
21 changes: 7 additions & 14 deletions src/commands/gate.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,9 @@ import { formatGate } from "../cli/formatters.js";
import type { GatePolicy } from "../cli/policy.js";
import { GATE_EXIT, coercePolicy, evaluatePolicy } from "../cli/policy.js";
import { extractResponse, isRecord, readRecord } from "../cli/records.js";
import { loadPack } from "./packs.js";
import { hashValue } from "../utils/hash.js";
import { loadPack, loadPackFile } from "./packs.js";
import { readJsonFile } from "../utils/io.js";
import { hashValue } from "../utils/hash.js";

export interface ResolvedPolicy {
policy: GatePolicy;
Expand All @@ -19,23 +19,16 @@ export interface ResolvedPolicy {
// Exactly one policy source. A pack carries its own policy, so `--pack verify` is
// enough; `--policy file.json` is for policies tuned on your own data.
export function resolvePolicy(global: GlobalOptions): ResolvedPolicy {
if (global.policy !== undefined && global.pack !== undefined) {
throw new Error("Conflicting inputs: --policy and --pack are mutually exclusive.");
}
if (global.pack !== undefined) {
const loaded = loadPack(global.pack);
return {
policy: loaded.pack.policy,
pack: { name: loaded.pack.name, pack_version: loaded.pack.pack_version, hash: loaded.hash },
questionsHash: `sha256:${hashValue(loaded.pack.questions)}`,
source: loaded.path,
};
if (global.policy !== undefined && (global.pack !== undefined || global.packFile !== undefined)) throw new Error("Conflicting inputs: --policy and pack options are mutually exclusive.");
if (global.pack !== undefined || global.packFile !== undefined) {
const loaded = global.pack !== undefined ? loadPack(global.pack) : loadPackFile(global.packFile as string);
return { policy: loaded.pack.policy, pack: { name: loaded.pack.name, pack_version: loaded.pack.pack_version, hash: loaded.hash }, questionsHash: `sha256:${hashValue(loaded.pack.questions)}`, source: loaded.path };
}
if (global.policy !== undefined) {
const policy = coercePolicy(readRecordOrJson(global.policy));
return { policy, pack: null, questionsHash: null, source: global.policy };
}
throw new Error("Missing policy: pass --pack <name> or --policy <file>. Run `jev packs` to list packs.");
throw new Error("Missing policy: pass --pack, --pack-file, or --policy. Run `jev packs` to list bundled packs.");
}

// A policy file may hold the policy itself or an object with a "policy" field.
Expand Down
2 changes: 1 addition & 1 deletion src/commands/ops.ts
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,8 @@ export async function handleBatch(global: GlobalOptions, opts: ClientOpts, forma
if (concurrency < 1) throw new Error(`Invalid --concurrency: "${global.concurrency}". Expected an integer in [1, 32].`);
const signal = abortSignal();
const client = createClient(opts);

const inFlight = new Map<number, Promise<SettledRow>>();

const buffered = new Map<number, SettledRow>();
let nextIndex = 0;
let failures = 0;
Expand Down
Loading
Loading