diff --git a/CHANGELOG.md b/CHANGELOG.md index ec2cabb..23db23a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,20 @@ Release tags use the form `vX.Y.Z` and match `package.json`. GitHub Releases car ## [Unreleased] +## [0.5.15] - 2026-09-23 + +### Changed + +- Provider-specific cache shaping now runs after route selection, so Claude receives stable and rolling cache breakpoints while ChatGPT and xAI receive stable request cache keys without leaking internal metadata to arbitrary OpenAI-compatible providers. +- Claude OAuth requests now use the current Claude Code 2.1.280 client fingerprint. +- Usage normalization now reports cache reads, cache writes, uncached input, and logical input consistently across provider accounting conventions. + +### Fixed + +- Manual provider model validation is bounded instead of hanging indefinitely when an upstream accepts a connection but never answers. +- Claude cache prefixes remain reusable across human turns when volatile invocation context changes. +- Parallel Claude tool results, including supplemental image blocks, are serialized into one immediately following user message with all `tool_result` blocks first, preventing Anthropic request-order failures. + ## [0.5.12] - 2026-08-12 ### Fixed diff --git a/package-lock.json b/package-lock.json index 4d468c5..bd4c26a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@gitcommit90/rerouted", - "version": "0.5.14", + "version": "0.5.15", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@gitcommit90/rerouted", - "version": "0.5.14", + "version": "0.5.15", "license": "MIT", "bin": { "rerouted": "src/cli/index.js" diff --git a/package.json b/package.json index 043afc5..9261b88 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@gitcommit90/rerouted", "productName": "ReRouted", - "version": "0.5.14", + "version": "0.5.15", "description": "A local AI router for connected accounts, models, named routes, and automatic fallback.", "author": "gitcommit90", "license": "MIT", diff --git a/src/lib/model-test.js b/src/lib/model-test.js index eb695fe..14a6232 100644 --- a/src/lib/model-test.js +++ b/src/lib/model-test.js @@ -3,6 +3,7 @@ const { redactString } = require("./logger"); const MAX_ERROR_BODY_LENGTH = 4096; +const DEFAULT_MODEL_TEST_TIMEOUT_MS = 60_000; function safeErrorBody(value, maxLength = MAX_ERROR_BODY_LENGTH) { const body = redactString(value); @@ -85,10 +86,22 @@ function logFailure(logger, label, status, body) { logger?.error?.(`Model test failed for ${label}`, { status, body: safeErrorBody(body) }); } -async function runProviderModelTest({ adapter, provider, model, onTokenRefresh, logger } = {}) { +async function runProviderModelTest({ adapter, provider, model, onTokenRefresh, logger, timeoutMs = DEFAULT_MODEL_TEST_TIMEOUT_MS } = {}) { const label = `${provider?.name || provider?.type || "provider"}/${model}`; + const controller = new AbortController(); + const boundedTimeoutMs = Math.max(1, Number(timeoutMs) || DEFAULT_MODEL_TEST_TIMEOUT_MS); + const timeoutError = new Error(`model test timed out after ${boundedTimeoutMs}ms`); + timeoutError.name = "TimeoutError"; + timeoutError.code = "ETIMEDOUT"; + let timer; + const timeout = new Promise((_resolve, reject) => { + timer = setTimeout(() => { + controller.abort(timeoutError); + reject(timeoutError); + }, boundedTimeoutMs); + }); try { - const result = await adapter.chat( + const request = adapter.chat( { ...provider }, { model, @@ -99,9 +112,11 @@ async function runProviderModelTest({ adapter, provider, model, onTokenRefresh, stream: false, }, stream: false, + signal: controller.signal, onTokenRefresh, } ); + const result = await Promise.race([request, timeout]); const response = result && result.response ? result.response : result; const inspection = await inspectModelTestResponse(response); if (!inspection.ok) { @@ -116,10 +131,13 @@ async function runProviderModelTest({ adapter, provider, model, onTokenRefresh, const message = safeErrorBody(error?.message || String(error)); logFailure(logger, label, error?.status || null, message); return { ok: false, error: `Model test failed: ${message}` }; + } finally { + clearTimeout(timer); } } module.exports = { + DEFAULT_MODEL_TEST_TIMEOUT_MS, MAX_ERROR_BODY_LENGTH, bodyHasUpstreamError, inspectModelTestResponse, diff --git a/src/lib/providers/chatgpt.js b/src/lib/providers/chatgpt.js index 79e7c13..24c9377 100644 --- a/src/lib/providers/chatgpt.js +++ b/src/lib/providers/chatgpt.js @@ -130,11 +130,30 @@ function toolArguments(value) { function toResponsesInput(messages, model, reasoningScope) { const input = []; const instructions = []; + const deferredDynamicContext = []; for (const message of messages || []) { if (!message || typeof message !== "object") continue; if (message.role === "system") { - instructions.push(textFromOpenAiContent(message.content)); + const cacheScope = message.extra_content?.openai?.cache_scope; + const text = textFromOpenAiContent(message.content); + if (cacheScope === "dynamic_context") { + deferredDynamicContext.push({ + type: "message", + role: "developer", + content: toResponsesContent(message.content, "developer"), + }); + } else if (cacheScope === "inline_context") { + input.push({ + type: "message", + role: "developer", + content: toResponsesContent(message.content, "developer"), + }); + } else { + // Unmarked clients retain the historical behavior. 1Helm explicitly + // marks only its durable identity/capability blocks as instructions. + instructions.push(text); + } continue; } if (message.role === "tool") { @@ -195,6 +214,21 @@ function toResponsesInput(messages, model, reasoningScope) { } } + if (deferredDynamicContext.length) { + // 1Helm's volatile time, recalled memory, session state, and invocation + // evidence belong immediately before the current user turn. Keeping them + // out of `instructions` leaves the durable instructions + append-only + // conversation as an exact provider-cache prefix across turns. + let insertionIndex = input.length; + for (let index = input.length - 1; index >= 0; index--) { + if (input[index]?.type === "message" && input[index]?.role === "user") { + insertionIndex = index; + break; + } + } + input.splice(insertionIndex, 0, ...deferredDynamicContext); + } + return { input, instructions }; } @@ -252,6 +286,9 @@ function toResponsesBody(body, model, stream, { reasoningScope } = {}) { if (body.parallel_tool_calls !== undefined) { out.parallel_tool_calls = body.parallel_tool_calls; } + if (body.prompt_cache_key !== undefined) { + out.prompt_cache_key = body.prompt_cache_key; + } const include = Array.isArray(body.include) ? [...body.include] : []; if (!include.includes("reasoning.encrypted_content")) { include.push("reasoning.encrypted_content"); diff --git a/src/lib/providers/claude.js b/src/lib/providers/claude.js index b9c710f..43701e8 100644 --- a/src/lib/providers/claude.js +++ b/src/lib/providers/claude.js @@ -9,7 +9,7 @@ const cfg = OAUTH.claude; const ANTHROPIC_METADATA = Symbol.for("rerouted.anthropic.metadata"); /** Provider-compatible request metadata and system-block shaping. */ -const CLAUDE_CLI_VERSION = "2.1.251"; +const CLAUDE_CLI_VERSION = "2.1.280"; const CC_ENTRYPOINT = "cli"; const ANTHROPIC_BETA = @@ -135,7 +135,7 @@ function wrapSystemReminder(text) { ); } -function prependToFirstUserMessage(messages, text) { +function prependToFirstUserMessage(messages, text, cacheControl) { if (!Array.isArray(messages) || !messages.length) return messages; const out = messages.map((m) => ({ ...m, @@ -144,15 +144,17 @@ function prependToFirstUserMessage(messages, text) { const idx = out.findIndex((m) => m.role === "user"); if (idx < 0) return out; const m = out[idx]; + const reminder = { + type: "text", + text, + ...(cacheControl ? { cache_control: cacheControl } : {}), + }; if (typeof m.content === "string") { - out[idx] = { ...m, content: text + m.content }; + out[idx] = { ...m, content: [reminder, { type: "text", text: m.content }] }; } else if (Array.isArray(m.content)) { - out[idx] = { - ...m, - content: [{ type: "text", text }, ...m.content], - }; + out[idx] = { ...m, content: [reminder, ...m.content] }; } else { - out[idx] = { ...m, content: [{ type: "text", text: text + String(m.content ?? "") }] }; + out[idx] = { ...m, content: [reminder, { type: "text", text: String(m.content ?? "") }] }; } return out; } @@ -179,11 +181,14 @@ function applyCloaking(body, accessToken, sessionId) { // Capture original client system for move to user message let userSystemText = ""; + let userSystemCacheControl; if (Array.isArray(result.system)) { userSystemText = result.system .map((b) => (typeof b === "string" ? b : b?.text || "")) .filter(Boolean) .join("\n\n"); + userSystemCacheControl = result.system.findLast?.((block) => block?.cache_control)?.cache_control + || [...result.system].reverse().find((block) => block?.cache_control)?.cache_control; } else if (typeof result.system === "string") { userSystemText = result.system; } @@ -198,7 +203,8 @@ function applyCloaking(body, accessToken, sessionId) { if (sanitized) { result.messages = prependToFirstUserMessage( result.messages || [], - wrapSystemReminder(sanitized) + wrapSystemReminder(sanitized), + userSystemCacheControl ); } } @@ -392,6 +398,84 @@ function contentBlocksFromMessage(msg) { return blocks; } +function hasExplicitCacheControl(payload) { + const system = Array.isArray(payload.system) ? payload.system : []; + const messageBlocks = (payload.messages || []).flatMap((message) => + Array.isArray(message.content) ? message.content : [] + ); + return [...system, ...messageBlocks].some((block) => block?.cache_control); +} + +/** Apply Claude's four-breakpoint policy only after the router has selected + * Claude. Explicit client-authored breakpoints remain authoritative. */ +function applyAutomaticCacheControl(payload, preferredBaseBlocks = []) { + if (hasExplicitCacheControl(payload)) return payload; + + const baseBlocks = [...new Set((Array.isArray(preferredBaseBlocks) ? preferredBaseBlocks : [preferredBaseBlocks]).filter(Boolean))]; + if (!baseBlocks.length) { + let baseBlock = null; + for (const message of payload.messages || []) { + if (message.role !== "user" || !Array.isArray(message.content)) continue; + if (message.content.some((block) => block?.type === "tool_result")) continue; + for (let index = message.content.length - 1; index >= 0; index--) { + const block = message.content[index]; + if (block?.type === "text" || block?.type === "image") { + baseBlock = block; + break; + } + } + } + if (baseBlock) baseBlocks.push(baseBlock); + } + for (const block of baseBlocks) block.cache_control = { type: "ephemeral" }; + + const toolResults = []; + for (const message of payload.messages || []) { + for (const block of Array.isArray(message.content) ? message.content : []) { + if (block?.type === "tool_result") toolResults.push(block); + } + } + const availableToolBreakpoints = Math.max(0, 4 - baseBlocks.length); + if (availableToolBreakpoints > 0) { + for (const block of toolResults.slice(-availableToolBreakpoints)) { + block.cache_control = { type: "ephemeral" }; + } + } + return payload; +} + +function systemCacheScope(message) { + return message?.extra_content?.openai?.cache_scope || ""; +} + +function systemTextBlocks(message) { + const nativeSystem = message?.[ANTHROPIC_METADATA]?.content; + if (Array.isArray(nativeSystem)) return JSON.parse(JSON.stringify(nativeSystem)); + if (typeof nativeSystem === "string") return [{ type: "text", text: nativeSystem }]; + if (typeof message?.content === "string") return [{ type: "text", text: message.content }]; + if (Array.isArray(message?.content)) { + return message.content + .filter((part) => part?.type === "text" && part.text != null) + .map((part) => ({ + type: "text", + text: String(part.text), + ...(part.cache_control ? { cache_control: part.cache_control } : {}), + })); + } + return message?.content != null ? [{ type: "text", text: String(message.content) }] : []; +} + +function lastCacheableMessageBlock(messages) { + for (let messageIndex = messages.length - 1; messageIndex >= 0; messageIndex--) { + const blocks = Array.isArray(messages[messageIndex]?.content) ? messages[messageIndex].content : []; + for (let blockIndex = blocks.length - 1; blockIndex >= 0; blockIndex--) { + const block = blocks[blockIndex]; + if (["text", "image", "tool_result"].includes(block?.type)) return block; + } + } + return null; +} + /** * Convert OpenAI chat messages → Anthropic messages + system + tools. * Critical: tool_result must be in a user message; tool_use only on assistant. @@ -400,13 +484,33 @@ function contentBlocksFromMessage(msg) { function toAnthropicBody(body, model, stream) { const systemParts = []; const messages = []; + const sourceMessages = body.messages || []; + const hasScopedSystem = sourceMessages.some((message) => message.role === "system" && systemCacheScope(message)); + const deferredDynamicSystemParts = []; + const lastDynamicSystemIndex = sourceMessages.findLastIndex?.( + (message) => message.role === "system" && systemCacheScope(message) === "dynamic_context" + ) ?? -1; + const preferredCacheBases = []; + + const userBlocksWithToolResultsFirst = (blocks) => { + const toolResults = blocks.filter((block) => block?.type === "tool_result"); + if (!toolResults.length) return blocks; + return [...toolResults, ...blocks.filter((block) => block?.type !== "tool_result")]; + }; const pushMessage = (role, blocks) => { if (!blocks.length) return; - // Merge consecutive same-role messages only when no tool blocks (simple text chats) - const hasTool = - blocks.some((b) => b.type === "tool_use" || b.type === "tool_result"); const last = messages[messages.length - 1]; + // Anthropic requires every result for one parallel assistant tool batch in + // the immediately following *single* user message, with tool_result blocks + // before text/images. OpenAI-compatible clients may emit each result and + // supplemental image evidence as separate consecutive user messages. + if (role === "user" && last?.role === "user" && Array.isArray(last.content)) { + last.content = userBlocksWithToolResultsFirst([...last.content, ...blocks]); + return; + } + // Keep the prior simple-text coalescing behavior for assistant messages. + const hasTool = blocks.some((b) => b.type === "tool_use" || b.type === "tool_result"); if ( last && last.role === role && @@ -417,35 +521,31 @@ function toAnthropicBody(body, model, stream) { last.content.push(...blocks); return; } - messages.push({ role, content: blocks }); + messages.push({ role, content: role === "user" ? userBlocksWithToolResultsFirst(blocks) : blocks }); }; - for (const m of body.messages || []) { + for (let sourceIndex = 0; sourceIndex < sourceMessages.length; sourceIndex++) { + const m = sourceMessages[sourceIndex]; if (m.role === "system") { - const nativeSystem = m[ANTHROPIC_METADATA]?.content; - if (Array.isArray(nativeSystem)) { - systemParts.push(...JSON.parse(JSON.stringify(nativeSystem))); + const parts = systemTextBlocks(m); + const scope = systemCacheScope(m); + if (!hasScopedSystem || scope === "stable_instruction" || !scope) { + systemParts.push(...parts); continue; } - if (typeof nativeSystem === "string") { - systemParts.push({ type: "text", text: nativeSystem }); + if (scope === "dynamic_context") { + deferredDynamicSystemParts.push(...parts); + if (sourceIndex !== lastDynamicSystemIndex) continue; + const stableBase = [...systemParts].reverse().find((block) => block?.type === "text" || block?.type === "image") || null; + const rollingBase = lastCacheableMessageBlock(messages); + if (stableBase) preferredCacheBases.push(stableBase); + if (rollingBase) preferredCacheBases.push(rollingBase); + const text = deferredDynamicSystemParts.map((part) => part?.text || "").filter(Boolean).join("\n\n"); + if (text) pushMessage("user", [{ type: "text", text: wrapSystemReminder(text) }]); continue; } - if (typeof m.content === "string") { - systemParts.push({ type: "text", text: m.content }); - } else if (Array.isArray(m.content)) { - for (const part of m.content) { - if (part?.type === "text" && part.text != null) { - systemParts.push({ - type: "text", - text: String(part.text), - ...(part.cache_control ? { cache_control: part.cache_control } : {}), - }); - } - } - } else if (m.content != null) { - systemParts.push({ type: "text", text: String(m.content) }); - } + const text = parts.map((part) => part?.text || "").filter(Boolean).join("\n\n"); + if (text) pushMessage("user", [{ type: "text", text: wrapSystemReminder(text) }]); continue; } const blocks = contentBlocksFromMessage(m); @@ -476,7 +576,7 @@ function toAnthropicBody(body, model, stream) { stream: !!stream, }; if (systemParts.length) { - out.system = systemParts.some((part) => part.cache_control) + out.system = hasScopedSystem || systemParts.some((part) => part.cache_control) ? systemParts : systemParts.map((part) => part.text).join("\n\n"); } @@ -521,7 +621,7 @@ function toAnthropicBody(body, model, stream) { for (const [key, value] of Object.entries(body[ANTHROPIC_METADATA]?.options || {})) { out[key] = JSON.parse(JSON.stringify(value)); } - return applyClaudeEffort(out, body, model); + return applyAutomaticCacheControl(applyClaudeEffort(out, body, model), preferredCacheBases); } function mapStopReason(stopReason) { @@ -613,6 +713,8 @@ async function pipeAnthropicSseToOpenAi( prompt_tokens: 0, completion_tokens: 0, cached_tokens: 0, + cache_read_tokens: 0, + cache_write_tokens: 0, total_tokens: 0, }; if (usage.input_tokens != null) current.prompt_tokens = Number(usage.input_tokens) || 0; @@ -621,6 +723,11 @@ async function pipeAnthropicSseToOpenAi( } if (usage.cache_read_input_tokens != null) { current.cached_tokens = Number(usage.cache_read_input_tokens) || 0; + current.cache_read_tokens = Number(usage.cache_read_input_tokens) || 0; + } + if (usage.cache_creation_input_tokens != null) { + current.cache_write_tokens = Number(usage.cache_creation_input_tokens) || 0; + current.cache_creation_input_tokens = Number(usage.cache_creation_input_tokens) || 0; } current.total_tokens = current.prompt_tokens + current.completion_tokens; streamUsage = current; @@ -861,5 +968,6 @@ module.exports = { generateFakeUserId, anthropicHeaders, stableSessionId, + applyAutomaticCacheControl, cfg, }; diff --git a/src/lib/providers/openai-compat.js b/src/lib/providers/openai-compat.js index 6054d51..4c071cb 100644 --- a/src/lib/providers/openai-compat.js +++ b/src/lib/providers/openai-compat.js @@ -186,8 +186,11 @@ async function listModels(provider, { fetchImpl = fetch, timeoutMs = MODELS_TIME */ async function chat(provider, { model, body, stream, signal, fetchImpl = fetch } = {}) { const url = joinUrl(provider.baseUrl, "chat/completions"); + // prompt_cache_key is provider-selected metadata for ChatGPT/xAI. Do not + // leak it to arbitrary OpenAI-compatible servers that may reject extensions. + const { prompt_cache_key: _promptCacheKey, ...compatibleBody } = body; const payload = { - ...body, + ...compatibleBody, ...(body.generationConfig ? { generationConfig: { ...body.generationConfig } } : {}), diff --git a/src/lib/usage.js b/src/lib/usage.js index e49b9d3..3da0e64 100644 --- a/src/lib/usage.js +++ b/src/lib/usage.js @@ -6,7 +6,7 @@ const { DatabaseSync } = require("node:sqlite"); const { KEYED_PRESETS, OAUTH } = require("./constants"); const RECENT_UI = 80; -const SCHEMA_VERSION = 1; +const SCHEMA_VERSION = 2; const LEGACY_MIGRATION_KEY = "legacy_usage_json_migrated"; const PERIODS = { @@ -32,8 +32,22 @@ function extractUsage(openAiJson) { u.cached_tokens || 0 ) || 0; + const cacheWrite = Number( + u.cache_creation_input_tokens || + u.prompt_tokens_details?.cache_creation_tokens || + u.input_tokens_details?.cache_creation_tokens || + u.cache_write_tokens || + 0 + ) || 0; const total = Number(u.total_tokens || prompt + completion) || prompt + completion; - return { prompt_tokens: prompt, completion_tokens: completion, cached_tokens: cached, total_tokens: total }; + return { + prompt_tokens: prompt, + completion_tokens: completion, + cached_tokens: cached, + cache_read_tokens: cached, + cache_write_tokens: cacheWrite, + total_tokens: total, + }; } function canonicalProviderType(type) { @@ -96,19 +110,41 @@ function numeric(value) { } function normalizedRow(entry, at = Date.now()) { + const providerType = canonicalProviderType(entry?.providerType) || null; + const promptTokens = numeric(entry?.prompt_tokens); + const cacheReadTokens = numeric(entry?.cache_read_tokens ?? entry?.cached_tokens); + const cacheWriteTokens = numeric(entry?.cache_write_tokens); + const excludesCacheTokens = providerType === "claude"; + const tokenSemantics = entry?.token_semantics || + (excludesCacheTokens ? "input_excludes_cache_read_write" : "input_includes_cache_read"); + const logicalInputTokens = entry?.logical_input_tokens != null + ? numeric(entry.logical_input_tokens) + : excludesCacheTokens + ? promptTokens + cacheReadTokens + cacheWriteTokens + : promptTokens; + const uncachedInputTokens = entry?.uncached_input_tokens != null + ? numeric(entry.uncached_input_tokens) + : excludesCacheTokens + ? promptTokens + cacheWriteTokens + : Math.max(0, promptTokens - cacheReadTokens); return { at, model: entry?.model || null, upstream: entry?.upstream || null, providerId: entry?.providerId || null, - providerType: entry?.providerType || null, + providerType, providerName: entry?.providerName || null, accountAlias: entry?.accountAlias || null, status: numeric(entry?.status), stream: !!entry?.stream, - prompt_tokens: numeric(entry?.prompt_tokens), + prompt_tokens: promptTokens, completion_tokens: numeric(entry?.completion_tokens), - cached_tokens: numeric(entry?.cached_tokens), + cached_tokens: cacheReadTokens, + cache_read_tokens: cacheReadTokens, + cache_write_tokens: cacheWriteTokens, + uncached_input_tokens: uncachedInputTokens, + logical_input_tokens: logicalInputTokens, + token_semantics: tokenSemantics, total_tokens: numeric(entry?.total_tokens), error: entry?.error || null, }; @@ -128,6 +164,11 @@ function insertValues(row, preservePayload = false) { numeric(row.prompt_tokens), numeric(row.completion_tokens), numeric(row.cached_tokens), + numeric(row.cache_read_tokens ?? row.cached_tokens), + numeric(row.cache_write_tokens), + numeric(row.uncached_input_tokens ?? (row.providerType === "claude" ? numeric(row.prompt_tokens) : Math.max(0, numeric(row.prompt_tokens) - numeric(row.cached_tokens)))), + numeric(row.logical_input_tokens ?? (row.providerType === "claude" ? numeric(row.prompt_tokens) + numeric(row.cached_tokens) : numeric(row.prompt_tokens))), + row.token_semantics || (row.providerType === "claude" ? "input_excludes_cache_read_write" : "input_includes_cache_read"), numeric(row.total_tokens), typeof row.error === "string" || row.error == null ? row.error : JSON.stringify(row.error), preservePayload ? JSON.stringify(row) : null, @@ -157,6 +198,11 @@ function decodeRow(row) { prompt_tokens: row.prompt_tokens, completion_tokens: row.completion_tokens, cached_tokens: row.cached_tokens, + cache_read_tokens: row.cache_read_tokens ?? row.cached_tokens, + cache_write_tokens: row.cache_write_tokens || 0, + uncached_input_tokens: row.uncached_input_tokens ?? Math.max(0, row.prompt_tokens - row.cached_tokens), + logical_input_tokens: row.logical_input_tokens ?? row.prompt_tokens, + token_semantics: row.token_semantics || "input_includes_cache_read", total_tokens: row.total_tokens, error: row.error, }; @@ -262,6 +308,11 @@ function initializeDatabase(db) { prompt_tokens INTEGER NOT NULL, completion_tokens INTEGER NOT NULL, cached_tokens INTEGER NOT NULL, + cache_read_tokens INTEGER NOT NULL DEFAULT 0, + cache_write_tokens INTEGER NOT NULL DEFAULT 0, + uncached_input_tokens INTEGER NOT NULL DEFAULT 0, + logical_input_tokens INTEGER NOT NULL DEFAULT 0, + token_semantics TEXT NOT NULL DEFAULT 'input_includes_cache_read', total_tokens INTEGER NOT NULL, error TEXT, payload_json TEXT, @@ -276,7 +327,35 @@ function initializeDatabase(db) { prompt_tokens, completion_tokens ); `); - db.prepare("INSERT OR IGNORE INTO usage_meta (key, value) VALUES ('schema_version', ?)").run( + const columns = new Set(db.prepare("PRAGMA table_info(usage_events)").all().map((column) => column.name)); + const additions = [ + ["cache_read_tokens", "INTEGER NOT NULL DEFAULT 0"], + ["cache_write_tokens", "INTEGER NOT NULL DEFAULT 0"], + ["uncached_input_tokens", "INTEGER NOT NULL DEFAULT 0"], + ["logical_input_tokens", "INTEGER NOT NULL DEFAULT 0"], + ["token_semantics", "TEXT NOT NULL DEFAULT 'input_includes_cache_read'"], + ]; + for (const [name, definition] of additions) { + if (!columns.has(name)) db.exec(`ALTER TABLE usage_events ADD COLUMN ${name} ${definition}`); + } + db.exec(` + UPDATE usage_events SET + cache_read_tokens = cached_tokens, + uncached_input_tokens = CASE + WHEN provider_type = 'claude' THEN prompt_tokens + ELSE MAX(prompt_tokens - cached_tokens, 0) + END, + logical_input_tokens = CASE + WHEN provider_type = 'claude' THEN prompt_tokens + cached_tokens + ELSE prompt_tokens + END, + token_semantics = CASE + WHEN provider_type = 'claude' THEN 'input_excludes_cache_read_write' + ELSE 'input_includes_cache_read' + END + WHERE logical_input_tokens = 0 AND (prompt_tokens > 0 OR cached_tokens > 0) + `); + db.prepare("INSERT INTO usage_meta (key, value) VALUES ('schema_version', ?) ON CONFLICT(key) DO UPDATE SET value=excluded.value").run( String(SCHEMA_VERSION) ); } @@ -327,9 +406,10 @@ function createUsageStore(databasePath, { legacyPath } = {}) { const insert = db.prepare(` INSERT INTO usage_events ( at, model, upstream, provider_id, provider_type, provider_name, account_alias, - status, stream, prompt_tokens, completion_tokens, cached_tokens, total_tokens, - error, payload_json, provider_key - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + status, stream, prompt_tokens, completion_tokens, cached_tokens, + cache_read_tokens, cache_write_tokens, uncached_input_tokens, logical_input_tokens, token_semantics, + total_tokens, error, payload_json, provider_key + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) `); migrateLegacyJson(db, insert, legacyPath); @@ -358,6 +438,10 @@ function createUsageStore(databasePath, { legacyPath } = {}) { COALESCE(SUM(prompt_tokens), 0) AS prompt_tokens, COALESCE(SUM(completion_tokens), 0) AS completion_tokens, COALESCE(SUM(cached_tokens), 0) AS cached_tokens, + COALESCE(SUM(cache_read_tokens), 0) AS cache_read_tokens, + COALESCE(SUM(cache_write_tokens), 0) AS cache_write_tokens, + COALESCE(SUM(uncached_input_tokens), 0) AS uncached_input_tokens, + COALESCE(SUM(logical_input_tokens), 0) AS logical_input_tokens, COALESCE(SUM(total_tokens), 0) AS total_tokens FROM usage_events ${period.sql} `) @@ -370,7 +454,11 @@ function createUsageStore(databasePath, { legacyPath } = {}) { COUNT(*) AS requests, COALESCE(SUM(prompt_tokens), 0) AS prompt_tokens, COALESCE(SUM(completion_tokens), 0) AS completion_tokens, - COALESCE(SUM(cached_tokens), 0) AS cached_tokens + COALESCE(SUM(cached_tokens), 0) AS cached_tokens, + COALESCE(SUM(cache_read_tokens), 0) AS cache_read_tokens, + COALESCE(SUM(cache_write_tokens), 0) AS cache_write_tokens, + COALESCE(SUM(uncached_input_tokens), 0) AS uncached_input_tokens, + COALESCE(SUM(logical_input_tokens), 0) AS logical_input_tokens FROM usage_events ${period.sql} GROUP BY COALESCE(model, 'unknown') ORDER BY requests DESC, MAX(id) DESC @@ -395,15 +483,22 @@ function createUsageStore(databasePath, { legacyPath } = {}) { COUNT(*) AS requests, COALESCE(SUM(prompt_tokens), 0) AS prompt_tokens, COALESCE(SUM(completion_tokens), 0) AS completion_tokens, + COALESCE(SUM(cache_read_tokens), 0) AS cache_read_tokens, + COALESCE(SUM(cache_write_tokens), 0) AS cache_write_tokens, + COALESCE(SUM(uncached_input_tokens), 0) AS uncached_input_tokens, + COALESCE(SUM(logical_input_tokens), 0) AS logical_input_tokens, + MIN(token_semantics) = MAX(token_semantics) AS uniform_semantics, + MIN(token_semantics) AS token_semantics_value, MAX(id) AS newest_id FROM usage_events ${period.sql} GROUP BY provider_key ORDER BY requests DESC, newest_id DESC `) .all(...period.params) - .map(({ provider_key: _providerKey, newest_id: _newestId, ...entry }) => ({ + .map(({ provider_key: _providerKey, newest_id: _newestId, uniform_semantics: uniform, token_semantics_value: semantics, ...entry }) => ({ provider: providerAggregateLabel(entry), ...entry, + token_semantics: uniform ? semantics : "mixed", })); const recentRows = db @@ -419,6 +514,10 @@ function createUsageStore(databasePath, { legacyPath } = {}) { prompt_tokens: totals.prompt_tokens, completion_tokens: totals.completion_tokens, cached_tokens: totals.cached_tokens, + cache_read_tokens: totals.cache_read_tokens, + cache_write_tokens: totals.cache_write_tokens, + uncached_input_tokens: totals.uncached_input_tokens, + logical_input_tokens: totals.logical_input_tokens, total_tokens: totals.total_tokens, byModel, byProvider, diff --git a/tests/cache-shaping.test.js b/tests/cache-shaping.test.js new file mode 100644 index 0000000..42974d2 --- /dev/null +++ b/tests/cache-shaping.test.js @@ -0,0 +1,133 @@ +"use strict"; + +const { describe, it } = require("node:test"); +const assert = require("node:assert/strict"); +const chatgpt = require("../src/lib/providers/chatgpt"); +const claude = require("../src/lib/providers/claude"); +const openaiCompat = require("../src/lib/providers/openai-compat"); + +const scoped = (content, cache_scope) => ({ + role: "system", + content, + extra_content: { openai: { cache_scope } }, +}); + +describe("provider-selected cache shaping", () => { + it("keeps ChatGPT instructions stable and moves volatile context behind durable history", () => { + const payload = chatgpt.toResponsesBody({ + prompt_cache_key: "scope-key", + messages: [ + scoped("stable identity", "stable_instruction"), + scoped("volatile 12:34:56", "dynamic_context"), + { role: "user", content: "old task" }, + { role: "assistant", content: "old answer" }, + scoped("prior invocation boundary", "inline_context"), + scoped("current invocation 42", "dynamic_context"), + { role: "user", content: "current task" }, + ], + }, "gpt-5.6-sol", true); + + assert.equal(payload.instructions, "stable identity"); + assert.equal(payload.prompt_cache_key, "scope-key"); + assert.deepEqual(payload.input.map((item) => [item.role, item.content?.[0]?.text]), [ + ["user", "old task"], + ["assistant", "old answer"], + ["developer", "prior invocation boundary"], + ["developer", "volatile 12:34:56"], + ["developer", "current invocation 42"], + ["user", "current task"], + ]); + }); + + it("adds Claude base and rolling tool-result breakpoints after Claude is selected", () => { + const messages = [ + { role: "system", content: "system" }, + { role: "user", content: "task" }, + ...[1, 2, 3, 4].flatMap((number) => [ + { role: "assistant", content: "", tool_calls: [{ id: `t${number}`, type: "function", function: { name: "shell", arguments: "{}" } }] }, + { role: "tool", tool_call_id: `t${number}`, content: `result ${number}` }, + ]), + ]; + const payload = claude.toAnthropicBody({ messages }, "claude-fable-5-1", true); + const blocks = payload.messages.flatMap((message) => message.content); + const marked = blocks.filter((block) => block.cache_control); + assert.equal(marked.length, 4); + assert.equal(blocks.find((block) => block.type === "text" && block.text === "task").cache_control.type, "ephemeral"); + assert.equal(blocks.find((block) => block.type === "tool_result" && block.tool_use_id === "t1").cache_control, undefined); + for (const id of ["t2", "t3", "t4"]) { + assert.equal(blocks.find((block) => block.type === "tool_result" && block.tool_use_id === id).cache_control.type, "ephemeral"); + } + }); + + it("keeps Claude volatile context behind durable history and preserves OAuth cache boundaries", () => { + const payload = claude.toAnthropicBody({ + messages: [ + scoped("stable identity", "stable_instruction"), + scoped("volatile 12:34:56", "dynamic_context"), + { role: "user", content: "old task" }, + { role: "assistant", content: "old answer" }, + scoped("prior invocation boundary", "inline_context"), + scoped("current invocation 42", "dynamic_context"), + { role: "user", content: "current task" }, + ], + }, "claude-opus-5-5", true); + + const cloaked = claude.applyCloaking(payload, "sk-ant-oat-test", "00000000-0000-4000-8000-000000000000"); + const blocks = cloaked.messages.flatMap((message) => message.content); + const stableIndex = blocks.findIndex((block) => block.text?.includes("stable identity")); + const oldTaskIndex = blocks.findIndex((block) => block.text === "old task"); + const oldAnswerIndex = blocks.findIndex((block) => block.text === "old answer"); + const inlineIndex = blocks.findIndex((block) => block.text?.includes("prior invocation boundary")); + const volatileIndex = blocks.findIndex((block) => block.text?.includes("volatile 12:34:56")); + const currentIndex = blocks.findIndex((block) => block.text === "current task"); + + assert.ok(stableIndex >= 0 && stableIndex < oldTaskIndex); + assert.ok(oldTaskIndex < oldAnswerIndex && oldAnswerIndex < inlineIndex); + assert.ok(inlineIndex < volatileIndex && volatileIndex < currentIndex); + assert.equal(blocks[stableIndex].cache_control.type, "ephemeral", "stable identity remains a reusable OAuth breakpoint"); + assert.equal(blocks[inlineIndex].cache_control.type, "ephemeral", "the durable history tail is cached before volatile context"); + assert.equal(blocks[volatileIndex].cache_control, undefined); + assert.doesNotMatch(blocks.slice(0, inlineIndex + 1).map((block) => block.text || "").join("\n"), /volatile 12:34:56|current invocation 42/); + }); + + it("reserves Claude breakpoints for stable/history prefixes before rolling tool results", () => { + const messages = [ + scoped("stable identity", "stable_instruction"), + scoped("volatile", "dynamic_context"), + { role: "user", content: "old task" }, + { role: "assistant", content: "old answer" }, + scoped("current invocation", "dynamic_context"), + { role: "user", content: "current task" }, + ...[1, 2, 3].flatMap((number) => [ + { role: "assistant", content: "", tool_calls: [{ id: `t${number}`, type: "function", function: { name: "shell", arguments: "{}" } }] }, + { role: "tool", tool_call_id: `t${number}`, content: `result ${number}` }, + ]), + ]; + const payload = claude.toAnthropicBody({ messages }, "claude-opus-5-5", true); + const blocks = [...(Array.isArray(payload.system) ? payload.system : []), ...payload.messages.flatMap((message) => message.content)]; + const marked = blocks.filter((block) => block.cache_control); + assert.equal(marked.length, 4); + assert.equal(blocks.find((block) => block.text === "stable identity").cache_control.type, "ephemeral"); + assert.equal(blocks.find((block) => block.text === "old answer").cache_control.type, "ephemeral"); + assert.equal(blocks.find((block) => block.type === "tool_result" && block.tool_use_id === "t1").cache_control, undefined); + for (const id of ["t2", "t3"]) { + assert.equal(blocks.find((block) => block.type === "tool_result" && block.tool_use_id === id).cache_control.type, "ephemeral"); + } + }); + + it("does not leak provider-selected cache keys to arbitrary compatible servers", async () => { + let sent; + await openaiCompat.chat({ baseUrl: "https://custom.test/v1", apiKey: "key" }, { + model: "custom-model", + body: { prompt_cache_key: "scoped-key", messages: [{ role: "user", content: "hi" }] }, + stream: false, + fetchImpl: async (_url, options) => { sent = JSON.parse(options.body); return new Response(JSON.stringify({ choices: [] }), { status: 200 }); }, + }); + assert.equal(sent.prompt_cache_key, undefined); + }); + + it("preserves explicit Claude cache policy without adding automatic breakpoints", () => { + const payload = claude.toAnthropicBody({ messages: [{ role: "user", content: [{ type: "text", text: "task", cache_control: { type: "ephemeral" } }] }] }, "claude-fable-5-1", false); + assert.equal(JSON.stringify(payload).match(/cache_control/g)?.length, 1); + }); +}); diff --git a/tests/gateway.test.js b/tests/gateway.test.js index 96a845a..c2bd88e 100644 --- a/tests/gateway.test.js +++ b/tests/gateway.test.js @@ -831,14 +831,14 @@ describe("claude oauth request shaping", () => { const h = seen.opts.headers; assert.equal(h["X-App"], "cli"); assert.ok(String(h["User-Agent"]).startsWith("claude-cli/")); - assert.ok(String(h["User-Agent"]).includes("2.1.251"), "new Claude models require the current supported CLI fingerprint"); + assert.ok(String(h["User-Agent"]).includes("2.1.280"), "new Claude models require the current supported CLI fingerprint"); assert.ok(String(h["User-Agent"]).includes("(external, cli)")); assert.ok(h["X-Stainless-Os"]); assert.ok(h["X-Claude-Code-Session-Id"]); assert.ok(String(h["Anthropic-Beta"]).includes("oauth-2025-04-20")); const body = JSON.parse(seen.opts.body); assert.ok(body.system?.[0]?.text?.startsWith("x-anthropic-billing-header:")); - assert.ok(body.system[0].text.includes("cc_version=2.1.251.")); + assert.ok(body.system[0].text.includes("cc_version=2.1.280.")); assert.equal(body.system.length, 3); assert.ok(body.metadata?.user_id); }); @@ -913,6 +913,41 @@ describe("format translation", () => { assert.equal(body.messages[2].content[0].tool_use_id, "call_1"); }); + it("groups parallel tool results and supplemental images into one Anthropic user message", () => { + const body = claude.toAnthropicBody( + { + messages: [ + { role: "user", content: "inspect both images" }, + { + role: "assistant", + content: null, + tool_calls: [ + { id: "call_a", type: "function", function: { name: "view_image", arguments: '{"path":"a.png"}' } }, + { id: "call_b", type: "function", function: { name: "view_image", arguments: '{"path":"b.png"}' } }, + { id: "call_c", type: "function", function: { name: "run_command", arguments: '{"command":"echo ok"}' } }, + ], + }, + { role: "tool", tool_call_id: "call_a", content: "viewed a" }, + { role: "user", content: [{ type: "text", text: "image a" }, { type: "image_url", image_url: { url: "data:image/png;base64,YQ==" } }] }, + { role: "tool", tool_call_id: "call_b", content: "viewed b" }, + { role: "user", content: [{ type: "text", text: "image b" }, { type: "image_url", image_url: { url: "data:image/png;base64,Yg==" } }] }, + { role: "tool", tool_call_id: "call_c", content: "ok" }, + ], + max_tokens: 64, + }, + "claude-opus-5-5", + false + ); + + assert.deepEqual(body.messages.map((message) => message.role), ["user", "assistant", "user"]); + assert.deepEqual( + body.messages[2].content.filter((block) => block.type === "tool_result").map((block) => block.tool_use_id), + ["call_a", "call_b", "call_c"] + ); + assert.deepEqual(body.messages[2].content.slice(0, 3).map((block) => block.type), ["tool_result", "tool_result", "tool_result"]); + assert.equal(body.messages[2].content.filter((block) => block.type === "image").length, 2); + }); + it("anthropic tool_use → openai tool_calls", () => { const out = claude.fromAnthropicJson( { @@ -1043,6 +1078,7 @@ describe("format translation", () => { { type: "image", source: { type: "url", url: "https://example.com/photo.jpg" }, + cache_control: { type: "ephemeral" }, }, ]); @@ -2033,7 +2069,7 @@ describe("OAuth → OpenAI SSE translation pipes", () => { it("pipeAnthropicSseToOpenAi emits OpenAI chunks", async () => { const { Readable } = require("node:stream"); const events = [ - 'event: message_start\ndata: {"type":"message_start","message":{"id":"m1","usage":{"input_tokens":7,"output_tokens":0,"cache_read_input_tokens":2}}}\n\n', + 'event: message_start\ndata: {"type":"message_start","message":{"id":"m1","usage":{"input_tokens":7,"output_tokens":0,"cache_read_input_tokens":2,"cache_creation_input_tokens":3}}}\n\n', 'event: content_block_delta\ndata: {"type":"content_block_delta","delta":{"type":"text_delta","text":"Hel"}}\n\n', 'event: content_block_delta\ndata: {"type":"content_block_delta","delta":{"type":"text_delta","text":"lo"}}\n\n', 'event: message_delta\ndata: {"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"output_tokens":4}}\n\n', @@ -2057,6 +2093,9 @@ describe("OAuth → OpenAI SSE translation pipes", () => { prompt_tokens: 7, completion_tokens: 4, cached_tokens: 2, + cache_read_tokens: 2, + cache_write_tokens: 3, + cache_creation_input_tokens: 3, total_tokens: 11, }); assert.doesNotMatch(joined, /extra_content/); diff --git a/tests/model-error.test.js b/tests/model-error.test.js index 5a69b86..4eafeed 100644 --- a/tests/model-error.test.js +++ b/tests/model-error.test.js @@ -116,3 +116,25 @@ it("bounds and redacts thrown adapter model-test failures", async () => { assert.equal(entries[0].meta.body.includes(secret), false); assert.match(entries[0].meta.body, /\[truncated \d+ chars\]/); }); + + +it("bounds and aborts a model test when the provider never responds", async () => { + let receivedSignal; + const started = Date.now(); + const result = await runProviderModelTest({ + adapter: { + chat: async (_provider, options) => { + receivedSignal = options.signal; + return await new Promise(() => {}); + }, + }, + provider: { type: "nvidia", name: "NVIDIA NIM" }, + model: "model-that-hangs", + timeoutMs: 25, + }); + + assert.equal(result.ok, false); + assert.equal(receivedSignal.aborted, true); + assert.match(result.error, /timed out after 25ms/); + assert.ok(Date.now() - started < 1000); +}); diff --git a/tests/usage.test.js b/tests/usage.test.js index 2eca318..a3e4a5c 100644 --- a/tests/usage.test.js +++ b/tests/usage.test.js @@ -90,6 +90,29 @@ describe("SQLite usage history", () => { }); }); + it("normalizes cache read, cache write, uncached, and provider token semantics", () => { + withTempUsage((directory) => { + const store = createUsageStore(path.join(directory, "usage.sqlite")); + try { + store.record({ providerType: "chatgpt", status: 200, prompt_tokens: 100, cached_tokens: 80, completion_tokens: 5, total_tokens: 105 }); + store.record({ providerType: "claude", status: 200, prompt_tokens: 20, cached_tokens: 70, cache_write_tokens: 10, completion_tokens: 5, total_tokens: 25 }); + const usage = store.aggregate("all"); + assert.equal(usage.cache_read_tokens, 150); + assert.equal(usage.cache_write_tokens, 10); + assert.equal(usage.uncached_input_tokens, 50); + assert.equal(usage.logical_input_tokens, 200); + const recent = store.recent(2); + assert.equal(recent[0].token_semantics, "input_excludes_cache_read_write"); + assert.equal(recent[0].logical_input_tokens, 100); + assert.equal(recent[0].uncached_input_tokens, 30); + assert.equal(recent[1].token_semantics, "input_includes_cache_read"); + assert.equal(recent[1].uncached_input_tokens, 20); + } finally { + store.close(); + } + }); + }); + it("preserves a corrupt database and starts a fresh usable history", () => { withTempUsage((directory) => { const databasePath = path.join(directory, "usage.sqlite");