diff --git a/.gitignore b/.gitignore index 388719d..e59fc60 100644 --- a/.gitignore +++ b/.gitignore @@ -41,3 +41,4 @@ tmp/ # ai-sdk-provider build output packages/ai-sdk-provider/dist/ +.env.bak-hillclimb diff --git a/nodemon.json b/nodemon.json new file mode 100644 index 0000000..2787a20 --- /dev/null +++ b/nodemon.json @@ -0,0 +1,3 @@ +{ + "ignore": ["data/*", ".lynkr/*", "*.db"] +} diff --git a/src/api/openai-router.js b/src/api/openai-router.js index cc9a565..0662b72 100644 --- a/src/api/openai-router.js +++ b/src/api/openai-router.js @@ -588,6 +588,22 @@ router.post("/chat/completions", async (req, res) => { const content = lynkrBadge(result.body) + (openaiResponse.choices[0].message.content || ""); let toolCalls = openaiResponse.choices[0].message.tool_calls; + // Guard: never serve a clean empty completion. Upstream failures + // (e.g. exhausted 429 retries) can surface here as a contentless + // message with finish_reason "stop"; agent clients read that as + // "task complete" and terminate mid-task. Killing the stream without + // a finish chunk makes the client retry instead. + if (!content && (!toolCalls || toolCalls.length === 0)) { + logger.error({ + finishReason: openaiResponse.choices[0]?.finish_reason, + terminationReason: result?.terminationReason, + status: result?.status, + }, "Empty completion reached serving path — aborting stream to force client retry"); + res.write(`data: ${JSON.stringify({ error: { message: "Upstream returned an empty completion; retry.", type: "server_error", code: "empty_completion" } })}\n\n`); + res.destroy(); + return; + } + if (clientType !== "unknown" && toolCalls && toolCalls.length > 0) { toolCalls = toolCalls.map(tc => { const mapped = mapToolForClient(tc.function?.name || "", tc.function?.arguments || "{}", clientType); diff --git a/src/clients/databricks.js b/src/clients/databricks.js index a44825b..9b9a63d 100644 --- a/src/clients/databricks.js +++ b/src/clients/databricks.js @@ -888,6 +888,23 @@ function detectAzureFormat(url) { } +/** + * Stable per-conversation cache key for GPT-5.6 declared prompt caching. + * Prefers the session id; falls back to hashing the first user message, + * which is identical across every turn of the same conversation. + */ +function derivePromptCacheKey(body) { + if (body._sessionId) return String(body._sessionId).slice(0, 64); + const first = (body.messages || []).find(m => m.role === "user"); + let text = ""; + if (first) { + text = typeof first.content === "string" + ? first.content + : JSON.stringify(first.content); + } + return "conv-" + crypto.createHash("sha1").update(text.slice(0, 4000)).digest("hex").slice(0, 32); +} + async function invokeAzureOpenAI(body, incomingHeaders = {}) { if (!config.azureOpenAI?.endpoint || !config.azureOpenAI?.apiKey) { throw new Error("Azure OpenAI endpoint or API key is not configured."); @@ -1004,6 +1021,16 @@ async function invokeAzureOpenAI(body, incomingHeaders = {}) { const responsesInput = []; // Track function call IDs for matching with outputs const pendingCallIds = []; + // Fallback IDs must be DETERMINISTIC: the same history must render + // byte-identically on every turn, or provider prompt caching never hits. + // (Was Date.now()+Math.random(), which re-randomized history each turn.) + let stableIdCounter = 0; + const stableCallId = (name, args) => { + const h = crypto.createHash("sha1") + .update(`${name || ""}|${args || ""}|${stableIdCounter++}`) + .digest("hex").slice(0, 16); + return `call_${h}`; + }; // Detect if this is a continuation request (has tool results) // Azure content filter triggers on full system prompt in continuations @@ -1065,36 +1092,32 @@ async function invokeAzureOpenAI(body, incomingHeaders = {}) { for (const msg of azureBody.messages) { if (msg.role === "system") { - // For continuation requests, use minimal system prompt to avoid content filter - // Azure's jailbreak detection triggers on security-related text in continuations - if (hasToolResults) { - responsesInput.push({ - type: "message", - role: "developer", - content: "You are a helpful coding assistant. Continue helping the user based on the tool results." - }); - } else { - // Initial request - use full system prompt - responsesInput.push({ - type: "message", - role: "developer", - content: typeof msg.content === 'string' ? msg.content : JSON.stringify(msg.content) - }); - } + // The system prompt must be IDENTICAL on every turn of a conversation: + // (a) swapping it per turn breaks provider prompt-cache prefixes, and + // (b) replacing it on continuations silently dropped the client + // agent's actual instructions mid-task. Strip system-reminder blocks + // uniformly (that was the content-filter trigger, not the prompt). + const sysText = typeof msg.content === 'string' ? msg.content : JSON.stringify(msg.content); + responsesInput.push({ + type: "message", + role: "developer", + content: stripSystemReminders(sysText) || sysText + }); } else if (msg.role === "user") { // Check if content contains tool_result blocks (Anthropic format) if (Array.isArray(msg.content)) { for (const block of msg.content) { if (block.type === "tool_result") { - const callId = block.tool_use_id || pendingCallIds.shift() || `call_${Date.now()}`; + const callId = block.tool_use_id || pendingCallIds.shift() || stableCallId("result", block.content); responsesInput.push({ type: "function_call_output", call_id: callId, output: typeof block.content === 'string' ? block.content : JSON.stringify(block.content || "") }); } else if (block.type === "text") { - // For continuation requests, strip system-reminder tags to avoid jailbreak filter - const textContent = hasToolResults ? stripSystemReminders(block.text || "") : (block.text || ""); + // Strip system-reminder tags on EVERY turn (uniformly), so the + // same message renders identically across turns (cache prefix). + const textContent = stripSystemReminders(block.text || ""); if (textContent) { // Only add if there's content after stripping responsesInput.push({ type: "message", @@ -1105,11 +1128,9 @@ async function invokeAzureOpenAI(body, incomingHeaders = {}) { } } } else { - // For continuation requests, strip system-reminder tags to avoid jailbreak filter + // Strip system-reminder tags uniformly on every turn (cache prefix). let userContent = typeof msg.content === 'string' ? msg.content : JSON.stringify(msg.content); - if (hasToolResults) { - userContent = stripSystemReminders(userContent); - } + userContent = stripSystemReminders(userContent); if (userContent) { // Only add if there's content after stripping responsesInput.push({ type: "message", @@ -1123,7 +1144,7 @@ async function invokeAzureOpenAI(body, incomingHeaders = {}) { if (msg.tool_calls && msg.tool_calls.length > 0) { // OpenAI format: tool_calls array for (const tc of msg.tool_calls) { - const callId = tc.id || `call_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`; + const callId = tc.id || stableCallId(tc.function?.name || tc.name, tc.function?.arguments); pendingCallIds.push(callId); responsesInput.push({ type: "function_call", @@ -1138,7 +1159,7 @@ async function invokeAzureOpenAI(body, incomingHeaders = {}) { // Anthropic format: content is array of blocks for (const block of msg.content) { if (block.type === "tool_use") { - const callId = block.id || `call_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`; + const callId = block.id || stableCallId(block.name, block.input); pendingCallIds.push(callId); responsesInput.push({ type: "function_call", @@ -1164,7 +1185,7 @@ async function invokeAzureOpenAI(body, incomingHeaders = {}) { } } else if (msg.role === "tool") { // Tool results become function_call_output - const callId = msg.tool_call_id || pendingCallIds.shift() || `call_${Date.now()}`; + const callId = msg.tool_call_id || pendingCallIds.shift() || stableCallId("tool", msg.content); responsesInput.push({ type: "function_call_output", call_id: callId, @@ -1173,12 +1194,26 @@ async function invokeAzureOpenAI(body, incomingHeaders = {}) { } } + // Reasoning effort: honor the client's request, else the env default. + // Without this, gpt-5.x reasoning models run at their shallowest setting. + const reasoningEffort = body.reasoning_effort + ?? body.reasoning?.effort + ?? process.env.AZURE_OPENAI_REASONING_EFFORT + ?? null; const responsesBody = { input: responsesInput, model: azureBody.model, - max_output_tokens: azureBody.max_tokens, + // gpt-5.x deployments store the cap under max_completion_tokens. + max_output_tokens: azureBody.max_completion_tokens ?? azureBody.max_tokens, tools: responsesTools, tool_choice: azureBody.tool_choice, + ...(isGpt5 && reasoningEffort ? { reasoning: { effort: reasoningEffort } } : {}), + // GPT-5.6 caching is declaration-based: a stable per-conversation key + // makes prefix cache matching reliable (~90% discount on agent loops). + // _sessionId is empty on the agentic path, so fall back to a hash of + // the conversation's first user message — stable across every turn of + // the same task. Keep per-key traffic under ~15 req/min. + ...(isGpt5 ? { prompt_cache_key: derivePromptCacheKey(body) } : {}), stream: false }; logger.debug({ @@ -1277,6 +1312,11 @@ async function invokeAzureOpenAI(body, incomingHeaders = {}) { completion_tokens: result.json.usage.completion_tokens ?? result.json.usage.output_tokens ?? 0, total_tokens: result.json.usage.total_tokens ?? ((result.json.usage.input_tokens ?? 0) + (result.json.usage.output_tokens ?? 0)), + // Provider-side prompt-cache hits (Responses API: input_tokens_details, + // Chat Completions: prompt_tokens_details) — telemetry reads this name. + cache_read_input_tokens: result.json.usage.input_tokens_details?.cached_tokens + ?? result.json.usage.prompt_tokens_details?.cached_tokens + ?? null, } : undefined }; @@ -2330,6 +2370,10 @@ function convertOpenAIToAnthropic(response) { usage: { input_tokens: response.usage?.prompt_tokens || 0, output_tokens: response.usage?.completion_tokens || 0, + // Provider-side prompt-cache hits — telemetry reads this field name. + cache_read_input_tokens: response.usage?.cache_read_input_tokens + ?? response.usage?.prompt_tokens_details?.cached_tokens + ?? null, } }; } diff --git a/src/config/index.js b/src/config/index.js index 74c88ce..cddf79d 100644 --- a/src/config/index.js +++ b/src/config/index.js @@ -673,7 +673,7 @@ var config = { fallbackProvider, }, toolResultCompression: { - enabled: true, + enabled: process.env.TOOL_RESULT_COMPRESSION_ENABLED !== "false", }, caveman: { enabled: cavemanEnabled, diff --git a/src/context/tool-result-compressor.js b/src/context/tool-result-compressor.js index 5a03caa..e5dbacc 100644 --- a/src/context/tool-result-compressor.js +++ b/src/context/tool-result-compressor.js @@ -9,20 +9,23 @@ */ const logger = require("../logger"); +const crypto = require("crypto"); // ── Tee Recovery Cache ─────────────────────────────────────────────── const teeCache = new Map(); const TEE_MAX_SIZE = 200; const TEE_TTL_MS = 5 * 60 * 1000; // 5 minutes -let teeCounter = 0; function teeStore(original) { - if (teeCache.size >= TEE_MAX_SIZE) { + // Content-derived id: the same tool result must compress to byte-identical + // output on every turn, or the tee marker breaks provider prompt-cache + // prefixes. (Was Date.now()+counter — new bytes in old messages each turn.) + const id = "tee_" + crypto.createHash("sha1").update(original).digest("hex").slice(0, 16); + if (!teeCache.has(id) && teeCache.size >= TEE_MAX_SIZE) { const oldest = teeCache.keys().next().value; teeCache.delete(oldest); } - const id = `tee_${Date.now()}_${teeCounter++}`; teeCache.set(id, { content: original, createdAt: Date.now() }); return id; } diff --git a/src/orchestrator/index.js b/src/orchestrator/index.js index b0093f4..e467f8d 100644 --- a/src/orchestrator/index.js +++ b/src/orchestrator/index.js @@ -1841,8 +1841,13 @@ IMPORTANT TOOL USAGE RULES: // before they reach the model (saves 60-90% on test/git/lint output) if (config.toolResultCompression?.enabled !== false) { const { compressToolResults } = require("../context/tool-result-compressor"); - const tier = cleanPayload._routingTier || "MEDIUM"; - compressToolResults(cleanPayload.messages, { tier }); + // Fixed threshold: compression must be deterministic per message — the + // routed tier flaps between turns, and re-compressing history differently + // breaks provider prompt-cache prefixes. COMPLEX (>2000 chars) compresses + // only bulky outputs: with prompt caching live, resending history is + // cheap, so lighter lossiness beats aggressive compression; still bounds + // context growth enough to stay clear of the token-budget compressor. + compressToolResults(cleanPayload.messages, { tier: "COMPLEX" }); } // MCP-aware tool dedup: drop built-in tools superseded by present MCP tools