From fa8469b6a4fbe48d4ae28b78336d6991bc12a96b Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Wed, 19 Aug 2026 00:27:02 -0600 Subject: [PATCH 1/2] fix(llm-client): retry Cloudflare edge timeouts 522 and 524 The retry classifier treated 524 as deterministic, so one edge timeout on a long non-streaming call killed a whole benchmark run. Cloudflare origin timeouts are transient exactly like 504. --- src/llm-client.ts | 5 ++++- tests/llm-transient-status.test.ts | 23 +++++++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 tests/llm-transient-status.test.ts diff --git a/src/llm-client.ts b/src/llm-client.ts index c78b4faa..5d4c71ba 100644 --- a/src/llm-client.ts +++ b/src/llm-client.ts @@ -349,7 +349,10 @@ function providerTokenCount(value: unknown): number | undefined { return typeof value === 'number' && Number.isSafeInteger(value) && value >= 0 ? value : undefined } -const RETRYABLE_STATUS = new Set([429, 502, 503, 504]) +// 522/524: Cloudflare edge timeouts (origin connect/respond) — transient +// exactly like 504. Router deployments serve through Cloudflare, so a long +// non-streaming call can draw these from the edge, not the origin. +const RETRYABLE_STATUS = new Set([429, 502, 503, 504, 522, 524]) /** * Transient transport/network error signatures, matched against an error's diff --git a/tests/llm-transient-status.test.ts b/tests/llm-transient-status.test.ts new file mode 100644 index 00000000..d90a10cf --- /dev/null +++ b/tests/llm-transient-status.test.ts @@ -0,0 +1,23 @@ +import { describe, expect, it } from 'vitest' +import { isTransientLlmError, LlmCallError } from '../src/llm-client' + +describe('isTransientLlmError — HTTP status classification', () => { + const call = (status: number) => new LlmCallError('upstream error', status, '', 'test-model') + + it.each([429, 502, 503, 504])('retries gateway-transient status %i', (status) => { + expect(isTransientLlmError(call(status))).toBe(true) + }) + + it.each([522, 524])('retries Cloudflare edge timeout %i like a 504', (status) => { + expect(isTransientLlmError(call(status))).toBe(true) + }) + + it.each([400, 401, 404, 422])('never retries deterministic status %i', (status) => { + expect(isTransientLlmError(call(status))).toBe(false) + }) + + it('reads a retryable numeric status off a foreign (non-LlmCallError) error', () => { + const err = Object.assign(new Error('edge timeout'), { status: 524 }) + expect(isTransientLlmError(err)).toBe(true) + }) +}) From d947bd0c94ab2985c4155ee14aa273e73e99067d Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Wed, 19 Aug 2026 00:38:37 -0600 Subject: [PATCH 2/2] fix(campaign): carry the thrown detail in the evaluation-failed 500 A silent catch returned an opaque 'evaluation failed' body, so a failed candidate evaluation erased its only diagnostic. Measured cost: one blind debugging round on a live benchmark run. --- src/campaign/external-optimizer-callback.ts | 8 ++++++-- .../external-optimizer-lifecycle.test.ts | 16 ++++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/src/campaign/external-optimizer-callback.ts b/src/campaign/external-optimizer-callback.ts index 3f2127b0..deb5eb38 100644 --- a/src/campaign/external-optimizer-callback.ts +++ b/src/campaign/external-optimizer-callback.ts @@ -179,7 +179,7 @@ async function handleCallback( let result: TResponse try { result = await args.evaluate({ candidate, exampleId: body.exampleId }, signal) - } catch { + } catch (error) { observe({ kind: 'refusal', reason: 'evaluation-failed', @@ -187,7 +187,11 @@ async function handleCallback( candidateHash, exampleId: body.exampleId, }) - sendJsonIfOpen(response, 500, { error: 'evaluation failed' }) + // The thrown detail is the only diagnostic for a failed evaluation — + // an opaque 500 costs the caller a blind debugging round. + sendJsonIfOpen(response, 500, { + error: `evaluation failed: ${String(error).slice(0, 400)}`, + }) return } let encoded: string diff --git a/tests/campaign/external-optimizer-lifecycle.test.ts b/tests/campaign/external-optimizer-lifecycle.test.ts index aabec74c..8ca750db 100644 --- a/tests/campaign/external-optimizer-lifecycle.test.ts +++ b/tests/campaign/external-optimizer-lifecycle.test.ts @@ -38,6 +38,22 @@ describe('external optimizer server lifecycle', () => { expect(activeEvaluations).toBe(0) }) + it('carries the thrown detail in the evaluation-failed 500 body', async () => { + const callback = await startExternalOptimizerCallback({ + token: 'secret', + maxEvaluations: 1, + evaluate: async () => { + throw new Error('router 503 after 3 attempts') + }, + }) + const response = await postEvaluation(callback.url, callback.token) + const body = (await response.json()) as { error: string } + await callback.close() + + expect(response.status).toBe(500) + expect(body.error).toContain('router 503 after 3 attempts') + }) + it('closes the callback and aborts active evaluation work with the owner signal', async () => { const owner = new AbortController() const started = deferred()