From 7c07ff215df2e7706cb8d68299ac9efded404b2c Mon Sep 17 00:00:00 2001 From: ishaan1124 Date: Mon, 7 Sep 2026 15:11:07 +0530 Subject: [PATCH] fix: wait for canary artifact delivery before verification --- CHANGELOG.md | 2 + backend/cli/src/science/capability/canary.ts | 7 +- .../test/science/capability-canary.test.ts | 97 +++++++++++++++++++ 3 files changed, 105 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 557946c1e..6d4658dc8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ tagged release also ships native binaries for Linux, macOS, and Windows. ### Changed +- Wait for scientific canary artifact delivery before validating a completed + remote computation, while preserving bounded waits and resource cleanup. - Start packaged macOS workers without loading application configuration or migrating storage, and verify their gated startup and piped input in release smokes. - Preserve optional/defaulted tool inputs in provider schemas and clarify inline diff --git a/backend/cli/src/science/capability/canary.ts b/backend/cli/src/science/capability/canary.ts index f35d3dd8d..2457cd17a 100644 --- a/backend/cli/src/science/capability/canary.ts +++ b/backend/cli/src/science/capability/canary.ts @@ -88,7 +88,7 @@ export async function runScientificCapabilityCanary(input: { try { const current = await input.tool.execute({ action: "status", id: input.id, job_id: jobID }, input.ctx) state = job(current) - while (!terminal.has(state.status)) { + while (!terminal.has(state.status) || state.lifecycle?.delivery === "pending") { const remaining = Math.ceil((deadline - Date.now()) / 1_000) if (remaining <= 0) throw new Error(`Scientific capability canary ${input.id}/${input.target} timed out`) const waited = await input.tool.execute( @@ -109,6 +109,11 @@ export async function runScientificCapabilityCanary(input: { } const logs = await input.tool.execute({ action: "logs", id: input.id, job_id: jobID, bytes: 64_000 }, input.ctx) + if (state.lifecycle?.delivery === "failed" || state.lifecycle?.delivery === "rejected" || state.capture_error) { + throw new Error( + `Scientific capability canary ${input.id}/${input.target} could not deliver its artifacts: ${state.capture_error ?? state.lifecycle?.delivery}.\n${logs.output}`, + ) + } const artifacts = await input.tool.execute({ action: "artifacts", id: input.id, job_id: jobID }, input.ctx) const verified = await input.tool.execute({ action: "verify", id: input.id, job_id: jobID }, input.ctx) const result = { diff --git a/backend/cli/test/science/capability-canary.test.ts b/backend/cli/test/science/capability-canary.test.ts index e3beb8870..5fd957346 100644 --- a/backend/cli/test/science/capability-canary.test.ts +++ b/backend/cli/test/science/capability-canary.test.ts @@ -58,6 +58,66 @@ function context() { } } +function deliveryResult( + status: "succeeded" | "failed", + delivery: "pending" | "complete" | "failed" | "rejected", + captureError?: string, +) { + const output = result(status, "{}", "modal") + return { + ...output, + metadata: { + compute_job: { + job: { + ...output.metadata.compute_job.job, + lifecycle: { + execution: status, + delivery, + resource: "active", + recoverable: delivery === "failed" || delivery === "rejected", + }, + ...(captureError ? { capture_error: captureError } : {}), + }, + }, + }, + } +} + +function deliveryCanary( + states: Array | ReturnType>, + timeoutSeconds = 30, +) { + const actions: string[] = [] + const tool = { + async execute(input: { action: string }) { + actions.push(input.action) + if (input.action === "doctor") return { title: "doctor", output: "{}", metadata: {} } + if (input.action === "smoke") return result("running", "{}", "modal") + if (input.action === "status" || input.action === "wait") { + const state = states.shift() + if (!state) throw new Error("unexpected additional wait") + return state + } + if (input.action === "logs") return { title: "logs", output: "bounded canary log", metadata: {} } + if (input.action === "artifacts" || input.action === "verify") + return { title: input.action, output: "{}", metadata: {} } + if (input.action === "release") return result("succeeded", "{}", "modal", true) + throw new Error(`unexpected ${input.action}`) + }, + } + return { + actions, + run: () => + runScientificCapabilityCanary({ + tool: tool as never, + ctx: context() as never, + id: "matplotlib", + target: "modal", + timeoutSeconds, + }), + } +} + describe("scientific capability release canary", () => { test("binds release evidence to the source embedded in the artifact", () => { const source = "a".repeat(40) @@ -191,6 +251,43 @@ describe("scientific capability release canary", () => { expect(actions).toEqual(["doctor", "smoke", "status", "wait", "logs", "release"]) }) + test.each(["status", "wait"])("waits for artifact delivery after %s reports success", async (first) => { + const pending = deliveryResult("succeeded", "pending") + const complete = deliveryResult("succeeded", "complete") + const states = [...(first === "wait" ? [result("running", "{}", "modal")] : []), pending, pending, complete] + const canary = deliveryCanary(states) + await expect(canary.run()).resolves.toMatchObject({ status: "succeeded", cleanup: { status: "closed" } }) + expect(canary.actions).toEqual([ + "doctor", + "smoke", + "status", + ...Array.from({ length: first === "wait" ? 3 : 2 }, () => "wait"), + "logs", + "artifacts", + "verify", + "release", + ]) + }) + + test.each([ + { delivery: "failed" as const, captureError: undefined, error: "failed" }, + { delivery: "rejected" as const, captureError: undefined, error: "rejected" }, + { delivery: "complete" as const, captureError: "immutable capture failed", error: "immutable capture failed" }, + ])("fails $delivery delivery without verification or retry", async ({ delivery, captureError, error }) => { + const canary = deliveryCanary([ + deliveryResult("succeeded", "pending"), + deliveryResult("succeeded", delivery, captureError), + ]) + await expect(canary.run()).rejects.toThrow(`could not deliver its artifacts: ${error}.\nbounded canary log`) + expect(canary.actions).toEqual(["doctor", "smoke", "status", "wait", "logs", "release"]) + }) + + test("times out pending delivery without cancelling completed execution", async () => { + const canary = deliveryCanary([deliveryResult("succeeded", "pending")], 0) + await expect(canary.run()).rejects.toThrow("timed out") + expect(canary.actions).toEqual(["doctor", "smoke", "status", "release"]) + }) + test("cancels and releases a running job when the bounded deadline expires", async () => { const actions: string[] = [] const tool = {