diff --git a/CHANGELOG.md b/CHANGELOG.md index 5e4335f3d..ebfb43798 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ tagged release also ships native binaries for Linux, macOS, and Windows. ### Changed +- Wait for scientific canary artifact delivery before validating a completed + remote computation, while preserving bounded waits and resource cleanup. - Start desktop onboarding with Synthetic Sciences sign-in and workspace selection, then continue to research project setup. A small Skip action allows local setup without an account; existing completed setups remain unchanged. diff --git a/backend/cli/src/science/capability/canary.ts b/backend/cli/src/science/capability/canary.ts index f35d3dd8d..2457cd17a 100644 --- a/backend/cli/src/science/capability/canary.ts +++ b/backend/cli/src/science/capability/canary.ts @@ -88,7 +88,7 @@ export async function runScientificCapabilityCanary(input: { try { const current = await input.tool.execute({ action: "status", id: input.id, job_id: jobID }, input.ctx) state = job(current) - while (!terminal.has(state.status)) { + while (!terminal.has(state.status) || state.lifecycle?.delivery === "pending") { const remaining = Math.ceil((deadline - Date.now()) / 1_000) if (remaining <= 0) throw new Error(`Scientific capability canary ${input.id}/${input.target} timed out`) const waited = await input.tool.execute( @@ -109,6 +109,11 @@ export async function runScientificCapabilityCanary(input: { } const logs = await input.tool.execute({ action: "logs", id: input.id, job_id: jobID, bytes: 64_000 }, input.ctx) + if (state.lifecycle?.delivery === "failed" || state.lifecycle?.delivery === "rejected" || state.capture_error) { + throw new Error( + `Scientific capability canary ${input.id}/${input.target} could not deliver its artifacts: ${state.capture_error ?? state.lifecycle?.delivery}.\n${logs.output}`, + ) + } const artifacts = await input.tool.execute({ action: "artifacts", id: input.id, job_id: jobID }, input.ctx) const verified = await input.tool.execute({ action: "verify", id: input.id, job_id: jobID }, input.ctx) const result = { diff --git a/backend/cli/test/science/capability-canary.test.ts b/backend/cli/test/science/capability-canary.test.ts index e3beb8870..5fd957346 100644 --- a/backend/cli/test/science/capability-canary.test.ts +++ b/backend/cli/test/science/capability-canary.test.ts @@ -58,6 +58,66 @@ function context() { } } +function deliveryResult( + status: "succeeded" | "failed", + delivery: "pending" | "complete" | "failed" | "rejected", + captureError?: string, +) { + const output = result(status, "{}", "modal") + return { + ...output, + metadata: { + compute_job: { + job: { + ...output.metadata.compute_job.job, + lifecycle: { + execution: status, + delivery, + resource: "active", + recoverable: delivery === "failed" || delivery === "rejected", + }, + ...(captureError ? { capture_error: captureError } : {}), + }, + }, + }, + } +} + +function deliveryCanary( + states: Array | ReturnType>, + timeoutSeconds = 30, +) { + const actions: string[] = [] + const tool = { + async execute(input: { action: string }) { + actions.push(input.action) + if (input.action === "doctor") return { title: "doctor", output: "{}", metadata: {} } + if (input.action === "smoke") return result("running", "{}", "modal") + if (input.action === "status" || input.action === "wait") { + const state = states.shift() + if (!state) throw new Error("unexpected additional wait") + return state + } + if (input.action === "logs") return { title: "logs", output: "bounded canary log", metadata: {} } + if (input.action === "artifacts" || input.action === "verify") + return { title: input.action, output: "{}", metadata: {} } + if (input.action === "release") return result("succeeded", "{}", "modal", true) + throw new Error(`unexpected ${input.action}`) + }, + } + return { + actions, + run: () => + runScientificCapabilityCanary({ + tool: tool as never, + ctx: context() as never, + id: "matplotlib", + target: "modal", + timeoutSeconds, + }), + } +} + describe("scientific capability release canary", () => { test("binds release evidence to the source embedded in the artifact", () => { const source = "a".repeat(40) @@ -191,6 +251,43 @@ describe("scientific capability release canary", () => { expect(actions).toEqual(["doctor", "smoke", "status", "wait", "logs", "release"]) }) + test.each(["status", "wait"])("waits for artifact delivery after %s reports success", async (first) => { + const pending = deliveryResult("succeeded", "pending") + const complete = deliveryResult("succeeded", "complete") + const states = [...(first === "wait" ? [result("running", "{}", "modal")] : []), pending, pending, complete] + const canary = deliveryCanary(states) + await expect(canary.run()).resolves.toMatchObject({ status: "succeeded", cleanup: { status: "closed" } }) + expect(canary.actions).toEqual([ + "doctor", + "smoke", + "status", + ...Array.from({ length: first === "wait" ? 3 : 2 }, () => "wait"), + "logs", + "artifacts", + "verify", + "release", + ]) + }) + + test.each([ + { delivery: "failed" as const, captureError: undefined, error: "failed" }, + { delivery: "rejected" as const, captureError: undefined, error: "rejected" }, + { delivery: "complete" as const, captureError: "immutable capture failed", error: "immutable capture failed" }, + ])("fails $delivery delivery without verification or retry", async ({ delivery, captureError, error }) => { + const canary = deliveryCanary([ + deliveryResult("succeeded", "pending"), + deliveryResult("succeeded", delivery, captureError), + ]) + await expect(canary.run()).rejects.toThrow(`could not deliver its artifacts: ${error}.\nbounded canary log`) + expect(canary.actions).toEqual(["doctor", "smoke", "status", "wait", "logs", "release"]) + }) + + test("times out pending delivery without cancelling completed execution", async () => { + const canary = deliveryCanary([deliveryResult("succeeded", "pending")], 0) + await expect(canary.run()).rejects.toThrow("timed out") + expect(canary.actions).toEqual(["doctor", "smoke", "status", "release"]) + }) + test("cancels and releases a running job when the bounded deadline expires", async () => { const actions: string[] = [] const tool = {