From 23529b56e681e1fd2232c6abd8deebd9c0c23983 Mon Sep 17 00:00:00 2001 From: Scott Lowe Date: Fri, 30 Oct 2026 22:56:38 -0700 Subject: [PATCH] feat(rollout): report each tick's ramps and fail the run when one is overdue Part of DEPLOY-205. Health-gate aborts are written as rollout-health-abort, so they are countable apart from an abort someone ran. Each advance and status run writes every ramp's state and the gate's reading to the step summary, and advance fails when a ramp is overdue: a step still waiting three intervals on, a stale pin left for a human, or a split changed under a step. Holds and aborts never fail. Co-Authored-By: Claude Opus 5.5 --- docs/guides/how-it-works.md | 5 ++- docs/guides/promotion.md | 23 ++++++++-- src/rollout-cmd.ts | 84 ++++++++++++++++++++++++++--------- src/rollout-summary.ts | 51 +++++++++++++++++++++ src/rollout-tick.ts | 79 ++++++++++++++++++++++++++------ src/user-agent.ts | 1 + tests/rollout-cmd.test.ts | 77 +++++++++++++++++++++++++++++--- tests/rollout-summary.test.ts | 62 ++++++++++++++++++++++++++ tests/rollout-tick.test.ts | 53 ++++++++++++++++++++-- 9 files changed, 385 insertions(+), 50 deletions(-) create mode 100644 src/rollout-summary.ts create mode 100644 tests/rollout-summary.test.ts diff --git a/docs/guides/how-it-works.md b/docs/guides/how-it-works.md index 6f4c7f2..e621a6b 100644 --- a/docs/guides/how-it-works.md +++ b/docs/guides/how-it-works.md @@ -162,8 +162,9 @@ or your machine. A [gradual rollout](promotion.md#gradual-rollout-optional)'s traffic-allocation writes each name their own command (`rollout-pin`, `rollout-step`, -`rollout-complete`, `rollout-hold`, `rollout-abort`, `rollout-release`, -`rollout-restore`), so Vapi can count ramps by step, completion and abort. +`rollout-complete`, `rollout-hold`, `rollout-abort`, `rollout-health-abort`, +`rollout-release`, `rollout-restore`), so Vapi can count ramps by step, +completion and abort, and tell a health-gate abort from one someone ran. Vapi uses these headers to count how the tool is used. Nothing else is sent beyond the requests themselves; there is no separate telemetry. diff --git a/docs/guides/promotion.md b/docs/guides/promotion.md index f7e2e3c..402eeb6 100644 --- a/docs/guides/promotion.md +++ b/docs/guides/promotion.md @@ -226,14 +226,28 @@ and commits nothing. Run it by hand from the Actions tab: An aborted assistant stays on its old version, and later publishes land at 0%, until the next promotion (which ramps again) or a `no-rollout` one. -The ramp stops, and `status` flags it, when the split changes under it (a -dashboard edit, say): it never overwrites someone else's change. A pin left +The ramp stops when the split changes under it (a dashboard edit, say): it +never overwrites someone else's change. A pin left by a promotion that was killed mid-run is picked up after 6 hours: carried on from step 1 if the run had published, released if it had replaced the default, otherwise left for you. The 6 hours assume the promotion step's 330-minute limit in CI; a promotion run on your machine must finish within them. +Each `advance` and `status` run writes every ramp's state, and what the health +gate read, to the run's summary page. An `advance` run fails, so GitHub's +failure notification reaches someone, when a ramp is overdue: + +- a step is still waiting three intervals after it was written (the health + gate has not seen `minCalls`; lower it, or abort); +- a pin from a killed promotion was left for you after 6 hours; +- the split changed under a step. Only the run that finds it fails, since the + next tick no longer sees a ramp there. + +Holds and aborts are deliberate, so they never fail a run. GitHub sends a +scheduled run's failure notification to whoever last changed its `cron` line, +so make sure that is someone who watches it. + Optionally, gate each step on the new version's calls: ```yaml @@ -245,8 +259,9 @@ Optionally, gate each step on the new version's calls: Before each step, the ticker reads the calls each version took since the current step (the same metrics as the assistant's traffic tab). It waits -until the new version has `minCalls`, then aborts the ramp, with the reason -in the allocation's note, if the new version's share of calls that ended in +until the new version has `minCalls`, then aborts the ramp (labelled +`rollout-health-abort`, so it is counted apart from an abort someone ran), +with the reason in the allocation's note, if the new version's share of calls that ended in an error is more than `maxErrorRateDelta` percentage points above the old version's. `healthGate: {}` takes those defaults. It runs after, never instead of, a `check:` gate. diff --git a/src/rollout-cmd.ts b/src/rollout-cmd.ts index cd99b6e..6f5b752 100644 --- a/src/rollout-cmd.ts +++ b/src/rollout-cmd.ts @@ -6,7 +6,7 @@ // `status` lists ramps. Each assistant's newest traffic allocation is the // ramp's whole state, so this commits nothing. -import { existsSync, readFileSync } from "node:fs"; +import { appendFileSync, existsSync, readFileSync } from "node:fs"; import { resolve } from "node:path"; import { fileURLToPath } from "node:url"; import type { OrgConnection } from "./org-connection.ts"; @@ -20,6 +20,11 @@ import type { } from "./rollout-api.ts"; import { allocationConflictGet, rolloutApiCreate } from "./rollout-api.ts"; import { rolloutMarkerParse } from "./rollout-marker.ts"; +import type { RolloutSummaryRow } from "./rollout-summary.ts"; +import { + rolloutOverdueMessage, + rolloutSummaryMarkdown, +} from "./rollout-summary.ts"; import type { RolloutTickWrite } from "./rollout-tick.ts"; import { rolloutAbortWrite, rolloutTickDecide } from "./rollout-tick.ts"; import { RUN_ID_ENV } from "./user-agent.ts"; @@ -32,6 +37,11 @@ interface RolloutArguments { assistant?: string; } +interface RolloutOrgResult { + failures: string[]; + rows: RolloutSummaryRow[]; +} + export interface RolloutDeps { rolloutApi: (connection: OrgConnection) => RolloutApi; now: () => number; @@ -97,25 +107,28 @@ function sameWrite(allocation: Allocation | null, tick: RolloutTickWrite): boole // Writes with compare-and-set. A 409 that turns out to be this same write // (a retried request that had committed) is success; any other means someone -// changed the split meanwhile, so the ramp stops for a human to look. +// changed the split meanwhile, so the ramp stops for a human to look. Returns +// the stop as a reason; the next tick no longer sees a ramp, so this is the +// one run that can report it. async function tickWrite( api: RolloutApi, label: string, tick: RolloutTickWrite, -): Promise { +): Promise { try { await api.allocationCreate(tick.write, tick.command); console.log(` ${label}: ${tick.summary}`); + return null; } catch (error) { if (allocationConflictGet(error) !== "stale_allocation") throw error; const newest = await api.allocationLatestGet(tick.write.assistantId); if (sameWrite(newest, tick)) { console.log(` ${label}: ${tick.summary}`); - return; + return null; } - console.log( - ` ${label}: the split changed meanwhile (${newest?.description ?? newest?.allocationIntent ?? "none"}); left for a human`, - ); + const stopped = `the split changed meanwhile (${newest?.description ?? newest?.allocationIntent ?? "none"}); left for a human`; + console.log(` ${label}: ${stopped}`); + return stopped; } } @@ -125,9 +138,10 @@ async function orgRun( args: RolloutArguments, deps: RolloutDeps, tokens: Map, -): Promise { +): Promise { + const result: RolloutOrgResult = { failures: [], rows: [] }; const rollout = config.orgs[org]?.rollout; - if (!rollout) return []; + if (!rollout) return result; const api = deps.rolloutApi( connectionLoad({ rootDir: ROOT_DIR, @@ -140,9 +154,8 @@ async function orgRun( console.log(`\n${org}`); if (!(await api.allocationsProbe())) { console.log(" skipped: traffic splitting is not enabled for this org"); - return []; + return result; } - const failures: string[] = []; const runId = process.env[RUN_ID_ENV]; for (const { id, uuid } of assistantsLoad(org)) { if (args.assistant && args.assistant !== id) continue; @@ -166,17 +179,27 @@ async function orgRun( runId, health: await healthRead(api, rollout, allocation, now), }); - if (args.action === "advance" && decision.write) - await tickWrite(api, id, decision.write); + const stopped = + args.action === "advance" && decision.write + ? await tickWrite(api, id, decision.write) + : null; if (args.action === "status" || !decision.write) console.log( ` ${id}: ${decision.note ?? decision.write?.summary ?? "no ramp"}${decision.stale ? " ⚠️" : ""}`, ); + result.rows.push({ + org, + assistant: id, + state: + stopped ?? decision.note ?? decision.write?.summary ?? "no ramp", + overdue: stopped !== null || decision.overdue === true, + ...(decision.gate ? { gate: decision.gate } : {}), + }); } catch (error) { - failures.push(`${org}/${id}: ${messageOf(error)}`); + result.failures.push(`${org}/${id}: ${messageOf(error)}`); } } - return failures; + return result; } // Per-version calls since the current step began, read only when a gated @@ -248,12 +271,31 @@ export async function rolloutCommandRun( const tokens = tokensParse(TOKENS_ENV); delete process.env[TOKENS_ENV]; const failures: string[] = []; - for (const org of orgs) - failures.push(...(await orgRun(config, org, parsed, deps, tokens))); - if (failures.length > 0) - throw new Error( - `Rollout ${parsed.action} failed for:\n${failures.map((failure) => ` - ${failure}`).join("\n")}`, - ); + const rows: RolloutSummaryRow[] = []; + for (const org of orgs) { + const result = await orgRun(config, org, parsed, deps, tokens); + failures.push(...result.failures); + rows.push(...result.rows); + } + if (parsed.action !== "abort") summaryWrite(parsed.action, rows); + // Only advance fails on an overdue ramp: it is the scheduled run, whose + // failure notification is the alert; status is read by a person already. + const overdue = + parsed.action === "advance" ? rolloutOverdueMessage(rows) : null; + const problems = [ + ...(failures.length > 0 + ? [ + `Rollout ${parsed.action} failed for:\n${failures.map((failure) => ` - ${failure}`).join("\n")}`, + ] + : []), + ...(overdue ? [overdue] : []), + ]; + if (problems.length > 0) throw new Error(problems.join("\n")); +} + +function summaryWrite(action: string, rows: RolloutSummaryRow[]): void { + const path = process.env.GITHUB_STEP_SUMMARY; + if (path) appendFileSync(path, rolloutSummaryMarkdown(action, rows)); } const isMainModule = diff --git a/src/rollout-summary.ts b/src/rollout-summary.ts new file mode 100644 index 0000000..e65b7d5 --- /dev/null +++ b/src/rollout-summary.ts @@ -0,0 +1,51 @@ +// The rollout ticker's per-run report, for $GITHUB_STEP_SUMMARY. A green run +// with no summary was the only trace of a stuck ramp; this puts every ramp's +// state and the health gate's reading where the run's page shows them. + +export interface RolloutSummaryRow { + org: string; + // The assistant's resource ID in the repo. + assistant: string; + // What the tick did, or why the ramp is waiting or stopped. + state: string; + // The ramp should have moved and has not; such a row fails the advance run. + overdue: boolean; + // What the health gate read; absent without a gate or before it runs. + gate?: string; +} + +// Table cells cannot hold a pipe or a line break. +function cell(text: string): string { + return text.replaceAll("|", "\\|").replaceAll("\n", " "); +} + +export function rolloutSummaryMarkdown( + action: string, + rows: RolloutSummaryRow[], +): string { + const title = `## Rollout ${action}`; + if (rows.length === 0) return `${title}\n\nNo ramp in progress.\n`; + const overdue = rows.filter((row) => row.overdue).length; + const lines = [ + title, + "", + overdue > 0 + ? `⚠️ ${overdue} of ${rows.length} ramps overdue.` + : `${rows.length} ramps, none overdue.`, + "", + "| Org | Assistant | State | Health gate |", + "| --- | --- | --- | --- |", + ...rows.map( + (row) => + `| ${cell(row.org)} | ${cell(row.assistant)} | ${row.overdue ? "⚠️ overdue: " : ""}${cell(row.state)} | ${cell(row.gate ?? "")} |`, + ), + ]; + return `${lines.join("\n")}\n`; +} + +// The advance run's failure, naming each overdue ramp; null when none is. +export function rolloutOverdueMessage(rows: RolloutSummaryRow[]): string | null { + const overdue = rows.filter((row) => row.overdue); + if (overdue.length === 0) return null; + return `${overdue.length} ramps overdue:\n${overdue.map((row) => ` - ${row.org}/${row.assistant}: ${row.state}`).join("\n")}`; +} diff --git a/src/rollout-tick.ts b/src/rollout-tick.ts index 455fd86..e066c3f 100644 --- a/src/rollout-tick.ts +++ b/src/rollout-tick.ts @@ -20,6 +20,11 @@ import type { RolloutCommand } from "./user-agent.ts"; // local promote has no cap. export const ROLLOUT_ORPHAN_AGE_MINUTES = 360; +// A step still waiting this many intervals after it was written is overdue, +// and fails the advance run. One interval is the normal wait; two more cover +// cron jitter and a skipped tick, so a failure means the ramp is stuck. +export const ROLLOUT_OVERDUE_INTERVALS = 3; + export interface RolloutTickInput { assistantId: string; allocation: Allocation | null; @@ -44,6 +49,18 @@ export interface RolloutTickDecision { note?: string; // A ramp a human should look at. stale?: boolean; + // The ramp should have moved and has not; fails the advance run. Holds and + // aborts are deliberate, so they are stale but never overdue. + overdue?: boolean; + // What the health gate read, for the run summary. + gate?: string; +} + +interface HealthGateResult { + // Set when the gate holds the step back or aborts it. + decision?: RolloutTickDecision; + // What it read; set whenever the gate ran. + gate?: string; } function ageMinutes(allocation: Allocation, now: number): number { @@ -84,18 +101,27 @@ function stepNext( const allocation = input.allocation!; const { steps, intervalMinutes } = input.rollout; if (!marker.from || !marker.to || marker.step === undefined) - return { note: "step without from/to; left alone", stale: true }; + return { + note: "step without from/to; left alone", + stale: true, + overdue: true, + }; const age = ageMinutes(allocation, input.now); if (age < intervalMinutes) return { note: `step ${marker.step}/${steps.length} at ${marker.to}; next in ${Math.ceil(intervalMinutes - age)} min`, }; - const gated = healthGateCheck(input, marker); - if (gated) return gated; + const { decision, gate } = healthGateCheck(input, marker, age); + if (decision) return decision; + const read = gate ? { gate } : {}; // The last step (100%) is written as `complete`, not as a 0/100 split. if (marker.step + 1 < steps.length) - return { write: stepWrite(input, marker.from, marker.to, marker.step + 1) }; + return { + write: stepWrite(input, marker.from, marker.to, marker.step + 1), + ...read, + }; return { + ...read, write: { command: "rollout-complete", write: { @@ -180,6 +206,7 @@ function pinNext( return { note: `pinned on ${marker.from} by a promotion that stopped before publishing; promote again, or abort`, stale: true, + overdue: true, }; } @@ -188,24 +215,43 @@ function errorRate(health: RolloutVersionHealth | undefined): number { return (health.errors / health.calls) * 100; } +function healthFormat( + label: string | undefined, + health: RolloutVersionHealth | undefined, +): string { + if (!health) return `${label}: 0 calls`; + return `${label}: ${health.calls} calls, ${errorRate(health).toFixed(1)}% errors`; +} + // Before a step moves on: wait for enough calls on the new version, then -// abort if its error rate is too far above the old version's. +// abort if its error rate is too far above the old version's. A gate abort +// is labelled rollout-health-abort so it is countable apart from a human's. function healthGateCheck( input: RolloutTickInput, marker: RolloutMarker, -): RolloutTickDecision | null { + age: number, +): HealthGateResult { const gate = input.rollout.healthGate; - if (!gate) return null; + if (!gate) return {}; const health = input.health ?? []; const to = health.find((version) => version.version === marker.to); const from = health.find((version) => version.version === marker.from); + const read = `${healthFormat(marker.to, to)}; ${healthFormat(marker.from, from)}`; const calls = to?.calls ?? 0; - if (calls < gate.minCalls) + if (calls < gate.minCalls) { + const overdue = + age >= input.rollout.intervalMinutes * ROLLOUT_OVERDUE_INTERVALS; return { - note: `step ${marker.step} at ${marker.to}: ${calls} of ${gate.minCalls} calls before the next step`, + gate: read, + decision: { + note: `step ${marker.step} at ${marker.to}: ${calls} of ${gate.minCalls} calls before the next step${overdue ? "; overdue, so lower minCalls or abort" : ""}`, + gate: read, + ...(overdue ? { overdue } : {}), + }, }; + } const delta = errorRate(to) - errorRate(from); - if (delta <= gate.maxErrorRateDelta) return null; + if (delta <= gate.maxErrorRateDelta) return { gate: read }; const reason = `error-rate+${delta.toFixed(1)}pp`; const abort = rolloutAbortWrite( input.assistantId, @@ -213,11 +259,16 @@ function healthGateCheck( input.runId, reason, ); - if (!abort) return null; + if (!abort) return { gate: read }; return { - write: { - ...abort, - summary: `aborted by the health gate: ${marker.to}'s error rate is ${delta.toFixed(1)} points above ${marker.from}'s`, + gate: read, + decision: { + gate: read, + write: { + ...abort, + command: "rollout-health-abort", + summary: `aborted by the health gate: ${marker.to}'s error rate is ${delta.toFixed(1)} points above ${marker.from}'s`, + }, }, }; } diff --git a/src/user-agent.ts b/src/user-agent.ts index 9ad42c6..8bc0de4 100644 --- a/src/user-agent.ts +++ b/src/user-agent.ts @@ -168,6 +168,7 @@ export type RolloutCommand = | "rollout-complete" | "rollout-hold" | "rollout-abort" + | "rollout-health-abort" | "rollout-release"; export function vapiClientGet( diff --git a/tests/rollout-cmd.test.ts b/tests/rollout-cmd.test.ts index 9a058c0..59c5e7f 100644 --- a/tests/rollout-cmd.test.ts +++ b/tests/rollout-cmd.test.ts @@ -1,9 +1,10 @@ import assert from "node:assert/strict"; -import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; import test from "node:test"; import { rolloutMarkerParse } from "../src/rollout-marker.ts"; +import { ROLLOUT_ORPHAN_AGE_MINUTES } from "../src/rollout-tick.ts"; import { RolloutFake } from "./rollout-fake.ts"; // rollout-cmd.ts binds its root at import. @@ -34,9 +35,22 @@ interface Run { output: string; } -async function rollout(args: string[], fake: RolloutFake): Promise { +interface RolloutOptions { + // How long after intake's newest row the run happens. + minutes?: number; + // A step summary file; unset otherwise, so CI's own summary stays clean. + summary?: string; +} + +async function rollout( + args: string[], + fake: RolloutFake, + { minutes = 31, summary }: RolloutOptions = {}, +): Promise { const run: Run = { output: "" }; process.env.VAPI_PROMOTION_TOKENS = JSON.stringify({ staging: "t", prod: "t" }); + if (summary) process.env.GITHUB_STEP_SUMMARY = summary; + else delete process.env.GITHUB_STEP_SUMMARY; const log = console.log; console.log = (...parts: unknown[]) => { run.output += `${parts.join(" ")}\n`; @@ -44,7 +58,7 @@ async function rollout(args: string[], fake: RolloutFake): Promise { try { await rolloutCommandRun(args, { rolloutApi: () => fake.api(), - now: () => Date.parse(fake.latest("intake-prod")?.createdAt ?? "2026-10-30T12:00:00Z") + 31 * 60_000, + now: () => Date.parse(fake.latest("intake-prod")?.createdAt ?? "2026-10-30T12:00:00Z") + minutes * 60_000, }); } catch (error) { run.error = error as Error; @@ -82,7 +96,7 @@ test("advance: an elapsed step moves on, with the rollout-complete label at 100% ]); }); -test("advance: a split changed by someone else stops that ramp without failing the run", async () => { +test("advance: a split changed by someone else stops that ramp and fails the run once", async () => { fixtureReset(); const fake = stepFake(); fake.beforeCreate = () => @@ -92,9 +106,62 @@ test("advance: a split changed by someone else stops that ramp without failing t description: "dashboard rollback", }); const run = await rollout(["advance"], fake); - assert.equal(run.error, undefined); assert.equal(fake.latest("intake-prod")?.description, "dashboard rollback"); assert.match(run.output, /intake: the split changed meanwhile \(dashboard rollback\); left for a human/); + assert.match(run.error?.message ?? "", /1 ramps overdue:\n - prod\/intake: the split changed meanwhile/); + // The next tick sees an unmarked row, so it no longer fails. + fake.beforeCreate = undefined; + assert.equal((await rollout(["advance"], fake)).error, undefined); +}); + +test("advance: writes each ramp's state to the step summary", async () => { + fixtureReset(); + const fake = stepFake(); + const summary = join(ROOT, "summary.md"); + const run = await rollout(["advance"], fake, { summary }); + assert.equal(run.error, undefined); + assert.equal( + readFileSync(summary, "utf8"), + "## Rollout advance\n\n1 ramps, none overdue.\n\n| Org | Assistant | State | Health gate |\n| --- | --- | --- | --- |\n| prod | intake | v2 at 100%, ramp complete | |\n", + ); +}); + +test("advance: a stale pin from a killed run fails the run; a hold does not", async () => { + fixtureReset(); + const fake = new RolloutFake(); + fake.assistantAdd("intake-prod", "v1"); + fake.manualWrite({ + assistantId: "intake-prod", + targets: [{ assistantVersion: "v1", percentage: 100 }], + description: "gitops-rollout pin from=v1 latest=v1 prev=marked run=r", + }); + fake.assistantAdd("billing-prod", "v3"); + fake.manualWrite({ + assistantId: "billing-prod", + targets: [{ assistantVersion: "v2", percentage: 100 }], + description: "gitops-rollout hold from=v1 to=v2 hold=v3", + }); + const run = await rollout(["advance"], fake, { + minutes: ROLLOUT_ORPHAN_AGE_MINUTES + 1, + }); + assert.match(run.error?.message ?? "", /^1 ramps overdue:\n - prod\/intake: pinned on v1/); + assert.doesNotMatch(run.error?.message ?? "", /billing/); +}); + +test("status: an overdue ramp is listed, not failed", async () => { + fixtureReset(); + const fake = new RolloutFake(); + fake.assistantAdd("intake-prod", "v1"); + fake.manualWrite({ + assistantId: "intake-prod", + targets: [{ assistantVersion: "v1", percentage: 100 }], + description: "gitops-rollout pin from=v1 latest=v1 prev=marked run=r", + }); + const run = await rollout(["status"], fake, { + minutes: ROLLOUT_ORPHAN_AGE_MINUTES + 1, + }); + assert.equal(run.error, undefined); + assert.match(run.output, /intake: pinned on v1/); }); test("advance: a 409 that is our own committed write counts as done", async () => { diff --git a/tests/rollout-summary.test.ts b/tests/rollout-summary.test.ts new file mode 100644 index 0000000..44ec03b --- /dev/null +++ b/tests/rollout-summary.test.ts @@ -0,0 +1,62 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { + rolloutOverdueMessage, + rolloutSummaryMarkdown, +} from "../src/rollout-summary.ts"; + +const STEPPED = { + org: "prod", + assistant: "intake", + state: "v1 → v2 at 5% (step 2/6)", + overdue: false, + gate: "v2: 40 calls, 0.0% errors; v1: 900 calls, 1.0% errors", +}; +const STUCK = { + org: "prod", + assistant: "billing", + state: "step 1 at v4: 3 of 20 calls before the next step", + overdue: true, +}; + +test("summary: no ramp says so", () => { + assert.equal( + rolloutSummaryMarkdown("advance", []), + "## Rollout advance\n\nNo ramp in progress.\n", + ); +}); + +test("summary: an overdue ramp is counted and marked", () => { + assert.equal( + rolloutSummaryMarkdown("advance", [STEPPED, STUCK]), + [ + "## Rollout advance", + "", + "⚠️ 1 of 2 ramps overdue.", + "", + "| Org | Assistant | State | Health gate |", + "| --- | --- | --- | --- |", + "| prod | intake | v1 → v2 at 5% (step 2/6) | v2: 40 calls, 0.0% errors; v1: 900 calls, 1.0% errors |", + "| prod | billing | ⚠️ overdue: step 1 at v4: 3 of 20 calls before the next step | |", + "", + ].join("\n"), + ); +}); + +test("summary: a pipe in a state cannot break the table", () => { + assert.match( + rolloutSummaryMarkdown("status", [{ ...STEPPED, state: "a | b" }]), + /\| a \\\| b \|/, + ); +}); + +test("overdue message: names each overdue ramp", () => { + assert.equal( + rolloutOverdueMessage([STEPPED, STUCK]), + "1 ramps overdue:\n - prod/billing: step 1 at v4: 3 of 20 calls before the next step", + ); +}); + +test("overdue message: none when nothing is overdue", () => { + assert.equal(rolloutOverdueMessage([STEPPED]), null); +}); diff --git a/tests/rollout-tick.test.ts b/tests/rollout-tick.test.ts index 30716cb..b2f85b2 100644 --- a/tests/rollout-tick.test.ts +++ b/tests/rollout-tick.test.ts @@ -5,6 +5,7 @@ import { rolloutMarkerParse } from "../src/rollout-marker.ts"; import type { RolloutTickInput } from "../src/rollout-tick.ts"; import { ROLLOUT_ORPHAN_AGE_MINUTES, + ROLLOUT_OVERDUE_INTERVALS, rolloutAbortWrite, rolloutTickDecide, } from "../src/rollout-tick.ts"; @@ -155,9 +156,28 @@ for (const [label, prev, latest, current] of [ ); assert.equal(decision.write, undefined); assert.equal(decision.stale, true); + assert.equal(decision.overdue, true); }); } +test("tick: a hold is deliberate, so it is flagged but never overdue", () => { + const decision = decide( + allocation("gitops-rollout hold from=v1 to=v2 hold=v3", 600), + "v3", + ); + assert.equal(decision.overdue, undefined); +}); + +test("tick: an abort is deliberate, so it is never overdue", () => { + const decision = decide(allocation("gitops-rollout abort from=v1 to=v2", 600)); + assert.equal(decision.overdue, undefined); +}); + +test("tick: a step with no from/to is overdue", () => { + const decision = decide(allocation("gitops-rollout step step=1/6", 31)); + assert.equal(decision.overdue, true); +}); + test("abort: an active step goes back to its old version", () => { const tick = rolloutAbortWrite("asst", allocation("gitops-rollout step from=v1 to=v2 step=3/6", 5)); assert.deepEqual( @@ -182,10 +202,10 @@ const GATED = { healthGate: { minCalls: 20, maxErrorRateDelta: 2 }, }; -function decideGated(health: RolloutTickInput["health"]) { +function decideGated(health: RolloutTickInput["health"], minutesAgo = 31) { return rolloutTickDecide({ assistantId: "asst", - allocation: allocation(STEP_1, 31), + allocation: allocation(STEP_1, minutesAgo), latestVersion: "v2", rollout: GATED, now: NOW, @@ -199,8 +219,31 @@ test("health gate: waits until the new version has taken enough calls", () => { { version: "v1", calls: 900, errors: 0 }, { version: "v2", calls: 12, errors: 0 }, ]), - { note: "step 1 at v2: 12 of 20 calls before the next step" }, + { + note: "step 1 at v2: 12 of 20 calls before the next step", + gate: "v2: 12 calls, 0.0% errors; v1: 900 calls, 0.0% errors", + }, + ); +}); + +test("health gate: a step still short of calls after three intervals is overdue", () => { + const decision = decideGated( + [{ version: "v2", calls: 12, errors: 0 }], + ROLLOUT.intervalMinutes * ROLLOUT_OVERDUE_INTERVALS, ); + assert.deepEqual(decision, { + note: "step 1 at v2: 12 of 20 calls before the next step; overdue, so lower minCalls or abort", + gate: "v2: 12 calls, 0.0% errors; v1: 0 calls", + overdue: true, + }); +}); + +test("health gate: a step that passes reports what the gate read", () => { + const decision = decideGated([ + { version: "v1", calls: 1000, errors: 10 }, + { version: "v2", calls: 100, errors: 2 }, + ]); + assert.equal(decision.gate, "v2: 100 calls, 2.0% errors; v1: 1000 calls, 1.0% errors"); }); test("health gate: moves on when the new version's error rate is within the margin", () => { @@ -220,7 +263,8 @@ test("health gate: aborts when the new version errors well above the old", () => { version: "v1", calls: 1000, errors: 10 }, { version: "v2", calls: 100, errors: 5 }, ]); - assert.equal(writeOf(decision), "rollout-abort abort v1:100"); + // Its own label, so gate aborts are countable apart from a human's. + assert.equal(writeOf(decision), "rollout-health-abort abort v1:100"); assert.equal( rolloutMarkerParse(decision.write?.write.description)?.reason, "error-rate+4.0pp", @@ -230,5 +274,6 @@ test("health gate: aborts when the new version errors well above the old", () => test("health gate: no calls at all waits rather than aborting", () => { assert.deepEqual(decideGated([]), { note: "step 1 at v2: 0 of 20 calls before the next step", + gate: "v2: 0 calls; v1: 0 calls", }); });