feat(rollout): report each tick's ramps and fail the run when one is overdue

docs/guides/how-it-works.md

@@ -162,8 +162,9 @@ or your machine.
 
 A [gradual rollout](promotion.md#gradual-rollout-optional)'s traffic-allocation
 writes each name their own command (`rollout-pin`, `rollout-step`,
-`rollout-complete`, `rollout-hold`, `rollout-abort`, `rollout-release`,
-`rollout-restore`), so Vapi can count ramps by step, completion and abort.
+`rollout-complete`, `rollout-hold`, `rollout-abort`, `rollout-health-abort`,
+`rollout-release`, `rollout-restore`), so Vapi can count ramps by step,
+completion and abort, and tell a health-gate abort from one someone ran.
 
 Vapi uses these headers to count how the tool is used. Nothing else is sent beyond the
 requests themselves; there is no separate telemetry.

docs/guides/promotion.md

@@ -226,14 +226,28 @@ and commits nothing. Run it by hand from the Actions tab:
 An aborted assistant stays on its old version, and later publishes land at
 0%, until the next promotion (which ramps again) or a `no-rollout` one.
 
-The ramp stops, and `status` flags it, when the split changes under it (a
-dashboard edit, say): it never overwrites someone else's change. A pin left
+The ramp stops when the split changes under it (a dashboard edit, say): it
+never overwrites someone else's change. A pin left
 by a promotion that was killed mid-run is picked up after 6 hours: carried on
 from step 1 if the run had published, released if it had replaced the
 default, otherwise left for you. The 6 hours assume the promotion step's
 330-minute limit in CI; a promotion run on your machine must finish within
 them.
 
+Each `advance` and `status` run writes every ramp's state, and what the health
+gate read, to the run's summary page. An `advance` run fails, so GitHub's
+failure notification reaches someone, when a ramp is overdue:
+
+- a step is still waiting three intervals after it was written (the health
+  gate has not seen `minCalls`; lower it, or abort);
+- a pin from a killed promotion was left for you after 6 hours;
+- the split changed under a step. Only the run that finds it fails, since the
+  next tick no longer sees a ramp there.
+
+Holds and aborts are deliberate, so they never fail a run. GitHub sends a
+scheduled run's failure notification to whoever last changed its `cron` line,
+so make sure that is someone who watches it.
+
 Optionally, gate each step on the new version's calls:
 
 ```yaml
@@ -245,8 +259,9 @@ Optionally, gate each step on the new version's calls:
 
 Before each step, the ticker reads the calls each version took since the
 current step (the same metrics as the assistant's traffic tab). It waits
-until the new version has `minCalls`, then aborts the ramp, with the reason
-in the allocation's note, if the new version's share of calls that ended in
+until the new version has `minCalls`, then aborts the ramp (labelled
+`rollout-health-abort`, so it is counted apart from an abort someone ran),
+with the reason in the allocation's note, if the new version's share of calls that ended in
 an error is more than `maxErrorRateDelta` percentage points above the old
 version's. `healthGate: {}` takes those defaults. It runs after, never
 instead of, a `check:` gate.

src/rollout-cmd.ts

@@ -6,7 +6,7 @@
 // `status` lists ramps. Each assistant's newest traffic allocation is the
 // ramp's whole state, so this commits nothing.
 
-import { existsSync, readFileSync } from "node:fs";
+import { appendFileSync, existsSync, readFileSync } from "node:fs";
 import { resolve } from "node:path";
 import { fileURLToPath } from "node:url";
 import type { OrgConnection } from "./org-connection.ts";
@@ -20,6 +20,11 @@ import type {
 } from "./rollout-api.ts";
 import { allocationConflictGet, rolloutApiCreate } from "./rollout-api.ts";
 import { rolloutMarkerParse } from "./rollout-marker.ts";
+import type { RolloutSummaryRow } from "./rollout-summary.ts";
+import {
+  rolloutOverdueMessage,
+  rolloutSummaryMarkdown,
+} from "./rollout-summary.ts";
 import type { RolloutTickWrite } from "./rollout-tick.ts";
 import { rolloutAbortWrite, rolloutTickDecide } from "./rollout-tick.ts";
 import { RUN_ID_ENV } from "./user-agent.ts";
@@ -32,6 +37,11 @@ interface RolloutArguments {
   assistant?: string;
 }
 
+interface RolloutOrgResult {
+  failures: string[];
+  rows: RolloutSummaryRow[];
+}
+
 export interface RolloutDeps {
   rolloutApi: (connection: OrgConnection) => RolloutApi;
   now: () => number;
@@ -97,25 +107,28 @@ function sameWrite(allocation: Allocation | null, tick: RolloutTickWrite): boole
 
 // Writes with compare-and-set. A 409 that turns out to be this same write
 // (a retried request that had committed) is success; any other means someone
-// changed the split meanwhile, so the ramp stops for a human to look.
+// changed the split meanwhile, so the ramp stops for a human to look. Returns
+// the stop as a reason; the next tick no longer sees a ramp, so this is the
+// one run that can report it.
 async function tickWrite(
   api: RolloutApi,
   label: string,
   tick: RolloutTickWrite,
-): Promise<void> {
+): Promise<string | null> {
   try {
     await api.allocationCreate(tick.write, tick.command);
     console.log(`  ${label}: ${tick.summary}`);
+    return null;
   } catch (error) {
     if (allocationConflictGet(error) !== "stale_allocation") throw error;
     const newest = await api.allocationLatestGet(tick.write.assistantId);
     if (sameWrite(newest, tick)) {
       console.log(`  ${label}: ${tick.summary}`);
-      return;
+      return null;
     }
-    console.log(
-      `  ${label}: the split changed meanwhile (${newest?.description ?? newest?.allocationIntent ?? "none"}); left for a human`,
-    );
+    const stopped = `the split changed meanwhile (${newest?.description ?? newest?.allocationIntent ?? "none"}); left for a human`;
+    console.log(`  ${label}: ${stopped}`);
+    return stopped;
   }
 }
 
@@ -125,9 +138,10 @@ async function orgRun(
   args: RolloutArguments,
   deps: RolloutDeps,
   tokens: Map<string, string>,
-): Promise<string[]> {
+): Promise<RolloutOrgResult> {
+  const result: RolloutOrgResult = { failures: [], rows: [] };
   const rollout = config.orgs[org]?.rollout;
-  if (!rollout) return [];
+  if (!rollout) return result;
   const api = deps.rolloutApi(
     connectionLoad({
       rootDir: ROOT_DIR,
@@ -140,9 +154,8 @@ async function orgRun(
   console.log(`\n${org}`);
   if (!(await api.allocationsProbe())) {
     console.log("  skipped: traffic splitting is not enabled for this org");
-    return [];
+    return result;
   }
-  const failures: string[] = [];
   const runId = process.env[RUN_ID_ENV];
   for (const { id, uuid } of assistantsLoad(org)) {
     if (args.assistant && args.assistant !== id) continue;
@@ -166,17 +179,27 @@ async function orgRun(
         runId,
         health: await healthRead(api, rollout, allocation, now),
       });
-      if (args.action === "advance" && decision.write)
-        await tickWrite(api, id, decision.write);
+      const stopped =
+        args.action === "advance" && decision.write
+          ? await tickWrite(api, id, decision.write)
+          : null;
       if (args.action === "status" || !decision.write)
         console.log(
           `  ${id}: ${decision.note ?? decision.write?.summary ?? "no ramp"}${decision.stale ? "  ⚠️" : ""}`,
         );
+      result.rows.push({
+        org,
+        assistant: id,
+        state:
+          stopped ?? decision.note ?? decision.write?.summary ?? "no ramp",
+        overdue: stopped !== null || decision.overdue === true,
+        ...(decision.gate ? { gate: decision.gate } : {}),
+      });
     } catch (error) {
-      failures.push(`${org}/${id}: ${messageOf(error)}`);
+      result.failures.push(`${org}/${id}: ${messageOf(error)}`);
     }
   }
-  return failures;
+  return result;
 }
 
 // Per-version calls since the current step began, read only when a gated
@@ -248,12 +271,31 @@ export async function rolloutCommandRun(
   const tokens = tokensParse(TOKENS_ENV);
   delete process.env[TOKENS_ENV];
   const failures: string[] = [];
-  for (const org of orgs)
-    failures.push(...(await orgRun(config, org, parsed, deps, tokens)));
-  if (failures.length > 0)
-    throw new Error(
-      `Rollout ${parsed.action} failed for:\n${failures.map((failure) => `  - ${failure}`).join("\n")}`,
-    );
+  const rows: RolloutSummaryRow[] = [];
+  for (const org of orgs) {
+    const result = await orgRun(config, org, parsed, deps, tokens);
+    failures.push(...result.failures);
+    rows.push(...result.rows);
+  }
+  if (parsed.action !== "abort") summaryWrite(parsed.action, rows);
+  // Only advance fails on an overdue ramp: it is the scheduled run, whose
+  // failure notification is the alert; status is read by a person already.
+  const overdue =
+    parsed.action === "advance" ? rolloutOverdueMessage(rows) : null;
+  const problems = [
+    ...(failures.length > 0
+      ? [
+          `Rollout ${parsed.action} failed for:\n${failures.map((failure) => `  - ${failure}`).join("\n")}`,
+        ]
+      : []),
+    ...(overdue ? [overdue] : []),
+  ];
+  if (problems.length > 0) throw new Error(problems.join("\n"));
+}
+
+function summaryWrite(action: string, rows: RolloutSummaryRow[]): void {
+  const path = process.env.GITHUB_STEP_SUMMARY;
+  if (path) appendFileSync(path, rolloutSummaryMarkdown(action, rows));
 }
 
 const isMainModule =

src/rollout-summary.ts

@@ -0,0 +1,51 @@
+// The rollout ticker's per-run report, for $GITHUB_STEP_SUMMARY. A green run
+// with no summary was the only trace of a stuck ramp; this puts every ramp's
+// state and the health gate's reading where the run's page shows them.
+
+export interface RolloutSummaryRow {
+  org: string;
+  // The assistant's resource ID in the repo.
+  assistant: string;
+  // What the tick did, or why the ramp is waiting or stopped.
+  state: string;
+  // The ramp should have moved and has not; such a row fails the advance run.
+  overdue: boolean;
+  // What the health gate read; absent without a gate or before it runs.
+  gate?: string;
+}
+
+// Table cells cannot hold a pipe or a line break.
+function cell(text: string): string {
+  return text.replaceAll("|", "\\|").replaceAll("\n", " ");
+}
+
+export function rolloutSummaryMarkdown(
+  action: string,
+  rows: RolloutSummaryRow[],
+): string {
+  const title = `## Rollout ${action}`;
+  if (rows.length === 0) return `${title}\n\nNo ramp in progress.\n`;
+  const overdue = rows.filter((row) => row.overdue).length;
+  const lines = [
+    title,
+    "",
+    overdue > 0
+      ? `⚠️ ${overdue} of ${rows.length} ramps overdue.`
+      : `${rows.length} ramps, none overdue.`,
+    "",
+    "| Org | Assistant | State | Health gate |",
+    "| --- | --- | --- | --- |",
+    ...rows.map(
+      (row) =>
+        `| ${cell(row.org)} | ${cell(row.assistant)} | ${row.overdue ? "⚠️ overdue: " : ""}${cell(row.state)} | ${cell(row.gate ?? "")} |`,
+    ),
+  ];
+  return `${lines.join("\n")}\n`;
+}
+
+// The advance run's failure, naming each overdue ramp; null when none is.
+export function rolloutOverdueMessage(rows: RolloutSummaryRow[]): string | null {
+  const overdue = rows.filter((row) => row.overdue);
+  if (overdue.length === 0) return null;
+  return `${overdue.length} ramps overdue:\n${overdue.map((row) => `  - ${row.org}/${row.assistant}: ${row.state}`).join("\n")}`;
+}