diff --git a/.github/workflows/promotion.yml b/.github/workflows/promotion.yml index 343295d..7943c7c 100644 --- a/.github/workflows/promotion.yml +++ b/.github/workflows/promotion.yml @@ -20,6 +20,15 @@ on: description: Destination org (requires a source org) required: false type: string + rollout: + description: >- + Gradual rollout orgs: default ramps; no-rollout publishes at once + (reverts); override pins over a split set outside gitops (needs + pipeline, from and to) + required: false + default: default + type: choice + options: [default, no-rollout, override] permissions: contents: write @@ -40,6 +49,7 @@ jobs: PIPELINE: ${{ inputs.pipeline }} SOURCE_ORG: ${{ inputs.from }} TARGET_ORG: ${{ inputs.to }} + ROLLOUT: ${{ inputs.rollout }} steps: - uses: actions/checkout@v4 @@ -84,6 +94,21 @@ jobs: exit 1 } args=(--apply) + case "${ROLLOUT:-default}" in + default) ;; + no-rollout) args+=(--no-rollout) ;; + override) + [[ "$supplied" == 3 ]] || { + echo "rollout: override needs pipeline, from and to." + exit 1 + } + args+=(--rollout-override) + ;; + *) + echo "Unknown rollout input: $ROLLOUT" + exit 1 + ;; + esac if [[ "$supplied" == 3 ]]; then args+=(--pipeline "$PIPELINE") args+=(--from "$SOURCE_ORG" --to "$TARGET_ORG") diff --git a/.github/workflows/rollout.yml b/.github/workflows/rollout.yml new file mode 100644 index 0000000..73fb2d2 --- /dev/null +++ b/.github/workflows/rollout.yml @@ -0,0 +1,69 @@ +name: Advance gradual rollouts + +# Moves each gradual rollout (orgs with rollout: in promotion.yml) one step +# once its interval has passed. The ramp's state lives in each assistant's +# traffic allocation, so this job commits nothing. See +# docs/guides/promotion.md#gradual-rollout-optional. +on: + schedule: + - cron: "*/15 * * * *" + workflow_dispatch: + inputs: + action: + description: advance, abort or status + required: false + default: advance + type: choice + options: [advance, abort, status] + org: + description: Only this org (required for abort) + required: false + type: string + assistant: + description: Only this assistant ID (e.g. support-agent) + required: false + type: string + +permissions: + contents: read + +# Its own group, never vapi-promotion: GitHub keeps one pending run per +# group, so a tick queued there would cancel a waiting promotion. An abort +# gets a group to itself, so it is never queued behind a tick. +concurrency: + group: ${{ inputs.action == 'abort' && format('vapi-rollout-abort-{0}', github.run_id) || 'vapi-rollout' }} + cancel-in-progress: false + +jobs: + rollout: + # Scheduled ticks need the opt-in variable; a manual run (an abort, most + # of all) always works. + if: >- + github.event_name == 'workflow_dispatch' || + vars.VAPI_ROLLOUT_ENABLED == 'true' + runs-on: ubuntu-latest + timeout-minutes: 15 + env: + VAPI_PROMOTION_TOKENS: ${{ secrets.VAPI_PROMOTION_TOKENS }} + ACTION: ${{ inputs.action }} + ORG: ${{ inputs.org }} + ASSISTANT: ${{ inputs.assistant }} + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: npm + + - run: npm ci + + - name: Run the rollout + shell: bash + run: | + set -euo pipefail + args=("${ACTION:-advance}") + if [[ -n "${ORG:-}" ]]; then args+=(--org "$ORG"); fi + if [[ -n "${ASSISTANT:-}" ]]; then args+=(--assistant "$ASSISTANT"); fi + npm run rollout -- "${args[@]}" diff --git a/README.md b/README.md index 9984b3d..30e45d3 100644 --- a/README.md +++ b/README.md @@ -217,6 +217,7 @@ More detail: [How the engine works](docs/guides/how-it-works.md). | `npm run sim -- ` | Run a simulation suite against deployed resources | | `npm run check` | Run PR simulation checks against local files | | `npm run promote` | Promote resources from one org to the next | +| `npm run rollout -- advance\|abort\|status` | Step, abort or list gradual rollouts | `setup`, `apply`, `pull`, `push`, `cleanup` and `call` prompt interactively when run without arguments. Full flags and examples: diff --git a/docs/guides/commands.md b/docs/guides/commands.md index 1cf2674..7a1e35a 100644 --- a/docs/guides/commands.md +++ b/docs/guides/commands.md @@ -20,6 +20,7 @@ The other commands are direct only. | `npm run call` | `npm run call -- -a ` or `-s ` | Talk to an assistant or squad from your terminal. | | `npm run sim` | `npm run sim -- --suite --target ` | Run a simulation suite against deployed resources. Exits 0 passed, 1 failed, 3 incomplete. | | `npm run check` | `npm run check -- ` or `--all`, `[--dry-run]` | Run [PR checks](pr-checks.md) against local files, nothing deployed. Exits 0 passed, 1 failed, 2 config or build error, 3 incomplete. | +| `npm run rollout` | `npm run rollout -- advance\|abort\|status [--org ] [--assistant ]` | Move [gradual rollouts](promotion.md#gradual-rollout-optional) a step, abort one (needs `--org`), or list them. The rollout workflow runs `advance` every 15 minutes. | | `npm run promote` | `npm run promote -- --pipeline

--from --to [--apply] [--no-rollout \| --rollout-override]` | Plan, or apply, a [promotion](promotion.md) between orgs. Into an org with a [gradual rollout](promotion.md#gradual-rollout-optional), `--no-rollout` publishes at once and `--rollout-override` pins over a split set outside gitops. | | `npm run build` | `npm run build` | Type-check the code and tests. | | `npm test` | `npm test` | Run the test suite. | diff --git a/docs/guides/how-it-works.md b/docs/guides/how-it-works.md index 5e1c969..6f4c7f2 100644 --- a/docs/guides/how-it-works.md +++ b/docs/guides/how-it-works.md @@ -160,6 +160,11 @@ a random UUID made once per command you run and shared by every process it starts, so Vapi can group one run's requests. It identifies nothing about you or your machine. +A [gradual rollout](promotion.md#gradual-rollout-optional)'s traffic-allocation +writes each name their own command (`rollout-pin`, `rollout-step`, +`rollout-complete`, `rollout-hold`, `rollout-abort`, `rollout-release`, +`rollout-restore`), so Vapi can count ramps by step, completion and abort. + Vapi uses these headers to count how the tool is used. Nothing else is sent beyond the requests themselves; there is no separate telemetry. diff --git a/docs/guides/promotion.md b/docs/guides/promotion.md index d2a0687..6976048 100644 --- a/docs/guides/promotion.md +++ b/docs/guides/promotion.md @@ -185,9 +185,11 @@ The rollout workflow then moves the ramp one step at a time (see below). Every write carries a `gitops-rollout …` note in the allocation's description, which is the ramp's only state; the traffic tab shows it. -Not everything is split. A new assistant goes live when it's created; squads, -tools and structured outputs aren't versioned per call, so a change to them -reaches 100% of calls on apply. The plan output lists each of these. +Not everything is split. A new assistant goes live when it's created. Only +calls placed directly to an assistant follow its split; calls through a squad +don't. Tools, structured outputs and squads aren't versioned per call, so a +change to them reaches 100% of calls on apply. The plan output lists each of +these. The promotion refuses, before the gate and with nothing written, when an assistant has a split someone set by hand (a manual split wins) or is managed @@ -200,6 +202,45 @@ for a revert: it also releases an assistant gitops was holding, so the reverted version takes every call. An assistant whose files didn't change, or a split set by hand, is left as it is. +### The rollout workflow + +`.github/workflows/rollout.yml` runs `npm run rollout -- advance` every 15 +minutes. For each assistant in a rollout org, it reads the newest traffic +allocation and, once `intervalMinutes` have passed since it was written, +moves to the next step. The last step is an explicit 100% on the new version; +the tick after that hands the assistant back to the default (every call to +the newest version). If someone published a newer version during the ramp, +that hand-back would send it every call un-ramped, so the ramp holds on the +ramped version instead and says so; promote again to ramp the newer one. + +To turn the schedule on, set the repository variable +`VAPI_ROLLOUT_ENABLED=true`. It uses the same `VAPI_PROMOTION_TOKENS` secret +and commits nothing. Run it by hand from the Actions tab: + +| `action` | What it does | +| --- | --- | +| `advance` | One step for every ramp whose interval has passed | +| `abort` | Puts the ramp back on its old version at 100% (needs `org`; `assistant` narrows it). Runs even with the schedule off, and never waits behind a tick | +| `status` | Lists ramps and anything waiting for a human, writes nothing | + +An aborted assistant stays on its old version, and later publishes land at +0%, until the next promotion (which ramps again) or a `no-rollout` one. + +The ramp stops, and `status` flags it, when the split changes under it (a +dashboard edit, say): it never overwrites someone else's change. A pin left +by a promotion that was killed mid-run is picked up after 6 hours: carried on +from step 1 if the run had published, released if it had replaced the +default, otherwise left for you. The 6 hours assume the promotion step's +330-minute limit in CI; a promotion run on your machine must finish within +them. + +Keep in mind: + +- GitHub runs scheduled workflows from the default branch only, may start + them minutes late, and disables them after 60 days without activity in a + public repository. Intervals are minimums. +- Each tick makes about two API reads per assistant in each rollout org. + ## Rolling Back a Promotion Treat a promotion rollback as a new, auditable Git change: revert the source diff --git a/package.json b/package.json index 50ad256..7c8d192 100644 --- a/package.json +++ b/package.json @@ -22,6 +22,7 @@ "check": "tsx src/check-cmd.ts", "rollback": "tsx src/rollback-cmd.ts", "promote": "tsx src/promote-cmd.ts", + "rollout": "tsx src/rollout-cmd.ts", "build": "tsc --noEmit", "test": "node --import tsx --import ./tests/no-vapi-api.ts --test tests/*.test.ts" }, diff --git a/src/rollout-cmd.ts b/src/rollout-cmd.ts new file mode 100644 index 0000000..34fbbaa --- /dev/null +++ b/src/rollout-cmd.ts @@ -0,0 +1,244 @@ +// npm run rollout -- advance|abort|status [--org ] [--assistant ] +// +// The ramp side of a gradual rollout. `advance` (the rollout workflow runs it +// every 15 minutes) moves each ramp in a rollout org one step once its +// interval has passed; `abort` puts an active ramp back on its old version; +// `status` lists ramps. Each assistant's newest traffic allocation is the +// ramp's whole state, so this commits nothing. + +import { existsSync, readFileSync } from "node:fs"; +import { resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import type { OrgConnection } from "./org-connection.ts"; +import { connectionLoad, tokensParse } from "./org-connection.ts"; +import type { PromotionConfig } from "./promotion.ts"; +import { promotionConfigParse, promotionStateParse } from "./promotion.ts"; +import type { Allocation, RolloutApi } from "./rollout-api.ts"; +import { allocationConflictGet, rolloutApiCreate } from "./rollout-api.ts"; +import { rolloutMarkerParse } from "./rollout-marker.ts"; +import type { RolloutTickWrite } from "./rollout-tick.ts"; +import { rolloutAbortWrite, rolloutTickDecide } from "./rollout-tick.ts"; +import { RUN_ID_ENV } from "./user-agent.ts"; + +type RolloutAction = "advance" | "abort" | "status"; + +interface RolloutArguments { + action: RolloutAction; + org?: string; + assistant?: string; +} + +export interface RolloutDeps { + rolloutApi: (connection: OrgConnection) => RolloutApi; + now: () => number; +} + +const ROOT_DIR = resolve( + process.env.VAPI_GITOPS_ROOT ?? fileURLToPath(new URL("..", import.meta.url)), +); +const TOKENS_ENV = "VAPI_PROMOTION_TOKENS"; +const USAGE = + "Usage: npm run rollout -- advance|abort|status [--org ] [--assistant ]"; +const ACTIONS: RolloutAction[] = ["advance", "abort", "status"]; + +function argumentsParse(args: string[]): RolloutArguments { + const [action, ...rest] = args; + if (!ACTIONS.includes(action as RolloutAction)) throw new Error(USAGE); + const parsed: RolloutArguments = { action: action as RolloutAction }; + for (let index = 0; index < rest.length; index++) { + const argument = rest[index]; + if (argument === "--org" && rest[index + 1]) { + parsed.org = rest[++index]; + continue; + } + if (argument === "--assistant" && rest[index + 1]) { + parsed.assistant = rest[++index]; + continue; + } + throw new Error(`Unrecognized or incomplete argument: ${argument}\n${USAGE}`); + } + // An abort is an emergency stop for a ramp someone chose, never a sweep. + if (parsed.action === "abort" && !parsed.org) + throw new Error("abort needs --org (and optionally --assistant)"); + return parsed; +} + +// Assistant resource ID → UUID in the org's state. +function assistantsLoad(org: string): Array<{ id: string; uuid: string }> { + const path = resolve(ROOT_DIR, `.vapi-state.${org}.json`); + if (!existsSync(path)) return []; + const state = promotionStateParse(readFileSync(path, "utf8")); + return Object.entries(state.assistants) + .filter(([, entry]) => entry?.uuid) + .map(([id, entry]) => ({ id, uuid: entry.uuid })); +} + +function messageOf(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +function sameWrite(allocation: Allocation | null, tick: RolloutTickWrite): boolean { + if (!allocation) return false; + const marker = rolloutMarkerParse(allocation.description); + const intended = rolloutMarkerParse(tick.write.description); + return ( + marker?.kind === intended?.kind && + marker?.from === intended?.from && + marker?.to === intended?.to && + marker?.step === intended?.step && + JSON.stringify(allocation.targets) === + JSON.stringify(tick.write.targets ?? []) + ); +} + +// Writes with compare-and-set. A 409 that turns out to be this same write +// (a retried request that had committed) is success; any other means someone +// changed the split meanwhile, so the ramp stops for a human to look. +async function tickWrite( + api: RolloutApi, + label: string, + tick: RolloutTickWrite, +): Promise { + try { + await api.allocationCreate(tick.write, tick.command); + console.log(` ${label}: ${tick.summary}`); + } catch (error) { + if (allocationConflictGet(error) !== "stale_allocation") throw error; + const newest = await api.allocationLatestGet(tick.write.assistantId); + if (sameWrite(newest, tick)) { + console.log(` ${label}: ${tick.summary}`); + return; + } + console.log( + ` ${label}: the split changed meanwhile (${newest?.description ?? newest?.allocationIntent ?? "none"}); left for a human`, + ); + } +} + +async function orgRun( + config: PromotionConfig, + org: string, + args: RolloutArguments, + deps: RolloutDeps, + tokens: Map, +): Promise { + const rollout = config.orgs[org]?.rollout; + if (!rollout) return []; + const api = deps.rolloutApi( + connectionLoad({ + rootDir: ROOT_DIR, + org, + tokens, + tokensEnvName: TOKENS_ENV, + baseUrl: config.orgs[org]?.baseUrl, + }), + ); + console.log(`\n${org}`); + if (!(await api.allocationsProbe())) { + console.log(" skipped: traffic splitting is not enabled for this org"); + return []; + } + const failures: string[] = []; + const runId = process.env[RUN_ID_ENV]; + for (const { id, uuid } of assistantsLoad(org)) { + if (args.assistant && args.assistant !== id) continue; + try { + const allocation = await api.allocationLatestGet(uuid); + if (!rolloutMarkerParse(allocation?.description)) continue; + if (args.action === "abort") { + const tick = rolloutAbortWrite(uuid, allocation, runId); + if (tick) await abortWrite(api, id, tick, runId); + else console.log(` ${id}: no active ramp to abort`); + continue; + } + const assistant = await api.assistantGet(uuid); + const decision = rolloutTickDecide({ + assistantId: uuid, + allocation, + latestVersion: assistant?.latestVersion ?? null, + rollout, + now: deps.now(), + runId, + }); + if (args.action === "advance" && decision.write) + await tickWrite(api, id, decision.write); + if (args.action === "status" || !decision.write) + console.log( + ` ${id}: ${decision.note ?? decision.write?.summary ?? "no ramp"}${decision.stale ? " ⚠️" : ""}`, + ); + } catch (error) { + failures.push(`${org}/${id}: ${messageOf(error)}`); + } + } + return failures; +} + +// An abort is idempotent in intent, so a lost race is retried once against +// the newest row. +async function abortWrite( + api: RolloutApi, + label: string, + tick: RolloutTickWrite, + runId?: string, +): Promise { + try { + await api.allocationCreate(tick.write, tick.command); + console.log(` ${label}: ${tick.summary}`); + return; + } catch (error) { + if (allocationConflictGet(error) !== "stale_allocation") throw error; + } + const newest = await api.allocationLatestGet(tick.write.assistantId); + const retry = rolloutAbortWrite(tick.write.assistantId, newest, runId); + if (!retry) { + console.log(` ${label}: no active ramp to abort any more`); + return; + } + await api.allocationCreate(retry.write, retry.command); + console.log(` ${label}: ${retry.summary}`); +} + +export async function rolloutCommandRun( + args = process.argv.slice(2), + overrides: Partial = {}, +): Promise { + const deps: RolloutDeps = { + rolloutApi: rolloutApiCreate, + now: Date.now, + ...overrides, + }; + const parsed = argumentsParse(args); + const configPath = resolve(ROOT_DIR, "promotion.yml"); + if (!existsSync(configPath)) { + console.log("No promotion.yml, so no rollout to run."); + return; + } + const config = promotionConfigParse(readFileSync(configPath, "utf8")); + const orgs = Object.keys(config.orgs).filter( + (org) => config.orgs[org]?.rollout && (!parsed.org || parsed.org === org), + ); + if (parsed.org && orgs.length === 0) + throw new Error(`${parsed.org} has no rollout: in promotion.yml`); + if (orgs.length === 0) { + console.log("No org in promotion.yml has rollout:, so nothing to do."); + return; + } + const tokens = tokensParse(TOKENS_ENV); + delete process.env[TOKENS_ENV]; + const failures: string[] = []; + for (const org of orgs) + failures.push(...(await orgRun(config, org, parsed, deps, tokens))); + if (failures.length > 0) + throw new Error( + `Rollout ${parsed.action} failed for:\n${failures.map((failure) => ` - ${failure}`).join("\n")}`, + ); +} + +const isMainModule = + resolve(process.argv[1] ?? "") === resolve(fileURLToPath(import.meta.url)); +if (isMainModule) { + rolloutCommandRun().catch((error: unknown) => { + console.error(`❌ Rollout failed: ${messageOf(error)}`); + process.exit(1); + }); +} diff --git a/src/rollout-tick.ts b/src/rollout-tick.ts new file mode 100644 index 0000000..67fc1ab --- /dev/null +++ b/src/rollout-tick.ts @@ -0,0 +1,228 @@ +// What the rollout ticker does next for one assistant, decided from its +// newest traffic allocation alone. Pure, so every row of the decision table +// is tested without an API. + +import type { PromotionRollout } from "./promotion.ts"; +import type { Allocation, AllocationWrite } from "./rollout-api.ts"; +import type { RolloutMarker } from "./rollout-marker.ts"; +import { rolloutMarkerFormat, rolloutMarkerParse } from "./rollout-marker.ts"; +import { rolloutStepTargets } from "./rollout-promote.ts"; +import type { RolloutCommand } from "./user-agent.ts"; + +// How old a pin must be before the ticker treats it as left behind by a +// killed promotion. Pins exist only inside the promotion step, which +// .github/workflows/promotion.yml caps at 330 minutes (a test ties the two), +// so no running promotion owns a pin older than this. Holds only in CI: a +// local promote has no cap. +export const ROLLOUT_ORPHAN_AGE_MINUTES = 360; + +export interface RolloutTickInput { + assistantId: string; + allocation: Allocation | null; + latestVersion: string | null; + rollout: PromotionRollout; + now: number; + runId?: string; +} + +export interface RolloutTickWrite { + command: RolloutCommand; + write: AllocationWrite; + // One line for the run's output. + summary: string; +} + +export interface RolloutTickDecision { + write?: RolloutTickWrite; + // A reason the ramp is waiting or stopped, for `status` and the log. + note?: string; + // A ramp a human should look at. + stale?: boolean; +} + +function ageMinutes(allocation: Allocation, now: number): number { + return (now - Date.parse(allocation.createdAt)) / 60_000; +} + +function stepWrite( + input: RolloutTickInput, + from: string, + to: string, + step: number, +): RolloutTickWrite { + const { steps } = input.rollout; + const percent = steps[step - 1]!; + return { + command: "rollout-step", + write: { + assistantId: input.assistantId, + targets: rolloutStepTargets(from, to, percent), + description: rolloutMarkerFormat({ + kind: "step", + from, + to, + step, + steps: steps.length, + run: input.runId, + }), + expectedCurrentAllocationId: input.allocation!.id, + }, + summary: `${from} → ${to} at ${percent}% (step ${step}/${steps.length})`, + }; +} + +function stepNext( + input: RolloutTickInput, + marker: RolloutMarker, +): RolloutTickDecision { + const allocation = input.allocation!; + const { steps, intervalMinutes } = input.rollout; + if (!marker.from || !marker.to || marker.step === undefined) + return { note: "step without from/to; left alone", stale: true }; + const age = ageMinutes(allocation, input.now); + if (age < intervalMinutes) + return { + note: `step ${marker.step}/${steps.length} at ${marker.to}; next in ${Math.ceil(intervalMinutes - age)} min`, + }; + // The last step (100%) is written as `complete`, not as a 0/100 split. + if (marker.step + 1 < steps.length) + return { write: stepWrite(input, marker.from, marker.to, marker.step + 1) }; + return { + write: { + command: "rollout-complete", + write: { + assistantId: input.assistantId, + targets: [{ assistantVersion: marker.to, percentage: 100 }], + description: rolloutMarkerFormat({ + kind: "complete", + from: marker.from, + to: marker.to, + run: input.runId, + }), + expectedCurrentAllocationId: allocation.id, + }, + summary: `${marker.to} at 100%, ramp complete`, + }, + }; +} + +function releaseWrite( + input: RolloutTickInput, + summary: string, +): RolloutTickWrite { + return { + command: "rollout-release", + write: { + assistantId: input.assistantId, + allocationIntent: "follow-latest", + description: rolloutMarkerFormat({ kind: "release", run: input.runId }), + expectedCurrentAllocationId: input.allocation!.id, + }, + summary, + }; +} + +// A complete ramp hands back to follow-latest, unless a newer version was +// published meanwhile: follow-latest would send it every call un-ramped. +function completeNext( + input: RolloutTickInput, + marker: RolloutMarker, +): RolloutTickDecision { + if (!marker.to || input.latestVersion === marker.to) + return { write: releaseWrite(input, "released to follow-latest") }; + return { + write: { + command: "rollout-hold", + write: { + assistantId: input.assistantId, + targets: [{ assistantVersion: marker.to, percentage: 100 }], + description: rolloutMarkerFormat({ + kind: "hold", + from: marker.from, + to: marker.to, + hold: input.latestVersion ?? undefined, + run: input.runId, + }), + expectedCurrentAllocationId: input.allocation!.id, + }, + summary: `held on ${marker.to}: ${input.latestVersion} was published during the ramp and has not been ramped; promote again to ramp it`, + }, + }; +} + +function pinNext( + input: RolloutTickInput, + marker: RolloutMarker, +): RolloutTickDecision { + const age = ageMinutes(input.allocation!, input.now); + if (age < ROLLOUT_ORPHAN_AGE_MINUTES) + return { note: "pinned by a promotion that may still be running" }; + // The killed run published: carry its ramp on from step 1. + if ( + marker.from && + input.latestVersion && + input.latestVersion !== marker.latest + ) + return { write: stepWrite(input, marker.from, input.latestVersion, 1) }; + // It was killed before publishing: undo the pin if it replaced the default. + if (marker.prev === "follow-latest") + return { + write: releaseWrite(input, "a stale pin from a killed promotion, released"), + }; + return { + note: `pinned on ${marker.from} by a promotion that stopped before publishing; promote again, or abort`, + stale: true, + }; +} + +export function rolloutTickDecide(input: RolloutTickInput): RolloutTickDecision { + const marker = rolloutMarkerParse(input.allocation?.description); + if (!input.allocation || !marker) return {}; + if (marker.kind === "step") return stepNext(input, marker); + if (marker.kind === "complete") return completeNext(input, marker); + if (marker.kind === "pin") return pinNext(input, marker); + if (marker.kind === "hold") + return { + note: `held on ${marker.to}; ${marker.hold} has not been ramped`, + stale: true, + }; + if (marker.kind === "abort") + return { + note: `aborted at ${marker.from} 100%; new publishes land at 0% until the next promotion`, + }; + return {}; +} + +// The abort write for an active ramp, or null when there is nothing to abort. +export function rolloutAbortWrite( + assistantId: string, + allocation: Allocation | null, + runId?: string, +): RolloutTickWrite | null { + const marker = rolloutMarkerParse(allocation?.description); + if (!allocation || !marker) return null; + if ( + marker.kind !== "step" && + marker.kind !== "complete" && + marker.kind !== "hold" + ) + return null; + if (!marker.from) return null; + return { + command: "rollout-abort", + write: { + assistantId, + targets: [{ assistantVersion: marker.from, percentage: 100 }], + description: rolloutMarkerFormat({ + kind: "abort", + from: marker.from, + to: marker.to, + step: marker.step, + steps: marker.steps, + run: runId, + }), + expectedCurrentAllocationId: allocation.id, + }, + summary: `aborted: ${marker.from} back to 100%`, + }; +} diff --git a/tests/rollout-cmd.test.ts b/tests/rollout-cmd.test.ts new file mode 100644 index 0000000..4511ae6 --- /dev/null +++ b/tests/rollout-cmd.test.ts @@ -0,0 +1,177 @@ +import assert from "node:assert/strict"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import test from "node:test"; +import { rolloutMarkerParse } from "../src/rollout-marker.ts"; +import { RolloutFake } from "./rollout-fake.ts"; + +// rollout-cmd.ts binds its root at import. +const ROOT = mkdtempSync(join(tmpdir(), "rollout-cmd-")); +process.env.VAPI_GITOPS_ROOT = ROOT; +const { rolloutCommandRun } = await import("../src/rollout-cmd.ts"); + +function write(path: string, content: string): void { + mkdirSync(dirname(join(ROOT, path)), { recursive: true }); + writeFileSync(join(ROOT, path), content); +} + +function fixtureReset(rollout = "{steps: [10, 100], intervalMinutes: 30}"): void { + rmSync(ROOT, { recursive: true, force: true }); + mkdirSync(ROOT, { recursive: true }); + write( + "promotion.yml", + `version: 1\norgs:\n staging: {}\n prod:${rollout ? `\n rollout: ${rollout}` : " {}"}\npipelines:\n release:\n orgs: [staging, prod]\n resources: ['**/*']\n`, + ); + write( + ".vapi-state.prod.json", + `${JSON.stringify({ assistants: { intake: { uuid: "intake-prod" }, billing: { uuid: "billing-prod" } } })}\n`, + ); +} + +interface Run { + error?: Error; + output: string; +} + +async function rollout(args: string[], fake: RolloutFake): Promise { + const run: Run = { output: "" }; + process.env.VAPI_PROMOTION_TOKENS = JSON.stringify({ staging: "t", prod: "t" }); + const log = console.log; + console.log = (...parts: unknown[]) => { + run.output += `${parts.join(" ")}\n`; + }; + try { + await rolloutCommandRun(args, { + rolloutApi: () => fake.api(), + now: () => Date.parse(fake.latest("intake-prod")?.createdAt ?? "2026-10-30T12:00:00Z") + 31 * 60_000, + }); + } catch (error) { + run.error = error as Error; + } finally { + console.log = log; + } + return run; +} + +function stepFake(): RolloutFake { + const fake = new RolloutFake(); + fake.assistantAdd("intake-prod", "v2"); + fake.manualWrite({ + assistantId: "intake-prod", + targets: [ + { assistantVersion: "v1", percentage: 90 }, + { assistantVersion: "v2", percentage: 10 }, + ], + description: "gitops-rollout step from=v1 to=v2 step=1/2", + }); + return fake; +} + +test("advance: an elapsed step moves on, with the rollout-complete label at 100%", async () => { + fixtureReset(); + const fake = stepFake(); + const run = await rollout(["advance"], fake); + assert.equal(run.error, undefined); + assert.deepEqual( + fake.writes.map((write) => write.command), + ["manual", "rollout-complete"], + ); + assert.deepEqual(fake.latest("intake-prod")?.targets, [ + { assistantVersion: "v2", percentage: 100 }, + ]); +}); + +test("advance: a split changed by someone else stops that ramp without failing the run", async () => { + fixtureReset(); + const fake = stepFake(); + fake.beforeCreate = () => + fake.manualWrite({ + assistantId: "intake-prod", + targets: [{ assistantVersion: "v1", percentage: 100 }], + description: "dashboard rollback", + }); + const run = await rollout(["advance"], fake); + assert.equal(run.error, undefined); + assert.equal(fake.latest("intake-prod")?.description, "dashboard rollback"); + assert.match(run.output, /intake: the split changed meanwhile \(dashboard rollback\); left for a human/); +}); + +test("advance: a 409 that is our own committed write counts as done", async () => { + fixtureReset(); + const fake = stepFake(); + // The write commits, then the response is lost as a 409 on the retry path. + fake.beforeCreate = (write) => { + fake.beforeCreate = undefined; + fake.manualWrite({ ...write, expectedCurrentAllocationId: undefined }); + }; + const run = await rollout(["advance"], fake); + assert.equal(run.error, undefined); + assert.match(run.output, /intake: v2 at 100%, ramp complete/); +}); + +test("advance: an org without traffic splitting is skipped, not failed", async () => { + fixtureReset(); + const fake = stepFake(); + fake.splittingEnabled = false; + const run = await rollout(["advance"], fake); + assert.equal(run.error, undefined); + assert.match(run.output, /skipped: traffic splitting is not enabled/); + assert.equal(fake.writes.length, 1); +}); + +test("advance: with no org rolling out, it does nothing and succeeds", async () => { + fixtureReset(""); + const fake = stepFake(); + const run = await rollout(["advance"], fake); + assert.equal(run.error, undefined); + assert.equal(fake.probes, 0); +}); + +test("abort: puts an active ramp back on its old version", async () => { + fixtureReset(); + const fake = stepFake(); + const run = await rollout(["abort", "--org", "prod", "--assistant", "intake"], fake); + assert.equal(run.error, undefined); + assert.deepEqual( + [ + fake.writes.at(-1)?.command, + rolloutMarkerParse(fake.latest("intake-prod")?.description)?.kind, + fake.latest("intake-prod")?.targets, + ], + ["rollout-abort", "abort", [{ assistantVersion: "v1", percentage: 100 }]], + ); +}); + +test("abort: retries once when a tick lands first", async () => { + fixtureReset(); + const fake = stepFake(); + fake.beforeCreate = () => { + fake.beforeCreate = undefined; + fake.manualWrite({ + assistantId: "intake-prod", + targets: [{ assistantVersion: "v2", percentage: 100 }], + description: "gitops-rollout complete from=v1 to=v2", + }); + }; + const run = await rollout(["abort", "--org", "prod"], fake); + assert.equal(run.error, undefined); + assert.equal(rolloutMarkerParse(fake.latest("intake-prod")?.description)?.kind, "abort"); +}); + +test("abort: needs --org", async () => { + fixtureReset(); + const run = await rollout(["abort"], stepFake()); + assert.match(run.error?.message ?? "", /abort needs --org/); +}); + +test("status: lists a ramp without writing", async () => { + fixtureReset(); + const fake = stepFake(); + const run = await rollout(["status"], fake); + assert.equal(run.error, undefined); + assert.equal(fake.writes.length, 1); + assert.match(run.output, /intake: v2 at 100%, ramp complete/); +}); + +test.after(() => rmSync(ROOT, { recursive: true, force: true })); diff --git a/tests/rollout-tick.test.ts b/tests/rollout-tick.test.ts new file mode 100644 index 0000000..c762ca1 --- /dev/null +++ b/tests/rollout-tick.test.ts @@ -0,0 +1,178 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import type { Allocation } from "../src/rollout-api.ts"; +import { rolloutMarkerParse } from "../src/rollout-marker.ts"; +import type { RolloutTickInput } from "../src/rollout-tick.ts"; +import { + ROLLOUT_ORPHAN_AGE_MINUTES, + rolloutAbortWrite, + rolloutTickDecide, +} from "../src/rollout-tick.ts"; + +const NOW = Date.parse("2026-10-30T12:00:00Z"); +const ROLLOUT = { steps: [1, 5, 10, 25, 50, 100], intervalMinutes: 30 }; + +function allocation( + description: string, + minutesAgo: number, + targets: Allocation["targets"] = [], +): Allocation { + return { + id: "alloc-1", + assistantId: "asst", + allocationIntent: targets.length > 0 ? "explicit" : "follow-latest", + createdAt: new Date(NOW - minutesAgo * 60_000).toISOString(), + description, + targets, + }; +} + +function decide( + row: Allocation | null, + latestVersion: string | null = "v2", +): ReturnType { + const input: RolloutTickInput = { + assistantId: "asst", + allocation: row, + latestVersion, + rollout: ROLLOUT, + now: NOW, + runId: "run-1", + }; + return rolloutTickDecide(input); +} + +// The decided write as `command kind targets`, for one-line assertions. +function writeOf(decision: ReturnType): string | undefined { + if (!decision.write) return undefined; + const { command, write } = decision.write; + return [ + command, + rolloutMarkerParse(write.description)?.kind, + (write.targets ?? []) + .map((target) => `${target.assistantVersion}:${target.percentage}`) + .join(","), + ] + .filter(Boolean) + .join(" "); +} + +const STEP_1 = "gitops-rollout step from=v1 to=v2 step=1/6"; + +test("tick: a step whose interval has passed moves to the next step", () => { + assert.equal(writeOf(decide(allocation(STEP_1, 31))), "rollout-step step v1:95,v2:5"); +}); + +test("tick: the next step's write is a compare-and-set on the current row", () => { + assert.equal( + decide(allocation(STEP_1, 31)).write?.write.expectedCurrentAllocationId, + "alloc-1", + ); +}); + +test("tick: a step inside its interval waits", () => { + assert.deepEqual(decide(allocation(STEP_1, 10)), { + note: "step 1/6 at v2; next in 20 min", + }); +}); + +test("tick: the step before the last completes on an explicit 100%, not follow-latest", () => { + assert.equal( + writeOf(decide(allocation("gitops-rollout step from=v1 to=v2 step=5/6", 31))), + "rollout-complete complete v2:100", + ); +}); + + +test("tick: a complete ramp is released when no newer version exists", () => { + assert.equal( + writeOf(decide(allocation("gitops-rollout complete from=v1 to=v2", 1))), + "rollout-release release", + ); +}); + +test("tick: a complete ramp is held when a newer, un-ramped version exists", () => { + const decision = decide(allocation("gitops-rollout complete from=v1 to=v2", 1), "v3"); + assert.equal(writeOf(decision), "rollout-hold hold v2:100"); + assert.equal(rolloutMarkerParse(decision.write?.write.description)?.hold, "v3"); +}); + +test("tick: a held ramp is left alone and flagged", () => { + assert.deepEqual(decide(allocation("gitops-rollout hold from=v1 to=v2 hold=v3", 600), "v3"), { + note: "held on v2; v3 has not been ramped", + stale: true, + }); +}); + +test("tick: an abort is left alone", () => { + assert.equal(decide(allocation("gitops-rollout abort from=v1 to=v2", 600)).write, undefined); +}); + +test("tick: a release is left alone", () => { + assert.deepEqual(decide(allocation("gitops-rollout release", 600)), {}); +}); + +test("tick: an unmarked row is ignored", () => { + assert.deepEqual(decide(allocation("customer note", 600)), {}); +}); + +test("tick: no allocation is ignored", () => { + assert.deepEqual(decide(null), {}); +}); + +const PIN = "gitops-rollout pin from=v1 latest=v1 prev=follow-latest run=r"; + +test("tick: a young pin may belong to a running promotion, so it waits", () => { + assert.equal(decide(allocation(PIN, ROLLOUT_ORPHAN_AGE_MINUTES - 1)).write, undefined); +}); + +test("tick: an old pin whose run published is adopted at step 1", () => { + assert.equal( + writeOf(decide(allocation(PIN, ROLLOUT_ORPHAN_AGE_MINUTES + 1), "v2")), + "rollout-step step v1:99,v2:1", + ); +}); + +test("tick: an old pin that replaced follow-latest, whose run never published, is released", () => { + assert.equal( + writeOf(decide(allocation(PIN, ROLLOUT_ORPHAN_AGE_MINUTES + 1), "v1")), + "rollout-release release", + ); +}); + +for (const [label, prev, latest, current] of [ + ["over an abort", "marked", "v2", "v2"], + ["over a hold", "marked", "v3", "v3"], + ["over an override of a manual split", "manual", "v2", "v2"], +] as const) { + test(`tick: an old pin ${label}, whose run never published, is not adopted`, () => { + const decision = decide( + allocation( + `gitops-rollout pin from=v1 latest=${latest} prev=${prev} run=r`, + ROLLOUT_ORPHAN_AGE_MINUTES + 1, + ), + current, + ); + assert.equal(decision.write, undefined); + assert.equal(decision.stale, true); + }); +} + +test("abort: an active step goes back to its old version", () => { + const tick = rolloutAbortWrite("asst", allocation("gitops-rollout step from=v1 to=v2 step=3/6", 5)); + assert.deepEqual( + [tick?.command, tick?.write.targets, rolloutMarkerParse(tick?.write.description)?.kind], + ["rollout-abort", [{ assistantVersion: "v1", percentage: 100 }], "abort"], + ); +}); + +for (const description of [ + "gitops-rollout pin from=v1 latest=v1 prev=follow-latest", + "gitops-rollout abort from=v1 to=v2", + "gitops-rollout release", + "a customer note", +]) { + test(`abort: nothing to abort for "${description}"`, () => { + assert.equal(rolloutAbortWrite("asst", allocation(description, 5)), null); + }); +} diff --git a/tests/rollout-workflow.test.ts b/tests/rollout-workflow.test.ts new file mode 100644 index 0000000..23a26ea --- /dev/null +++ b/tests/rollout-workflow.test.ts @@ -0,0 +1,133 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { + chmodSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; +import { parse as parseYaml } from "yaml"; +import { ROLLOUT_ORPHAN_AGE_MINUTES } from "../src/rollout-tick.ts"; + +// Runs the workflows' real shell steps with an `npm` stub that records its +// arguments, so the dispatch inputs are checked by what they run. + +const REPO = fileURLToPath(new URL("..", import.meta.url)); +const WORK = mkdtempSync(join(tmpdir(), "rollout-workflow-")); + +interface Step { + name?: string; + run?: string; + "timeout-minutes"?: number; +} + +function stepsRead(file: string, job: string): Step[] { + const workflow = parseYaml( + readFileSync(join(REPO, ".github/workflows", file), "utf8"), + ) as { jobs: Record }; + return workflow.jobs[job]!.steps; +} + +const RECONCILE = stepsRead("promotion.yml", "promote").find( + (step) => step.name === "Reconcile configured promotions", +)!; +const ROLLOUT_RUN = stepsRead("rollout.yml", "rollout").find( + (step) => step.name === "Run the rollout", +)!.run!; + +// Runs `script` with bash in a directory holding promotion.yml and an npm +// stub; returns the npm arguments, or the failure output. +function runStep( + script: string, + env: Record, +): { args?: string; failure?: string } { + rmSync(WORK, { recursive: true, force: true }); + mkdirSync(join(WORK, "bin"), { recursive: true }); + writeFileSync(join(WORK, "promotion.yml"), "version: 1\n"); + const stub = join(WORK, "bin", "npm"); + writeFileSync(stub, `#!/bin/bash\necho "$*" > "${join(WORK, "npm-args")}"\n`); + chmodSync(stub, 0o755); + const result = spawnSync("bash", ["-e", "-c", script], { + cwd: WORK, + encoding: "utf8", + env: { + PATH: `${join(WORK, "bin")}:${process.env.PATH}`, + VAPI_PROMOTION_TOKENS: '{"prod":"t"}', + ...env, + }, + }); + if (result.status !== 0) return { failure: result.stdout + result.stderr }; + return { args: readFileSync(join(WORK, "npm-args"), "utf8").trim() }; +} + +// GitHub sets every variable in the step's env:, empty when no input was given. +const INPUTS = { PIPELINE: "", SOURCE_ORG: "", TARGET_ORG: "", ROLLOUT: "" }; +const DISPATCH = { GITHUB_EVENT_NAME: "workflow_dispatch", ...INPUTS }; +const ONE = { PIPELINE: "release", SOURCE_ORG: "staging", TARGET_ORG: "prod" }; + +for (const [label, env, expected] of [ + ["default ramps", { ROLLOUT: "default" }, "run promote -- --apply --all"], + ["an empty input ramps", {}, "run promote -- --apply --all"], + ["no-rollout publishes at once", { ROLLOUT: "no-rollout" }, "run promote -- --apply --no-rollout --all"], + [ + "override pins over one transition's manual split", + { ROLLOUT: "override", ...ONE }, + "run promote -- --apply --rollout-override --pipeline release --from staging --to prod", + ], +] as const) { + test(`promotion dispatch: ${label}`, () => { + assert.deepEqual(runStep(RECONCILE.run!, { ...DISPATCH, ...env }), { + args: expected, + }); + }); +} + +test("promotion dispatch: override without pipeline, from and to is refused", () => { + assert.match( + runStep(RECONCILE.run!, { ...DISPATCH, ROLLOUT: "override" }).failure ?? "", + /override needs pipeline, from and to/, + ); +}); + +test("a push-triggered promotion never passes a rollout flag", () => { + assert.deepEqual( + runStep(RECONCILE.run!, { + ...INPUTS, + GITHUB_EVENT_NAME: "push", + ROLLOUT: "override", + }), + { args: "run promote -- --all --apply" }, + ); +}); + +test("the orphan-pin age outlasts the promotion step's timeout", () => { + // Pins exist only inside this step, so no running promotion owns a pin + // older than its timeout; raising one means raising the other. + assert.ok(RECONCILE["timeout-minutes"]! < ROLLOUT_ORPHAN_AGE_MINUTES); +}); + +for (const [label, env, expected] of [ + [ + "a scheduled tick advances", + { ACTION: "", ORG: "", ASSISTANT: "" }, + "run rollout -- advance", + ], + [ + "an abort names its org and assistant", + { ACTION: "abort", ORG: "prod", ASSISTANT: "intake" }, + "run rollout -- abort --org prod --assistant intake", + ], + ["status for one org", { ACTION: "status", ORG: "prod" }, "run rollout -- status --org prod"], +] as const) { + test(`rollout workflow: ${label}`, () => { + assert.deepEqual(runStep(ROLLOUT_RUN, env), { args: expected }); + }); +} + +test.after(() => rmSync(WORK, { recursive: true, force: true }));