diff --git a/docs/guides/commands.md b/docs/guides/commands.md index 299b12f..1cf2674 100644 --- a/docs/guides/commands.md +++ b/docs/guides/commands.md @@ -20,7 +20,7 @@ The other commands are direct only. | `npm run call` | `npm run call -- -a ` or `-s ` | Talk to an assistant or squad from your terminal. | | `npm run sim` | `npm run sim -- --suite --target ` | Run a simulation suite against deployed resources. Exits 0 passed, 1 failed, 3 incomplete. | | `npm run check` | `npm run check -- ` or `--all`, `[--dry-run]` | Run [PR checks](pr-checks.md) against local files, nothing deployed. Exits 0 passed, 1 failed, 2 config or build error, 3 incomplete. | -| `npm run promote` | `npm run promote -- --pipeline

--from --to [--apply]` | Plan, or apply, a [promotion](promotion.md) between orgs. | +| `npm run promote` | `npm run promote -- --pipeline

--from --to [--apply] [--no-rollout \| --rollout-override]` | Plan, or apply, a [promotion](promotion.md) between orgs. Into an org with a [gradual rollout](promotion.md#gradual-rollout-optional), `--no-rollout` publishes at once and `--rollout-override` pins over a split set outside gitops. | | `npm run build` | `npm run build` | Type-check the code and tests. | | `npm test` | `npm test` | Run the test suite. | diff --git a/docs/guides/promotion.md b/docs/guides/promotion.md index 9d02cb5..d2a0687 100644 --- a/docs/guides/promotion.md +++ b/docs/guides/promotion.md @@ -170,6 +170,36 @@ The org needs traffic splitting, which Vapi enables per org during its beta. An `--apply` run checks every rollout org first and stops, with nothing promoted, if one doesn't have it. +### What a promotion into a rollout org does + +For each changed assistant that already exists in the org: + +1. **Pin.** Before publishing, it writes a traffic allocation holding the + version that serves calls now at 100%. Without it, the publish would move + every call at once. +2. **Publish.** The apply publishes the new version, which lands at 0%. +3. **Step 1.** It moves the first step's share (1% by default) to the new + version. If nothing was published (the files matched), it undoes the pin. + +The rollout workflow then moves the ramp one step at a time (see below). +Every write carries a `gitops-rollout …` note in the allocation's +description, which is the ramp's only state; the traffic tab shows it. + +Not everything is split. A new assistant goes live when it's created; squads, +tools and structured outputs aren't versioned per call, so a change to them +reaches 100% of calls on apply. The plan output lists each of these. + +The promotion refuses, before the gate and with nothing written, when an +assistant has a split someone set by hand (a manual split wins) or is managed +by Vapi for CX. To pin over a manual split on purpose, promote that one +transition with `--rollout-override` (the `override` dispatch input): it pins +the version with the most traffic, never the canary. + +`--no-rollout` (the `no-rollout` dispatch input) publishes at once. Use it +for a revert: it also releases an assistant gitops was holding, so the +reverted version takes every call. An assistant whose files didn't change, +or a split set by hand, is left as it is. + ## Rolling Back a Promotion Treat a promotion rollback as a new, auditable Git change: revert the source @@ -180,6 +210,10 @@ git revert npm run promote -- --pipeline release --from dev --to staging --apply ``` +Into an org with a gradual rollout, add `--no-rollout` so the revert goes +live at once instead of ramping. To stop a ramp in progress without a new +promotion, abort it (see the rollout workflow below). + This is distinct from `npm run rollback`, which restores a single org from a local pre-deploy snapshot. Because promotion uses the pipeline's scoped mirror boundary, reverting a resource creation also removes that promoted resource diff --git a/src/promote-cmd.ts b/src/promote-cmd.ts index 108191a..6800058 100644 --- a/src/promote-cmd.ts +++ b/src/promote-cmd.ts @@ -14,12 +14,28 @@ import { } from "./promotion-gate.ts"; import type { RolloutApi } from "./rollout-api.ts"; import { rolloutApiCreate } from "./rollout-api.ts"; +import type { + RolloutAssistantRef, + RolloutPin, + RolloutRead, +} from "./rollout-promote.ts"; +import { + rolloutFinish, + rolloutPinAll, + rolloutPreflight, + rolloutRelease, + rolloutReleaseBaseline, + rolloutSnapshot, + rolloutUnpinnedReport, +} from "./rollout-promote.ts"; +import type { PromotionChange } from "./promotion.ts"; import { promotionConfigParse, promotionPlanApply, promotionPlanBuild, promotionStateParse, promotionTransitionValidate, + resourceType, } from "./promotion.ts"; interface PromotionArguments { @@ -28,6 +44,10 @@ interface PromotionArguments { target?: string; all: boolean; apply: boolean; + // Skip the gradual rollout: publish at once, as a revert should. + noRollout: boolean; + // Pin over a split set outside gitops instead of refusing. + rolloutOverride: boolean; } interface PromotionTransition { @@ -67,7 +87,12 @@ interface PromotionGates { } function argumentsParse(args: string[]): PromotionArguments { - const parsed: PromotionArguments = { all: false, apply: false }; + const parsed: PromotionArguments = { + all: false, + apply: false, + noRollout: false, + rolloutOverride: false, + }; for (let index = 0; index < args.length; index++) { const argument = args[index]; if (argument === "--all") { @@ -78,6 +103,14 @@ function argumentsParse(args: string[]): PromotionArguments { parsed.apply = true; continue; } + if (argument === "--no-rollout") { + parsed.noRollout = true; + continue; + } + if (argument === "--rollout-override") { + parsed.rolloutOverride = true; + continue; + } if (argument === "--pipeline" && args[index + 1]) { parsed.pipeline = args[++index]; continue; @@ -98,7 +131,15 @@ function argumentsParse(args: string[]): PromotionArguments { ); if (!parsed.all && (!parsed.pipeline || !parsed.source || !parsed.target)) throw new Error( - "Usage: npm run promote -- --pipeline --from --to [--apply]", + "Usage: npm run promote -- --pipeline --from --to [--apply] [--no-rollout | --rollout-override]", + ); + if (parsed.noRollout && parsed.rolloutOverride) + throw new Error("--no-rollout cannot be combined with --rollout-override"); + // Overriding a manual split is a decision about one transition, never a + // blanket one across every org. + if (parsed.rolloutOverride && parsed.all) + throw new Error( + "--rollout-override needs --pipeline, --from and --to; it cannot be combined with --all", ); return parsed; } @@ -254,6 +295,49 @@ function stateLoad(org: string) { return promotionStateParse(readFileSync(path, "utf8")); } +// The changed assistants a rollout can ramp: creates and updates with a UUID +// in the target's state. Deletes have nothing to ramp, and an assistant with +// no UUID is created by the apply, so it goes live on creation. +function rolloutRefsGet( + target: string, + changes: PromotionChange[], +): RolloutAssistantRef[] { + const assistants = stateLoad(target).assistants; + const refs: RolloutAssistantRef[] = []; + for (const change of changes) { + if (change.kind === "delete") continue; + const resource = resourceType(change.path); + if (resource?.type !== "assistants") continue; + const uuid = assistants[resource.id]?.uuid; + if (!uuid) { + console.log(` rollout ${resource.id}: new in ${target}, live on create`); + continue; + } + refs.push({ id: resource.id, uuid }); + } + return refs; +} + +// What a promotion publishes at 100% even under a rollout: only direct +// assistant calls are split, and tools and structured outputs aren't +// versioned. +function rolloutUnsplitPrint(changes: PromotionChange[]): void { + for (const change of changes) { + const type = resourceType(change.path)?.type; + if (type && type !== "assistants") + console.log(` rollout ${change.path}: not split, live at 100% on apply`); + } +} + +// The target's other assistants, whose versions the unpinned-publish report +// compares across the apply. +function othersGet(target: string, pins: RolloutPin[]): RolloutAssistantRef[] { + const pinned = new Set(pins.map((pin) => pin.read.ref.uuid)); + return Object.entries(stateLoad(target).assistants) + .filter(([, entry]) => entry?.uuid && !pinned.has(entry.uuid)) + .map(([id, entry]) => ({ id, uuid: entry.uuid })); +} + async function transitionRun( config: PromotionConfig, transition: PromotionTransition, @@ -262,6 +346,7 @@ async function transitionRun( allowEmptySourceDeletion: boolean, deps: PromotionDeps, gates: PromotionGates, + args: PromotionArguments, ): Promise { const run = deps.childRun; if (apply) { @@ -299,6 +384,24 @@ async function transitionRun( if (check && !apply) console.log(await promotionGatePlanLine(ROOT_DIR, check)); if (!apply) return false; + const rollout = config.orgs[transition.target]?.rollout; + const api = rollout + ? deps.rolloutApi(orgConnection(config, transition.target, tokens)) + : undefined; + const refs = rollout ? rolloutRefsGet(transition.target, plan.changes) : []; + const ramp = api && rollout && !args.noRollout ? { api, rollout } : undefined; + if (ramp) rolloutUnsplitPrint(plan.changes); + // A fast refusal before the gate's simulations, which cost minutes. + let reads: RolloutRead[] = ramp + ? await rolloutPreflight( + ramp.api, + transition.target, + refs, + args.rolloutOverride, + ) + : []; + // Decided before the gate block, which marks the source passed. + const gateRuns = Boolean(check && !gates.passed.has(transition.source)); if (check && !gates.passed.has(transition.source)) { console.log(` check running ${check.name} in ${transition.source}…`); const result = await deps.checkRun( @@ -311,19 +414,85 @@ async function transitionRun( ); gates.passed.add(transition.source); } + // The gate can run for hours while the ticker or an abort writes, so read + // again: pins and undos must start from what is current now. + if (ramp && gateRuns) + reads = await rolloutPreflight( + ramp.api, + transition.target, + refs, + args.rolloutOverride, + ); await promotionPlanApply(plan); const changedPaths = plan.changes.map( (change) => `resources/${transition.target}/${change.path}`, ); - run( - "src/apply.ts", - transition.target, - orgConnection(config, transition.target, tokens), - ["--force", "--allow-new-files", "--resolve=ours", ...changedPaths], - ); - appliedPathsRecord(transition.target); - // The target's files just changed, so an earlier pass no longer covers it. - gates.passed.delete(transition.target); + const releaseBaseline = + api && rollout && args.noRollout + ? await rolloutReleaseBaseline(api, refs) + : undefined; + const pins: RolloutPin[] = ramp + ? await rolloutPinAll( + ramp.api, + transition.target, + reads, + args.rolloutOverride, + ) + : []; + const others = ramp ? othersGet(transition.target, pins) : []; + let childError: unknown; + const failures: string[] = []; + try { + const before = ramp ? await rolloutSnapshot(ramp.api, others) : undefined; + try { + run( + "src/apply.ts", + transition.target, + orgConnection(config, transition.target, tokens), + ["--force", "--allow-new-files", "--resolve=ours", ...changedPaths], + ); + } catch (error) { + childError = error; + } + if (ramp && before) await rolloutUnpinnedReport(ramp.api, others, before); + if (api && releaseBaseline) + failures.push(...(await rolloutRelease(api, refs, releaseBaseline))); + } finally { + if (ramp) + failures.push( + ...(await rolloutFinish({ + api: ramp.api, + pins, + steps: ramp.rollout.steps, + childFailed: childError !== undefined, + })), + ); + } + if (childError === undefined) { + // Recorded before any rollout failure is thrown: the org has the changes, + // so the committed state must have its files. With no rollout failure to + // report alongside it, a recording failure throws as it is. + if (failures.length === 0) appliedPathsRecord(transition.target); + else + try { + appliedPathsRecord(transition.target); + } catch (error) { + failures.unshift( + error instanceof Error ? error.message : String(error), + ); + } + // The target's files just changed, so an earlier pass no longer covers it. + gates.passed.delete(transition.target); + } + if (childError !== undefined && failures.length === 0) throw childError; + if (failures.length > 0) + throw new Error( + [ + ...(childError instanceof Error ? [childError.message] : []), + `Gradual rollout into ${transition.target}:`, + ...failures.map((failure) => ` - ${failure}`), + ].join("\n"), + ); return plan.changes.some((change) => change.kind === "delete"); } @@ -360,7 +529,8 @@ export async function promotionCommandRun( // reviewed authorization forward so the same deletion can reach later orgs. const deletionAuthorizedSources = new Set(); const transitions = transitionsBuild(config, parsed); - if (parsed.apply) await rolloutOrgsProbe(config, transitions, tokens, deps); + if (parsed.apply && !parsed.noRollout) + await rolloutOrgsProbe(config, transitions, tokens, deps); for (const transition of transitions) { const sourceKey = `${transition.pipeline}:${transition.source}`; const deleted = await transitionRun( @@ -371,6 +541,7 @@ export async function promotionCommandRun( deletionAuthorizedSources.has(sourceKey), deps, gates, + parsed, ); if (deleted) deletionAuthorizedSources.add( diff --git a/src/promotion.ts b/src/promotion.ts index 4bb46fb..fbceb50 100644 --- a/src/promotion.ts +++ b/src/promotion.ts @@ -354,7 +354,7 @@ function ignored(path: string, patterns: string[]): boolean { return patterns.some((pattern) => glob(pattern, stem)); } -function resourceType(path: string): { type: ResourceType; id: string } | null { +export function resourceType(path: string): { type: ResourceType; id: string } | null { for (const [type, folder] of Object.entries(FOLDER_MAP)) { if (!path.startsWith(`${folder}/`)) continue; const id = path.slice(folder.length + 1).replace(/\.(yml|yaml|md|ts)$/, ""); diff --git a/src/rollout-marker.ts b/src/rollout-marker.ts new file mode 100644 index 0000000..03c2f95 --- /dev/null +++ b/src/rollout-marker.ts @@ -0,0 +1,98 @@ +// The marker a gradual rollout writes into each traffic allocation's +// description. The allocations are the rollout's only state: the ticker reads +// the newest one per assistant and decides the next step from its marker, so +// nothing about a ramp is committed to git. +// +// gitops-rollout [from=vN] [to=vM] [step=i/K] [latest=vX] +// [prev=follow-latest|marked|manual] [hold=vK] [run=] + +export type RolloutMarkerKind = + | "pin" + | "step" + | "complete" + | "hold" + | "abort" + | "release"; + +// What a pin replaced: the default (follow-latest or nothing), a row this +// tool wrote, or a split someone made by hand. +export type RolloutPrevious = "follow-latest" | "marked" | "manual"; + +export interface RolloutMarker { + kind: RolloutMarkerKind; + // The version the ramp moves traffic away from. + from?: string; + // The version the ramp moves traffic to. + to?: string; + // 1-based step number and the number of steps. + step?: number; + steps?: number; + // A pin's latestVersion when it was written; a different latestVersion + // later means the run that wrote the pin published. + latest?: string; + prev?: RolloutPrevious; + // A newer, un-ramped version that kept a complete ramp from releasing. + hold?: string; + run?: string; +} + +const PREFIX = "gitops-rollout"; +const KINDS: RolloutMarkerKind[] = [ + "pin", + "step", + "complete", + "hold", + "abort", + "release", +]; +const PREVIOUS: RolloutPrevious[] = ["follow-latest", "marked", "manual"]; + +export function rolloutMarkerFormat(marker: RolloutMarker): string { + const fields: string[] = [PREFIX, marker.kind]; + if (marker.from) fields.push(`from=${marker.from}`); + if (marker.to) fields.push(`to=${marker.to}`); + if (marker.step !== undefined && marker.steps !== undefined) + fields.push(`step=${marker.step}/${marker.steps}`); + if (marker.latest) fields.push(`latest=${marker.latest}`); + if (marker.prev) fields.push(`prev=${marker.prev}`); + if (marker.hold) fields.push(`hold=${marker.hold}`); + if (marker.run) fields.push(`run=${marker.run}`); + return fields.join(" "); +} + +// null for a description this tool didn't write (or none at all). +export function rolloutMarkerParse( + description: string | undefined, +): RolloutMarker | null { + const [prefix, kind, ...fields] = (description ?? "").trim().split(/\s+/); + if (prefix !== PREFIX || !KINDS.includes(kind as RolloutMarkerKind)) + return null; + const marker: RolloutMarker = { kind: kind as RolloutMarkerKind }; + for (const field of fields) { + const equals = field.indexOf("="); + if (equals <= 0) continue; + const key = field.slice(0, equals); + const value = field.slice(equals + 1); + if (key === "from") marker.from = value; + if (key === "to") marker.to = value; + if (key === "latest") marker.latest = value; + if (key === "hold") marker.hold = value; + if (key === "run") marker.run = value; + if (key === "prev" && PREVIOUS.includes(value as RolloutPrevious)) + marker.prev = value as RolloutPrevious; + if (key === "step") { + const match = /^(\d+)\/(\d+)$/.exec(value); + if (match) { + marker.step = Number(match[1]); + marker.steps = Number(match[2]); + } + } + } + return marker; +} + +// The N of a "vN" label, or null for any other label. +export function versionNumber(label: string): number | null { + const match = /^v(\d+)$/.exec(label); + return match ? Number(match[1]) : null; +} diff --git a/src/rollout-promote.ts b/src/rollout-promote.ts new file mode 100644 index 0000000..0c229e8 --- /dev/null +++ b/src/rollout-promote.ts @@ -0,0 +1,485 @@ +// The promote side of a gradual rollout: around the one apply child that +// publishes a transition's assistants, pin each changed assistant to the +// version it serves now, so the publish lands at 0%, then write step 1 (or +// undo the pin when nothing was published). The rollout ticker +// (rollout-cmd.ts) takes the ramp from there. +// +// Order is the whole safety argument. A publish under follow-latest moves +// 100% of calls at once, so every pin is written before the child runs, the +// child never runs with a pin missing, and step 1 or the undo runs in a +// `finally` so a failed child can't strand an assistant at 0%. + +import { millipercentage } from "./promotion.ts"; +import type { + Allocation, + AllocationTarget, + AllocationWrite, + RolloutApi, +} from "./rollout-api.ts"; +import { allocationConflictGet } from "./rollout-api.ts"; +import type { RolloutMarker, RolloutPrevious } from "./rollout-marker.ts"; +import { + rolloutMarkerFormat, + rolloutMarkerParse, + versionNumber, +} from "./rollout-marker.ts"; +import { RUN_ID_ENV } from "./user-agent.ts"; +import { VapiApiError } from "./vapi-client.ts"; + +export interface RolloutAssistantRef { + // The resource ID: its path under assistants/ without the extension. + id: string; + uuid: string; +} + +// One assistant as a preflight read found it. +export interface RolloutRead { + ref: RolloutAssistantRef; + latestVersion: string; + allocation: Allocation | null; +} + +export interface RolloutPin { + read: RolloutRead; + // The version the pin holds at 100%, which the ramp moves traffic from. + from: string; + pin: Allocation; + // The row the pin's compare-and-set replaced: what an undo writes back. + replaced: Allocation | null; +} + +// A rollout refused before anything was written. +export class RolloutRefusal extends Error { + constructor(message: string) { + super(message); + this.name = "RolloutRefusal"; + } +} + +// A write that succeeded without a compare-and-set (there was no allocation +// to compare against) but was no longer the newest row when read back. +class RolloutNotOurs extends Error {} + +function runId(): string | undefined { + return process.env[RUN_ID_ENV]; +} + +function splitDescribe(targets: AllocationTarget[]): string { + return targets + .map((target) => `${target.assistantVersion} ${target.percentage}%`) + .join(" / "); +} + +function markerGet(allocation: Allocation | null): RolloutMarker | null { + return rolloutMarkerParse(allocation?.description); +} + +// A split nobody can tell was this tool's: refused unless overridden, because +// a manual split wins. +function manualSplit(allocation: Allocation | null): boolean { + return ( + allocation?.allocationIntent === "explicit" && markerGet(allocation) === null + ); +} + +function refusalGet( + read: RolloutRead, + override: boolean, +): string | null { + if (!override && manualSplit(read.allocation)) + return `${read.ref.id}: a split set outside gitops (${splitDescribe(read.allocation!.targets)}). Promote with the rollout: override dispatch input to pin over it.`; + return null; +} + +async function readOne( + api: RolloutApi, + ref: RolloutAssistantRef, +): Promise<{ read?: RolloutRead; skip?: string; managed?: boolean }> { + const assistant = await api.assistantGet(ref.uuid); + if (!assistant) + return { + skip: `${ref.id}: not found (deleted in the dashboard?); the apply removes its mapping and publishes nothing`, + }; + if (!assistant.latestVersion) + return { + skip: `${ref.id}: no published version (versioning is off), so there is nothing to ramp`, + }; + if (assistant.managed) return { managed: true }; + return { + read: { + ref, + latestVersion: assistant.latestVersion, + allocation: await api.allocationLatestGet(ref.uuid), + }, + }; +} + +// Reads every assistant and refuses, writing nothing, when any has a split set +// by hand (unless overridden) or is locked by Vapi for CX. +export async function rolloutPreflight( + api: RolloutApi, + org: string, + refs: RolloutAssistantRef[], + override: boolean, +): Promise { + const reads: RolloutRead[] = []; + const offenders: string[] = []; + for (const ref of refs) { + const result = await readOne(api, ref); + if (result.skip) console.log(` rollout skip ${result.skip}`); + if (result.managed) + offenders.push( + `${ref.id}: managed by Vapi for CX, which locks its traffic allocation`, + ); + if (!result.read) continue; + const refusal = refusalGet(result.read, override); + if (refusal) offenders.push(refusal); + reads.push(result.read); + } + if (offenders.length > 0) + throw new RolloutRefusal( + `Gradual rollout into ${org} refused; nothing was promoted:\n${offenders.map((line) => ` - ${line}`).join("\n")}`, + ); + return reads; +} + +function olderLabelFirst(left: string, right: string): number { + return (versionNumber(left) ?? 0) - (versionNumber(right) ?? 0); +} + +// The version a pin holds: whatever serves now, so the publish moves nothing. +export function rolloutPinVersionGet(read: RolloutRead): string { + const { allocation, latestVersion } = read; + if (!allocation || allocation.allocationIntent === "follow-latest") + return latestVersion; + const marker = markerGet(allocation); + if (marker?.kind === "complete" || marker?.kind === "hold") + return marker.to ?? latestVersion; + if (marker) return marker.from ?? latestVersion; + // A manual split under override: the version with the most traffic, so the + // canary never jumps to 100%. A tie goes to the older version. + const [largest] = [...allocation.targets].sort( + (left, right) => + right.percentage - left.percentage || + olderLabelFirst(left.assistantVersion, right.assistantVersion), + ); + return largest?.assistantVersion ?? latestVersion; +} + +function previousGet(allocation: Allocation | null): RolloutPrevious { + if (!allocation || allocation.allocationIntent === "follow-latest") + return "follow-latest"; + return markerGet(allocation) ? "marked" : "manual"; +} + +async function pinOne( + api: RolloutApi, + read: RolloutRead, + override: boolean, +): Promise { + const from = rolloutPinVersionGet(read); + if (manualSplit(read.allocation) && override) + console.log( + ` rollout override ${read.ref.id}: replacing ${splitDescribe(read.allocation!.targets)} with ${from} 100%`, + ); + const pin = await api.allocationCreate( + { + assistantId: read.ref.uuid, + targets: [{ assistantVersion: from, percentage: 100 }], + description: rolloutMarkerFormat({ + kind: "pin", + from, + latest: read.latestVersion, + prev: previousGet(read.allocation), + run: runId(), + }), + ...(read.allocation + ? { expectedCurrentAllocationId: read.allocation.id } + : {}), + }, + "rollout-pin", + ); + if (!read.allocation) { + const newest = await api.allocationLatestGet(read.ref.uuid); + if (newest?.id !== pin.id) + throw new RolloutNotOurs(`${read.ref.id}: another write landed with the pin`); + } + return { read, from, pin, replaced: read.allocation }; +} + +function retryable(error: unknown): boolean { + if (error instanceof RolloutNotOurs) return true; + if (allocationConflictGet(error) === "stale_allocation") return true; + return error instanceof VapiApiError && error.statusCode >= 500; +} + +function messageOf(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +export function rolloutRestoreWrite(pin: RolloutPin): AllocationWrite { + const replaced = pin.replaced; + const description = replaced?.description + ? { description: replaced.description } + : {}; + if (!replaced || replaced.allocationIntent === "follow-latest") + return { + assistantId: pin.read.ref.uuid, + allocationIntent: "follow-latest", + ...description, + expectedCurrentAllocationId: pin.pin.id, + }; + return { + assistantId: pin.read.ref.uuid, + targets: replaced.targets.map((target) => ({ + assistantVersion: target.assistantVersion, + percentage: target.percentage, + })), + ...description, + expectedCurrentAllocationId: pin.pin.id, + }; +} + +async function restoreAll(api: RolloutApi, pins: RolloutPin[]): Promise { + const failures: string[] = []; + for (const pin of pins) + await api + .allocationCreate(rolloutRestoreWrite(pin), "rollout-restore") + .catch((error: unknown) => + failures.push( + `${pin.read.ref.id}: couldn't undo its pin (${messageOf(error)})`, + ), + ); + return failures; +} + +// Pins every assistant, or none: on a failure that a fresh read and one +// retry don't clear, the pins already written are undone and nothing is +// published. +export async function rolloutPinAll( + api: RolloutApi, + org: string, + reads: RolloutRead[], + override: boolean, +): Promise { + const pins: RolloutPin[] = []; + for (const read of reads) { + try { + pins.push(await pinOne(api, read, override)); + continue; + } catch (error) { + if (!retryable(error)) + return rolloutPinAbandon(api, org, pins, messageOf(error)); + } + const fresh = await readOne(api, read.ref).catch((error: unknown) => ({ + error: messageOf(error), + })); + if ("error" in fresh || !fresh.read) + return rolloutPinAbandon( + api, + org, + pins, + "error" in fresh + ? fresh.error + : `${read.ref.id}: ${fresh.skip ?? "managed by Vapi for CX"}`, + ); + const refusal = refusalGet(fresh.read, override); + if (refusal) return rolloutPinAbandon(api, org, pins, refusal); + try { + pins.push(await pinOne(api, fresh.read, override)); + } catch (error) { + return rolloutPinAbandon(api, org, pins, messageOf(error)); + } + } + return pins; +} + +async function rolloutPinAbandon( + api: RolloutApi, + org: string, + pins: RolloutPin[], + reason: string, +): Promise { + const failures = await restoreAll(api, pins); + throw new RolloutRefusal( + [ + `Gradual rollout into ${org} stopped before publishing: ${reason}`, + ...failures, + ].join("\n"), + ); +} + +export function rolloutStepTargets( + from: string, + to: string, + percent: number, +): AllocationTarget[] { + const share = millipercentage(percent); + return [ + { assistantVersion: from, percentage: (100_000 - share) / 1000 }, + { assistantVersion: to, percentage: share / 1000 }, + ]; +} + +function sameWrite(allocation: Allocation | null, write: AllocationWrite): boolean { + if (!allocation) return false; + const marker = markerGet(allocation); + const intended = rolloutMarkerParse(write.description); + return ( + marker?.kind === intended?.kind && + marker?.from === intended?.from && + marker?.to === intended?.to && + JSON.stringify(allocation.targets) === JSON.stringify(write.targets ?? []) + ); +} + +export interface RolloutFinishArgs { + api: RolloutApi; + pins: RolloutPin[]; + steps: number[]; + childFailed: boolean; +} + +// After the apply child, successful or not: step 1 where a new version was +// published, the undo where nothing was. Returns per-assistant failures. +export async function rolloutFinish(args: RolloutFinishArgs): Promise { + const { api, pins, steps, childFailed } = args; + const failures: string[] = []; + for (const pin of pins) { + const { ref } = pin.read; + try { + const assistant = await api.assistantGet(ref.uuid); + const latest = assistant?.latestVersion ?? null; + const published = latest !== null && latest !== pin.read.latestVersion; + const write: AllocationWrite = published + ? { + assistantId: ref.uuid, + targets: rolloutStepTargets(pin.from, latest, steps[0]!), + description: rolloutMarkerFormat({ + kind: "step", + from: pin.from, + to: latest, + step: 1, + steps: steps.length, + run: runId(), + }), + expectedCurrentAllocationId: pin.pin.id, + } + : rolloutRestoreWrite(pin); + if (published && childFailed) + console.warn( + ` rollout ${ref.id}: ${latest} was published while the apply failed; it may be a dashboard publish rather than this promotion's`, + ); + try { + await api.allocationCreate(write, published ? "rollout-step" : "rollout-restore"); + console.log( + published + ? ` rollout ${ref.id}: ${pin.from} → ${latest} at ${steps[0]}%` + : ` rollout ${ref.id}: nothing published, pin undone`, + ); + } catch (error) { + if (allocationConflictGet(error) !== "stale_allocation") throw error; + const newest = await api.allocationLatestGet(ref.uuid); + if (markerGet(newest)?.kind === "abort") { + console.log(` rollout ${ref.id}: aborted meanwhile; left aborted`); + continue; + } + if (!sameWrite(newest, write)) throw error; + } + } catch (error) { + failures.push(`${ref.id}: ${messageOf(error)}`); + } + } + return failures; +} + +// latestVersion for each assistant, skipping any that can't be read: the +// unpinned-publish report is advice, so it never fails a run. +export async function rolloutSnapshot( + api: RolloutApi, + refs: RolloutAssistantRef[], +): Promise> { + const snapshot = new Map(); + for (const ref of refs) + try { + const assistant = await api.assistantGet(ref.uuid); + if (assistant) snapshot.set(ref.uuid, assistant.latestVersion); + } catch (error) { + console.warn(` rollout ${ref.id}: couldn't read its version (${messageOf(error)})`); + } + return snapshot; +} + +// Assistants the apply published without a pin (a squad member it re-applied, +// say), which went live at 100%. +export async function rolloutUnpinnedReport( + api: RolloutApi, + refs: RolloutAssistantRef[], + before: Map, +): Promise { + const after = await rolloutSnapshot( + api, + refs.filter((ref) => before.has(ref.uuid)), + ); + for (const ref of refs) { + const was = before.get(ref.uuid); + const now = after.get(ref.uuid); + if (before.has(ref.uuid) && after.has(ref.uuid) && was !== now) + console.warn( + ` rollout ${ref.id}: published ${now} without a pin, so it went live at 100%`, + ); + } +} + +// --no-rollout: after the child, a gitops-held assistant whose new version +// was published goes back to follow-latest, so a revert goes live at once. +export async function rolloutRelease( + api: RolloutApi, + refs: RolloutAssistantRef[], + before: Map, +): Promise { + const failures: string[] = []; + for (const ref of refs) { + if (!before.has(ref.uuid)) continue; + try { + const assistant = await api.assistantGet(ref.uuid); + const latest = assistant?.latestVersion ?? null; + if (!latest || latest === before.get(ref.uuid)) continue; + const allocation = await api.allocationLatestGet(ref.uuid); + if (allocation?.allocationIntent !== "explicit") continue; + if (!markerGet(allocation)) { + console.warn( + ` rollout ${ref.id}: kept the split set outside gitops (${splitDescribe(allocation.targets)}); ${latest} was published at 0%`, + ); + continue; + } + await api.allocationCreate( + { + assistantId: ref.uuid, + allocationIntent: "follow-latest", + description: rolloutMarkerFormat({ kind: "release", run: runId() }), + expectedCurrentAllocationId: allocation.id, + }, + "rollout-release", + ); + console.log(` rollout ${ref.id}: released; ${latest} takes every call`); + } catch (error) { + failures.push(`${ref.id}: ${messageOf(error)}`); + } + } + return failures; +} + +// latestVersion before the child, for --no-rollout; 404s and assistants +// without versions are left out, as preflight skips them. +export async function rolloutReleaseBaseline( + api: RolloutApi, + refs: RolloutAssistantRef[], +): Promise> { + const baseline = new Map(); + for (const ref of refs) { + const assistant = await api.assistantGet(ref.uuid); + if (assistant?.latestVersion) baseline.set(ref.uuid, assistant.latestVersion); + } + return baseline; +} diff --git a/tests/promote-rollout.test.ts b/tests/promote-rollout.test.ts index 870a2c3..3d5ccd0 100644 --- a/tests/promote-rollout.test.ts +++ b/tests/promote-rollout.test.ts @@ -1,16 +1,26 @@ import assert from "node:assert/strict"; import { execFileSync } from "node:child_process"; -import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; import test from "node:test"; +import { rolloutMarkerParse } from "../src/rollout-marker.ts"; import { RolloutFake } from "./rollout-fake.ts"; // promote-cmd.ts binds its root at import, so one fixture repo serves the // file; each test resets it. const ROOT = mkdtempSync(join(tmpdir(), "promote-rollout-")); process.env.VAPI_GITOPS_ROOT = ROOT; -const { promotionCommandRun } = await import("../src/promote-cmd.ts"); +const { APPLIED_PATHS_FILE, promotionCommandRun } = await import( + "../src/promote-cmd.ts" +); function git(...args: string[]): string { return execFileSync( @@ -25,78 +35,567 @@ function write(path: string, content: string): void { writeFileSync(join(ROOT, path), content); } -// staging → prod, where prod rolls out gradually unless `rollout` is false. -function fixtureReset(rollout = "{}"): void { +const CHECK_FILES: Record = { + "structuredOutputs/ok.yml": "name: ok\nschema:\n type: boolean\n", + "simulations/scenarios/s1.yml": + "name: S1\ninstructions: Hi.\nevaluations:\n - structuredOutputId: ok\n comparator: '='\n value: true\n required: true\n", + "simulations/personalities/calm.yml": "name: Calm\n", + "simulations/tests/t1.yml": "name: T1\npersonalityId: calm\nscenarioId: s1\n", +}; + +interface FixtureArgs { + // promotion.yml's prod rollout: value; "" for none. + rollout?: string; + // Assistant IDs changed in staging, and whether prod's state maps each. + assistants?: Record; + // Gate promotions out of staging on a check. + gated?: boolean; +} + +// staging → prod. Each assistant differs between the orgs, so the plan +// updates it (or creates it in prod when unmapped). +function fixtureReset(args: FixtureArgs = {}): void { + const rollout = args.rollout ?? "{}"; + const assistants = args.assistants ?? { intake: { mapped: true } }; rmSync(ROOT, { recursive: true, force: true }); mkdirSync(ROOT, { recursive: true }); + const staging = args.gated ? "\n check: staging-core" : " {}"; + const prod = rollout ? `\n rollout: ${rollout}` : " {}"; write( "promotion.yml", - `version: 1\norgs:\n staging: {}\n prod:\n${rollout ? ` rollout: ${rollout}\n` : " {}\n"}pipelines:\n release:\n orgs: [staging, prod]\n resources: ['**/*']\n`, + `version: 1\norgs:\n staging:${staging}\n prod:${prod}\npipelines:\n release:\n orgs: [staging, prod]\n resources: ['**/*']\n`, ); - write("resources/staging/assistants/intake.yml", "name: Intake v2\n"); - write("resources/prod/assistants/intake.yml", "name: Intake v1\n"); + if (args.gated) { + write( + "vapi-checks.yml", + "version: 1\nchecks:\n staging-core:\n org: staging\n targets: [assistants/intake]\n simulations: [t1]\n", + ); + for (const [path, content] of Object.entries(CHECK_FILES)) { + write(`resources/staging/${path}`, content); + write(`resources/prod/${path}`, content); + } + } + const state: Record = {}; + for (const [id, { mapped }] of Object.entries(assistants)) { + write(`resources/staging/assistants/${id}.yml`, `name: ${id} new\n`); + if (!mapped) continue; + write(`resources/prod/assistants/${id}.yml`, `name: ${id} old\n`); + state[id] = { uuid: `${id}-prod` }; + } write(".vapi-state.staging.json", "{}\n"); - write( - ".vapi-state.prod.json", - `${JSON.stringify({ assistants: { intake: { uuid: "intake-prod" } } })}\n`, - ); + write(".vapi-state.prod.json", `${JSON.stringify({ assistants: state })}\n`); write(".gitignore", "tmp/\n.env.*\n"); git("init", "-q", "-b", "main"); git("add", "-A"); git("commit", "-qm", "base"); } +interface Run { + error?: Error; + scripts: string[]; + checks: number; + output: string; +} + +interface ApplyFake { + // uuid → version the apply publishes. + publishes?: Record; + // Throw after publishing. + fail?: boolean; + // Runs inside the apply, after publishing. + during?: () => void; +} + async function promote( args: string[], fake: RolloutFake, - childRun: (script: string, org: string) => void = () => {}, -): Promise { + apply: ApplyFake = {}, + gate?: () => void, +): Promise { + const run: Run = { scripts: [], checks: 0, output: "" }; // promote reads the tokens, then deletes the variable. - process.env.VAPI_PROMOTION_TOKENS = JSON.stringify({ - staging: "t", - prod: "t", - }); + process.env.VAPI_PROMOTION_TOKENS = JSON.stringify({ staging: "t", prod: "t" }); const log = console.log; - console.log = () => {}; + const warn = console.warn; + console.log = (...parts: unknown[]) => { + run.output += `${parts.join(" ")}\n`; + }; + console.warn = console.log; try { - await promotionCommandRun(args, { childRun, rolloutApi: () => fake.api() }); - return undefined; + await promotionCommandRun(args, { + childRun: (script, org) => { + run.scripts.push(`${script}@${org}`); + if (script !== "src/apply.ts") return; + for (const [uuid, version] of Object.entries(apply.publishes ?? {})) + fake.publish(uuid, version); + apply.during?.(); + if (apply.fail) throw new Error(`${script} failed for ${org}`); + }, + checkRun: async () => { + run.checks++; + gate?.(); + return { outcome: "passed", reason: "1 of 1 simulations passed" }; + }, + rolloutApi: () => fake.api(), + }); } catch (error) { - return error as Error; + run.error = error as Error; } finally { console.log = log; + console.warn = warn; } + return run; +} + +const ALL = ["--all", "--apply"]; + +// Each write as `command kind targets`, e.g. `rollout-pin pin v1:100`. +function writesOf(fake: RolloutFake, uuid: string): string[] { + return fake.writes + .filter((write) => write.allocation.assistantId === uuid) + .map(({ command, allocation }) => + [ + command, + rolloutMarkerParse(allocation.description)?.kind ?? + allocation.allocationIntent, + allocation.targets + .map((target) => `${target.assistantVersion}:${target.percentage}`) + .join(","), + ] + .filter(Boolean) + .join(" "), + ); +} + +function appliedPaths(): string[] { + const file = join(ROOT, APPLIED_PATHS_FILE); + if (!existsSync(file)) return []; + return readFileSync(file, "utf8") + .split("\n") + .filter(Boolean) + .map((line) => line.split("\t")[1]!); } test("probe: a rollout org without traffic splitting stops the run before anything applies", async () => { fixtureReset(); const fake = new RolloutFake(); fake.splittingEnabled = false; - const scripts: string[] = []; - const failure = await promote(["--all", "--apply"], fake, (script) => { - scripts.push(script); - }); - assert.match( - failure?.message ?? "", - /Traffic splitting is not enabled for prod/, - ); - assert.deepEqual(scripts, []); + const run = await promote(ALL, fake); + assert.match(run.error?.message ?? "", /Traffic splitting is not enabled for prod/); + assert.deepEqual(run.scripts, []); }); test("probe: an org without rollout: is never probed", async () => { - fixtureReset(""); + fixtureReset({ rollout: "" }); const fake = new RolloutFake(); - const failure = await promote(["--all", "--apply"], fake); - assert.equal(failure, undefined); + const run = await promote(ALL, fake); + assert.equal(run.error, undefined); assert.equal(fake.probes, 0); }); test("probe: a plan without --apply makes no API call", async () => { fixtureReset(); const fake = new RolloutFake(); - const failure = await promote(["--all"], fake); - assert.equal(failure, undefined); + const run = await promote(["--all"], fake); + assert.equal(run.error, undefined); + assert.equal(fake.probes + fake.reads, 0); +}); + +test("a publish under follow-latest is pinned first, lands at 0%, then takes step 1", async () => { + fixtureReset({ rollout: "{steps: [2, 100]}" }); + const fake = new RolloutFake(); + fake.assistantAdd("intake-prod", "v1"); + const run = await promote(ALL, fake, { publishes: { "intake-prod": "v2" } }); + assert.equal(run.error, undefined); + // No system follow-latest row: the publish found the pin, not follow-latest. + assert.deepEqual(writesOf(fake, "intake-prod"), [ + "rollout-pin pin v1:100", + "rollout-step step v1:98,v2:2", + ]); + assert.deepEqual(rolloutMarkerParse(fake.latest("intake-prod")?.description), { + kind: "step", + from: "v1", + to: "v2", + step: 1, + steps: 2, + run: process.env.VAPI_GITOPS_RUN_ID, + }); +}); + +test("a promotion that publishes nothing undoes its pin back to follow-latest, with no targets", async () => { + fixtureReset(); + const fake = new RolloutFake(); + fake.assistantAdd("intake-prod", "v1"); + const run = await promote(ALL, fake); + assert.equal(run.error, undefined); + assert.deepEqual(writesOf(fake, "intake-prod"), [ + "rollout-pin pin v1:100", + "rollout-restore follow-latest", + ]); +}); + +test("when the apply fails after publishing one assistant, it still takes step 1 and the other's pin is undone", async () => { + fixtureReset({ + assistants: { intake: { mapped: true }, billing: { mapped: true } }, + }); + const fake = new RolloutFake(); + fake.assistantAdd("intake-prod", "v1"); + fake.assistantAdd("billing-prod", "v4"); + const run = await promote(ALL, fake, { + publishes: { "intake-prod": "v2" }, + fail: true, + }); + assert.match(run.error?.message ?? "", /src\/apply.ts failed for prod/); + assert.deepEqual(writesOf(fake, "intake-prod"), [ + "rollout-pin pin v1:100", + "rollout-step step v1:99,v2:1", + ]); + assert.deepEqual(writesOf(fake, "billing-prod"), [ + "rollout-pin pin v4:100", + "rollout-restore follow-latest", + ]); + // The failed transition's files aren't recorded as applied. + assert.deepEqual(appliedPaths(), []); +}); + +test("a step-1 failure after a successful apply records the applied files, then fails", async () => { + fixtureReset({ + assistants: { billing: { mapped: true }, intake: { mapped: true } }, + }); + const fake = new RolloutFake(); + fake.assistantAdd("billing-prod", "v4"); + fake.assistantAdd("intake-prod", "v1"); + // Writes: pin billing, pin intake, step billing, then intake's step fails. + fake.failCreate = { nth: 4, status: 400, error: "bad request" }; + const run = await promote(ALL, fake, { + publishes: { "billing-prod": "v5", "intake-prod": "v2" }, + }); + assert.match(run.error?.message ?? "", /Gradual rollout into prod:\n {2}- intake:/); + assert.deepEqual(appliedPaths().sort(), [ + "resources/prod/assistants/billing.yml", + "resources/prod/assistants/intake.yml", + ]); +}); + +test("a split set outside gitops is refused before the gate, the apply and any write", async () => { + fixtureReset({ gated: true }); + const fake = new RolloutFake(); + fake.assistantAdd("intake-prod", "v2"); + fake.manualWrite({ + assistantId: "intake-prod", + targets: [ + { assistantVersion: "v1", percentage: 95 }, + { assistantVersion: "v2", percentage: 5 }, + ], + }); + const run = await promote(ALL, fake); + assert.match( + run.error?.message ?? "", + /refused; nothing was promoted:\n {2}- intake: a split set outside gitops \(v1 95% \/ v2 5%\)/, + ); + assert.equal(run.checks, 0); + // Only the read-only bindings pulls ran. + assert.deepEqual(run.scripts, ["src/pull.ts@staging", "src/pull.ts@prod"]); + assert.deepEqual(writesOf(fake, "intake-prod"), ["manual explicit v1:95,v2:5"]); + assert.equal( + readFileSync(join(ROOT, "resources/prod/assistants/intake.yml"), "utf8"), + "name: intake old\n", + ); +}); + +test("an override pins a manual split on the version with the most traffic, not the canary", async () => { + fixtureReset(); + const fake = new RolloutFake(); + fake.assistantAdd("intake-prod", "v2"); + fake.manualWrite({ + assistantId: "intake-prod", + targets: [ + { assistantVersion: "v1", percentage: 95 }, + { assistantVersion: "v2", percentage: 5 }, + ], + }); + const run = await promote( + [ + "--pipeline", + "release", + "--from", + "staging", + "--to", + "prod", + "--apply", + "--rollout-override", + ], + fake, + { publishes: { "intake-prod": "v3" } }, + ); + assert.equal(run.error, undefined); + assert.deepEqual(writesOf(fake, "intake-prod"), [ + "manual explicit v1:95,v2:5", + "rollout-pin pin v1:100", + "rollout-step step v1:99,v3:1", + ]); + assert.match(run.output, /replacing v1 95% \/ v2 5% with v1 100%/); +}); + +test("--rollout-override is refused with --all", async () => { + fixtureReset(); + const run = await promote(["--all", "--apply", "--rollout-override"], new RolloutFake()); + assert.match(run.error?.message ?? "", /cannot be combined with --all/); +}); + +test("a pin that keeps failing on the second of three assistants undoes the first and never applies", async () => { + fixtureReset({ + assistants: { + a1: { mapped: true }, + a2: { mapped: true }, + a3: { mapped: true }, + }, + }); + const fake = new RolloutFake(); + for (const id of ["a1", "a2", "a3"]) fake.assistantAdd(`${id}-prod`, "v1"); + fake.failCreate = { nth: 2, status: 400, error: "bad request" }; + const run = await promote(ALL, fake); + assert.match(run.error?.message ?? "", /stopped before publishing/); + assert.deepEqual(run.scripts.filter((script) => script.startsWith("src/apply")), []); + assert.deepEqual(writesOf(fake, "a1-prod"), [ + "rollout-pin pin v1:100", + "rollout-restore follow-latest", + ]); + assert.deepEqual(writesOf(fake, "a2-prod"), []); +}); + +test("a pin that loses a race is retried once against a fresh read", async () => { + fixtureReset(); + const fake = new RolloutFake(); + fake.assistantAdd("intake-prod", "v1"); + fake.manualWrite({ assistantId: "intake-prod", allocationIntent: "follow-latest" }); + let raced = false; + fake.beforeCreate = () => { + if (raced) return; + raced = true; + fake.manualWrite({ assistantId: "intake-prod", allocationIntent: "follow-latest" }); + }; + const run = await promote(ALL, fake, { publishes: { "intake-prod": "v2" } }); + assert.equal(run.error, undefined); + assert.deepEqual(writesOf(fake, "intake-prod"), [ + "manual follow-latest", + "manual follow-latest", + "rollout-pin pin v1:100", + "rollout-step step v1:99,v2:1", + ]); +}); + +test("an assistant managed by Vapi for CX is refused in preflight", async () => { + fixtureReset(); + const fake = new RolloutFake(); + fake.assistantAdd("intake-prod", "v1", true); + const run = await promote(ALL, fake); + assert.match(run.error?.message ?? "", /intake: managed by Vapi for CX/); + assert.deepEqual(run.scripts.filter((script) => script.startsWith("src/apply")), []); +}); + +test("an abort written during the apply is left in place", async () => { + fixtureReset(); + const fake = new RolloutFake(); + fake.assistantAdd("intake-prod", "v1"); + const run = await promote(ALL, fake, { + publishes: { "intake-prod": "v2" }, + during: () => + fake.manualWrite({ + assistantId: "intake-prod", + targets: [{ assistantVersion: "v1", percentage: 100 }], + description: "gitops-rollout abort from=v1 to=v2 step=1/6", + }), + }); + assert.equal(run.error, undefined); + assert.equal(rolloutMarkerParse(fake.latest("intake-prod")?.description)?.kind, "abort"); + assert.match(run.output, /aborted meanwhile; left aborted/); +}); + +test("a deleted assistant gets no pin and the run succeeds", async () => { + fixtureReset(); + // The source state's mapping is the tombstone that authorizes the deletion. + write(".vapi-state.staging.json", `${JSON.stringify({ assistants: { gone: { uuid: "gone-staging" } } })}\n`); + write( + ".vapi-state.prod.json", + `${JSON.stringify({ assistants: { intake: { uuid: "intake-prod" }, gone: { uuid: "gone-prod" } } })}\n`, + ); + write("resources/prod/assistants/gone.yml", "name: gone\n"); + git("add", "-A"); + git("commit", "-qm", "gone deleted from staging"); + const fake = new RolloutFake(); + fake.assistantAdd("intake-prod", "v1"); + fake.assistantAdd("gone-prod", "v3"); + const run = await promote(ALL, fake, { publishes: { "intake-prod": "v2" } }); + assert.equal(run.error, undefined); + assert.deepEqual(writesOf(fake, "gone-prod"), []); + assert.deepEqual(writesOf(fake, "intake-prod"), [ + "rollout-pin pin v1:100", + "rollout-step step v1:99,v2:1", + ]); +}); + +test("an assistant with no state UUID goes live on create and isn't pinned", async () => { + fixtureReset({ assistants: { fresh: { mapped: false } } }); + const fake = new RolloutFake(); + const run = await promote(ALL, fake); + assert.equal(run.error, undefined); + assert.equal(fake.writes.length, 0); + assert.match(run.output, /fresh: new in prod, live on create/); +}); + +test("an assistant deleted in the dashboard is skipped with a message", async () => { + fixtureReset(); + const fake = new RolloutFake(); + const run = await promote(ALL, fake); + assert.equal(run.error, undefined); + assert.equal(fake.writes.length, 0); + assert.match(run.output, /intake: not found/); +}); + +test("the gate's window is re-read: a split set during the gate is refused with nothing pinned", async () => { + fixtureReset({ gated: true }); + const fake = new RolloutFake(); + fake.assistantAdd("intake-prod", "v1"); + const run = await promote(ALL, fake, {}, () => + fake.manualWrite({ + assistantId: "intake-prod", + targets: [ + { assistantVersion: "v1", percentage: 50 }, + { assistantVersion: "v0", percentage: 50 }, + ], + }), + ); + assert.equal(run.checks, 1); + assert.match(run.error?.message ?? "", /a split set outside gitops/); + assert.deepEqual(writesOf(fake, "intake-prod"), ["manual explicit v1:50,v0:50"]); + assert.equal( + readFileSync(join(ROOT, "resources/prod/assistants/intake.yml"), "utf8"), + "name: intake old\n", + ); +}); + +test("an abort written during the gate survives a promotion that publishes nothing", async () => { + fixtureReset({ gated: true }); + const fake = new RolloutFake(); + fake.assistantAdd("intake-prod", "v2"); + const abort = "gitops-rollout abort from=v1 to=v2 step=3/6"; + const run = await promote(ALL, fake, {}, () => + fake.manualWrite({ + assistantId: "intake-prod", + targets: [{ assistantVersion: "v1", percentage: 100 }], + description: abort, + }), + ); + assert.equal(run.error, undefined); + assert.deepEqual(writesOf(fake, "intake-prod"), [ + "manual abort v1:100", + "rollout-pin pin v1:100", + "rollout-restore abort v1:100", + ]); + assert.equal(fake.latest("intake-prod")?.description, abort); +}); + +test("without a gate, preflight reads each assistant once", async () => { + fixtureReset(); + const fake = new RolloutFake(); + fake.assistantAdd("intake-prod", "v1"); + await promote(ALL, fake); + // Preflight 2 (assistant, allocation), the pin's read-back 1 (no prior row + // to compare-and-set against), finish 1. A gate adds a second round of 2. + assert.equal(fake.reads, 4); +}); + +test("--no-rollout releases a published gitops-held assistant and leaves an unpublished abort alone", async () => { + fixtureReset({ + assistants: { intake: { mapped: true }, billing: { mapped: true } }, + }); + const fake = new RolloutFake(); + fake.assistantAdd("intake-prod", "v2"); + fake.assistantAdd("billing-prod", "v7"); + for (const [uuid, from] of [ + ["intake-prod", "v1"], + ["billing-prod", "v6"], + ] as const) + fake.manualWrite({ + assistantId: uuid, + targets: [{ assistantVersion: from, percentage: 100 }], + description: `gitops-rollout abort from=${from}`, + }); + const run = await promote([...ALL, "--no-rollout"], fake, { + publishes: { "intake-prod": "v3" }, + }); + assert.equal(run.error, undefined); + assert.deepEqual(writesOf(fake, "intake-prod"), [ + "manual abort v1:100", + "rollout-release release", + ]); + assert.deepEqual(writesOf(fake, "billing-prod"), ["manual abort v6:100"]); assert.equal(fake.probes, 0); }); +test("--no-rollout over a finished ramp publishes and writes nothing", async () => { + fixtureReset(); + const fake = new RolloutFake(); + fake.assistantAdd("intake-prod", "v2"); + fake.manualWrite({ + assistantId: "intake-prod", + allocationIntent: "follow-latest", + description: "gitops-rollout release", + }); + const run = await promote([...ALL, "--no-rollout"], fake, { + publishes: { "intake-prod": "v3" }, + }); + assert.equal(run.error, undefined); + assert.deepEqual(writesOf(fake, "intake-prod"), [ + "manual release", + "system follow-latest", + ]); +}); + +test("--no-rollout keeps a split set outside gitops and says the publish landed at 0%", async () => { + fixtureReset(); + const fake = new RolloutFake(); + fake.assistantAdd("intake-prod", "v2"); + fake.manualWrite({ + assistantId: "intake-prod", + targets: [ + { assistantVersion: "v1", percentage: 50 }, + { assistantVersion: "v2", percentage: 50 }, + ], + }); + const run = await promote([...ALL, "--no-rollout"], fake, { + publishes: { "intake-prod": "v3" }, + }); + assert.equal(run.error, undefined); + assert.equal(fake.writes.length, 1); + assert.match(run.output, /kept the split set outside gitops .* v3 was published at 0%/); +}); + +test("--no-rollout skips an assistant that 404s", async () => { + fixtureReset(); + const fake = new RolloutFake(); + const run = await promote([...ALL, "--no-rollout"], fake); + assert.equal(run.error, undefined); + assert.equal(fake.writes.length, 0); +}); + +test("an assistant the apply publishes without a pin is reported", async () => { + fixtureReset(); + write( + ".vapi-state.prod.json", + `${JSON.stringify({ assistants: { intake: { uuid: "intake-prod" }, member: { uuid: "member-prod" } } })}\n`, + ); + git("add", "-A"); + git("commit", "-qm", "squad member"); + const fake = new RolloutFake(); + fake.assistantAdd("intake-prod", "v1"); + fake.assistantAdd("member-prod", "v8"); + const run = await promote(ALL, fake, { + publishes: { "intake-prod": "v2", "member-prod": "v9" }, + }); + assert.equal(run.error, undefined); + assert.match(run.output, /member: published v9 without a pin, so it went live at 100%/); +}); + test.after(() => rmSync(ROOT, { recursive: true, force: true }));