diff --git a/docs/guides/promotion.md b/docs/guides/promotion.md index e5b9efc..9d02cb5 100644 --- a/docs/guides/promotion.md +++ b/docs/guides/promotion.md @@ -149,6 +149,27 @@ dependency is missing from source". Use personality files under `simulations/personalities/` in any org you promote out of, gated or not (`improvements.md` #38). +## Gradual rollout (optional) + +Promote into an org one traffic step at a time instead of all at once. Add +`rollout:` to the destination org: + +```yaml +# promotion.yml +orgs: + example-production: + rollout: + steps: [1, 5, 10, 25, 50, 100] # percent of calls on the new version + intervalMinutes: 30 # at least this long between steps +``` + +`rollout: {}` takes those defaults. Steps must increase, end at 100 and have +at most three decimal places; `intervalMinutes` is at least 15. + +The org needs traffic splitting, which Vapi enables per org during its beta. +An `--apply` run checks every rollout org first and stops, with nothing +promoted, if one doesn't have it. + ## Rolling Back a Promotion Treat a promotion rollback as a new, auditable Git change: revert the source diff --git a/promotion.example.yml b/promotion.example.yml index 84165e0..668e095 100644 --- a/promotion.example.yml +++ b/promotion.example.yml @@ -21,6 +21,13 @@ orgs: support-line: bind example-production: baseUrl: https://api.vapi.ai + # Optional gradual rollout: a promotion into this org publishes each + # changed assistant at 0%, then moves its calls to the new version in + # these steps (percent), at most one step per intervalMinutes. Needs + # traffic splitting enabled for the org. See docs/guides/promotion.md. + # rollout: + # steps: [1, 5, 10, 25, 50, 100] + # intervalMinutes: 30 bindings: credentials: default: bind diff --git a/src/check-cmd.ts b/src/check-cmd.ts index 494194d..e9bc44c 100644 --- a/src/check-cmd.ts +++ b/src/check-cmd.ts @@ -34,6 +34,7 @@ import type { OrgConnection } from "./org-connection.ts"; import { childRun, connectionLoad, tokensParse } from "./org-connection.ts"; import { userAgentGet, vapiClientGet } from "./user-agent.ts"; import type { VapiConnection } from "./vapi-client.ts"; +import { DEFAULT_BASE_URL } from "./vapi-client.ts"; const USAGE = [ "Usage:", @@ -58,7 +59,6 @@ const USAGE = [ const DEFAULT_PAYLOAD_DIR = "tmp/check-payloads"; const DEFAULT_BUDGET_MINUTES = 60; -const DEFAULT_BASE_URL = "https://api.vapi.ai"; const TOKENS_ENV = "VAPI_CHECK_TOKENS"; interface CheckArgs { diff --git a/src/promote-cmd.ts b/src/promote-cmd.ts index a219b17..108191a 100644 --- a/src/promote-cmd.ts +++ b/src/promote-cmd.ts @@ -12,6 +12,8 @@ import { promotionGatePlanLine, promotionGateRun, } from "./promotion-gate.ts"; +import type { RolloutApi } from "./rollout-api.ts"; +import { rolloutApiCreate } from "./rollout-api.ts"; import { promotionConfigParse, promotionPlanApply, @@ -53,6 +55,8 @@ export interface PromotionDeps { check: CheckDefinition, connection: OrgConnection, ) => Promise; + // The traffic-allocation client for an org with a gradual rollout. + rolloutApi: (connection: OrgConnection) => RolloutApi; } // Gated orgs' checks, and the orgs whose check already passed in this run. @@ -148,6 +152,29 @@ function orgConnection( }); } +// Every org a transition here would roll out into must have traffic +// splitting, a beta the API gates per org. Checked once, before anything is +// promoted, so a missing flag stops the run instead of one transition in. +async function rolloutOrgsProbe( + config: PromotionConfig, + transitions: PromotionTransition[], + tokens: Map, + deps: PromotionDeps, +): Promise { + const orgs = new Set( + transitions + .map((transition) => transition.target) + .filter((target) => config.orgs[target]?.rollout), + ); + for (const org of orgs) { + const api = deps.rolloutApi(orgConnection(config, org, tokens)); + if (!(await api.allocationsProbe())) + throw new Error( + `Traffic splitting is not enabled for ${org}, so its rollout: setting in promotion.yml can't run. Ask Vapi to enable traffic splitting for the org, or remove rollout: from ${org}.`, + ); + } +} + function orgScriptRun( script: string, org: string, @@ -308,6 +335,7 @@ export async function promotionCommandRun( childRun: orgScriptRun, checkRun: (check, connection) => promotionGateRun(ROOT_DIR, check, connection), + rolloutApi: rolloutApiCreate, ...overrides, }; const parsed = argumentsParse(args); @@ -331,7 +359,9 @@ export async function promotionCommandRun( // Applying a deletion removes the intermediate org's state entry. Carry the // reviewed authorization forward so the same deletion can reach later orgs. const deletionAuthorizedSources = new Set(); - for (const transition of transitionsBuild(config, parsed)) { + const transitions = transitionsBuild(config, parsed); + if (parsed.apply) await rolloutOrgsProbe(config, transitions, tokens, deps); + for (const transition of transitions) { const sourceKey = `${transition.pipeline}:${transition.source}`; const deleted = await transitionRun( config, diff --git a/src/promotion-gate.ts b/src/promotion-gate.ts index 22e42c2..3951140 100644 --- a/src/promotion-gate.ts +++ b/src/promotion-gate.ts @@ -11,6 +11,7 @@ import { checkRunAll, MAX_CONCURRENT, MIN_START_MS } from "./check-run.ts"; import type { OrgConnection } from "./org-connection.ts"; import type { PromotionConfig } from "./promotion.ts"; import { userAgentGet, vapiClientGet } from "./user-agent.ts"; +import { DEFAULT_BASE_URL } from "./vapi-client.ts"; export interface PromotionGateResult { outcome: CheckOutcome; @@ -18,7 +19,6 @@ export interface PromotionGateResult { url?: string; } -const DEFAULT_BASE_URL = "https://api.vapi.ai"; // Gate checks must fit well inside the promotion step's timeout, so a check // that can't finish is rejected when the config loads, not killed mid-run. export const GATE_BUDGET_MINUTES = 300; diff --git a/src/promotion.ts b/src/promotion.ts index eb13bae..4bb46fb 100644 --- a/src/promotion.ts +++ b/src/promotion.ts @@ -26,12 +26,23 @@ export interface PromotionBindings { phoneNumbers: PromotionBindingMap; } +// A gradual rollout: a promotion into the org publishes changed assistants at +// 0% and then moves their traffic to the new version step by step, one step +// per `intervalMinutes`, instead of all at once. +export interface PromotionRollout { + // Percent of calls on the new version at each step; the last is 100. + steps: number[]; + // Minimum minutes between steps. The rollout workflow ticks every 15. + intervalMinutes: number; +} + export interface PromotionOrg { baseUrl?: string; bindings: PromotionBindings; // A vapi-checks.yml check that must pass in this org before anything is // promoted out of it. check?: string; + rollout?: PromotionRollout; } export interface PromotionPipeline { @@ -145,7 +156,66 @@ export function promotionBindingsParse(value: unknown): PromotionBindings { }; } -const PROMOTION_ORG_KEYS = ["baseUrl", "bindings", "check"]; +const PROMOTION_ORG_KEYS = ["baseUrl", "bindings", "check", "rollout"]; + +export const ROLLOUT_STEPS_DEFAULT = [1, 5, 10, 25, 50, 100]; +export const ROLLOUT_INTERVAL_MINUTES_DEFAULT = 30; +// The rollout workflow's cron period; a shorter interval could not be honored. +export const ROLLOUT_INTERVAL_MINUTES_MIN = 15; +const ROLLOUT_KEYS = ["steps", "intervalMinutes"]; + +// The API stores shares as integer thousandths of a percent. +export function millipercentage(percent: number): number { + return Math.round(percent * 1000); +} + +export function promotionRolloutParse( + value: unknown, + slug: string, +): PromotionRollout { + const raw = object(value ?? {}, `org ${slug}.rollout`); + for (const key of Object.keys(raw)) + if (!ROLLOUT_KEYS.includes(key)) + throw new Error( + `org ${slug}.rollout has unknown key "${key}" (allowed: ${ROLLOUT_KEYS.join(", ")})`, + ); + const steps = raw.steps ?? ROLLOUT_STEPS_DEFAULT; + if ( + !Array.isArray(steps) || + steps.length === 0 || + !steps.every((step) => typeof step === "number" && Number.isFinite(step)) + ) + throw new Error(`org ${slug}.rollout.steps must be a list of percentages`); + for (const [index, step] of steps.entries()) { + if ( + step < 0.001 || + step > 100 || + // 1.005 * 1000 is 1004.9999999999999 in floating point, so compare + // with a tolerance rather than exactly. + Math.abs(step * 1000 - millipercentage(step)) > 1e-6 + ) + throw new Error( + `org ${slug}.rollout.steps: ${step} must be between 0.001 and 100, with at most three decimal places`, + ); + if (index > 0 && step <= steps[index - 1]!) + throw new Error( + `org ${slug}.rollout.steps must increase: ${step} follows ${steps[index - 1]}`, + ); + } + if (steps[steps.length - 1] !== 100) + throw new Error(`org ${slug}.rollout.steps must end at 100`); + const intervalMinutes = + raw.intervalMinutes ?? ROLLOUT_INTERVAL_MINUTES_DEFAULT; + if ( + typeof intervalMinutes !== "number" || + !Number.isInteger(intervalMinutes) || + intervalMinutes < ROLLOUT_INTERVAL_MINUTES_MIN + ) + throw new Error( + `org ${slug}.rollout.intervalMinutes must be a whole number of at least ${ROLLOUT_INTERVAL_MINUTES_MIN}`, + ); + return { steps: [...steps], intervalMinutes }; +} export function promotionConfigParse(content: string): PromotionConfig { const raw = object(parseYaml(content), "promotion.yml"); @@ -174,6 +244,9 @@ export function promotionConfigParse(content: string): PromotionConfig { baseUrl: typeof org.baseUrl === "string" ? org.baseUrl : undefined, bindings: promotionBindingsParse(org.bindings), ...(typeof org.check === "string" ? { check: org.check } : {}), + ...("rollout" in org + ? { rollout: promotionRolloutParse(org.rollout, slug) } + : {}), }; } const pipelinesRaw = object(raw.pipelines, "promotion.yml pipelines"); diff --git a/src/rollout-api.ts b/src/rollout-api.ts new file mode 100644 index 0000000..7492b9e --- /dev/null +++ b/src/rollout-api.ts @@ -0,0 +1,170 @@ +// The traffic-allocation calls a gradual rollout makes, against one org. +// Config-free (like vapi-client.ts) because promotion and the rollout ticker +// both work with several orgs in one process. + +import type { OrgConnection } from "./org-connection.ts"; +import type { RolloutCommand } from "./user-agent.ts"; +import { userAgentGet, vapiClientGet } from "./user-agent.ts"; +import type { VapiConnection } from "./vapi-client.ts"; +import { + DEFAULT_BASE_URL, + VapiApiError, + vapiFetchJson, +} from "./vapi-client.ts"; + +export interface AllocationTarget { + // A published version label, such as "v7". + assistantVersion: string; + // Decimal percent with at most three decimal places. + percentage: number; +} + +// GET /traffic-allocations/latest's allocation. Follow-latest rows carry no +// targets: every call goes to the newest published version. +export interface Allocation { + id: string; + assistantId: string; + allocationIntent: "explicit" | "follow-latest"; + createdAt: string; + description?: string; + targets: AllocationTarget[]; +} + +// POST /traffic-allocations. Allocations are append-only: a write replaces +// the one in effect. expectedCurrentAllocationId makes it a compare-and-set +// (409 stale_allocation when another write landed first); omitted, it never +// conflicts. +export interface AllocationWrite { + assistantId: string; + allocationIntent?: "explicit" | "follow-latest"; + targets?: AllocationTarget[]; + description?: string; + expectedCurrentAllocationId?: string; +} + +export interface RolloutAssistant { + // The newest published version label; null when versioning is off. + latestVersion: string | null; + // Vapi for CX manages it, so the API refuses allocation writes with a 409. + managed: boolean; +} + +// Why a write was refused with 409, from the response body's `error`. +export type AllocationConflict = + | "stale_allocation" + | "assistant_managed_by_vapi_agents" + | "other"; + +export interface RolloutApi { + // null when the assistant no longer exists (deleted in the dashboard). + assistantGet(assistantId: string): Promise; + // null when traffic splitting was never configured for the assistant. + allocationLatestGet(assistantId: string): Promise; + allocationCreate( + write: AllocationWrite, + command: RolloutCommand, + ): Promise; + // false when traffic splitting is not enabled for the org (a beta flag). + allocationsProbe(): Promise; +} + +// The platform's test for an assistant managed by Vapi for CX: a UUID in +// metadata.vapiAgentId. Anything else there isn't a lock. +const VAPI_AGENT_ID_PATTERN = + /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; + +interface AssistantResponse { + latestVersion?: string | null; + metadata?: { vapiAgentId?: unknown } | null; +} + +interface AllocationLatestResponse { + allocation?: Allocation; +} + +function notFound(error: unknown): boolean { + return error instanceof VapiApiError && error.statusCode === 404; +} + +export function allocationConflictGet( + error: unknown, +): AllocationConflict | null { + if (!(error instanceof VapiApiError) || error.statusCode !== 409) return null; + try { + const code: unknown = JSON.parse(error.rawBody)?.error; + if ( + code === "stale_allocation" || + code === "assistant_managed_by_vapi_agents" + ) + return code; + } catch { + /* not JSON: an unrecognized conflict */ + } + return "other"; +} + +// Reads retry 429 and 5xx. Writes retry only 429: a 5xx may come after the +// write committed, and a blind retry would then 409 against its own write. +export function rolloutApiCreate(connection: OrgConnection): RolloutApi { + const base = { + token: connection.token, + baseUrl: connection.baseUrl ?? DEFAULT_BASE_URL, + userAgent: userAgentGet(), + }; + const reads: VapiConnection = { ...base, vapiClient: vapiClientGet() }; + return { + async assistantGet(assistantId) { + try { + const assistant = await vapiFetchJson( + reads, + "GET", + `/assistant/${encodeURIComponent(assistantId)}`, + ); + const agentId = assistant.metadata?.vapiAgentId; + return { + latestVersion: assistant.latestVersion ?? null, + managed: + typeof agentId === "string" && VAPI_AGENT_ID_PATTERN.test(agentId), + }; + } catch (error) { + if (notFound(error)) return null; + throw error; + } + }, + async allocationLatestGet(assistantId) { + const response = await vapiFetchJson( + reads, + "GET", + `/traffic-allocations/latest?assistantId=${encodeURIComponent(assistantId)}`, + ); + return response.allocation ?? null; + }, + allocationCreate(write, command) { + const writes: VapiConnection = { + ...base, + vapiClient: vapiClientGet(command), + }; + return vapiFetchJson( + writes, + "POST", + "/traffic-allocations", + write, + { retry: "rate-limit-only" }, + ); + }, + async allocationsProbe() { + try { + await vapiFetchJson( + reads, + "GET", + "/traffic-allocations?limit=1", + ); + return true; + } catch (error) { + if (error instanceof VapiApiError && error.statusCode === 403) + return false; + throw error; + } + }, + }; +} diff --git a/src/user-agent.ts b/src/user-agent.ts index b3dbffb..9ad42c6 100644 --- a/src/user-agent.ts +++ b/src/user-agent.ts @@ -23,6 +23,9 @@ // parses for every first-party client, plus a run id shared by every request // of one gitops run, so the platform can group them. The run id is pinned in // VAPI_GITOPS_RUN_ID exactly as the command is pinned in VAPI_GITOPS_COMMAND. +// A gradual rollout's traffic-allocation writes override the command per +// request (`rollout-pin`, `rollout-step`, …; see RolloutCommand), so each +// write type is countable; the process's own reads keep its command. // // Config-free on purpose (like api-key.ts): importing config.ts would parse // argv and exit, which breaks importing this from sim.ts and tests. @@ -90,6 +93,7 @@ const COMMANDS = new Set([ "check", "rollback", "promote", + "rollout", ]); export const COMMAND_ENV = "VAPI_GITOPS_COMMAND"; @@ -153,8 +157,21 @@ if (!RUN_ID_PATTERN.test(process.env[RUN_ID_ENV] ?? "")) { process.env[RUN_ID_ENV] = randomUUID(); } +// The traffic-allocation writes a gradual rollout makes, one label each, so +// analytics can tell a pin from a step, a restore or an abort by command +// alone. A per-request label, never a process label: they are not in +// COMMANDS, so VAPI_GITOPS_COMMAND can't name one. +export type RolloutCommand = + | "rollout-pin" + | "rollout-step" + | "rollout-restore" + | "rollout-complete" + | "rollout-hold" + | "rollout-abort" + | "rollout-release"; + export function vapiClientGet( - product?: "sim" | "check" | "promote", + product?: "sim" | "check" | "promote" | RolloutCommand, context: UserAgentContext = { env: process.env, scriptPath: process.argv[1], diff --git a/src/vapi-client.ts b/src/vapi-client.ts index f1e76be..3c66476 100644 --- a/src/vapi-client.ts +++ b/src/vapi-client.ts @@ -6,6 +6,11 @@ // runner) uses this client instead. api.ts imports the error class and retry // policy from here, so both clients fail and retry the same way. +// The API host when neither promotion.yml nor `.env.` names one. The +// engine's child processes fall back to the same host (config.ts), so a +// parent that talks to the API directly must too. +export const DEFAULT_BASE_URL = "https://api.vapi.ai"; + export interface VapiConnection { token: string; baseUrl: string; diff --git a/tests/promote-rollout.test.ts b/tests/promote-rollout.test.ts new file mode 100644 index 0000000..870a2c3 --- /dev/null +++ b/tests/promote-rollout.test.ts @@ -0,0 +1,102 @@ +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import test from "node:test"; +import { RolloutFake } from "./rollout-fake.ts"; + +// promote-cmd.ts binds its root at import, so one fixture repo serves the +// file; each test resets it. +const ROOT = mkdtempSync(join(tmpdir(), "promote-rollout-")); +process.env.VAPI_GITOPS_ROOT = ROOT; +const { promotionCommandRun } = await import("../src/promote-cmd.ts"); + +function git(...args: string[]): string { + return execFileSync( + "git", + ["-c", "user.name=t", "-c", "user.email=t@example.com", ...args], + { cwd: ROOT, encoding: "utf8" }, + ); +} + +function write(path: string, content: string): void { + mkdirSync(dirname(join(ROOT, path)), { recursive: true }); + writeFileSync(join(ROOT, path), content); +} + +// staging → prod, where prod rolls out gradually unless `rollout` is false. +function fixtureReset(rollout = "{}"): void { + rmSync(ROOT, { recursive: true, force: true }); + mkdirSync(ROOT, { recursive: true }); + write( + "promotion.yml", + `version: 1\norgs:\n staging: {}\n prod:\n${rollout ? ` rollout: ${rollout}\n` : " {}\n"}pipelines:\n release:\n orgs: [staging, prod]\n resources: ['**/*']\n`, + ); + write("resources/staging/assistants/intake.yml", "name: Intake v2\n"); + write("resources/prod/assistants/intake.yml", "name: Intake v1\n"); + write(".vapi-state.staging.json", "{}\n"); + write( + ".vapi-state.prod.json", + `${JSON.stringify({ assistants: { intake: { uuid: "intake-prod" } } })}\n`, + ); + write(".gitignore", "tmp/\n.env.*\n"); + git("init", "-q", "-b", "main"); + git("add", "-A"); + git("commit", "-qm", "base"); +} + +async function promote( + args: string[], + fake: RolloutFake, + childRun: (script: string, org: string) => void = () => {}, +): Promise { + // promote reads the tokens, then deletes the variable. + process.env.VAPI_PROMOTION_TOKENS = JSON.stringify({ + staging: "t", + prod: "t", + }); + const log = console.log; + console.log = () => {}; + try { + await promotionCommandRun(args, { childRun, rolloutApi: () => fake.api() }); + return undefined; + } catch (error) { + return error as Error; + } finally { + console.log = log; + } +} + +test("probe: a rollout org without traffic splitting stops the run before anything applies", async () => { + fixtureReset(); + const fake = new RolloutFake(); + fake.splittingEnabled = false; + const scripts: string[] = []; + const failure = await promote(["--all", "--apply"], fake, (script) => { + scripts.push(script); + }); + assert.match( + failure?.message ?? "", + /Traffic splitting is not enabled for prod/, + ); + assert.deepEqual(scripts, []); +}); + +test("probe: an org without rollout: is never probed", async () => { + fixtureReset(""); + const fake = new RolloutFake(); + const failure = await promote(["--all", "--apply"], fake); + assert.equal(failure, undefined); + assert.equal(fake.probes, 0); +}); + +test("probe: a plan without --apply makes no API call", async () => { + fixtureReset(); + const fake = new RolloutFake(); + const failure = await promote(["--all"], fake); + assert.equal(failure, undefined); + assert.equal(fake.probes, 0); +}); + +test.after(() => rmSync(ROOT, { recursive: true, force: true })); diff --git a/tests/promotion-gate.test.ts b/tests/promotion-gate.test.ts index ede1899..a380588 100644 --- a/tests/promotion-gate.test.ts +++ b/tests/promotion-gate.test.ts @@ -320,7 +320,7 @@ test("a typo'd org key is rejected, so a misspelled check: can't silently drop t errors, ["checks", "Check", "gate"].map( (key) => - `org a has unknown key "${key}" (allowed: baseUrl, bindings, check)`, + `org a has unknown key "${key}" (allowed: baseUrl, bindings, check, rollout)`, ), ); }); diff --git a/tests/rollout-api.test.ts b/tests/rollout-api.test.ts new file mode 100644 index 0000000..3e1badb --- /dev/null +++ b/tests/rollout-api.test.ts @@ -0,0 +1,175 @@ +import assert from "node:assert/strict"; +import type { IncomingMessage, ServerResponse } from "node:http"; +import { createServer } from "node:http"; +import type { AddressInfo } from "node:net"; +import test from "node:test"; +import { allocationConflictGet, rolloutApiCreate } from "../src/rollout-api.ts"; +import { vapiClientGet } from "../src/user-agent.ts"; +import { VapiApiError } from "../src/vapi-client.ts"; + +interface Seen { + method?: string; + url?: string; + vapiClient?: string; + body?: string; +} + +// A local stand-in for the API: answers every request with `status` and +// `body`, and records the last request. +async function withServer( + status: number, + body: unknown, + fn: (baseUrl: string, seen: Seen, count: () => number) => Promise, +): Promise { + const seen: Seen = {}; + let requests = 0; + const server = createServer((req: IncomingMessage, res: ServerResponse) => { + requests++; + let data = ""; + req.on("data", (chunk) => (data += chunk)); + req.on("end", () => { + seen.method = req.method; + seen.url = req.url; + seen.vapiClient = String(req.headers["x-vapi-client"]); + seen.body = data; + res.writeHead(status, { "Content-Type": "application/json" }); + res.end(JSON.stringify(body)); + }); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const { port } = server.address() as AddressInfo; + try { + return await fn(`http://127.0.0.1:${port}`, seen, () => requests); + } finally { + await new Promise((resolve) => server.close(() => resolve())); + } +} + +const ASSISTANT = "9d1d2f6e-3b0a-4c7b-9f4e-1a2b3c4d5e6f"; + +test("rolloutApi: an allocation write carries its own command label", async () => { + await withServer(201, { id: "a1" }, async (baseUrl, seen) => { + const api = rolloutApiCreate({ token: "t", baseUrl }); + await api.allocationCreate( + { + assistantId: ASSISTANT, + targets: [{ assistantVersion: "v7", percentage: 100 }], + description: "gitops-rollout pin from=v7", + }, + "rollout-pin", + ); + assert.deepEqual(seen, { + method: "POST", + url: "/traffic-allocations", + vapiClient: vapiClientGet("rollout-pin"), + body: JSON.stringify({ + assistantId: ASSISTANT, + targets: [{ assistantVersion: "v7", percentage: 100 }], + description: "gitops-rollout pin from=v7", + }), + }); + }); +}); + +test("rolloutApi: a write is not retried after a 5xx", async () => { + await withServer( + 502, + { message: "bad gateway" }, + async (baseUrl, _seen, count) => { + const api = rolloutApiCreate({ token: "t", baseUrl }); + await assert.rejects( + api.allocationCreate({ assistantId: ASSISTANT }, "rollout-release"), + VapiApiError, + ); + assert.equal(count(), 1); + }, + ); +}); + +test("rolloutApi: reads the latest version and the Vapi for CX lock", async () => { + await withServer( + 200, + { + latestVersion: "v8", + metadata: { vapiAgentId: "0b8f1a52-4c2e-4f6a-9d3b-7e1c5a9f2d40" }, + }, + async (baseUrl, seen) => { + const api = rolloutApiCreate({ token: "t", baseUrl }); + assert.deepEqual(await api.assistantGet(ASSISTANT), { + latestVersion: "v8", + managed: true, + }); + assert.equal(seen.url, `/assistant/${ASSISTANT}`); + }, + ); +}); + +test("rolloutApi: a non-UUID vapiAgentId is not a lock", async () => { + await withServer( + 200, + { latestVersion: null, metadata: { vapiAgentId: "my-agent" } }, + async (baseUrl) => { + const api = rolloutApiCreate({ token: "t", baseUrl }); + assert.deepEqual(await api.assistantGet(ASSISTANT), { + latestVersion: null, + managed: false, + }); + }, + ); +}); + +test("rolloutApi: a deleted assistant reads as null", async () => { + await withServer(404, { message: "not found" }, async (baseUrl) => { + const api = rolloutApiCreate({ token: "t", baseUrl }); + assert.equal(await api.assistantGet(ASSISTANT), null); + }); +}); + +test("rolloutApi: no allocation ever configured reads as null", async () => { + await withServer(200, {}, async (baseUrl, seen) => { + const api = rolloutApiCreate({ token: "t", baseUrl }); + assert.equal(await api.allocationLatestGet(ASSISTANT), null); + assert.equal( + seen.url, + `/traffic-allocations/latest?assistantId=${ASSISTANT}`, + ); + }); +}); + +test("rolloutApi: the probe reports a 403 as traffic splitting off", async () => { + await withServer(403, { message: "not enabled" }, async (baseUrl, seen) => { + const api = rolloutApiCreate({ token: "t", baseUrl }); + assert.equal(await api.allocationsProbe(), false); + assert.equal(seen.url, "/traffic-allocations?limit=1"); + }); +}); + +test("rolloutApi: the probe reports a 200 as traffic splitting on", async () => { + await withServer(200, { results: [] }, async (baseUrl) => { + const api = rolloutApiCreate({ token: "t", baseUrl }); + assert.equal(await api.allocationsProbe(), true); + }); +}); + +for (const [label, status, body, expected] of [ + ["a stale write", 409, { error: "stale_allocation" }, "stale_allocation"], + [ + "a managed assistant", + 409, + { error: "assistant_managed_by_vapi_agents" }, + "assistant_managed_by_vapi_agents", + ], + ["an unknown 409", 409, { error: "something_else" }, "other"], + ["a 400", 400, { error: "stale_allocation" }, null], +] as const) { + test(`allocationConflictGet: classifies ${label}`, () => { + const error = new VapiApiError( + "POST", + "/traffic-allocations", + status, + "conflict", + JSON.stringify(body), + ); + assert.equal(allocationConflictGet(error), expected); + }); +} diff --git a/tests/rollout-config.test.ts b/tests/rollout-config.test.ts new file mode 100644 index 0000000..597ee83 --- /dev/null +++ b/tests/rollout-config.test.ts @@ -0,0 +1,64 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { promotionConfigParse } from "../src/promotion.ts"; + +function configWithRollout(rollout: string): string { + return `version: 1 +orgs: + staging: {} + prod: + rollout: ${rollout} +pipelines: + release: + orgs: [staging, prod] + resources: [assistants/**] +`; +} + +test("rollout: an empty mapping takes the default ramp and interval", () => { + assert.deepEqual(promotionConfigParse(configWithRollout("{}")).orgs.prod, { + baseUrl: undefined, + bindings: { + credentials: { default: "bind", aliases: {} }, + phoneNumbers: { default: "omit", aliases: {} }, + }, + rollout: { steps: [1, 5, 10, 25, 50, 100], intervalMinutes: 30 }, + }); +}); + +test("rollout: custom steps and interval are kept", () => { + assert.deepEqual( + promotionConfigParse( + configWithRollout("{steps: [0.5, 1.005, 20, 100], intervalMinutes: 60}"), + ).orgs.prod?.rollout, + { steps: [0.5, 1.005, 20, 100], intervalMinutes: 60 }, + ); +}); + +test("rollout: an org without it has none", () => { + assert.equal( + promotionConfigParse(configWithRollout("{}")).orgs.staging?.rollout, + undefined, + ); +}); + +for (const [label, rollout, message] of [ + ["steps that don't end at 100", "{steps: [1, 50]}", /must end at 100/], + ["steps that don't increase", "{steps: [5, 5, 100]}", /must increase/], + ["a step above 100", "{steps: [1, 101]}", /between 0.001 and 100/], + ["a step of 0", "{steps: [0, 100]}", /between 0.001 and 100/], + ["four decimal places", "{steps: [0.0005, 100]}", /three decimal places/], + ["an empty list", "{steps: []}", /list of percentages/], + ["a non-number step", "{steps: [ten, 100]}", /list of percentages/], + ["an interval under 15 minutes", "{intervalMinutes: 10}", /at least 15/], + ["a fractional interval", "{intervalMinutes: 20.5}", /whole number/], + ["a misspelled key", "{step: [1, 100]}", /unknown key "step"/], + ["a list instead of a mapping", "[1, 100]", /must be a mapping/], +] as const) { + test(`rollout: rejects ${label}`, () => { + assert.throws( + () => promotionConfigParse(configWithRollout(rollout)), + message, + ); + }); +} diff --git a/tests/rollout-fake.ts b/tests/rollout-fake.ts new file mode 100644 index 0000000..87d0da4 --- /dev/null +++ b/tests/rollout-fake.ts @@ -0,0 +1,134 @@ +// An in-memory stand-in for the traffic-allocation API, with the platform's +// semantics: append-only allocations, compare-and-set writes, and the +// publish-time rule (a publish under follow-latest or no allocation appends a +// system follow-latest row; under an explicit split it appends nothing, so +// the new version lands at 0%). + +import type { + Allocation, + AllocationWrite, + RolloutApi, + RolloutAssistant, +} from "../src/rollout-api.ts"; +import type { RolloutCommand } from "../src/user-agent.ts"; +import { VapiApiError } from "../src/vapi-client.ts"; + +export interface FakeWrite { + command: RolloutCommand | "system" | "manual"; + allocation: Allocation; +} + +export class RolloutFake { + assistants = new Map(); + history = new Map(); + writes: FakeWrite[] = []; + splittingEnabled = true; + probes = 0; + reads = 0; + // Fails the Nth allocationCreate (1-based) with this status and body. + failCreate?: { nth: number; status: number; error: string }; + // Runs before each allocationCreate, to simulate a concurrent writer. + beforeCreate?: (write: AllocationWrite) => void; + private creates = 0; + private clock = Date.parse("2026-10-30T12:00:00Z"); + private ids = 0; + + assistantAdd(id: string, latestVersion: string | null, managed = false) { + this.assistants.set(id, { latestVersion, managed }); + } + + latest(assistantId: string): Allocation | null { + const rows = this.history.get(assistantId) ?? []; + return rows[rows.length - 1] ?? null; + } + + // Moves the clock, so createdAt-based intervals can elapse. + advance(minutes: number) { + this.clock += minutes * 60_000; + } + + // A write made outside gitops (the dashboard, another tool). + manualWrite(write: AllocationWrite) { + this.append(write, "manual"); + } + + publish(assistantId: string, version: string) { + const assistant = this.assistants.get(assistantId)!; + assistant.latestVersion = version; + const latest = this.latest(assistantId); + if (!latest || latest.allocationIntent === "follow-latest") + this.append({ assistantId, allocationIntent: "follow-latest" }, "system"); + } + + private append(write: AllocationWrite, command: FakeWrite["command"]) { + const explicit = write.targets !== undefined; + const allocation: Allocation = { + id: `alloc-${++this.ids}`, + assistantId: write.assistantId, + allocationIntent: explicit ? "explicit" : "follow-latest", + createdAt: new Date(this.clock).toISOString(), + ...(write.description ? { description: write.description } : {}), + targets: (write.targets ?? []).map((target) => ({ ...target })), + }; + const rows = this.history.get(write.assistantId) ?? []; + rows.push(allocation); + this.history.set(write.assistantId, rows); + this.writes.push({ command, allocation }); + return allocation; + } + + api(): RolloutApi { + return { + assistantGet: async (id) => { + this.reads++; + const assistant = this.assistants.get(id); + return assistant ? { ...assistant } : null; + }, + allocationLatestGet: async (id) => { + this.reads++; + const latest = this.latest(id); + return latest ? structuredClone(latest) : null; + }, + allocationCreate: async (write, command) => { + this.creates++; + this.beforeCreate?.(write); + if (this.failCreate?.nth === this.creates) + throw new VapiApiError( + "POST", + "/traffic-allocations", + this.failCreate.status, + this.failCreate.error, + JSON.stringify({ error: this.failCreate.error }), + ); + if (this.assistants.get(write.assistantId)?.managed) + throw new VapiApiError( + "POST", + "/traffic-allocations", + 409, + "managed", + JSON.stringify({ error: "assistant_managed_by_vapi_agents" }), + ); + const current = this.latest(write.assistantId)?.id; + if ( + write.expectedCurrentAllocationId !== undefined && + write.expectedCurrentAllocationId !== current + ) + throw new VapiApiError( + "POST", + "/traffic-allocations", + 409, + "stale", + JSON.stringify({ + error: "stale_allocation", + currentAllocationId: current ?? null, + }), + ); + return structuredClone(this.append(write, command)); + }, + allocationsProbe: async () => { + this.probes++; + return this.splittingEnabled; + }, + }; + } +} diff --git a/tests/user-agent.test.ts b/tests/user-agent.test.ts index d060e24..b9b4b49 100644 --- a/tests/user-agent.test.ts +++ b/tests/user-agent.test.ts @@ -150,6 +150,13 @@ test("vapiClientGet: names the command, run and CI flag", () => { ); }); +test("vapiClientGet: labels a rollout write by its own command", () => { + assert.equal( + vapiClientGet("rollout-pin", { env: { [RUN_ID_ENV]: RUN_ID } }), + `gitops/${packageVersion}; command=rollout-pin; run=${RUN_ID}`, + ); +}); + test("vapiClientGet: leaves out a run id the platform would reject", () => { assert.equal( vapiClientGet("check", { env: { [RUN_ID_ENV]: "bad id; ci" } }),