diff --git a/apps/cli-e2e/src/tests/env.ts b/apps/cli-e2e/src/tests/env.ts index e8803530cc..5936cef911 100644 --- a/apps/cli-e2e/src/tests/env.ts +++ b/apps/cli-e2e/src/tests/env.ts @@ -50,6 +50,59 @@ export const TARGET_API_URL = export const PROJECT_HOST = process.env["CLI_E2E_PROJECT_HOST"] ?? (TARGET_ENV === "staging" ? "supabase.red" : ""); +// Runtime capabilities a live target can offer the cli. Live tests declare what +// they need (see `testLiveRequires`) and are skipped when the target can't +// provide it — so the same suite runs everywhere, skipping only what a given +// environment genuinely can't do: +// - docker control a container / has a Docker socket +// - internet reach 3rd-party network at runtime (jsr.io, npm, image pulls) +// - external-tool native pg_dump/psql, diff engine (SUPABASE_DB_USE_LOCAL_TOOLS) +// The project data plane (its Postgres via the pooler, the Storage API) is NOT a +// capability — any live target is assumed to provide a provisioned project, so +// data-plane access is a baseline, not something a test opts into. +const ALL_CAPABILITIES = ["docker", "internet", "external-tool"] as const; +export type Capability = (typeof ALL_CAPABILITIES)[number]; + +// Per-target defaults for what the environment provides. `staging` provides +// everything — it is the oracle: every probe must pass there, so a supabox +// skip/red is a genuine gap. `supabox` starts empty (locked down) and is opened +// up via CLI_E2E_CAPABILITIES as it gains real support; a probe only *runs* on +// supabox once its capability is declared, and must then pass (vs staging). +const DEFAULT_CAPABILITIES: Record = { + staging: ALL_CAPABILITIES, + supabox: [], +}; + +function isCapability(value: string): value is Capability { + return (ALL_CAPABILITIES as readonly string[]).includes(value); +} + +// Parse the comma list, rejecting unknown tokens: silently dropping a typo (e.g. +// `external_tools`) would skip every test for that capability and still leave the +// run green, hiding coverage the environment thought it had enabled. +function parseCapabilities(raw: string): Capability[] { + const tokens = raw + .split(",") + .map((entry) => entry.trim()) + .filter((entry) => entry.length > 0); + const unknown = tokens.filter((token) => !isCapability(token)); + if (unknown.length > 0) { + throw new Error( + `Unknown CLI_E2E_CAPABILITIES: ${unknown.join(", ")}. Valid: ${ALL_CAPABILITIES.join(", ")}.`, + ); + } + return tokens.filter(isCapability); +} + +// Override the per-target defaults with an explicit comma list, e.g. +// CLI_E2E_CAPABILITIES=docker,external-tool (an Antithesis run never sets `internet`). +const CAPABILITIES_OVERRIDE = process.env["CLI_E2E_CAPABILITIES"]; +export const PROVIDED_CAPABILITIES: ReadonlySet = new Set( + CAPABILITIES_OVERRIDE === undefined + ? DEFAULT_CAPABILITIES[TARGET_ENV] + : parseCapabilities(CAPABILITIES_OVERRIDE), +); + // In replay mode the token never reaches a real API, but the Go CLI validates // the format before making any request (must match sbp_[a-f0-9]{40}). // In record/live mode it must be a valid token for the target env. Falls back to diff --git a/apps/cli-e2e/src/tests/live/capabilities.live.e2e.test.ts b/apps/cli-e2e/src/tests/live/capabilities.live.e2e.test.ts new file mode 100644 index 0000000000..c496931b76 --- /dev/null +++ b/apps/cli-e2e/src/tests/live/capabilities.live.e2e.test.ts @@ -0,0 +1,121 @@ +import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { describe, expect } from "vitest"; +import { expectFunctionOk } from "./invoke.ts"; +import { seedFunctions, testLiveRequires } from "./live-context.ts"; + +// Capability probes: one minimal test per runtime-capability combination the cli +// needs from its target. Each is a smoke check that FORCES its capability, so a +// failure (or a target-driven skip) is a precise signal of what an environment +// can or cannot do. Distilled from the richer feature tests so each isolates a +// single capability. +// +// Staging is the oracle: with CLI_E2E_TARGET_ENV=staging (provides all +// capabilities) every probe runs and must pass — proving the probe is sound, so +// any supabox skip/red is a genuine gap. On supabox each probe only runs once +// its capability is declared via CLI_E2E_CAPABILITIES, then must match staging. +// See `testLiveRequires` + PROVIDED_CAPABILITIES in ../env.ts. +describe("capability probes (live)", () => { + // C1 — mgmt-api only (no docker, no internet, no external tool). The + // provisioned project shows up in `projects list`: a pure control-plane read. + testLiveRequires([])( + "[C1] mgmt-api only: projects list includes the project", + async ({ run, projectRef }) => { + const res = await run(["projects", "list", "--output", "json"]); + expect(res.exitCode, res.stderr).toBe(0); + const refs = (JSON.parse(res.stdout) as Array<{ id?: string; ref?: string }>).map( + (project) => project.ref ?? project.id, + ); + expect(refs).toContain(projectRef); + }, + ); + + // C2 — docker control, no runtime internet. `db pull`'s schema diff starts a + // shadow postgres *server* (DockerStart) and runs the diff engine in a + // container; both use pre-built images (no 3rd-party network). Push first so + // local history matches the shared per-run project, then pull. A missing + // Docker socket makes DockerStart fail — a genuine docker gate. + testLiveRequires(["docker"])( + "[C2] docker (offline): db push then db pull round-trips", + async ({ run, dbUrl, workspace }) => { + const migrations = join(workspace.path, "supabase", "migrations"); + mkdirSync(migrations, { recursive: true }); + // Unique timestamp so this never collides with db-sync.live.e2e.test.ts's + // migration on the shared per-run project (history compares timestamps). + writeFileSync( + join(migrations, "20240202020202_capability_probe_push.sql"), + "create table if not exists capability_probe (id int);\n", + ); + + const pushed = await run(["db", "push", "--db-url", dbUrl, "--yes"]); + expect(pushed.exitCode, pushed.stderr).toBe(0); + + const pulled = await run(["db", "pull", "--db-url", dbUrl, "--yes"]); + const output = `${pulled.stdout}${pulled.stderr}`; + // Distinguish a real docker/connection failure from a benign "no changes". + expect(output, "db pull hit a docker/connection error").not.toMatch( + /cannot connect to the docker daemon|is the docker daemon running|dial|connection refused|could not connect/i, + ); + expect(pulled.exitCode === 0 || /No schema changes found/i.test(output), pulled.stderr).toBe( + true, + ); + }, + ); + + // C3 — external tool, no docker/internet. `db dump` normally runs `pg_dump` in a + // supabase/postgres CONTAINER; `SUPABASE_DB_USE_LOCAL_TOOLS=1` switches it to the + // native pg_dump/psql on PATH, so this exercises the external-tool path (no + // Docker socket) rather than the container path. Fails if the tool is absent. + testLiveRequires(["external-tool"])( + "[C3] external tool: db dump exports the remote schema via native pg_dump", + async ({ run, dbUrl, workspace }) => { + const file = join(workspace.path, "dump.sql"); + const res = await run(["db", "dump", "--db-url", dbUrl, "-f", file], { + env: { SUPABASE_DB_USE_LOCAL_TOOLS: "1" }, + }); + expect(res.exitCode, res.stderr).toBe(0); + expect(existsSync(file)).toBe(true); + expect(readFileSync(file, "utf8")).toMatch(/CREATE|PostgreSQL database dump|SCHEMA/i); + }, + ); + + // C4 — runtime 3rd-party internet, no docker. Deploy a function that imports + // from jsr.io with the default (server-side) bundler and invoke it; the bundle + // path must fetch the import over the network. Fails offline. + testLiveRequires(["internet"])( + "[C4] internet: deploy a jsr-importing function and invoke", + async ({ run, invoke, workspace, projectRef }) => { + seedFunctions(workspace.path); + const slug = "deploy-e2e-jsr"; + const deployed = await run(["functions", "deploy", slug, "--project-ref", projectRef]); + expect(deployed.exitCode, deployed.stderr).toBe(0); + expect(deployed.stdout).toContain("Deployed Functions"); + expectFunctionOk(await invoke(slug), slug); + }, + ); + + // C5 — docker AND runtime internet. Bundle an npm-importing function locally in + // a Docker container (`--use-docker`): needs a Docker socket AND the in-container + // bundler needs internet to fetch the npm import. A DISTINCT slug from C4 so the + // invoke proves THIS (--use-docker) deploy produced the running function, not + // C4's earlier server-bundled one on the shared project. Fails if either is + // missing. + testLiveRequires(["docker", "internet"])( + "[C5] docker + internet: --use-docker deploy of an npm-importing function", + async ({ run, invoke, workspace, projectRef }) => { + seedFunctions(workspace.path); + const slug = "deploy-e2e-npm"; + const deployed = await run([ + "functions", + "deploy", + slug, + "--project-ref", + projectRef, + "--use-docker", + ]); + expect(deployed.exitCode, deployed.stderr).toBe(0); + expect(deployed.stdout).toContain("Deployed Functions"); + expectFunctionOk(await invoke(slug), slug); + }, + ); +}); diff --git a/apps/cli-e2e/src/tests/live/database.live.e2e.test.ts b/apps/cli-e2e/src/tests/live/database.live.e2e.test.ts index 6a99f7131f..4e9dea664c 100644 --- a/apps/cli-e2e/src/tests/live/database.live.e2e.test.ts +++ b/apps/cli-e2e/src/tests/live/database.live.e2e.test.ts @@ -1,13 +1,14 @@ import { existsSync, readFileSync } from "node:fs"; import { join } from "node:path"; import { describe, expect } from "vitest"; -import { testLive } from "./live-context.ts"; +import { testLive, testLiveRequires } from "./live-context.ts"; // DB-connectivity commands against the fresh project's Postgres via the IPv4 // session-mode Supavisor pooler (`dbUrl` from live-setup). The direct host // (db..supabase.red) is IPv6-only and unreachable from IPv4-only CI // runners; the pooler is IPv4, and session mode is required for pg_dump. -// A non-zero exit here means the connection itself failed. +// A non-zero exit here means the connection itself failed. The pooler is baseline +// data plane, so the connect-only cases need no runtime capability. describe("database (live, session pooler --db-url)", () => { testLive("inspect db db-stats connects and reports stats", async ({ run, dbUrl }) => { const res = await run(["inspect", "db", "db-stats", "--db-url", dbUrl]); @@ -22,11 +23,16 @@ describe("database (live, session pooler --db-url)", () => { expect(res.exitCode, res.stderr).toBe(0); }); - testLive("db dump exports the remote schema", async ({ run, dbUrl, workspace }) => { - const file = join(workspace.path, "dump.sql"); - const res = await run(["db", "dump", "--db-url", dbUrl, "-f", file]); - expect(res.exitCode, res.stderr).toBe(0); - expect(existsSync(file)).toBe(true); - expect(readFileSync(file, "utf8")).toMatch(/CREATE|PostgreSQL database dump|SCHEMA/i); - }); + // Needs `docker`: without SUPABASE_DB_USE_LOCAL_TOOLS this `db dump` runs + // pg_dump in a supabase/postgres container (see the C3 probe for the native path). + testLiveRequires(["docker"])( + "db dump exports the remote schema", + async ({ run, dbUrl, workspace }) => { + const file = join(workspace.path, "dump.sql"); + const res = await run(["db", "dump", "--db-url", dbUrl, "-f", file]); + expect(res.exitCode, res.stderr).toBe(0); + expect(existsSync(file)).toBe(true); + expect(readFileSync(file, "utf8")).toMatch(/CREATE|PostgreSQL database dump|SCHEMA/i); + }, + ); }); diff --git a/apps/cli-e2e/src/tests/live/db-sync.live.e2e.test.ts b/apps/cli-e2e/src/tests/live/db-sync.live.e2e.test.ts index 4780b56537..b29053ee87 100644 --- a/apps/cli-e2e/src/tests/live/db-sync.live.e2e.test.ts +++ b/apps/cli-e2e/src/tests/live/db-sync.live.e2e.test.ts @@ -1,7 +1,7 @@ import { mkdirSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { describe, expect } from "vitest"; -import { testLive } from "./live-context.ts"; +import { testLiveRequires } from "./live-context.ts"; // Local↔remote schema sync (workflows 1-2) over the IPv4 session pooler. Done as // one round-trip in a single workspace: pushing first makes the local migration @@ -9,8 +9,12 @@ import { testLive } from "./live-context.ts"; // (a separate fresh-workspace pull would see a history mismatch on the shared // per-run project). db push/pull confirm via a prompt that only auto-accepts // with --yes. Mutates the throwaway project's schema — deleted on teardown. +// +// Requires `docker`: `db push` is a plain pooler connection, but `db pull`'s +// schema diff starts a shadow postgres server (PrepareRawShadow → DockerStart) + +// diff container, so the round-trip skips on targets without a Docker socket. describe("db push + pull (live, session pooler)", () => { - testLive( + testLiveRequires(["docker"])( "pushes a local migration and pulls the remote schema back", async ({ run, dbUrl, workspace }) => { const migrations = join(workspace.path, "supabase", "migrations"); diff --git a/apps/cli-e2e/src/tests/live/functions-deploy.live.e2e.test.ts b/apps/cli-e2e/src/tests/live/functions-deploy.live.e2e.test.ts index e9101cc1be..cd4375c9f1 100644 --- a/apps/cli-e2e/src/tests/live/functions-deploy.live.e2e.test.ts +++ b/apps/cli-e2e/src/tests/live/functions-deploy.live.e2e.test.ts @@ -2,7 +2,7 @@ import { readdirSync } from "node:fs"; import { join } from "node:path"; import { describe, expect } from "vitest"; import { expectFunctionOk } from "./invoke.ts"; -import { seedFunctions, testLive } from "./live-context.ts"; +import { seedFunctions, testLiveRequires } from "./live-context.ts"; // Pilot (ADR-0013): deploy with the real CLI across the three bundler paths, // then invoke the deployed function over HTTP and assert the body it returns. @@ -10,35 +10,47 @@ import { seedFunctions, testLive } from "./live-context.ts"; // produced a running function — the shared project means a single slug could // otherwise be served by an earlier mode's deploy. Negative/arg-validation // cases live in apps/cli integration tests. +// `requires` per mode: `--use-docker` bundles locally in a container (needs a +// Docker socket); default/`--use-api` bundle server-side (no local docker). The +// fixtures here are import-free, so no runtime internet is required. const MODES = [ - { name: "default", slug: "deploy-e2e-mode-default", flags: [] as string[] }, - { name: "use-api", slug: "deploy-e2e-mode-api", flags: ["--use-api"] }, - { name: "use-docker", slug: "deploy-e2e-mode-docker", flags: ["--use-docker"] }, + { name: "default", slug: "deploy-e2e-mode-default", flags: [] as string[], requires: [] }, + { name: "use-api", slug: "deploy-e2e-mode-api", flags: ["--use-api"], requires: [] }, + { + name: "use-docker", + slug: "deploy-e2e-mode-docker", + flags: ["--use-docker"], + requires: ["docker"], + }, ] as const; -describe.each(MODES)("functions deploy ($name)", ({ slug, flags }) => { - testLive("deploys and the function responds", async ({ run, invoke, workspace, projectRef }) => { - seedFunctions(workspace.path); - const deployed = await run([ - "functions", - "deploy", - slug, - "--project-ref", - projectRef, - ...flags, - ]); - expect(deployed.exitCode, deployed.stderr).toBe(0); - expect(deployed.stdout).toContain("Deployed Functions"); +describe.each(MODES)("functions deploy ($name)", ({ slug, flags, requires }) => { + testLiveRequires(requires)( + "deploys and the function responds", + async ({ run, invoke, workspace, projectRef }) => { + seedFunctions(workspace.path); + const deployed = await run([ + "functions", + "deploy", + slug, + "--project-ref", + projectRef, + ...flags, + ]); + expect(deployed.exitCode, deployed.stderr).toBe(0); + expect(deployed.stdout).toContain("Deployed Functions"); - const res = await invoke(slug); - expectFunctionOk(res, slug); - }); + const res = await invoke(slug); + expectFunctionOk(res, slug); + }, + ); }); // No slug → the CLI walks every function declared under supabase/functions and // deploys them all. Assert each declared function appears in the deploy output, -// then smoke-invoke a representative one. -testLive( +// then smoke-invoke a representative one. Requires `internet`: the fixture set +// includes functions that import from jsr.io / npm, resolved at bundle time. +testLiveRequires(["internet"])( "deploys every declared function when no slug is given", async ({ run, invoke, workspace, projectRef }) => { seedFunctions(workspace.path); diff --git a/apps/cli-e2e/src/tests/live/gen-types.live.e2e.test.ts b/apps/cli-e2e/src/tests/live/gen-types.live.e2e.test.ts index e75455989b..ef17d91ef7 100644 --- a/apps/cli-e2e/src/tests/live/gen-types.live.e2e.test.ts +++ b/apps/cli-e2e/src/tests/live/gen-types.live.e2e.test.ts @@ -1,13 +1,15 @@ import { describe, expect } from "vitest"; -import { testLive } from "./live-context.ts"; +import { testLiveRequires } from "./live-context.ts"; // gen types introspects the remote schema over the IPv4 session pooler and emits -// TypeScript types. It pulls the postgres-meta Docker image, so it needs Docker -// (present in the CI live job alongside the --use-docker bundler cell). +// TypeScript types by running the postgres-meta Docker image (needs `docker`). describe("gen types (live, session pooler)", () => { - testLive("generates TypeScript types from the remote schema", async ({ run, dbUrl }) => { - const res = await run(["gen", "types", "--db-url", dbUrl, "--lang", "typescript"]); - expect(res.exitCode, res.stderr).toBe(0); - expect(res.stdout).toMatch(/export type (Database|Json)/); - }); + testLiveRequires(["docker"])( + "generates TypeScript types from the remote schema", + async ({ run, dbUrl }) => { + const res = await run(["gen", "types", "--db-url", dbUrl, "--lang", "typescript"]); + expect(res.exitCode, res.stderr).toBe(0); + expect(res.stdout).toMatch(/export type (Database|Json)/); + }, + ); }); diff --git a/apps/cli-e2e/src/tests/live/live-context.ts b/apps/cli-e2e/src/tests/live/live-context.ts index 2cabe016e3..c4c9eac255 100644 --- a/apps/cli-e2e/src/tests/live/live-context.ts +++ b/apps/cli-e2e/src/tests/live/live-context.ts @@ -8,7 +8,15 @@ import { type CLIResult, type TempDir, } from "@supabase/cli-test-helpers"; -import { ACCESS_TOKEN, isLive, PROJECT_HOST, TARGET, TARGET_API_URL } from "../env.ts"; +import { + ACCESS_TOKEN, + type Capability, + isLive, + PROJECT_HOST, + PROVIDED_CAPABILITIES, + TARGET, + TARGET_API_URL, +} from "../env.ts"; import { invokeFunction, type InvokeResult } from "./invoke.ts"; type ExecOptions = NonNullable[2]>; @@ -91,7 +99,12 @@ const base = test.extend({ }, workspace: async ({ task }, use) => { - const dir = makeTempDir(`cli-e2e-live-${task.name.slice(0, 30)}-`); + // Sanitize the task name: it becomes part of the temp-dir path, which the cli + // mounts as a Docker volume for docker-backed commands (functions bundling, + // db diff shadow). A `:` or space in the path breaks the `src:dst:mode` + // volume spec ("too many colons"), so collapse anything non-alphanumeric. + const safeName = task.name.replace(/[^a-zA-Z0-9]+/g, "-").slice(0, 30); + const dir = makeTempDir(`cli-e2e-live-${safeName}-`); // Generate config.toml via `supabase init` so the golden paths run against a // freshly-generated config (functions tests add functions via seedFunctions). const init = await exec(liveHarness(dir.path), ["init"]); @@ -120,3 +133,14 @@ const base = test.extend({ /** Live test API — skipped unless CLI_E2E_MODE=live, so files are inert on * replay/PR runs (and globalSetup provisions nothing). */ export const testLive = base.skipIf(!isLive); + +/** Live test API that additionally skips unless the target env provides every + * required runtime capability (docker / internet / external-tool). Lets one + * suite run against staging (all capabilities → runs everything, the oracle), + * supabox (only what it currently supports), and Antithesis (offline subset), + * each skipping only what it genuinely can't do. Put the requirement in the test + * name (e.g. "[C5] … (docker+internet)") so a skip reads clearly in the report. */ +export function testLiveRequires(required: readonly Capability[]): typeof testLive { + const missing = required.filter((capability) => !PROVIDED_CAPABILITIES.has(capability)); + return missing.length === 0 ? testLive : testLive.skip; +} diff --git a/apps/cli-e2e/src/tests/live/storage.live.e2e.test.ts b/apps/cli-e2e/src/tests/live/storage.live.e2e.test.ts index 2d705f690d..56a7ca3be6 100644 --- a/apps/cli-e2e/src/tests/live/storage.live.e2e.test.ts +++ b/apps/cli-e2e/src/tests/live/storage.live.e2e.test.ts @@ -8,6 +8,7 @@ import { testLive } from "./live-context.ts"; // IPv6-only (unreachable from IPv4-only CI), so we `link` first (with the db // password) to persist the IPv4 pooler connection that storage then reuses. // The bucket is pre-seeded by live-setup; storage is gated behind --experimental. +// No runtime capability needed (pooler + Storage API are baseline data plane). const STORAGE_FLAGS = ["--linked", "--experimental"]; describe("storage (live --linked)", () => { testLive( diff --git a/apps/cli-e2e/tests/staging-project.ts b/apps/cli-e2e/tests/staging-project.ts index e0d54017fb..fd4a16d905 100644 --- a/apps/cli-e2e/tests/staging-project.ts +++ b/apps/cli-e2e/tests/staging-project.ts @@ -206,20 +206,29 @@ export async function getServiceRoleKey( /** Create a private storage bucket via the project's Storage API (host derived * from projectHost, IPv4-reachable). Idempotent — treats an existing bucket as - * success. */ + * success. Retries on `TenantNotFound`: storage tenant registration completes + * asynchronously, *after* the project reports ACTIVE_HEALTHY, so the first + * bucket call can race it (400 TenantNotFound). Other errors fail fast. */ export async function createStorageBucket( projectHost: string, projectRef: string, serviceRoleKey: string, bucket: string, + attempts = 12, ): Promise { - const res = await fetch(`https://${projectRef}.${projectHost}/storage/v1/bucket`, { - method: "POST", - headers: { Authorization: `Bearer ${serviceRoleKey}`, "Content-Type": "application/json" }, - body: JSON.stringify({ id: bucket, name: bucket, public: false }), - }); - if (!res.ok && res.status !== 409) { - throw new Error(`Failed to create bucket ${bucket}: ${res.status} ${await res.text()}`); + for (let attempt = 1; attempt <= attempts; attempt++) { + const res = await fetch(`https://${projectRef}.${projectHost}/storage/v1/bucket`, { + method: "POST", + headers: { Authorization: `Bearer ${serviceRoleKey}`, "Content-Type": "application/json" }, + body: JSON.stringify({ id: bucket, name: bucket, public: false }), + }); + if (res.ok || res.status === 409) return; // created, or already exists + const body = await res.text(); + // TenantNotFound is the eventual-consistency window — retry; anything else is real. + if (res.status !== 400 || !body.includes("TenantNotFound") || attempt === attempts) { + throw new Error(`Failed to create bucket ${bucket}: ${res.status} ${body}`); + } + await new Promise((r) => setTimeout(r, 10_000)); } }