From 26e70380d24ba3ac920b987842da439f25f25c6a Mon Sep 17 00:00:00 2001 From: avallete Date: Wed, 1 Jul 2026 11:21:00 +0200 Subject: [PATCH 1/6] test(cli-e2e): add capability probes + capability-based live gating MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a declarative way to mark live e2e tests by the runtime capabilities they need (docker / internet / external-tool) and skip them when the target env can't provide it — so one suite runs against staging (all capabilities, the oracle), supabox (whatever it currently supports), and Antithesis (offline subset), each skipping only what it genuinely can't do. - env.ts: `Capability` type + `PROVIDED_CAPABILITIES`, from `CLI_E2E_CAPABILITIES` with per-target defaults (staging = all; supabox = none until opened up). - live-context.ts: `testLiveRequires(caps)` → `testLive` or `testLive.skip`. - capabilities.live.e2e.test.ts: one minimal probe per category, each forcing its capability so a failure/skip is a precise supabox-gap signal — C1 mgmt-api (projects list), C2 docker/offline (db push+pull shadow), C3 external-tool (db dump via native pg_dump), C4 internet (deploy jsr fn), C5 docker+internet (--use-docker deploy of a jsr fn). Reuses existing fixtures. Staging is the oracle: probes green there prove soundness, so any supabox skip/red is a real gap for the CLI-in-supabox work (supabox#106) to close. Co-Authored-By: Claude Opus 4.8 --- apps/cli-e2e/src/tests/env.ts | 35 ++++++ .../tests/live/capabilities.live.e2e.test.ts | 115 ++++++++++++++++++ apps/cli-e2e/src/tests/live/live-context.ts | 21 +++- 3 files changed, 170 insertions(+), 1 deletion(-) create mode 100644 apps/cli-e2e/src/tests/live/capabilities.live.e2e.test.ts diff --git a/apps/cli-e2e/src/tests/env.ts b/apps/cli-e2e/src/tests/env.ts index e8803530cc..db7722b45e 100644 --- a/apps/cli-e2e/src/tests/env.ts +++ b/apps/cli-e2e/src/tests/env.ts @@ -50,6 +50,41 @@ export const TARGET_API_URL = export const PROJECT_HOST = process.env["CLI_E2E_PROJECT_HOST"] ?? (TARGET_ENV === "staging" ? "supabase.red" : ""); +// Runtime capabilities a live target can offer the cli. Live tests declare what +// they need (see `testLiveRequires`) and are skipped when the target can't +// provide it — so the same suite runs everywhere, skipping only what a given +// environment genuinely can't do: +// - docker control a container / has a Docker socket +// - internet reach 3rd-party network at runtime (jsr.io, npm, image pulls) +// - external-tool native pg_dump/psql, diff engine (SUPABASE_DB_USE_LOCAL_TOOLS) +const ALL_CAPABILITIES = ["docker", "internet", "external-tool"] as const; +export type Capability = (typeof ALL_CAPABILITIES)[number]; + +// Per-target defaults for what the environment provides. `staging` provides +// everything — it is the oracle: every probe must pass there, so a supabox +// skip/red is a genuine gap. `supabox` starts empty (locked down) and is opened +// up via CLI_E2E_CAPABILITIES as it gains real support; a probe only *runs* on +// supabox once its capability is declared, and must then pass (vs staging). +const DEFAULT_CAPABILITIES: Record = { + staging: ALL_CAPABILITIES, + supabox: [], +}; + +function isCapability(value: string): value is Capability { + return (ALL_CAPABILITIES as readonly string[]).includes(value); +} + +// Override the per-target defaults with an explicit comma list, e.g. +// CLI_E2E_CAPABILITIES=docker,external-tool (an Antithesis run never sets `internet`). +const CAPABILITIES_OVERRIDE = process.env["CLI_E2E_CAPABILITIES"]; +export const PROVIDED_CAPABILITIES: ReadonlySet = new Set( + CAPABILITIES_OVERRIDE === undefined + ? DEFAULT_CAPABILITIES[TARGET_ENV] + : CAPABILITIES_OVERRIDE.split(",") + .map((entry) => entry.trim()) + .filter(isCapability), +); + // In replay mode the token never reaches a real API, but the Go CLI validates // the format before making any request (must match sbp_[a-f0-9]{40}). // In record/live mode it must be a valid token for the target env. Falls back to diff --git a/apps/cli-e2e/src/tests/live/capabilities.live.e2e.test.ts b/apps/cli-e2e/src/tests/live/capabilities.live.e2e.test.ts new file mode 100644 index 0000000000..a84f93b838 --- /dev/null +++ b/apps/cli-e2e/src/tests/live/capabilities.live.e2e.test.ts @@ -0,0 +1,115 @@ +import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { describe, expect } from "vitest"; +import { expectFunctionOk } from "./invoke.ts"; +import { seedFunctions, testLiveRequires } from "./live-context.ts"; + +// Capability probes: one minimal test per runtime-capability combination the cli +// needs from its target. Each is a smoke check that FORCES its capability, so a +// failure (or a target-driven skip) is a precise signal of what an environment +// can or cannot do. Distilled from the richer feature tests so each isolates a +// single capability. +// +// Staging is the oracle: with CLI_E2E_TARGET_ENV=staging (provides all +// capabilities) every probe runs and must pass — proving the probe is sound, so +// any supabox skip/red is a genuine gap. On supabox each probe only runs once +// its capability is declared via CLI_E2E_CAPABILITIES, then must match staging. +// See `testLiveRequires` + PROVIDED_CAPABILITIES in ../env.ts. +describe("capability probes (live)", () => { + // C1 — mgmt-api only (no docker, no internet, no external tool). The + // provisioned project shows up in `projects list`: a pure control-plane read. + testLiveRequires([])( + "[C1] mgmt-api only: projects list includes the project", + async ({ run, projectRef }) => { + const res = await run(["projects", "list", "--output", "json"]); + expect(res.exitCode, res.stderr).toBe(0); + const refs = (JSON.parse(res.stdout) as Array<{ id?: string; ref?: string }>).map( + (project) => project.ref ?? project.id, + ); + expect(refs).toContain(projectRef); + }, + ); + + // C3 — external tool, no docker/internet. `db dump` of the remote schema over + // the IPv4 pooler using the native pg_dump/psql on PATH + // (SUPABASE_DB_USE_LOCAL_TOOLS), i.e. without spawning a supabase/postgres + // container. Fails if the external tool is absent. + testLiveRequires(["external-tool"])( + "[C3] external tool: db dump exports the remote schema", + async ({ run, dbUrl, workspace }) => { + const file = join(workspace.path, "dump.sql"); + const res = await run(["db", "dump", "--db-url", dbUrl, "-f", file]); + expect(res.exitCode, res.stderr).toBe(0); + expect(existsSync(file)).toBe(true); + expect(readFileSync(file, "utf8")).toMatch(/CREATE|PostgreSQL database dump|SCHEMA/i); + }, + ); + + // C2 — docker control, no runtime internet. `db pull`'s schema diff starts a + // shadow postgres *server* (DockerStart) and runs the diff engine in a + // container; both use pre-built images (no 3rd-party network). Push first so + // local history matches the shared per-run project, then pull. A missing + // Docker socket makes DockerStart fail — a genuine docker gate. + testLiveRequires(["docker"])( + "[C2] docker (offline): db push then db pull round-trips", + async ({ run, dbUrl, workspace }) => { + const migrations = join(workspace.path, "supabase", "migrations"); + mkdirSync(migrations, { recursive: true }); + writeFileSync( + join(migrations, "20240101000000_probe_push.sql"), + "create table if not exists capability_probe (id int);\n", + ); + + const pushed = await run(["db", "push", "--db-url", dbUrl, "--yes"]); + expect(pushed.exitCode, pushed.stderr).toBe(0); + + const pulled = await run(["db", "pull", "--db-url", dbUrl, "--yes"]); + const output = `${pulled.stdout}${pulled.stderr}`; + // Distinguish a real docker/connection failure from a benign "no changes". + expect(output, "db pull hit a docker/connection error").not.toMatch( + /cannot connect to the docker daemon|is the docker daemon running|dial|connection refused|could not connect/i, + ); + expect(pulled.exitCode === 0 || /No schema changes found/i.test(output), pulled.stderr).toBe( + true, + ); + }, + ); + + // C4 — runtime 3rd-party internet, no docker. Deploy a function that imports + // from jsr.io with the default (server-side) bundler and invoke it; the bundle + // path must fetch the import over the network. Fails offline. + testLiveRequires(["internet"])( + "[C4] internet: deploy a jsr-importing function and invoke", + async ({ run, invoke, workspace, projectRef }) => { + seedFunctions(workspace.path); + const slug = "deploy-e2e-jsr"; + const deployed = await run(["functions", "deploy", slug, "--project-ref", projectRef]); + expect(deployed.exitCode, deployed.stderr).toBe(0); + expect(deployed.stdout).toContain("Deployed Functions"); + expectFunctionOk(await invoke(slug), slug); + }, + ); + + // C5 — docker AND runtime internet. Same jsr-importing function, but bundled + // locally in a Docker container (`--use-docker`): needs a Docker socket AND the + // in-container bundler needs internet to fetch the jsr import. Fails if either + // is missing. + testLiveRequires(["docker", "internet"])( + "[C5] docker + internet: --use-docker deploy of a jsr function", + async ({ run, invoke, workspace, projectRef }) => { + seedFunctions(workspace.path); + const slug = "deploy-e2e-jsr"; + const deployed = await run([ + "functions", + "deploy", + slug, + "--project-ref", + projectRef, + "--use-docker", + ]); + expect(deployed.exitCode, deployed.stderr).toBe(0); + expect(deployed.stdout).toContain("Deployed Functions"); + expectFunctionOk(await invoke(slug), slug); + }, + ); +}); diff --git a/apps/cli-e2e/src/tests/live/live-context.ts b/apps/cli-e2e/src/tests/live/live-context.ts index 2cabe016e3..03fa907e6e 100644 --- a/apps/cli-e2e/src/tests/live/live-context.ts +++ b/apps/cli-e2e/src/tests/live/live-context.ts @@ -8,7 +8,15 @@ import { type CLIResult, type TempDir, } from "@supabase/cli-test-helpers"; -import { ACCESS_TOKEN, isLive, PROJECT_HOST, TARGET, TARGET_API_URL } from "../env.ts"; +import { + ACCESS_TOKEN, + type Capability, + isLive, + PROJECT_HOST, + PROVIDED_CAPABILITIES, + TARGET, + TARGET_API_URL, +} from "../env.ts"; import { invokeFunction, type InvokeResult } from "./invoke.ts"; type ExecOptions = NonNullable[2]>; @@ -120,3 +128,14 @@ const base = test.extend({ /** Live test API — skipped unless CLI_E2E_MODE=live, so files are inert on * replay/PR runs (and globalSetup provisions nothing). */ export const testLive = base.skipIf(!isLive); + +/** Live test API that additionally skips unless the target env provides every + * required runtime capability (docker / internet / external-tool). Lets one + * suite run against staging (all capabilities → runs everything, the oracle), + * supabox (only what it currently supports), and Antithesis (offline subset), + * each skipping only what it genuinely can't do. Put the requirement in the test + * name (e.g. "[C5] … (docker+internet)") so a skip reads clearly in the report. */ +export function testLiveRequires(required: readonly Capability[]): typeof testLive { + const missing = required.filter((capability) => !PROVIDED_CAPABILITIES.has(capability)); + return missing.length === 0 ? testLive : testLive.skip; +} From 8bcc653850e531302957c3d7e70e5019642018a5 Mon Sep 17 00:00:00 2001 From: avallete Date: Wed, 1 Jul 2026 11:51:41 +0200 Subject: [PATCH 2/6] =?UTF-8?q?test(cli-e2e):=20address=20review=20?= =?UTF-8?q?=E2=80=94=20tighten=20capability=20probes=20+=20gating?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - env.ts: reject unknown CLI_E2E_CAPABILITIES tokens (a typo like `external_tools` silently skipped tests and left the run green). - C3 probe: pass SUPABASE_DB_USE_LOCAL_TOOLS=1 so `db dump` exercises the native pg_dump path (external-tool), not the default container path. - C2 probe: unique migration timestamp so it can't collide with db-sync's on the shared per-run project. - C5 probe: deploy a distinct slug (deploy-e2e-npm) so the invoke proves the --use-docker deploy produced the function, not C4's earlier server-bundled one. - Tag the existing docker/internet live tests so CLI_E2E_CAPABILITIES gates them too: db-sync (docker), functions deploy --use-docker (docker), deploy-all (internet). Co-Authored-By: Claude Opus 4.8 --- apps/cli-e2e/src/tests/env.ts | 21 ++++++- .../tests/live/capabilities.live.e2e.test.ts | 32 ++++++----- .../src/tests/live/db-sync.live.e2e.test.ts | 7 ++- .../live/functions-deploy.live.e2e.test.ts | 56 +++++++++++-------- 4 files changed, 76 insertions(+), 40 deletions(-) diff --git a/apps/cli-e2e/src/tests/env.ts b/apps/cli-e2e/src/tests/env.ts index db7722b45e..a3de611726 100644 --- a/apps/cli-e2e/src/tests/env.ts +++ b/apps/cli-e2e/src/tests/env.ts @@ -74,15 +74,30 @@ function isCapability(value: string): value is Capability { return (ALL_CAPABILITIES as readonly string[]).includes(value); } +// Parse the comma list, rejecting unknown tokens: silently dropping a typo (e.g. +// `external_tools`) would skip every test for that capability and still leave the +// run green, hiding coverage the environment thought it had enabled. +function parseCapabilities(raw: string): Capability[] { + const tokens = raw + .split(",") + .map((entry) => entry.trim()) + .filter((entry) => entry.length > 0); + const unknown = tokens.filter((token) => !isCapability(token)); + if (unknown.length > 0) { + throw new Error( + `Unknown CLI_E2E_CAPABILITIES: ${unknown.join(", ")}. Valid: ${ALL_CAPABILITIES.join(", ")}.`, + ); + } + return tokens.filter(isCapability); +} + // Override the per-target defaults with an explicit comma list, e.g. // CLI_E2E_CAPABILITIES=docker,external-tool (an Antithesis run never sets `internet`). const CAPABILITIES_OVERRIDE = process.env["CLI_E2E_CAPABILITIES"]; export const PROVIDED_CAPABILITIES: ReadonlySet = new Set( CAPABILITIES_OVERRIDE === undefined ? DEFAULT_CAPABILITIES[TARGET_ENV] - : CAPABILITIES_OVERRIDE.split(",") - .map((entry) => entry.trim()) - .filter(isCapability), + : parseCapabilities(CAPABILITIES_OVERRIDE), ); // In replay mode the token never reaches a real API, but the Go CLI validates diff --git a/apps/cli-e2e/src/tests/live/capabilities.live.e2e.test.ts b/apps/cli-e2e/src/tests/live/capabilities.live.e2e.test.ts index a84f93b838..2a07c11b7b 100644 --- a/apps/cli-e2e/src/tests/live/capabilities.live.e2e.test.ts +++ b/apps/cli-e2e/src/tests/live/capabilities.live.e2e.test.ts @@ -30,15 +30,17 @@ describe("capability probes (live)", () => { }, ); - // C3 — external tool, no docker/internet. `db dump` of the remote schema over - // the IPv4 pooler using the native pg_dump/psql on PATH - // (SUPABASE_DB_USE_LOCAL_TOOLS), i.e. without spawning a supabase/postgres - // container. Fails if the external tool is absent. + // C3 — external tool, no docker/internet. `db dump` normally runs `pg_dump` in a + // supabase/postgres CONTAINER; `SUPABASE_DB_USE_LOCAL_TOOLS=1` switches it to the + // native pg_dump/psql on PATH, so this exercises the external-tool path (no + // Docker socket) rather than the container path. Fails if the tool is absent. testLiveRequires(["external-tool"])( - "[C3] external tool: db dump exports the remote schema", + "[C3] external tool: db dump exports the remote schema via native pg_dump", async ({ run, dbUrl, workspace }) => { const file = join(workspace.path, "dump.sql"); - const res = await run(["db", "dump", "--db-url", dbUrl, "-f", file]); + const res = await run(["db", "dump", "--db-url", dbUrl, "-f", file], { + env: { SUPABASE_DB_USE_LOCAL_TOOLS: "1" }, + }); expect(res.exitCode, res.stderr).toBe(0); expect(existsSync(file)).toBe(true); expect(readFileSync(file, "utf8")).toMatch(/CREATE|PostgreSQL database dump|SCHEMA/i); @@ -55,8 +57,10 @@ describe("capability probes (live)", () => { async ({ run, dbUrl, workspace }) => { const migrations = join(workspace.path, "supabase", "migrations"); mkdirSync(migrations, { recursive: true }); + // Unique timestamp so this never collides with db-sync.live.e2e.test.ts's + // migration on the shared per-run project (history compares timestamps). writeFileSync( - join(migrations, "20240101000000_probe_push.sql"), + join(migrations, "20240202020202_capability_probe_push.sql"), "create table if not exists capability_probe (id int);\n", ); @@ -90,15 +94,17 @@ describe("capability probes (live)", () => { }, ); - // C5 — docker AND runtime internet. Same jsr-importing function, but bundled - // locally in a Docker container (`--use-docker`): needs a Docker socket AND the - // in-container bundler needs internet to fetch the jsr import. Fails if either - // is missing. + // C5 — docker AND runtime internet. Bundle an npm-importing function locally in + // a Docker container (`--use-docker`): needs a Docker socket AND the in-container + // bundler needs internet to fetch the npm import. A DISTINCT slug from C4 so the + // invoke proves THIS (--use-docker) deploy produced the running function, not + // C4's earlier server-bundled one on the shared project. Fails if either is + // missing. testLiveRequires(["docker", "internet"])( - "[C5] docker + internet: --use-docker deploy of a jsr function", + "[C5] docker + internet: --use-docker deploy of an npm-importing function", async ({ run, invoke, workspace, projectRef }) => { seedFunctions(workspace.path); - const slug = "deploy-e2e-jsr"; + const slug = "deploy-e2e-npm"; const deployed = await run([ "functions", "deploy", diff --git a/apps/cli-e2e/src/tests/live/db-sync.live.e2e.test.ts b/apps/cli-e2e/src/tests/live/db-sync.live.e2e.test.ts index 4780b56537..ad58625b06 100644 --- a/apps/cli-e2e/src/tests/live/db-sync.live.e2e.test.ts +++ b/apps/cli-e2e/src/tests/live/db-sync.live.e2e.test.ts @@ -1,7 +1,7 @@ import { mkdirSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { describe, expect } from "vitest"; -import { testLive } from "./live-context.ts"; +import { testLiveRequires } from "./live-context.ts"; // Local↔remote schema sync (workflows 1-2) over the IPv4 session pooler. Done as // one round-trip in a single workspace: pushing first makes the local migration @@ -9,8 +9,11 @@ import { testLive } from "./live-context.ts"; // (a separate fresh-workspace pull would see a history mismatch on the shared // per-run project). db push/pull confirm via a prompt that only auto-accepts // with --yes. Mutates the throwaway project's schema — deleted on teardown. +// +// Requires `docker`: `db pull`'s schema diff starts a shadow postgres server +// (DockerStart) + diff container, so it skips on targets without a Docker socket. describe("db push + pull (live, session pooler)", () => { - testLive( + testLiveRequires(["docker"])( "pushes a local migration and pulls the remote schema back", async ({ run, dbUrl, workspace }) => { const migrations = join(workspace.path, "supabase", "migrations"); diff --git a/apps/cli-e2e/src/tests/live/functions-deploy.live.e2e.test.ts b/apps/cli-e2e/src/tests/live/functions-deploy.live.e2e.test.ts index e9101cc1be..cd4375c9f1 100644 --- a/apps/cli-e2e/src/tests/live/functions-deploy.live.e2e.test.ts +++ b/apps/cli-e2e/src/tests/live/functions-deploy.live.e2e.test.ts @@ -2,7 +2,7 @@ import { readdirSync } from "node:fs"; import { join } from "node:path"; import { describe, expect } from "vitest"; import { expectFunctionOk } from "./invoke.ts"; -import { seedFunctions, testLive } from "./live-context.ts"; +import { seedFunctions, testLiveRequires } from "./live-context.ts"; // Pilot (ADR-0013): deploy with the real CLI across the three bundler paths, // then invoke the deployed function over HTTP and assert the body it returns. @@ -10,35 +10,47 @@ import { seedFunctions, testLive } from "./live-context.ts"; // produced a running function — the shared project means a single slug could // otherwise be served by an earlier mode's deploy. Negative/arg-validation // cases live in apps/cli integration tests. +// `requires` per mode: `--use-docker` bundles locally in a container (needs a +// Docker socket); default/`--use-api` bundle server-side (no local docker). The +// fixtures here are import-free, so no runtime internet is required. const MODES = [ - { name: "default", slug: "deploy-e2e-mode-default", flags: [] as string[] }, - { name: "use-api", slug: "deploy-e2e-mode-api", flags: ["--use-api"] }, - { name: "use-docker", slug: "deploy-e2e-mode-docker", flags: ["--use-docker"] }, + { name: "default", slug: "deploy-e2e-mode-default", flags: [] as string[], requires: [] }, + { name: "use-api", slug: "deploy-e2e-mode-api", flags: ["--use-api"], requires: [] }, + { + name: "use-docker", + slug: "deploy-e2e-mode-docker", + flags: ["--use-docker"], + requires: ["docker"], + }, ] as const; -describe.each(MODES)("functions deploy ($name)", ({ slug, flags }) => { - testLive("deploys and the function responds", async ({ run, invoke, workspace, projectRef }) => { - seedFunctions(workspace.path); - const deployed = await run([ - "functions", - "deploy", - slug, - "--project-ref", - projectRef, - ...flags, - ]); - expect(deployed.exitCode, deployed.stderr).toBe(0); - expect(deployed.stdout).toContain("Deployed Functions"); +describe.each(MODES)("functions deploy ($name)", ({ slug, flags, requires }) => { + testLiveRequires(requires)( + "deploys and the function responds", + async ({ run, invoke, workspace, projectRef }) => { + seedFunctions(workspace.path); + const deployed = await run([ + "functions", + "deploy", + slug, + "--project-ref", + projectRef, + ...flags, + ]); + expect(deployed.exitCode, deployed.stderr).toBe(0); + expect(deployed.stdout).toContain("Deployed Functions"); - const res = await invoke(slug); - expectFunctionOk(res, slug); - }); + const res = await invoke(slug); + expectFunctionOk(res, slug); + }, + ); }); // No slug → the CLI walks every function declared under supabase/functions and // deploys them all. Assert each declared function appears in the deploy output, -// then smoke-invoke a representative one. -testLive( +// then smoke-invoke a representative one. Requires `internet`: the fixture set +// includes functions that import from jsr.io / npm, resolved at bundle time. +testLiveRequires(["internet"])( "deploys every declared function when no slug is given", async ({ run, invoke, workspace, projectRef }) => { seedFunctions(workspace.path); From 45bbf0d409b2ad6e4f42aea160a10f39f89d1745 Mon Sep 17 00:00:00 2001 From: avallete Date: Wed, 1 Jul 2026 12:15:21 +0200 Subject: [PATCH 3/6] test(cli-e2e): gate the whole live suite by capability requirements MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extends the capability set with the data-plane axes and tags every live test with its full requirements, so CLI_E2E_CAPABILITIES gates the entire suite (not just the probes): - Add `database` (project Postgres reachable via the pooler) and `storage` (the Storage API) to the capability set; staging provides all, supabox opts in. - Tag data-plane tests: database db-stats/migration-list → [database], db dump → [database, docker]; db-sync → [database, docker]; gen-types → [database, docker]; storage → [database, storage]; capability probes C2/C3 gain [database]. - Management-API-only tests (projects, link, secrets, branches, functions-lifecycle) need no extra capability and stay on the `testLive` baseline. Co-Authored-By: Claude Opus 4.8 --- apps/cli-e2e/src/tests/env.ts | 14 +++-- .../tests/live/capabilities.live.e2e.test.ts | 4 +- .../src/tests/live/database.live.e2e.test.ts | 52 ++++++++++++------- .../src/tests/live/gen-types.live.e2e.test.ts | 21 ++++---- .../src/tests/live/storage.live.e2e.test.ts | 6 ++- 5 files changed, 59 insertions(+), 38 deletions(-) diff --git a/apps/cli-e2e/src/tests/env.ts b/apps/cli-e2e/src/tests/env.ts index a3de611726..e66ac14e64 100644 --- a/apps/cli-e2e/src/tests/env.ts +++ b/apps/cli-e2e/src/tests/env.ts @@ -50,14 +50,18 @@ export const TARGET_API_URL = export const PROJECT_HOST = process.env["CLI_E2E_PROJECT_HOST"] ?? (TARGET_ENV === "staging" ? "supabase.red" : ""); -// Runtime capabilities a live target can offer the cli. Live tests declare what -// they need (see `testLiveRequires`) and are skipped when the target can't -// provide it — so the same suite runs everywhere, skipping only what a given -// environment genuinely can't do: +// Capabilities a live target can offer the cli. Live tests declare what they +// need (see `testLiveRequires`) and are skipped when the target can't provide +// it — so the same suite runs everywhere, skipping only what a given environment +// genuinely can't do: // - docker control a container / has a Docker socket // - internet reach 3rd-party network at runtime (jsr.io, npm, image pulls) // - external-tool native pg_dump/psql, diff engine (SUPABASE_DB_USE_LOCAL_TOOLS) -const ALL_CAPABILITIES = ["docker", "internet", "external-tool"] as const; +// - database the project's own Postgres is reachable (session pooler / dbUrl) +// - storage the project's Storage API is reachable +// The last two are the data plane: present on a full stack (staging, a complete +// supabox), absent on a Management-API-only target. +const ALL_CAPABILITIES = ["docker", "internet", "external-tool", "database", "storage"] as const; export type Capability = (typeof ALL_CAPABILITIES)[number]; // Per-target defaults for what the environment provides. `staging` provides diff --git a/apps/cli-e2e/src/tests/live/capabilities.live.e2e.test.ts b/apps/cli-e2e/src/tests/live/capabilities.live.e2e.test.ts index 2a07c11b7b..e3eafc7f55 100644 --- a/apps/cli-e2e/src/tests/live/capabilities.live.e2e.test.ts +++ b/apps/cli-e2e/src/tests/live/capabilities.live.e2e.test.ts @@ -34,7 +34,7 @@ describe("capability probes (live)", () => { // supabase/postgres CONTAINER; `SUPABASE_DB_USE_LOCAL_TOOLS=1` switches it to the // native pg_dump/psql on PATH, so this exercises the external-tool path (no // Docker socket) rather than the container path. Fails if the tool is absent. - testLiveRequires(["external-tool"])( + testLiveRequires(["database", "external-tool"])( "[C3] external tool: db dump exports the remote schema via native pg_dump", async ({ run, dbUrl, workspace }) => { const file = join(workspace.path, "dump.sql"); @@ -52,7 +52,7 @@ describe("capability probes (live)", () => { // container; both use pre-built images (no 3rd-party network). Push first so // local history matches the shared per-run project, then pull. A missing // Docker socket makes DockerStart fail — a genuine docker gate. - testLiveRequires(["docker"])( + testLiveRequires(["database", "docker"])( "[C2] docker (offline): db push then db pull round-trips", async ({ run, dbUrl, workspace }) => { const migrations = join(workspace.path, "supabase", "migrations"); diff --git a/apps/cli-e2e/src/tests/live/database.live.e2e.test.ts b/apps/cli-e2e/src/tests/live/database.live.e2e.test.ts index 6a99f7131f..b98491458f 100644 --- a/apps/cli-e2e/src/tests/live/database.live.e2e.test.ts +++ b/apps/cli-e2e/src/tests/live/database.live.e2e.test.ts @@ -1,32 +1,44 @@ import { existsSync, readFileSync } from "node:fs"; import { join } from "node:path"; import { describe, expect } from "vitest"; -import { testLive } from "./live-context.ts"; +import { testLiveRequires } from "./live-context.ts"; // DB-connectivity commands against the fresh project's Postgres via the IPv4 // session-mode Supavisor pooler (`dbUrl` from live-setup). The direct host // (db..supabase.red) is IPv6-only and unreachable from IPv4-only CI // runners; the pooler is IPv4, and session mode is required for pg_dump. -// A non-zero exit here means the connection itself failed. +// A non-zero exit here means the connection itself failed. All require the +// `database` capability (the project Postgres reachable over the pooler). describe("database (live, session pooler --db-url)", () => { - testLive("inspect db db-stats connects and reports stats", async ({ run, dbUrl }) => { - const res = await run(["inspect", "db", "db-stats", "--db-url", dbUrl]); - expect(res.exitCode, res.stderr).toBe(0); - expect(res.stdout).toContain("Database Size"); - }); + testLiveRequires(["database"])( + "inspect db db-stats connects and reports stats", + async ({ run, dbUrl }) => { + const res = await run(["inspect", "db", "db-stats", "--db-url", dbUrl]); + expect(res.exitCode, res.stderr).toBe(0); + expect(res.stdout).toContain("Database Size"); + }, + ); - testLive("migration list connects to the remote migration history", async ({ run, dbUrl }) => { - const res = await run(["migration", "list", "--db-url", dbUrl]); - // Fresh project has no migrations, but exit 0 proves it connected and - // queried the remote history table. - expect(res.exitCode, res.stderr).toBe(0); - }); + testLiveRequires(["database"])( + "migration list connects to the remote migration history", + async ({ run, dbUrl }) => { + const res = await run(["migration", "list", "--db-url", dbUrl]); + // Fresh project has no migrations, but exit 0 proves it connected and + // queried the remote history table. + expect(res.exitCode, res.stderr).toBe(0); + }, + ); - testLive("db dump exports the remote schema", async ({ run, dbUrl, workspace }) => { - const file = join(workspace.path, "dump.sql"); - const res = await run(["db", "dump", "--db-url", dbUrl, "-f", file]); - expect(res.exitCode, res.stderr).toBe(0); - expect(existsSync(file)).toBe(true); - expect(readFileSync(file, "utf8")).toMatch(/CREATE|PostgreSQL database dump|SCHEMA/i); - }); + // Also needs `docker`: without SUPABASE_DB_USE_LOCAL_TOOLS this `db dump` runs + // pg_dump in a supabase/postgres container (see the C3 probe for the native path). + testLiveRequires(["database", "docker"])( + "db dump exports the remote schema", + async ({ run, dbUrl, workspace }) => { + const file = join(workspace.path, "dump.sql"); + const res = await run(["db", "dump", "--db-url", dbUrl, "-f", file]); + expect(res.exitCode, res.stderr).toBe(0); + expect(existsSync(file)).toBe(true); + expect(readFileSync(file, "utf8")).toMatch(/CREATE|PostgreSQL database dump|SCHEMA/i); + }, + ); }); diff --git a/apps/cli-e2e/src/tests/live/gen-types.live.e2e.test.ts b/apps/cli-e2e/src/tests/live/gen-types.live.e2e.test.ts index e75455989b..a41a991c16 100644 --- a/apps/cli-e2e/src/tests/live/gen-types.live.e2e.test.ts +++ b/apps/cli-e2e/src/tests/live/gen-types.live.e2e.test.ts @@ -1,13 +1,16 @@ import { describe, expect } from "vitest"; -import { testLive } from "./live-context.ts"; +import { testLiveRequires } from "./live-context.ts"; -// gen types introspects the remote schema over the IPv4 session pooler and emits -// TypeScript types. It pulls the postgres-meta Docker image, so it needs Docker -// (present in the CI live job alongside the --use-docker bundler cell). +// gen types introspects the remote schema over the IPv4 session pooler (needs +// `database`) and emits TypeScript types by running the postgres-meta Docker +// image (needs `docker`). describe("gen types (live, session pooler)", () => { - testLive("generates TypeScript types from the remote schema", async ({ run, dbUrl }) => { - const res = await run(["gen", "types", "--db-url", dbUrl, "--lang", "typescript"]); - expect(res.exitCode, res.stderr).toBe(0); - expect(res.stdout).toMatch(/export type (Database|Json)/); - }); + testLiveRequires(["database", "docker"])( + "generates TypeScript types from the remote schema", + async ({ run, dbUrl }) => { + const res = await run(["gen", "types", "--db-url", dbUrl, "--lang", "typescript"]); + expect(res.exitCode, res.stderr).toBe(0); + expect(res.stdout).toMatch(/export type (Database|Json)/); + }, + ); }); diff --git a/apps/cli-e2e/src/tests/live/storage.live.e2e.test.ts b/apps/cli-e2e/src/tests/live/storage.live.e2e.test.ts index 2d705f690d..689f3738bb 100644 --- a/apps/cli-e2e/src/tests/live/storage.live.e2e.test.ts +++ b/apps/cli-e2e/src/tests/live/storage.live.e2e.test.ts @@ -1,16 +1,18 @@ import { writeFileSync } from "node:fs"; import { join } from "node:path"; import { describe, expect } from "vitest"; -import { testLive } from "./live-context.ts"; +import { testLiveRequires } from "./live-context.ts"; // Storage object round-trip against the project's real Storage API. `storage // --linked` opens a DB connection to resolve storage config; the direct host is // IPv6-only (unreachable from IPv4-only CI), so we `link` first (with the db // password) to persist the IPv4 pooler connection that storage then reuses. // The bucket is pre-seeded by live-setup; storage is gated behind --experimental. +// Requires `database` (the pooler, to resolve storage config) + `storage` (the +// Storage API endpoint). const STORAGE_FLAGS = ["--linked", "--experimental"]; describe("storage (live --linked)", () => { - testLive( + testLiveRequires(["database", "storage"])( "uploads, lists, and removes an object", async ({ run, workspace, projectRef, storageBucket, dbPassword }) => { const linked = await run(["link", "--project-ref", projectRef], { From 70389ddf268bb899f40e7a4b94d4d1db5c2c2883 Mon Sep 17 00:00:00 2001 From: avallete Date: Wed, 1 Jul 2026 12:28:59 +0200 Subject: [PATCH 4/6] =?UTF-8?q?test(cli-e2e):=20drop=20database/storage=20?= =?UTF-8?q?capabilities=20=E2=80=94=20data=20plane=20is=20baseline?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The project data plane (Postgres via the pooler, the Storage API) is always present on a live target, so it isn't a capability a test opts into. Revert to the three runtime capabilities (docker / internet / external-tool) and re-tag: - storage, database db-stats/migration-list → back to the bare `testLive` baseline (pooler connection needs no runtime capability). - database db dump, gen-types, db-sync, probe C2 → `docker` only (db pull's shadow DB / gen-types' postgres-meta / db dump's pg_dump container); probe C3 → `external-tool` only. db push+pull keeps `docker` because `db pull` starts a shadow postgres container (PrepareRawShadow → DockerStart), not because of the --db-url connection. Co-Authored-By: Claude Opus 4.8 --- apps/cli-e2e/src/tests/env.ts | 17 ++++----- .../tests/live/capabilities.live.e2e.test.ts | 4 +- .../src/tests/live/database.live.e2e.test.ts | 38 ++++++++----------- .../src/tests/live/db-sync.live.e2e.test.ts | 5 ++- .../src/tests/live/gen-types.live.e2e.test.ts | 7 ++-- .../src/tests/live/storage.live.e2e.test.ts | 7 ++-- 6 files changed, 35 insertions(+), 43 deletions(-) diff --git a/apps/cli-e2e/src/tests/env.ts b/apps/cli-e2e/src/tests/env.ts index e66ac14e64..5936cef911 100644 --- a/apps/cli-e2e/src/tests/env.ts +++ b/apps/cli-e2e/src/tests/env.ts @@ -50,18 +50,17 @@ export const TARGET_API_URL = export const PROJECT_HOST = process.env["CLI_E2E_PROJECT_HOST"] ?? (TARGET_ENV === "staging" ? "supabase.red" : ""); -// Capabilities a live target can offer the cli. Live tests declare what they -// need (see `testLiveRequires`) and are skipped when the target can't provide -// it — so the same suite runs everywhere, skipping only what a given environment -// genuinely can't do: +// Runtime capabilities a live target can offer the cli. Live tests declare what +// they need (see `testLiveRequires`) and are skipped when the target can't +// provide it — so the same suite runs everywhere, skipping only what a given +// environment genuinely can't do: // - docker control a container / has a Docker socket // - internet reach 3rd-party network at runtime (jsr.io, npm, image pulls) // - external-tool native pg_dump/psql, diff engine (SUPABASE_DB_USE_LOCAL_TOOLS) -// - database the project's own Postgres is reachable (session pooler / dbUrl) -// - storage the project's Storage API is reachable -// The last two are the data plane: present on a full stack (staging, a complete -// supabox), absent on a Management-API-only target. -const ALL_CAPABILITIES = ["docker", "internet", "external-tool", "database", "storage"] as const; +// The project data plane (its Postgres via the pooler, the Storage API) is NOT a +// capability — any live target is assumed to provide a provisioned project, so +// data-plane access is a baseline, not something a test opts into. +const ALL_CAPABILITIES = ["docker", "internet", "external-tool"] as const; export type Capability = (typeof ALL_CAPABILITIES)[number]; // Per-target defaults for what the environment provides. `staging` provides diff --git a/apps/cli-e2e/src/tests/live/capabilities.live.e2e.test.ts b/apps/cli-e2e/src/tests/live/capabilities.live.e2e.test.ts index e3eafc7f55..2a07c11b7b 100644 --- a/apps/cli-e2e/src/tests/live/capabilities.live.e2e.test.ts +++ b/apps/cli-e2e/src/tests/live/capabilities.live.e2e.test.ts @@ -34,7 +34,7 @@ describe("capability probes (live)", () => { // supabase/postgres CONTAINER; `SUPABASE_DB_USE_LOCAL_TOOLS=1` switches it to the // native pg_dump/psql on PATH, so this exercises the external-tool path (no // Docker socket) rather than the container path. Fails if the tool is absent. - testLiveRequires(["database", "external-tool"])( + testLiveRequires(["external-tool"])( "[C3] external tool: db dump exports the remote schema via native pg_dump", async ({ run, dbUrl, workspace }) => { const file = join(workspace.path, "dump.sql"); @@ -52,7 +52,7 @@ describe("capability probes (live)", () => { // container; both use pre-built images (no 3rd-party network). Push first so // local history matches the shared per-run project, then pull. A missing // Docker socket makes DockerStart fail — a genuine docker gate. - testLiveRequires(["database", "docker"])( + testLiveRequires(["docker"])( "[C2] docker (offline): db push then db pull round-trips", async ({ run, dbUrl, workspace }) => { const migrations = join(workspace.path, "supabase", "migrations"); diff --git a/apps/cli-e2e/src/tests/live/database.live.e2e.test.ts b/apps/cli-e2e/src/tests/live/database.live.e2e.test.ts index b98491458f..4e9dea664c 100644 --- a/apps/cli-e2e/src/tests/live/database.live.e2e.test.ts +++ b/apps/cli-e2e/src/tests/live/database.live.e2e.test.ts @@ -1,37 +1,31 @@ import { existsSync, readFileSync } from "node:fs"; import { join } from "node:path"; import { describe, expect } from "vitest"; -import { testLiveRequires } from "./live-context.ts"; +import { testLive, testLiveRequires } from "./live-context.ts"; // DB-connectivity commands against the fresh project's Postgres via the IPv4 // session-mode Supavisor pooler (`dbUrl` from live-setup). The direct host // (db..supabase.red) is IPv6-only and unreachable from IPv4-only CI // runners; the pooler is IPv4, and session mode is required for pg_dump. -// A non-zero exit here means the connection itself failed. All require the -// `database` capability (the project Postgres reachable over the pooler). +// A non-zero exit here means the connection itself failed. The pooler is baseline +// data plane, so the connect-only cases need no runtime capability. describe("database (live, session pooler --db-url)", () => { - testLiveRequires(["database"])( - "inspect db db-stats connects and reports stats", - async ({ run, dbUrl }) => { - const res = await run(["inspect", "db", "db-stats", "--db-url", dbUrl]); - expect(res.exitCode, res.stderr).toBe(0); - expect(res.stdout).toContain("Database Size"); - }, - ); + testLive("inspect db db-stats connects and reports stats", async ({ run, dbUrl }) => { + const res = await run(["inspect", "db", "db-stats", "--db-url", dbUrl]); + expect(res.exitCode, res.stderr).toBe(0); + expect(res.stdout).toContain("Database Size"); + }); - testLiveRequires(["database"])( - "migration list connects to the remote migration history", - async ({ run, dbUrl }) => { - const res = await run(["migration", "list", "--db-url", dbUrl]); - // Fresh project has no migrations, but exit 0 proves it connected and - // queried the remote history table. - expect(res.exitCode, res.stderr).toBe(0); - }, - ); + testLive("migration list connects to the remote migration history", async ({ run, dbUrl }) => { + const res = await run(["migration", "list", "--db-url", dbUrl]); + // Fresh project has no migrations, but exit 0 proves it connected and + // queried the remote history table. + expect(res.exitCode, res.stderr).toBe(0); + }); - // Also needs `docker`: without SUPABASE_DB_USE_LOCAL_TOOLS this `db dump` runs + // Needs `docker`: without SUPABASE_DB_USE_LOCAL_TOOLS this `db dump` runs // pg_dump in a supabase/postgres container (see the C3 probe for the native path). - testLiveRequires(["database", "docker"])( + testLiveRequires(["docker"])( "db dump exports the remote schema", async ({ run, dbUrl, workspace }) => { const file = join(workspace.path, "dump.sql"); diff --git a/apps/cli-e2e/src/tests/live/db-sync.live.e2e.test.ts b/apps/cli-e2e/src/tests/live/db-sync.live.e2e.test.ts index ad58625b06..b29053ee87 100644 --- a/apps/cli-e2e/src/tests/live/db-sync.live.e2e.test.ts +++ b/apps/cli-e2e/src/tests/live/db-sync.live.e2e.test.ts @@ -10,8 +10,9 @@ import { testLiveRequires } from "./live-context.ts"; // per-run project). db push/pull confirm via a prompt that only auto-accepts // with --yes. Mutates the throwaway project's schema — deleted on teardown. // -// Requires `docker`: `db pull`'s schema diff starts a shadow postgres server -// (DockerStart) + diff container, so it skips on targets without a Docker socket. +// Requires `docker`: `db push` is a plain pooler connection, but `db pull`'s +// schema diff starts a shadow postgres server (PrepareRawShadow → DockerStart) + +// diff container, so the round-trip skips on targets without a Docker socket. describe("db push + pull (live, session pooler)", () => { testLiveRequires(["docker"])( "pushes a local migration and pulls the remote schema back", diff --git a/apps/cli-e2e/src/tests/live/gen-types.live.e2e.test.ts b/apps/cli-e2e/src/tests/live/gen-types.live.e2e.test.ts index a41a991c16..ef17d91ef7 100644 --- a/apps/cli-e2e/src/tests/live/gen-types.live.e2e.test.ts +++ b/apps/cli-e2e/src/tests/live/gen-types.live.e2e.test.ts @@ -1,11 +1,10 @@ import { describe, expect } from "vitest"; import { testLiveRequires } from "./live-context.ts"; -// gen types introspects the remote schema over the IPv4 session pooler (needs -// `database`) and emits TypeScript types by running the postgres-meta Docker -// image (needs `docker`). +// gen types introspects the remote schema over the IPv4 session pooler and emits +// TypeScript types by running the postgres-meta Docker image (needs `docker`). describe("gen types (live, session pooler)", () => { - testLiveRequires(["database", "docker"])( + testLiveRequires(["docker"])( "generates TypeScript types from the remote schema", async ({ run, dbUrl }) => { const res = await run(["gen", "types", "--db-url", dbUrl, "--lang", "typescript"]); diff --git a/apps/cli-e2e/src/tests/live/storage.live.e2e.test.ts b/apps/cli-e2e/src/tests/live/storage.live.e2e.test.ts index 689f3738bb..56a7ca3be6 100644 --- a/apps/cli-e2e/src/tests/live/storage.live.e2e.test.ts +++ b/apps/cli-e2e/src/tests/live/storage.live.e2e.test.ts @@ -1,18 +1,17 @@ import { writeFileSync } from "node:fs"; import { join } from "node:path"; import { describe, expect } from "vitest"; -import { testLiveRequires } from "./live-context.ts"; +import { testLive } from "./live-context.ts"; // Storage object round-trip against the project's real Storage API. `storage // --linked` opens a DB connection to resolve storage config; the direct host is // IPv6-only (unreachable from IPv4-only CI), so we `link` first (with the db // password) to persist the IPv4 pooler connection that storage then reuses. // The bucket is pre-seeded by live-setup; storage is gated behind --experimental. -// Requires `database` (the pooler, to resolve storage config) + `storage` (the -// Storage API endpoint). +// No runtime capability needed (pooler + Storage API are baseline data plane). const STORAGE_FLAGS = ["--linked", "--experimental"]; describe("storage (live --linked)", () => { - testLiveRequires(["database", "storage"])( + testLive( "uploads, lists, and removes an object", async ({ run, workspace, projectRef, storageBucket, dbPassword }) => { const linked = await run(["link", "--project-ref", projectRef], { From 6a0a0d388b783479f74299fafd3a49faeff16d5c Mon Sep 17 00:00:00 2001 From: avallete Date: Wed, 1 Jul 2026 12:30:43 +0200 Subject: [PATCH 5/6] =?UTF-8?q?test(cli-e2e):=20order=20capability=20probe?= =?UTF-8?q?s=20C1=E2=86=92C5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move the C2 (docker) probe before C3 (external-tool) so the probes read in numeric order. Co-Authored-By: Claude Opus 4.8 --- .../tests/live/capabilities.live.e2e.test.ts | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/apps/cli-e2e/src/tests/live/capabilities.live.e2e.test.ts b/apps/cli-e2e/src/tests/live/capabilities.live.e2e.test.ts index 2a07c11b7b..c496931b76 100644 --- a/apps/cli-e2e/src/tests/live/capabilities.live.e2e.test.ts +++ b/apps/cli-e2e/src/tests/live/capabilities.live.e2e.test.ts @@ -30,23 +30,6 @@ describe("capability probes (live)", () => { }, ); - // C3 — external tool, no docker/internet. `db dump` normally runs `pg_dump` in a - // supabase/postgres CONTAINER; `SUPABASE_DB_USE_LOCAL_TOOLS=1` switches it to the - // native pg_dump/psql on PATH, so this exercises the external-tool path (no - // Docker socket) rather than the container path. Fails if the tool is absent. - testLiveRequires(["external-tool"])( - "[C3] external tool: db dump exports the remote schema via native pg_dump", - async ({ run, dbUrl, workspace }) => { - const file = join(workspace.path, "dump.sql"); - const res = await run(["db", "dump", "--db-url", dbUrl, "-f", file], { - env: { SUPABASE_DB_USE_LOCAL_TOOLS: "1" }, - }); - expect(res.exitCode, res.stderr).toBe(0); - expect(existsSync(file)).toBe(true); - expect(readFileSync(file, "utf8")).toMatch(/CREATE|PostgreSQL database dump|SCHEMA/i); - }, - ); - // C2 — docker control, no runtime internet. `db pull`'s schema diff starts a // shadow postgres *server* (DockerStart) and runs the diff engine in a // container; both use pre-built images (no 3rd-party network). Push first so @@ -79,6 +62,23 @@ describe("capability probes (live)", () => { }, ); + // C3 — external tool, no docker/internet. `db dump` normally runs `pg_dump` in a + // supabase/postgres CONTAINER; `SUPABASE_DB_USE_LOCAL_TOOLS=1` switches it to the + // native pg_dump/psql on PATH, so this exercises the external-tool path (no + // Docker socket) rather than the container path. Fails if the tool is absent. + testLiveRequires(["external-tool"])( + "[C3] external tool: db dump exports the remote schema via native pg_dump", + async ({ run, dbUrl, workspace }) => { + const file = join(workspace.path, "dump.sql"); + const res = await run(["db", "dump", "--db-url", dbUrl, "-f", file], { + env: { SUPABASE_DB_USE_LOCAL_TOOLS: "1" }, + }); + expect(res.exitCode, res.stderr).toBe(0); + expect(existsSync(file)).toBe(true); + expect(readFileSync(file, "utf8")).toMatch(/CREATE|PostgreSQL database dump|SCHEMA/i); + }, + ); + // C4 — runtime 3rd-party internet, no docker. Deploy a function that imports // from jsr.io with the default (server-side) bundler and invoke it; the bundle // path must fetch the import over the network. Fails offline. From 1ba3539c1a1bc22c07aea3d396016ad1fd5fd734 Mon Sep 17 00:00:00 2001 From: avallete Date: Wed, 1 Jul 2026 13:56:58 +0200 Subject: [PATCH 6/6] =?UTF-8?q?test(cli-e2e):=20harden=20live=20setup=20?= =?UTF-8?q?=E2=80=94=20storage-bucket=20retry=20+=20temp-dir=20sanitize?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Validated the capability probes against real staging (all 5 pass); two harness flakes surfaced and are fixed: - createStorageBucket retries on 400 TenantNotFound: the storage tenant is registered asynchronously, after the project reports ACTIVE_HEALTHY, so the first bucket call races it and aborted global setup (upstreams the supabox cli-storage-bucket-retry patch into staging-project.ts). - Sanitize the workspace temp-dir name: a test title with ':' or spaces leaks into the temp path, which the cli mounts as a Docker volume for docker-backed commands (functions bundling, db diff shadow), breaking the src:dst:mode spec ("too many colons"). Collapse non-alphanumerics so any test name is volume-safe. Co-Authored-By: Claude Opus 4.8 --- apps/cli-e2e/src/tests/live/live-context.ts | 7 +++++- apps/cli-e2e/tests/staging-project.ts | 25 ++++++++++++++------- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/apps/cli-e2e/src/tests/live/live-context.ts b/apps/cli-e2e/src/tests/live/live-context.ts index 03fa907e6e..c4c9eac255 100644 --- a/apps/cli-e2e/src/tests/live/live-context.ts +++ b/apps/cli-e2e/src/tests/live/live-context.ts @@ -99,7 +99,12 @@ const base = test.extend({ }, workspace: async ({ task }, use) => { - const dir = makeTempDir(`cli-e2e-live-${task.name.slice(0, 30)}-`); + // Sanitize the task name: it becomes part of the temp-dir path, which the cli + // mounts as a Docker volume for docker-backed commands (functions bundling, + // db diff shadow). A `:` or space in the path breaks the `src:dst:mode` + // volume spec ("too many colons"), so collapse anything non-alphanumeric. + const safeName = task.name.replace(/[^a-zA-Z0-9]+/g, "-").slice(0, 30); + const dir = makeTempDir(`cli-e2e-live-${safeName}-`); // Generate config.toml via `supabase init` so the golden paths run against a // freshly-generated config (functions tests add functions via seedFunctions). const init = await exec(liveHarness(dir.path), ["init"]); diff --git a/apps/cli-e2e/tests/staging-project.ts b/apps/cli-e2e/tests/staging-project.ts index e0d54017fb..fd4a16d905 100644 --- a/apps/cli-e2e/tests/staging-project.ts +++ b/apps/cli-e2e/tests/staging-project.ts @@ -206,20 +206,29 @@ export async function getServiceRoleKey( /** Create a private storage bucket via the project's Storage API (host derived * from projectHost, IPv4-reachable). Idempotent — treats an existing bucket as - * success. */ + * success. Retries on `TenantNotFound`: storage tenant registration completes + * asynchronously, *after* the project reports ACTIVE_HEALTHY, so the first + * bucket call can race it (400 TenantNotFound). Other errors fail fast. */ export async function createStorageBucket( projectHost: string, projectRef: string, serviceRoleKey: string, bucket: string, + attempts = 12, ): Promise { - const res = await fetch(`https://${projectRef}.${projectHost}/storage/v1/bucket`, { - method: "POST", - headers: { Authorization: `Bearer ${serviceRoleKey}`, "Content-Type": "application/json" }, - body: JSON.stringify({ id: bucket, name: bucket, public: false }), - }); - if (!res.ok && res.status !== 409) { - throw new Error(`Failed to create bucket ${bucket}: ${res.status} ${await res.text()}`); + for (let attempt = 1; attempt <= attempts; attempt++) { + const res = await fetch(`https://${projectRef}.${projectHost}/storage/v1/bucket`, { + method: "POST", + headers: { Authorization: `Bearer ${serviceRoleKey}`, "Content-Type": "application/json" }, + body: JSON.stringify({ id: bucket, name: bucket, public: false }), + }); + if (res.ok || res.status === 409) return; // created, or already exists + const body = await res.text(); + // TenantNotFound is the eventual-consistency window — retry; anything else is real. + if (res.status !== 400 || !body.includes("TenantNotFound") || attempt === attempts) { + throw new Error(`Failed to create bucket ${bucket}: ${res.status} ${body}`); + } + await new Promise((r) => setTimeout(r, 10_000)); } }