From 7abd16efd3dcfcfe07d6ae58aa4d4d286ef5b85a Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 18 Aug 2026 17:01:47 +0200 Subject: [PATCH 01/20] feat(stack): prepare slim resources on demand --- apps/cli/src/next/config/stack-config.ts | 1 - .../src/next/config/stack-config.unit.test.ts | 5 +- packages/stack/docs/architecture.md | 33 +- .../src/BinaryResolver.integration.test.ts | 519 ++++++++++----- packages/stack/src/BinaryResolver.ts | 627 ++++++++++++++---- .../stack/src/BinaryResolver.unit.test.ts | 113 +--- packages/stack/src/LocalStack.ts | 240 ++++--- packages/stack/src/Platform.ts | 40 +- packages/stack/src/Platform.unit.test.ts | 73 +- packages/stack/src/ServiceActivation.ts | 7 +- .../stack/src/ServiceActivation.unit.test.ts | 3 +- packages/stack/src/ServiceCatalog.ts | 325 +++++---- packages/stack/src/ServicePorts.ts | 6 +- packages/stack/src/Stack.unit.test.ts | 292 ++++---- packages/stack/src/StackBuilder.ts | 7 +- packages/stack/src/StackBuilder.unit.test.ts | 115 +--- packages/stack/src/StackConfig.ts | 10 +- .../StackConfigResolver.policy.unit.test.ts | 41 ++ packages/stack/src/StackConfigResolver.ts | 142 +++- packages/stack/src/StackPreparation.ts | 402 +++++------ .../stack/src/createStack.integration.test.ts | 1 - packages/stack/src/createStack.unit.test.ts | 17 +- packages/stack/src/effect.ts | 17 +- packages/stack/src/errors.ts | 21 + packages/stack/src/index.ts | 4 +- packages/stack/src/prefetch.ts | 5 +- packages/stack/src/prefetch.unit.test.ts | 220 ++---- packages/stack/src/services/auth.ts | 2 +- packages/stack/src/services/postgrest.ts | 2 +- .../stack/src/services/services.unit.test.ts | 4 +- packages/stack/src/version-plan.unit.test.ts | 20 +- packages/stack/src/versions.ts | 50 +- packages/stack/src/versions.unit.test.ts | 47 +- .../tests/createStack-docker.e2e.test.ts | 18 +- packages/stack/tests/createStack.e2e.test.ts | 60 +- packages/stack/tests/helpers/mocks.ts | 12 +- 36 files changed, 2001 insertions(+), 1500 deletions(-) create mode 100644 packages/stack/src/StackConfigResolver.policy.unit.test.ts diff --git a/apps/cli/src/next/config/stack-config.ts b/apps/cli/src/next/config/stack-config.ts index 8cc9880552..0469f96b51 100644 --- a/apps/cli/src/next/config/stack-config.ts +++ b/apps/cli/src/next/config/stack-config.ts @@ -27,7 +27,6 @@ export function toStartStackConfig( const excluded = new Set(exclude); return { mode, - startupMode: "lazy", realtime: excluded.has("realtime") ? false : {}, storage: excluded.has("storage") ? false : {}, imgproxy: excluded.has("imgproxy") || excluded.has("storage") ? false : {}, diff --git a/apps/cli/src/next/config/stack-config.unit.test.ts b/apps/cli/src/next/config/stack-config.unit.test.ts index d60e7d20fa..f11f460a1a 100644 --- a/apps/cli/src/next/config/stack-config.unit.test.ts +++ b/apps/cli/src/next/config/stack-config.unit.test.ts @@ -2,18 +2,15 @@ import { describe, expect, it } from "vitest"; import { toStartStackConfig, withServiceVersions } from "./stack-config.ts"; describe("toStartStackConfig", () => { - it("uses lazy service startup with the requested runtime mode", () => { + it("uses the requested runtime mode and catalog service defaults", () => { expect(toStartStackConfig([], "auto")).toMatchObject({ mode: "auto", - startupMode: "lazy", }); expect(toStartStackConfig([], "docker")).toMatchObject({ mode: "docker", - startupMode: "lazy", }); expect(toStartStackConfig([], "native")).toMatchObject({ mode: "native", - startupMode: "lazy", }); }); diff --git a/packages/stack/docs/architecture.md b/packages/stack/docs/architecture.md index 742cc3bbd6..3ff0b4e3bd 100644 --- a/packages/stack/docs/architecture.md +++ b/packages/stack/docs/architecture.md @@ -79,8 +79,11 @@ prepares/builds a scoped runtime and handle. It does **not** start service processes. Asset preparation and process-compose graph construction happen when the handle is first started or a service is activated. -`stack.start()` starts services according to the configured startup mode and -waits for the selected readiness policy. The handle also exposes status, logs, +`stack.start()` prepares and starts Postgres plus the services whose resource +policy is `eager`, then waits for the selected readiness policy. Services whose +policy is `lazy` remain dormant until a proxy request or explicit service +operation activates them; activation prepares the service and its required +dependencies before starting it. The handle also exposes status, logs, per-service operations, and graceful `stop()`/`dispose()` methods. Its scope owns service processes and releases the lease when disposed. A direct stack never reads or writes managed documents and never coordinates with a sibling @@ -217,15 +220,23 @@ status, service operations, logs, graceful stop, and launch-update routes; ## Service execution and `ApiProxy` -`StackPreparation` resolves each enabled service to a verified native binary or -a Docker image. `mode: "native"` uses the supported native services and rejects -Docker-only services; `mode: "docker"` resolves every service to an image; -`mode: "auto"` prefers native artifacts and falls back to Docker. The -`StackBuilder` turns those resolutions into one process-compose graph, so a -stack can run native and Docker-backed services together. Docker resources are -namespaced with the managed stack id; when native Postgres is combined with -Docker services, the graph supplies the platform-specific host address so the -containers can reach it. +`StackPreparation` plans resources without materializing them, then prepares +only the requested services and their required dependency closure. Concurrent +requests for the same resource share one installation or pull. Native assets +come from the pinned slim-services release contract and are checksum- and +manifest-verified before atomic publication. Docker assets use one canonical +`ghcr.io/supabase/cli/:` reference; a locally cached image is +reused and a missing image is pulled without registry fallback. `mode: +"native"` rejects Docker-only services, `mode: "docker"` resolves every service +to Docker, and `mode: "auto"` selects the catalog-supported runtime without +falling back after a preparation failure. + +The `StackBuilder` turns planned resolutions into one process-compose graph, +so a stack can run native and Docker-backed services together without eagerly +materializing every graph resource. Docker resources are namespaced with the +managed stack id; when native Postgres is combined with Docker services, the +graph supplies the platform-specific host address so the containers can reach +it. `ApiProxy` listens on the configured public `apiPort` and routes Supabase API paths (`/auth`, `/rest`, `/functions`, `/realtime`, `/storage`, `/pg`, diff --git a/packages/stack/src/BinaryResolver.integration.test.ts b/packages/stack/src/BinaryResolver.integration.test.ts index 5dc58fe38d..3c1e5149f2 100644 --- a/packages/stack/src/BinaryResolver.integration.test.ts +++ b/packages/stack/src/BinaryResolver.integration.test.ts @@ -1,3 +1,4 @@ +import { createHash } from "node:crypto"; import { execFileSync } from "node:child_process"; import { existsSync, @@ -5,44 +6,105 @@ import { mkdtempSync, readFileSync, rmSync, + symlinkSync, utimesSync, writeFileSync, } from "node:fs"; import { tmpdir } from "node:os"; -import { basename, dirname, join } from "node:path"; +import { dirname, join } from "node:path"; import { NodeServices } from "@effect/platform-node"; import { describe, expect, it } from "@effect/vitest"; import { Effect, Layer } from "effect"; import { HttpClient, HttpClientResponse } from "effect/unstable/http"; -import { afterEach } from "vitest"; import { BinaryResolver } from "./BinaryResolver.ts"; -import { DownloadError } from "./errors.ts"; +import { + BinaryManifestError, + BinaryRuntimeError, + ChecksumMismatchError, + DownloadError, +} from "./errors.ts"; import { detectPlatform } from "./Platform.ts"; import { nativeReleaseForService } from "./ServiceCatalog.ts"; import { DEFAULT_VERSIONS } from "./versions.ts"; -const tempRoots: string[] = []; +const makeRoot = (): string => mkdtempSync(join(tmpdir(), "stack-slim-services-")); -const makeTempRoot = (): string => { - const root = mkdtempSync(join(tmpdir(), "stack-binary-resolver-")); - tempRoots.push(root); - return root; +const makeFixture = ( + root: string, + manifestOverride: Record = {}, + includePostgrest = true, +) => { + const source = join(root, "source"); + const archive = join(root, "postgrest.tar.zst"); + rmSync(source, { recursive: true, force: true }); + mkdirSync(join(source, "bin"), { recursive: true }); + if (includePostgrest) { + writeFileSync(join(source, "bin", "postgrest"), "#!/bin/sh\necho postgrest\n"); + } + execFileSync("tar", ["--zstd", "-cf", archive, "-C", source, "."]); + return { archive: readFileSync(archive), manifestOverride }; }; -const makeArchive = (root: string): Uint8Array => { - const source = join(root, "source"); - const archive = join(root, "auth.tar.gz"); - execFileSync("mkdir", ["-p", source]); - writeFileSync(join(source, "auth"), "#!/bin/sh\necho auth\n"); - execFileSync("tar", ["czf", archive, "-C", source, "."]); - return readFileSync(archive); +const makeTraversalFixture = (root: string) => { + const source = join(root, "traversal-source"); + const archive = join(root, "postgrest-traversal.tar.zst"); + rmSync(source, { recursive: true, force: true }); + mkdirSync(source, { recursive: true }); + writeFileSync(join(root, "outside.txt"), "must not extract\n"); + execFileSync("tar", ["--zstd", "-cf", archive, "-C", source, "../outside.txt"]); + return { archive: readFileSync(archive), manifestOverride: {} }; }; -const makeResolverLayer = (cacheRoot: string, archive: Uint8Array, onRequest: () => void) => { +const makeEscapingSymlinkFixture = (root: string) => { + const source = join(root, "symlink-source"); + const archive = join(root, "postgrest-symlink.tar.zst"); + rmSync(source, { recursive: true, force: true }); + mkdirSync(join(source, "bin"), { recursive: true }); + symlinkSync("/bin/sh", join(source, "bin/postgrest")); + execFileSync("tar", ["--zstd", "-cf", archive, "-C", source, "."]); + return { archive: readFileSync(archive), manifestOverride: {} }; +}; + +const makeResolverLayer = ( + cacheRoot: string, + fixture: ReturnType, + options: { readonly checksum?: string } = {}, +) => { const client = HttpClient.make((request) => Effect.sync(() => { - onRequest(); - return HttpClientResponse.fromWeb(request, new Response(archive, { status: 200 })); + if (request.url.endsWith(".manifest.json")) { + return HttpClientResponse.fromWeb( + request, + new Response( + JSON.stringify({ + service: "postgrest", + version: DEFAULT_VERSIONS.postgrest, + target: process.platform === "darwin" ? "darwin-arm64" : "linux-amd64", + entrypoint: [], + cmd: ["/bin/postgrest"], + runtime_requires: null, + libc: process.platform === "linux" ? "glibc" : null, + os_floor: + process.platform === "linux" + ? { kind: "glibc", floor: null, scanned: 1 } + : { kind: "macos", floor: null, scanned: 1 }, + ...fixture.manifestOverride, + }), + { status: 200 }, + ), + ); + } + if (request.url.endsWith("SHA256SUMS")) { + const hash = options.checksum ?? createHash("sha256").update(fixture.archive).digest("hex"); + return HttpClientResponse.fromWeb( + request, + new Response( + `${hash} postgrest-${DEFAULT_VERSIONS.postgrest}-${process.platform === "darwin" ? "darwin-arm64" : "linux-amd64"}.tar.zst\n`, + { status: 200 }, + ), + ); + } + return HttpClientResponse.fromWeb(request, new Response(fixture.archive, { status: 200 })); }), ); return BinaryResolver.make(cacheRoot).pipe( @@ -51,160 +113,277 @@ const makeResolverLayer = (cacheRoot: string, archive: Uint8Array, onRequest: () ); }; -const makeUnavailableResolverLayer = (cacheRoot: string) => { - const client = HttpClient.make((request) => - Effect.succeed( - HttpClientResponse.fromWeb(request, new Response("unavailable", { status: 503 })), - ), +describe("BinaryResolver slim-services installer", () => { + it.live("installs a tar.zst archive into an empty cache and reuses it", () => + Effect.gen(function* () { + const root = makeRoot(); + try { + const fixture = makeFixture(root); + const resolverLayer = makeResolverLayer(root, fixture); + const release = nativeReleaseForService( + "postgrest", + DEFAULT_VERSIONS.postgrest, + yield* detectPlatform, + ); + if (release === undefined) return; + const result = yield* Effect.gen(function* () { + const resolver = yield* BinaryResolver; + const first = yield* resolver.resolveWithMetadata({ + service: "postgrest", + version: DEFAULT_VERSIONS.postgrest, + }); + const second = yield* resolver.resolveWithMetadata({ + service: "postgrest", + version: DEFAULT_VERSIONS.postgrest, + }); + return { first, second }; + }).pipe(Effect.provide(resolverLayer)); + expect(result.first.downloaded).toBe(true); + expect(result.second.downloaded).toBe(false); + expect(readFileSync(join(result.first.path, "bin/postgrest"), "utf8")).toContain( + "postgrest", + ); + expect(existsSync(join(result.first.path, ".complete"))).toBe(true); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }), ); - return BinaryResolver.make(cacheRoot).pipe( - Layer.provide(Layer.succeed(HttpClient.HttpClient, client)), - Layer.provide(NodeServices.layer), + + it.live("accepts the Mailpit-style glibc runtime requirement", () => + Effect.gen(function* () { + if (process.platform !== "linux") return; + const root = makeRoot(); + try { + const fixture = makeFixture(root, { runtime_requires: "glibc" }); + const resolverLayer = makeResolverLayer(root, fixture); + const result = yield* Effect.gen(function* () { + const resolver = yield* BinaryResolver; + return yield* resolver.resolve({ + service: "postgrest", + version: DEFAULT_VERSIONS.postgrest, + }); + }).pipe(Effect.provide(resolverLayer)); + expect(existsSync(join(result, "bin/postgrest"))).toBe(true); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }), ); -}; -const authCachePath = (cacheRoot: string) => - Effect.gen(function* () { - const platform = yield* detectPlatform; - const release = nativeReleaseForService("auth", DEFAULT_VERSIONS.auth, platform); - if (release === undefined) { - return yield* Effect.die(`unsupported test platform: ${platform.os}-${platform.arch}`); - } - return BinaryResolver.cachePath(join(cacheRoot, "bin"), { - service: "auth", - provider: release.provider, - version: DEFAULT_VERSIONS.auth, - assetName: release.assetName, - }); - }); - -const legacyAuthCachePath = (cacheRoot: string) => - Effect.gen(function* () { - const platform = yield* detectPlatform; - const release = nativeReleaseForService("auth", DEFAULT_VERSIONS.auth, platform); - if (release === undefined) { - return yield* Effect.die(`unsupported test platform: ${platform.os}-${platform.arch}`); - } - return join(cacheRoot, "bin", "auth", DEFAULT_VERSIONS.auth, release.assetName); - }); - -afterEach(() => { - for (const root of tempRoots.splice(0)) { - rmSync(root, { recursive: true, force: true }); - } -}); + it.live("reclaims interrupted staging while preserving complete cache entries", () => + Effect.gen(function* () { + const root = makeRoot(); + try { + const fixture = makeFixture(root); + const resolverLayer = makeResolverLayer(root, fixture); + const release = nativeReleaseForService( + "postgrest", + DEFAULT_VERSIONS.postgrest, + yield* detectPlatform, + ); + if (release === undefined) return; + const cacheDir = BinaryResolver.cachePath(join(root, "bin"), { + service: "postgrest", + releaseSet: "slim-services", + version: DEFAULT_VERSIONS.postgrest, + runtime: "native", + target: release.target, + }); + const stale = join(dirname(cacheDir), `.${release.assetName}.partial-interrupted`); + mkdirSync(stale, { recursive: true }); + const old = new Date(Date.now() - 2 * 24 * 60 * 60 * 1_000); + utimesSync(stale, old, old); + + const resolved = yield* Effect.gen(function* () { + const resolver = yield* BinaryResolver; + return yield* resolver.resolve({ + service: "postgrest", + version: DEFAULT_VERSIONS.postgrest, + }); + }).pipe(Effect.provide(resolverLayer)); + + expect(existsSync(stale)).toBe(false); + expect(existsSync(join(resolved, ".complete"))).toBe(true); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }), + ); + + it.live("replaces caches with invalid identity markers or missing required paths", () => + Effect.gen(function* () { + const root = makeRoot(); + try { + const fixture = makeFixture(root); + const resolverLayer = makeResolverLayer(root, fixture); + const result = yield* Effect.gen(function* () { + const resolver = yield* BinaryResolver; + return yield* resolver.resolveWithMetadata({ + service: "postgrest", + version: DEFAULT_VERSIONS.postgrest, + }); + }).pipe(Effect.provide(resolverLayer)); + expect(result.downloaded).toBe(true); + + const markerPath = join(result.path, ".complete"); + writeFileSync(markerPath, JSON.stringify({ service: "postgrest" })); + const replacedMarker = yield* Effect.gen(function* () { + const resolver = yield* BinaryResolver; + return yield* resolver.resolveWithMetadata({ + service: "postgrest", + version: DEFAULT_VERSIONS.postgrest, + }); + }).pipe(Effect.provide(resolverLayer)); + expect(replacedMarker.downloaded).toBe(true); + expect(JSON.parse(readFileSync(markerPath, "utf8"))).toMatchObject({ + service: "postgrest", + version: DEFAULT_VERSIONS.postgrest, + runtime: "native", + }); + + rmSync(join(replacedMarker.path, "bin/postgrest")); + const restoredPath = yield* Effect.gen(function* () { + const resolver = yield* BinaryResolver; + return yield* resolver.resolveWithMetadata({ + service: "postgrest", + version: DEFAULT_VERSIONS.postgrest, + }); + }).pipe(Effect.provide(resolverLayer)); + expect(restoredPath.downloaded).toBe(true); + expect(existsSync(join(restoredPath.path, "bin/postgrest"))).toBe(true); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }), + ); + + it.live("rejects archive members that escape the private staging directory", () => + Effect.gen(function* () { + const root = makeRoot(); + try { + const fixture = makeTraversalFixture(root); + const resolverLayer = makeResolverLayer(root, fixture); + const failure = yield* Effect.gen(function* () { + const resolver = yield* BinaryResolver; + return yield* resolver.resolve({ + service: "postgrest", + version: DEFAULT_VERSIONS.postgrest, + }); + }).pipe(Effect.provide(resolverLayer), Effect.flip); + expect(failure).toBeInstanceOf(DownloadError); + const release = nativeReleaseForService( + "postgrest", + DEFAULT_VERSIONS.postgrest, + yield* detectPlatform, + ); + if (release !== undefined) { + const cache = BinaryResolver.cachePath(join(root, "bin"), { + service: "postgrest", + releaseSet: "slim-services", + version: DEFAULT_VERSIONS.postgrest, + runtime: "native", + target: release.target, + }); + expect(existsSync(join(dirname(cache), "outside.txt"))).toBe(false); + expect(existsSync(join(cache, ".complete"))).toBe(false); + } + } finally { + rmSync(root, { recursive: true, force: true }); + } + }), + ); + + it.live("rejects archive symlinks that resolve outside private staging", () => + Effect.gen(function* () { + const root = makeRoot(); + try { + const resolverLayer = makeResolverLayer(root, makeEscapingSymlinkFixture(root)); + const failure = yield* Effect.gen(function* () { + const resolver = yield* BinaryResolver; + return yield* resolver.resolve({ + service: "postgrest", + version: DEFAULT_VERSIONS.postgrest, + }); + }).pipe(Effect.provide(resolverLayer), Effect.flip); + expect(failure).toBeInstanceOf(BinaryRuntimeError); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }), + ); + + it.live("rejects checksum and manifest/runtime validation failures", () => + Effect.gen(function* () { + const root = makeRoot(); + try { + const fixture = makeFixture(root); + const resolverLayer = makeResolverLayer(root, fixture, { checksum: "0".repeat(64) }); + const checksum = yield* Effect.gen(function* () { + const resolver = yield* BinaryResolver; + return yield* resolver.resolve({ + service: "postgrest", + version: DEFAULT_VERSIONS.postgrest, + }); + }).pipe(Effect.provide(resolverLayer), Effect.flip); + expect(checksum).toBeInstanceOf(ChecksumMismatchError); -describe("BinaryResolver cache publication", () => { - it.live("publishes one complete cache entry for concurrent resolvers", () => { - const root = makeTempRoot(); - const archive = makeArchive(root); - let requestCount = 0; - const layer = makeResolverLayer(root, archive, () => { - requestCount += 1; - }); - - return Effect.gen(function* () { - const resolver = yield* BinaryResolver; - const results = yield* Effect.all( - [ - resolver.resolveWithMetadata({ service: "auth", version: DEFAULT_VERSIONS.auth }), - resolver.resolveWithMetadata({ service: "auth", version: DEFAULT_VERSIONS.auth }), - ], - { concurrency: "unbounded" }, - ); - - expect(requestCount).toBe(2); - expect(results.filter((result) => result.downloaded)).toHaveLength(1); - expect(results[0]?.path).toBe(results[1]?.path); - expect(readFileSync(join(results[0]!.path, "auth"), "utf8")).toContain("echo auth"); - expect(readFileSync(join(results[0]!.path, ".complete"), "utf8")).toContain( - "github.com/supabase/auth", - ); - }).pipe(Effect.provide(layer)); - }); - - it.live("reuses a complete cache from the legacy layout without downloading", () => { - const root = makeTempRoot(); - const layer = makeUnavailableResolverLayer(root); - - return Effect.gen(function* () { - const resolver = yield* BinaryResolver; - const legacyCacheDir = yield* legacyAuthCachePath(root); - mkdirSync(legacyCacheDir, { recursive: true }); - writeFileSync(join(legacyCacheDir, "auth"), "legacy auth binary"); - - const result = yield* resolver.resolveWithMetadata({ - service: "auth", - version: DEFAULT_VERSIONS.auth, - }); - - expect(result).toEqual({ path: legacyCacheDir, downloaded: false }); - }).pipe(Effect.provide(layer)); - }); - - it.live("preserves a markerless provider cache when replacement download fails", () => { - const root = makeTempRoot(); - const layer = makeUnavailableResolverLayer(root); - - return Effect.gen(function* () { - const resolver = yield* BinaryResolver; - const cacheDir = yield* authCachePath(root); - const legacyBinary = join(cacheDir, "auth"); - mkdirSync(cacheDir, { recursive: true }); - writeFileSync(legacyBinary, "legacy auth binary"); - - const error = yield* resolver - .resolveWithMetadata({ service: "auth", version: DEFAULT_VERSIONS.auth }) - .pipe(Effect.flip); - - expect(error).toBeInstanceOf(DownloadError); - expect(readFileSync(legacyBinary, "utf8")).toBe("legacy auth binary"); - }).pipe(Effect.provide(layer)); - }); - - it.live("replaces an incomplete provider cache after staging succeeds", () => { - const root = makeTempRoot(); - const archive = makeArchive(root); - const layer = makeResolverLayer(root, archive, () => {}); - - return Effect.gen(function* () { - const resolver = yield* BinaryResolver; - const cacheDir = yield* authCachePath(root); - mkdirSync(cacheDir, { recursive: true }); - writeFileSync(join(cacheDir, ".complete"), "orphaned marker"); - - const result = yield* resolver.resolveWithMetadata({ - service: "auth", - version: DEFAULT_VERSIONS.auth, - }); - - expect(result).toEqual({ path: cacheDir, downloaded: true }); - expect(readFileSync(join(cacheDir, "auth"), "utf8")).toContain("echo auth"); - expect(readFileSync(join(cacheDir, ".complete"), "utf8")).toContain( - "github.com/supabase/auth", - ); - }).pipe(Effect.provide(layer)); - }); - - it.live("reaps stale staging directories even when the artifact is cached", () => { - const root = makeTempRoot(); - const archive = makeArchive(root); - const layer = makeResolverLayer(root, archive, () => {}); - - return Effect.gen(function* () { - const resolver = yield* BinaryResolver; - const spec = { service: "auth", version: DEFAULT_VERSIONS.auth } as const; - const first = yield* resolver.resolveWithMetadata(spec); - const staleStaging = join(dirname(first.path), `.${basename(first.path)}.partial-abandoned`); - mkdirSync(staleStaging); - writeFileSync(join(staleStaging, "partial"), "partial artifact"); - const staleTime = new Date(Date.now() - 25 * 60 * 60 * 1_000); - utimesSync(staleStaging, staleTime, staleTime); - - const second = yield* resolver.resolveWithMetadata(spec); - - expect(second.downloaded).toBe(false); - expect(existsSync(staleStaging)).toBe(false); - }).pipe(Effect.provide(layer)); - }); + const manifestLayer = makeResolverLayer( + root, + makeFixture(root, { + target: process.platform === "darwin" ? "linux-amd64" : "darwin-arm64", + }), + ); + const manifest = yield* Effect.gen(function* () { + const resolver = yield* BinaryResolver; + return yield* resolver.resolve({ + service: "postgrest", + version: DEFAULT_VERSIONS.postgrest, + }); + }).pipe(Effect.provide(manifestLayer), Effect.flip); + expect(manifest).toBeInstanceOf(BinaryManifestError); + expect(manifest).not.toBeInstanceOf(BinaryRuntimeError); + + const unsafeCommandLayer = makeResolverLayer( + root, + makeFixture(root, { cmd: ["../bin/postgrest"] }), + ); + const unsafeCommand = yield* Effect.gen(function* () { + const resolver = yield* BinaryResolver; + return yield* resolver.resolve({ + service: "postgrest", + version: DEFAULT_VERSIONS.postgrest, + }); + }).pipe(Effect.provide(unsafeCommandLayer), Effect.flip); + expect(unsafeCommand).toBeInstanceOf(BinaryManifestError); + + const runtimeLayer = makeResolverLayer(root, makeFixture(root, {}, false)); + const runtime = yield* Effect.gen(function* () { + const resolver = yield* BinaryResolver; + return yield* resolver.resolve({ + service: "postgrest", + version: DEFAULT_VERSIONS.postgrest, + }); + }).pipe(Effect.provide(runtimeLayer), Effect.flip); + expect(runtime).toBeInstanceOf(BinaryRuntimeError); + const release = nativeReleaseForService( + "postgrest", + DEFAULT_VERSIONS.postgrest, + yield* detectPlatform, + ); + if (release !== undefined) { + const cache = BinaryResolver.cachePath(join(root, "bin"), { + service: "postgrest", + releaseSet: "slim-services", + version: DEFAULT_VERSIONS.postgrest, + runtime: "native", + target: release.target, + }); + expect(existsSync(join(cache, ".complete"))).toBe(false); + } + } finally { + rmSync(root, { recursive: true, force: true }); + } + }), + ); }); diff --git a/packages/stack/src/BinaryResolver.ts b/packages/stack/src/BinaryResolver.ts index f6ee4070d4..5ba3b59887 100644 --- a/packages/stack/src/BinaryResolver.ts +++ b/packages/stack/src/BinaryResolver.ts @@ -2,13 +2,16 @@ import { createHash } from "node:crypto"; import { Context, Effect, FileSystem, Layer, Option, Path, Result } from "effect"; import { HttpClient } from "effect/unstable/http"; import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; -import { BinaryNotFoundError, ChecksumMismatchError, DownloadError } from "./errors.ts"; -import { detectPlatform } from "./Platform.ts"; import { - nativeReleaseForService, - type ArchiveFormat, - type NativeReleaseArtifact, -} from "./ServiceCatalog.ts"; + BinaryHostCompatibilityError, + BinaryManifestError, + BinaryNotFoundError, + BinaryRuntimeError, + ChecksumMismatchError, + DownloadError, +} from "./errors.ts"; +import { detectPlatform, type NativeTarget } from "./Platform.ts"; +import { nativeReleaseForService, type NativeReleaseArtifact } from "./ServiceCatalog.ts"; import type { ServiceName } from "./ServiceName.ts"; export interface BinarySpec { @@ -22,91 +25,77 @@ interface ResolveBinaryResult { readonly downloaded: boolean; } +interface SlimServiceManifest { + readonly service: string; + readonly version: string; + readonly target: string; + readonly entrypoint: ReadonlyArray; + readonly cmd: ReadonlyArray; + readonly runtime_requires?: null | "glibc"; + readonly libc: null | "glibc"; + readonly os_floor: null | { + readonly kind: string; + readonly floor: string | null; + readonly offender?: string | null; + readonly scanned: number; + readonly bundled_glibc?: boolean; + }; +} + export interface ResolveBinaryOptions { readonly onDownloadStart?: Effect.Effect; } interface AssetInfo { readonly service: ServiceName; + readonly releaseSet: "slim-services"; + readonly version: string; + readonly runtime: "native"; + readonly target: NativeTarget; +} + +interface CacheCompleteMarker { readonly provider: string; + readonly service: string; readonly version: string; - readonly assetName: string; + readonly asset: string; + readonly url: string; + readonly target: NativeTarget; + readonly releaseSet: "slim-services"; + readonly runtime: "native"; } const cachePath = (baseDir: string, info: AssetInfo): string => - `${baseDir}/${info.service}/${info.provider.replaceAll("/", "_")}/${info.version}/${info.assetName}`; - -const LEGACY_NATIVE_PROVIDERS: Partial> = { - postgres: "github.com/supabase/postgres", - postgrest: "github.com/PostgREST/postgrest", - auth: "github.com/supabase/auth", - "edge-runtime": "github.com/supabase/edge-runtime", -}; - -const legacyCachePath = (baseDir: string, info: AssetInfo): string | undefined => - LEGACY_NATIVE_PROVIDERS[info.service] === info.provider - ? `${baseDir}/${info.service}/${info.version}/${info.assetName}` - : undefined; - -const legacyExecutablePath = ( - directory: string, - service: ServiceName, - platformOs: string, -): string | undefined => { - const executableSuffix = platformOs === "win32" ? ".exe" : ""; - switch (service) { - case "postgres": - return `${directory}/bin/postgres${executableSuffix}`; - case "postgrest": - return `${directory}/postgrest${executableSuffix}`; - case "auth": - return `${directory}/auth${executableSuffix}`; - case "edge-runtime": - return `${directory}/bin/edge-runtime${executableSuffix}`; - default: - return undefined; - } -}; - -const legacyCacheRequiredPaths = ( - directory: string, - service: ServiceName, - platformOs: string, -): ReadonlyArray => { - const executable = legacyExecutablePath(directory, service, platformOs); - if (executable === undefined) return []; - return service === "postgres" - ? [ - executable, - `${directory}/bin/pg_isready${platformOs === "win32" ? ".exe" : ""}`, - `${directory}/bin/psql${platformOs === "win32" ? ".exe" : ""}`, - `${directory}/share/supabase-cli/bin/supabase-postgres-init.sh`, - `${directory}/lib`, - ] - : [executable]; -}; + `${baseDir}/${info.releaseSet}/${info.service}/${info.version}/${info.runtime}/${info.target}`; const CACHE_COMPLETE_MARKER = ".complete"; const STALE_STAGING_AGE_MS = 24 * 60 * 60 * 1_000; -const extractCommand = ( - archive: ArchiveFormat, - archivePath: string, - destDir: string, - os: string, - stripComponents: boolean, -): string[] => { - if (archive === "zip") { - return os === "win32" - ? ["tar", "xf", archivePath, "-C", destDir] - : ["unzip", "-o", archivePath, "-d", destDir]; - } - const flag = archive === "tar.gz" ? "xzf" : "xf"; - const args = ["tar", flag, archivePath, "-C", destDir]; - if (stripComponents) args.push("--strip-components=1"); +const extractCommand = (archivePath: string, destDir: string): string[] => { + const args = ["tar", "--zstd", "-xf", archivePath, "-C", destDir]; return args; }; +const hasTraversalSegment = (value: string): boolean => + value.split(/[\\/]/).some((segment) => segment === ".."); + +const isUnsafeArchiveMember = (member: string): boolean => { + const normalized = member.trim(); + if (normalized.length === 0) return false; + if (normalized.startsWith("/") || /^[A-Za-z]:[\\/]/.test(normalized)) return true; + let depth = 0; + for (const segment of normalized.split(/[\\/]/)) { + if (segment.length === 0 || segment === ".") continue; + if (segment === "..") { + if (depth === 0) return true; + depth -= 1; + } else { + depth += 1; + } + } + return false; +}; + const verifyChecksum = ( data: ArrayBuffer, expected: string, @@ -126,25 +115,263 @@ const verifyChecksum = ( }), ); +const checksumForArchive = (contents: string, archiveName: string): string | undefined => { + for (const line of contents.split(/\r?\n/)) { + const match = line.trim().match(/^([a-f0-9]{64})\s+[* ]?(.+)$/i); + if (match?.[2] === archiveName || match?.[2]?.endsWith(`/${archiveName}`)) return match[1]; + } + const single = contents.trim().match(/^([a-f0-9]{64})(?:\s|$)/i); + return single?.[1]; +}; + +const manifestError = (url: string, detail: string): BinaryManifestError => + new BinaryManifestError({ url, detail }); + +const isSlimServiceManifest = (value: unknown): value is SlimServiceManifest => { + if (typeof value !== "object" || value === null) return false; + if (!("service" in value) || typeof value.service !== "string") return false; + if (!("version" in value) || typeof value.version !== "string") return false; + if (!("target" in value) || typeof value.target !== "string") return false; + if (!("entrypoint" in value) || !Array.isArray(value.entrypoint)) return false; + if (!("cmd" in value) || !Array.isArray(value.cmd)) return false; + if ( + "runtime_requires" in value && + value.runtime_requires !== null && + value.runtime_requires !== "glibc" + ) + return false; + if (!("libc" in value) || (value.libc !== null && value.libc !== "glibc")) return false; + if (!("os_floor" in value)) return false; + if (value.os_floor !== null) { + if (typeof value.os_floor !== "object") return false; + if (!("kind" in value.os_floor) || typeof value.os_floor.kind !== "string") return false; + if (!("floor" in value.os_floor)) return false; + if (value.os_floor.floor !== null && typeof value.os_floor.floor !== "string") return false; + if (!("scanned" in value.os_floor) || typeof value.os_floor.scanned !== "number") return false; + if ( + "offender" in value.os_floor && + value.os_floor.offender !== null && + typeof value.os_floor.offender !== "string" + ) + return false; + if ("bundled_glibc" in value.os_floor && typeof value.os_floor.bundled_glibc !== "boolean") + return false; + } + return true; +}; + +const validateManifest = ( + release: NativeReleaseArtifact, + raw: unknown, + platform: { readonly os: string; readonly arch: string }, + spawner: ChildProcessSpawner.ChildProcessSpawner["Service"], +): Effect.Effect => + Effect.gen(function* () { + if (typeof raw !== "object" || raw === null) { + return yield* Effect.fail(manifestError(release.manifestUrl, "Manifest must be an object")); + } + if (!isSlimServiceManifest(raw)) { + return yield* Effect.fail(manifestError(release.manifestUrl, "Manifest schema is invalid")); + } + const manifest = raw; + if ( + manifest.service !== release.service || + manifest.version !== release.version || + manifest.target !== release.target + ) { + return yield* Effect.fail( + manifestError( + release.manifestUrl, + "Manifest service/version/target does not match release", + ), + ); + } + if ( + !Array.isArray(manifest.entrypoint) || + !manifest.entrypoint.every((value) => typeof value === "string") || + !Array.isArray(manifest.cmd) || + !manifest.cmd.every((value) => typeof value === "string") + ) { + return yield* Effect.fail( + manifestError(release.manifestUrl, "Manifest entrypoint/cmd must be string arrays"), + ); + } + if (manifest.entrypoint.length === 0 && manifest.cmd.length === 0) { + return yield* Effect.fail(manifestError(release.manifestUrl, "Manifest has no command")); + } + const runtimeRequires = manifest.runtime_requires ?? null; + const commandPaths = [...manifest.entrypoint, ...manifest.cmd].filter( + (entry) => + entry.startsWith("/") || + entry.includes("/") || + entry.includes("\\") || + entry === "." || + entry === "..", + ); + if (commandPaths.some((entry) => hasTraversalSegment(entry))) { + return yield* Effect.fail( + manifestError(release.manifestUrl, "Manifest command path is unsafe"), + ); + } + const osFloor = manifest.os_floor; + if (osFloor !== null && typeof osFloor !== "object") { + return yield* Effect.fail(manifestError(release.manifestUrl, "Manifest os_floor is invalid")); + } + const requiresGlibc = manifest.libc === "glibc" || runtimeRequires === "glibc"; + if (requiresGlibc && platform.os !== "linux") { + return yield* Effect.fail( + new BinaryHostCompatibilityError({ + target: release.target, + detail: "Manifest requires Linux/glibc", + }), + ); + } + if (osFloor !== null && osFloor.kind === "macos" && platform.os !== "darwin") { + return yield* Effect.fail( + new BinaryHostCompatibilityError({ + target: release.target, + detail: "Manifest requires macOS", + }), + ); + } + if (osFloor !== null && osFloor.kind === "glibc" && platform.os !== "linux") { + return yield* Effect.fail( + new BinaryHostCompatibilityError({ + target: release.target, + detail: "Manifest requires Linux/glibc", + }), + ); + } + if (osFloor !== null && osFloor.kind !== "macos" && osFloor.kind !== "glibc") { + return yield* Effect.fail( + new BinaryHostCompatibilityError({ + target: release.target, + detail: `Unsupported manifest host kind ${osFloor.kind}`, + }), + ); + } + const floor = osFloor?.floor; + if ( + platform.os === "linux" && + osFloor?.kind === "glibc" && + floor !== null && + floor !== undefined + ) { + const host = yield* Effect.sync(() => { + try { + const report = process.report?.getReport?.(); + if (typeof report !== "object" || report === null || !("header" in report)) { + return undefined; + } + const header = report.header; + if ( + typeof header !== "object" || + header === null || + !("glibcVersionRuntime" in header) || + typeof header.glibcVersionRuntime !== "string" + ) { + return undefined; + } + return header.glibcVersionRuntime; + } catch { + return undefined; + } + }); + if (typeof host !== "string" || host.trim().length === 0) { + return yield* Effect.fail( + new BinaryHostCompatibilityError({ + target: release.target, + detail: "Unable to determine host glibc version", + }), + ); + } + if (compareVersions(host, floor) < 0) { + return yield* Effect.fail( + new BinaryHostCompatibilityError({ + target: release.target, + detail: `Host glibc ${host} is below manifest floor ${floor}`, + }), + ); + } + } + if ( + platform.os === "darwin" && + osFloor?.kind === "macos" && + floor !== null && + floor !== undefined + ) { + const host = yield* spawner.string(ChildProcess.make("sw_vers", ["-productVersion"])).pipe( + Effect.mapError( + (cause) => + new BinaryHostCompatibilityError({ + target: release.target, + detail: `Unable to determine macOS version: ${String(cause)}`, + }), + ), + ); + const hostVersion = host.trim().split(/\s+/)[0] ?? ""; + if (hostVersion.length === 0) { + return yield* Effect.fail( + new BinaryHostCompatibilityError({ + target: release.target, + detail: "Unable to determine macOS version", + }), + ); + } + if (compareVersions(hostVersion, floor) < 0) { + return yield* Effect.fail( + new BinaryHostCompatibilityError({ + target: release.target, + detail: `Host macOS ${hostVersion} is below manifest floor ${floor}`, + }), + ); + } + } + return manifest; + }); + +const compareVersions = (left: string, right: string): number => { + const a = left.split(".").map((part) => Number(part) || 0); + const b = right.split(".").map((part) => Number(part) || 0); + for (let index = 0; index < Math.max(a.length, b.length); index += 1) { + const diff = (a[index] ?? 0) - (b[index] ?? 0); + if (diff !== 0) return diff; + } + return 0; +}; + export class BinaryResolver extends Context.Service< BinaryResolver, { + /** Computes the immutable cache identity without inspecting or changing the filesystem. */ + readonly plan: (spec: BinarySpec) => Effect.Effect; readonly resolveWithMetadata: ( spec: BinarySpec, options?: ResolveBinaryOptions, ) => Effect.Effect< ResolveBinaryResult, - BinaryNotFoundError | DownloadError | ChecksumMismatchError + | BinaryNotFoundError + | DownloadError + | ChecksumMismatchError + | BinaryManifestError + | BinaryRuntimeError + | BinaryHostCompatibilityError >; readonly resolve: ( spec: BinarySpec, - ) => Effect.Effect; + ) => Effect.Effect< + string, + | BinaryNotFoundError + | DownloadError + | ChecksumMismatchError + | BinaryManifestError + | BinaryRuntimeError + | BinaryHostCompatibilityError + >; } >()("local/BinaryResolver") { // Static pure functions — tested in unit tests static cachePath = cachePath; - static legacyExecutablePath = legacyExecutablePath; - static legacyCacheRequiredPaths = legacyCacheRequiredPaths; static make( cacheRoot: string, @@ -165,27 +392,80 @@ export class BinaryResolver extends Context.Service< const httpClient = (yield* HttpClient.HttpClient).pipe(HttpClient.filterStatusOk); const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; - const isCompleteCache = (directory: string) => + const isCacheCompleteMarker = (value: unknown): value is CacheCompleteMarker => { + if (typeof value !== "object" || value === null) return false; + if (!("provider" in value) || typeof value.provider !== "string") return false; + if (!("service" in value) || typeof value.service !== "string") return false; + if (!("version" in value) || typeof value.version !== "string") return false; + if (!("asset" in value) || typeof value.asset !== "string") return false; + if (!("url" in value) || typeof value.url !== "string") return false; + if ( + !("target" in value) || + (value.target !== "darwin-arm64" && + value.target !== "linux-amd64" && + value.target !== "linux-arm64") + ) + return false; + if (!("releaseSet" in value) || value.releaseSet !== "slim-services") return false; + if (!("runtime" in value) || value.runtime !== "native") return false; + return true; + }; + + const isCompleteCache = ( + directory: string, + release: NativeReleaseArtifact, + info: AssetInfo, + ) => Effect.gen(function* () { - if (!(yield* fs.exists(path.join(directory, CACHE_COMPLETE_MARKER)))) { + const marker = yield* fs + .readFileString(path.join(directory, CACHE_COMPLETE_MARKER)) + .pipe(Effect.option); + if (Option.isNone(marker)) return false; + const parsed = yield* Effect.try({ + try: () => JSON.parse(marker.value), + catch: () => undefined, + }); + if (!isCacheCompleteMarker(parsed)) return false; + if ( + parsed.provider !== release.provider || + parsed.service !== info.service || + parsed.version !== info.version || + parsed.asset !== release.assetName || + parsed.url !== release.downloadUrl || + parsed.target !== info.target || + parsed.releaseSet !== info.releaseSet || + parsed.runtime !== info.runtime + ) { return false; } - const entries = yield* fs.readDirectory(directory); - return entries.some((entry) => entry !== CACHE_COMPLETE_MARKER); - }); + const requiredPaths = [...new Set(release.requiredRuntimePaths)]; + const present = yield* Effect.forEach(requiredPaths, (entry) => + fs.exists(path.join(directory, entry)), + ); + return present.every(Boolean); + }).pipe(Effect.catch(() => Effect.succeed(false))); - const isReusableLegacyCache = ( - directory: string, - service: ServiceName, - platformOs: string, - ) => { - const requiredPaths = legacyCacheRequiredPaths(directory, service, platformOs); - return requiredPaths.length === 0 - ? Effect.succeed(false) - : Effect.forEach(requiredPaths, fs.exists).pipe( - Effect.map((results) => results.every(Boolean)), + const plan = (spec: BinarySpec): Effect.Effect => + Effect.gen(function* () { + const platform = yield* detectPlatform; + const release = nativeReleaseForService(spec.service, spec.version, platform); + if (release === undefined) { + return yield* Effect.fail( + new BinaryNotFoundError({ + service: spec.service, + platform: `${platform.os}-${platform.arch}`, + }), ); - }; + } + const info: AssetInfo = { + service: spec.service, + releaseSet: "slim-services", + version: spec.version, + runtime: "native", + target: release.target, + }; + return cachePath(spec.cacheDir ?? binDir, info); + }); const cleanupStaleStaging = (directory: string, prefix: string) => fs.readDirectory(directory).pipe( @@ -213,12 +493,64 @@ export class BinaryResolver extends Context.Service< Effect.ignore, ); + const validateExtractedTree = (directory: string) => + Effect.gen(function* () { + const root = yield* fs.realPath(directory); + const entries = yield* fs.readDirectory(directory, { recursive: true }); + for (const entry of entries) { + const candidate = path.join(directory, entry); + const resolved = yield* fs.realPath(candidate).pipe( + Effect.mapError( + () => + new BinaryRuntimeError({ + path: candidate, + detail: "Extracted path cannot be resolved inside private staging", + }), + ), + ); + const relative = path.relative(root, resolved); + if ( + path.isAbsolute(relative) || + relative === ".." || + relative.startsWith(`..${path.sep}`) + ) { + return yield* Effect.fail( + new BinaryRuntimeError({ + path: candidate, + detail: `Extracted path resolves outside private staging: ${entry}`, + }), + ); + } + } + }); + const extractRelease = ( release: NativeReleaseArtifact, destination: string, - platformOs: string, + platform: { readonly os: string; readonly arch: string }, ) => Effect.gen(function* () { + const manifestResponse = yield* httpClient + .get(release.manifestUrl) + .pipe( + Effect.catchTag("HttpClientError", (cause) => + Effect.fail(new DownloadError({ url: release.manifestUrl, cause })), + ), + ); + const manifestText = yield* manifestResponse.text.pipe( + Effect.catchTag("HttpClientError", (cause) => + Effect.fail(new DownloadError({ url: release.manifestUrl, cause })), + ), + ); + yield* Effect.try({ + try: () => { + const parsed: unknown = JSON.parse(manifestText); + return parsed; + }, + catch: (cause) => + manifestError(release.manifestUrl, `Invalid JSON: ${String(cause)}`), + }).pipe(Effect.flatMap((value) => validateManifest(release, value, platform, spawner))); + const tarballResponse = yield* httpClient .get(release.downloadUrl) .pipe( @@ -232,33 +564,55 @@ export class BinaryResolver extends Context.Service< ), ); - const checksumUrl = release.checksumUrl; - if (checksumUrl !== null) { - const checksumResponse = yield* httpClient - .get(checksumUrl) - .pipe( - Effect.catchTag("HttpClientError", (cause) => - Effect.fail(new DownloadError({ url: checksumUrl, cause })), - ), - ); - const checksumText = yield* checksumResponse.text.pipe( + const checksumResponse = yield* httpClient + .get(release.checksumUrl) + .pipe( Effect.catchTag("HttpClientError", (cause) => - Effect.fail(new DownloadError({ url: checksumUrl, cause })), + Effect.fail(new DownloadError({ url: release.checksumUrl, cause })), ), ); - yield* verifyChecksum(tarball, checksumText, checksumUrl); + const checksumText = yield* checksumResponse.text.pipe( + Effect.catchTag("HttpClientError", (cause) => + Effect.fail(new DownloadError({ url: release.checksumUrl, cause })), + ), + ); + const expected = checksumForArchive(checksumText, `${release.assetName}.tar.zst`); + if (expected === undefined) { + return yield* Effect.fail( + manifestError(release.checksumUrl, "SHA256SUMS has no entry for the archive"), + ); } + yield* verifyChecksum(tarball, expected, release.checksumUrl); const archivePath = path.join(destination, `_download.${release.archive}`); yield* fs.writeFile(archivePath, new Uint8Array(tarball)); - const [command, ...args] = extractCommand( - release.archive, - archivePath, - destination, - platformOs, - release.stripComponents, - ); + const members = yield* spawner + .string(ChildProcess.make("tar", ["--zstd", "-tf", archivePath])) + .pipe( + Effect.catch((cause) => + Effect.fail( + new DownloadError({ + url: release.downloadUrl, + cause, + }), + ), + ), + ); + const unsafeMember = members + .split(/\r?\n/) + .map((member) => member.trim()) + .find(isUnsafeArchiveMember); + if (unsafeMember !== undefined) { + return yield* Effect.fail( + new DownloadError({ + url: release.downloadUrl, + cause: new Error(`archive member is unsafe: ${unsafeMember}`), + }), + ); + } + + const [command, ...args] = extractCommand(archivePath, destination); if (command === undefined) { return yield* Effect.fail( new DownloadError({ @@ -283,15 +637,17 @@ export class BinaryResolver extends Context.Service< ); } + yield* validateExtractedTree(destination); + yield* fs.remove(archivePath).pipe(Effect.ignore); - if (platformOs !== "win32") { + if (platform.os !== "win32") { yield* spawner .exitCode(ChildProcess.make("chmod", ["-R", "u+x", destination])) .pipe(Effect.ignore); } - if (platformOs === "darwin") { + if (platform.os === "darwin") { yield* spawner .exitCode( ChildProcess.make("find", [ @@ -316,6 +672,19 @@ export class BinaryResolver extends Context.Service< ) .pipe(Effect.ignore); } + + const requiredPaths = [...new Set(release.requiredRuntimePaths)]; + const missing = yield* Effect.forEach(requiredPaths, (entry) => + fs.exists(path.join(destination, entry)), + ).pipe(Effect.map((exists) => requiredPaths.filter((_entry, index) => !exists[index]))); + if (missing.length > 0) { + return yield* Effect.fail( + new BinaryRuntimeError({ + path: destination, + detail: `Manifest runtime paths are missing: ${missing.join(", ")}`, + }), + ); + } }); const resolveWithMetadata = (spec: BinarySpec, options?: ResolveBinaryOptions) => { @@ -333,32 +702,22 @@ export class BinaryResolver extends Context.Service< const info: AssetInfo = { service: spec.service, - provider: release.provider, + releaseSet: "slim-services", version: spec.version, - assetName: release.assetName, + runtime: "native", + target: release.target, }; const baseDir = spec.cacheDir ?? binDir; const cacheDir = cachePath(baseDir, info); - const legacyDir = legacyCachePath(baseDir, info); const parentDir = path.dirname(cacheDir); const stagingPrefix = `.${release.assetName}.partial-`; yield* cleanupStaleStaging(parentDir, stagingPrefix); - if (yield* isCompleteCache(cacheDir)) { + if (yield* isCompleteCache(cacheDir, release, info)) { return { path: cacheDir, downloaded: false, } satisfies ResolveBinaryResult; } - if ( - legacyDir !== undefined && - (yield* isReusableLegacyCache(legacyDir, spec.service, platform.os)) - ) { - return { - path: legacyDir, - downloaded: false, - } satisfies ResolveBinaryResult; - } - yield* fs.makeDirectory(parentDir, { recursive: true }); yield* options?.onDownloadStart ?? Effect.void; @@ -367,7 +726,7 @@ export class BinaryResolver extends Context.Service< prefix: stagingPrefix, }); return yield* Effect.gen(function* () { - yield* extractRelease(release, stagingDir, platform.os); + yield* extractRelease(release, stagingDir, platform); yield* fs.writeFile( path.join(stagingDir, CACHE_COMPLETE_MARKER), new TextEncoder().encode( @@ -377,6 +736,9 @@ export class BinaryResolver extends Context.Service< version: spec.version, asset: release.assetName, url: release.downloadUrl, + target: info.target, + releaseSet: info.releaseSet, + runtime: info.runtime, }), ), ); @@ -388,7 +750,7 @@ export class BinaryResolver extends Context.Service< if (Result.isSuccess(publication)) { return { path: cacheDir, downloaded: true } satisfies ResolveBinaryResult; } - if (yield* isCompleteCache(cacheDir)) { + if (yield* isCompleteCache(cacheDir, release, info)) { return { path: cacheDir, downloaded: false } satisfies ResolveBinaryResult; } @@ -401,7 +763,7 @@ export class BinaryResolver extends Context.Service< if (Result.isSuccess(retry)) { return { path: cacheDir, downloaded: true } satisfies ResolveBinaryResult; } - if (yield* isCompleteCache(cacheDir)) { + if (yield* isCompleteCache(cacheDir, release, info)) { return { path: cacheDir, downloaded: false } satisfies ResolveBinaryResult; } return yield* Effect.fail(retry.failure); @@ -423,6 +785,7 @@ export class BinaryResolver extends Context.Service< }; return { + plan, resolveWithMetadata, resolve: (spec: BinarySpec) => { return Effect.map(resolveWithMetadata(spec), ({ path }) => path); diff --git a/packages/stack/src/BinaryResolver.unit.test.ts b/packages/stack/src/BinaryResolver.unit.test.ts index d0576010bd..5a6c930596 100644 --- a/packages/stack/src/BinaryResolver.unit.test.ts +++ b/packages/stack/src/BinaryResolver.unit.test.ts @@ -3,111 +3,46 @@ import { BinaryResolver } from "./BinaryResolver.ts"; import { nativeReleaseForService } from "./ServiceCatalog.ts"; import { DEFAULT_VERSIONS } from "./versions.ts"; -const postgresVersion = DEFAULT_VERSIONS.postgres; -const postgrestVersion = DEFAULT_VERSIONS.postgrest; -const authVersion = DEFAULT_VERSIONS.auth; -const authRcVersion = "2.188.0-rc.15"; -const edgeRuntimeVersion = DEFAULT_VERSIONS["edge-runtime"]; - -describe("nativeReleaseForService", () => { - it("constructs postgres URL (appends -cli suffix for native binaries)", () => { - const release = nativeReleaseForService("postgres", postgresVersion, { +describe("slim native release descriptors", () => { + it("uses the frozen slim-services archive, manifest, and checksum names", () => { + const release = nativeReleaseForService("postgrest", DEFAULT_VERSIONS.postgrest, { os: "darwin", arch: "arm64", }); - expect(release?.downloadUrl).toBe( - `https://github.com/supabase/postgres/releases/download/v${postgresVersion}-cli/supabase-postgres-v${postgresVersion}-cli-darwin-arm64.tar.gz`, - ); - expect(release?.checksumUrl).toBe(`${release?.downloadUrl}.sha256`); - expect(release?.stripComponents).toBe(true); - }); - - it("constructs postgrest URL", () => { - const release = nativeReleaseForService("postgrest", postgrestVersion, { - os: "darwin", - arch: "arm64", - }); - expect(release?.downloadUrl).toBe( - `https://github.com/PostgREST/postgrest/releases/download/v${postgrestVersion}/postgrest-v${postgrestVersion}-macos-aarch64.tar.xz`, - ); - }); - - it("constructs postgrest Windows URL with .zip extension", () => { - const release = nativeReleaseForService("postgrest", postgrestVersion, { - os: "win32", - arch: "x64", - }); - expect(release?.downloadUrl).toBe( - `https://github.com/PostgREST/postgrest/releases/download/v${postgrestVersion}/postgrest-v${postgrestVersion}-windows-x86-64.zip`, - ); - expect(release?.archive).toBe("zip"); - }); - - it("constructs auth URL for rc releases", () => { - const release = nativeReleaseForService("auth", authRcVersion, { - os: "linux", - arch: "arm64", - }); - expect(release?.downloadUrl).toBe( - `https://github.com/supabase/auth/releases/download/rc${authRcVersion}/auth-v${authRcVersion}-arm64.tar.gz`, - ); - }); - - it("constructs edge-runtime URL", () => { - const release = nativeReleaseForService("edge-runtime", edgeRuntimeVersion, { - os: "darwin", - arch: "arm64", + expect(release).toMatchObject({ + releaseTag: "postgrest-v16.1", + target: "darwin-arm64", + archive: "tar.zst", + downloadUrl: + "https://github.com/supabase/slim-services/releases/download/postgrest-v16.1/postgrest-v16.1-darwin-arm64.tar.zst", + manifestUrl: + "https://github.com/supabase/slim-services/releases/download/postgrest-v16.1/postgrest-v16.1-darwin-arm64.manifest.json", + checksumUrl: + "https://github.com/supabase/slim-services/releases/download/postgrest-v16.1/SHA256SUMS", }); - expect(release?.downloadUrl).toBe( - `https://github.com/supabase/edge-runtime/releases/download/v${edgeRuntimeVersion}/edge-runtime-v${edgeRuntimeVersion}-aarch64-darwin.tar.gz`, - ); }); - it("returns no native release for unsupported platforms", () => { + it("only exposes the three supported native targets", () => { expect( - nativeReleaseForService("auth", authVersion, { os: "win32", arch: "arm64" }), + nativeReleaseForService("auth", DEFAULT_VERSIONS.auth, { os: "win32", arch: "x64" }), ).toBeUndefined(); + expect( + nativeReleaseForService("auth", DEFAULT_VERSIONS.auth, { os: "linux", arch: "x64" })?.target, + ).toBe("linux-amd64"); }); }); describe("BinaryResolver.cachePath", () => { - it("constructs cache path", () => { + it("includes service, release provider, version, and target identity", () => { const path = BinaryResolver.cachePath("/home/user/.supabase/bin", { service: "postgres", - provider: "github.com/supabase/postgres", - version: postgresVersion, - assetName: "darwin-arm64", + releaseSet: "slim-services", + version: DEFAULT_VERSIONS.postgres, + runtime: "native", + target: "linux-amd64", }); expect(path).toBe( - `/home/user/.supabase/bin/postgres/github.com_supabase_postgres/${postgresVersion}/darwin-arm64`, - ); - }); -}); - -describe("BinaryResolver.legacyExecutablePath", () => { - it("recognizes the executable suffix used by Windows archives", () => { - expect(BinaryResolver.legacyExecutablePath("C:/cache/postgrest", "postgrest", "win32")).toBe( - "C:/cache/postgrest/postgrest.exe", - ); - }); - - it("keeps Unix executable names unchanged", () => { - expect(BinaryResolver.legacyExecutablePath("/cache/postgrest", "postgrest", "linux")).toBe( - "/cache/postgrest/postgrest", - ); - }); -}); - -describe("BinaryResolver.legacyCacheRequiredPaths", () => { - it("requires the Postgres initialization payload as well as the executable", () => { - expect(BinaryResolver.legacyCacheRequiredPaths("/cache/postgres", "postgres", "linux")).toEqual( - [ - "/cache/postgres/bin/postgres", - "/cache/postgres/bin/pg_isready", - "/cache/postgres/bin/psql", - "/cache/postgres/share/supabase-cli/bin/supabase-postgres-init.sh", - "/cache/postgres/lib", - ], + `/home/user/.supabase/bin/slim-services/postgres/${DEFAULT_VERSIONS.postgres}/native/linux-amd64`, ); }); }); diff --git a/packages/stack/src/LocalStack.ts b/packages/stack/src/LocalStack.ts index 04f7e2884a..f212a457d1 100644 --- a/packages/stack/src/LocalStack.ts +++ b/packages/stack/src/LocalStack.ts @@ -41,14 +41,13 @@ import { StackServiceActivator, } from "./ServiceActivation.ts"; import { portFieldsForService } from "./ServicePorts.ts"; -import { StackPreparation } from "./StackPreparation.ts"; -import type { PreparedStackArtifacts } from "./StackPreparation.ts"; import { - enabledServicesForConfig, - StackBuilder, - validateResolvedConfig, - versionsForConfig, -} from "./StackBuilder.ts"; + PreparationCompleted, + ServiceDownloadStarted, + StackPreparation, +} from "./StackPreparation.ts"; +import type { PreparedStackArtifacts } from "./StackPreparation.ts"; +import { enabledServicesForConfig, StackBuilder, versionsForConfig } from "./StackBuilder.ts"; import { resolveReadinessPolicy } from "./StackConfig.ts"; import type { ReadinessPolicy, ReadyOptions, ResolvedStackConfig } from "./StackConfig.ts"; import { projectStackStates, type StackServiceProjectionCatalog } from "./StackStateProjection.ts"; @@ -123,7 +122,7 @@ const stackInfoFor = (config: ResolvedStackConfig): StackInfo => { storage: `${apiUrl}/storage/v1`, storage_s3: `${apiUrl}/storage/v1/s3`, }), - ...(config.imgproxy === false || config.startupMode === "lazy" + ...(config.imgproxy === false || config.servicePolicies.imgproxy !== "eager" ? {} : { imgproxy: `http://127.0.0.1:${config.imgproxy.port}` }), ...(config.mailpit === false @@ -241,112 +240,133 @@ export const localStackLayer = ( return service; }); - let preparedArtifacts: PreparedStackArtifacts | undefined; - let prepareDeferred: Deferred.Deferred | undefined; + let plannedArtifacts: PreparedStackArtifacts | undefined; + let planDeferred: Deferred.Deferred | undefined; + const preparedResolutions: Partial = {}; + const preparationInFlight = new Map< + string, + Deferred.Deferred + >(); let runtimeState: RuntimeState | undefined; let runtimeDeferred: Deferred.Deferred | undefined; let exactCleanupTargets: CleanupTargets | undefined; - const ensurePrepared = Effect.suspend(() => { - if (preparedArtifacts !== undefined) { - return Effect.succeed(preparedArtifacts); - } - if (prepareDeferred !== undefined) { - return Deferred.await(prepareDeferred); - } - + const ensurePlanned = Effect.suspend(() => { + if (plannedArtifacts !== undefined) return Effect.succeed(plannedArtifacts); + if (planDeferred !== undefined) return Deferred.await(planDeferred); const deferred = Deferred.makeUnsafe(); - prepareDeferred = deferred; - - const effect = Effect.gen(function* () { - yield* validateResolvedConfig(config); - yield* Ref.set(phaseRef, "preparing"); - - let prepared: PreparedStackArtifacts | undefined; - yield* preparation - .prepareEvents({ - mode: config.mode, - services: enabledServicesForConfig(config), - versions: versionsForConfig(config), - }) - .pipe( - Stream.mapError( - (cause) => - new StackBuildError({ - detail: "Failed to prepare stack assets", - cause, - reason: - cause instanceof DockerPullError && cause.daemonDown - ? "docker_not_running" - : "asset_preparation", - }), - ), - ) - .pipe( - Stream.runForEach((event) => { - switch (event._tag) { - case "ServiceDownloadStarted": - return updateState( - new StackServiceState({ - name: event.service, - status: "Downloading", - pid: null, - exitCode: null, - restartCount: 0, - startedAt: null, - error: null, - }), - ); - case "ServiceDownloadFinished": - return updateState( - new StackServiceState({ - name: event.service, - status: "Pending", - pid: null, - exitCode: null, - restartCount: 0, - startedAt: null, - error: null, - }), - ); - case "PreparationCompleted": - return Effect.sync(() => { - prepared = event.artifacts; - }); - } - }), - ); - - if (prepared === undefined) { - return yield* Effect.fail( - new StackBuildError({ - detail: "Stack preparation completed without prepared artifacts", + planDeferred = deferred; + const effect = preparation + .plan({ + mode: config.mode, + services: enabledServicesForConfig(config), + versions: versionsForConfig(config), + }) + .pipe( + Effect.mapError( + (cause) => + new StackBuildError({ + detail: "Failed to plan stack assets", + cause, + reason: "asset_preparation", + }), + ), + Effect.tap((value) => + Effect.sync(() => { + plannedArtifacts = value; }), - ); - } - - yield* Ref.set(phaseRef, "prepared"); - return prepared; - }).pipe( - Effect.tap((value) => - Effect.sync(() => { - preparedArtifacts = value; - }), - ), - Effect.onError(() => Ref.set(phaseRef, "idle")), - Effect.ensuring( - Effect.sync(() => { - prepareDeferred = undefined; - }), - ), - ); - + ), + Effect.ensuring(Effect.sync(() => (planDeferred = undefined))), + ); return Effect.gen(function* () { yield* Effect.forkIn(effect.pipe(Deferred.into(deferred)), scope); return yield* Deferred.await(deferred); }); }); + const prepareServices = (services: ReadonlyArray) => + Effect.suspend(() => { + const targets = [ + ...new Set( + services.flatMap((service) => activationTargetsForService(enabledServices, service)), + ), + ]; + const pending = targets.filter((service) => preparedResolutions[service] === undefined); + if (pending.length === 0) { + return Effect.succeed({ + resolutions: preparedResolutions, + } satisfies PreparedStackArtifacts); + } + const key = pending.toSorted().join(","); + const existing = preparationInFlight.get(key); + if (existing !== undefined) return Deferred.await(existing); + const previousStates = new Map( + pending.flatMap((service) => { + const state = SubscriptionRef.getUnsafe(stateRef).find( + (entry) => entry.name === service, + ); + return state === undefined ? [] : [[service, state] as const]; + }), + ); + const deferred = Deferred.makeUnsafe(); + preparationInFlight.set(key, deferred); + const effect = Stream.runFoldEffect( + preparation.prepareEvents({ + mode: config.mode, + services: pending, + versions: versionsForConfig(config), + }), + () => ({ resolutions: {} }) satisfies PreparedStackArtifacts, + (current, event) => + Effect.gen(function* () { + if (event instanceof ServiceDownloadStarted) { + const state = SubscriptionRef.getUnsafe(stateRef).find( + (entry) => entry.name === event.service, + ); + if (state !== undefined && state.status !== "Downloading") { + yield* updateState(new StackServiceState({ ...state, status: "Downloading" })); + } + } + return event instanceof PreparationCompleted ? event.artifacts : current; + }), + ).pipe( + Effect.mapError( + (cause) => + new StackBuildError({ + detail: "Failed to prepare stack assets", + cause, + reason: + cause instanceof DockerPullError && cause.daemonDown + ? "docker_not_running" + : "asset_preparation", + }), + ), + Effect.tapError(() => + Effect.forEach( + pending, + (service) => { + const previous = previousStates.get(service); + const current = SubscriptionRef.getUnsafe(stateRef).find( + (entry) => entry.name === service, + ); + return previous !== undefined && current?.status === "Downloading" + ? updateState(previous) + : Effect.void; + }, + { discard: true, concurrency: "unbounded" }, + ), + ), + Effect.tap((value) => + Effect.sync(() => Object.assign(preparedResolutions, value.resolutions)), + ), + Effect.ensuring(Effect.sync(() => preparationInFlight.delete(key))), + ); + return Effect.gen(function* () { + yield* Effect.forkIn(effect.pipe(Deferred.into(deferred)), scope); + return yield* Deferred.await(deferred); + }); + }); + const ensureRuntime = Effect.suspend(() => { if (runtimeState !== undefined) { return Effect.succeed(runtimeState); @@ -359,7 +379,7 @@ export const localStackLayer = ( runtimeDeferred = deferred; const effect = Effect.gen(function* () { - const prepared = yield* ensurePrepared; + const prepared = yield* ensurePlanned; const { graph, serviceProjection, cleanupTargets } = yield* builder.build( config, prepared, @@ -407,7 +427,7 @@ export const localStackLayer = ( let disposed = false; let disposing = false; const runtimeHost = Effect.gen(function* () { - const prepared = yield* ensurePrepared; + const prepared = yield* ensurePlanned; const platform = yield* detectPlatform; const edgeRuntimeResolution = prepared.resolutions["edge-runtime"]; return { @@ -688,6 +708,7 @@ export const localStackLayer = ( yield* requireRunningPhase; const concurrentlyStarted = yield* inspectStartedTargets(service); if (concurrentlyStarted !== undefined) return concurrentlyStarted; + yield* prepareServices([service]); return yield* beginStartTargets(service, new Set()); }).pipe(withLifecycleLock); yield* waitForTargets(started).pipe((effect) => @@ -710,8 +731,11 @@ export const localStackLayer = ( yield* configureFunctions(config, yield* Ref.get(functionsBundleRef)); serviceStartupBegan = true; - if (config.startupMode === "lazy") { + const eager = eagerServices(enabledServices, config.servicePolicies); + const allServicesEager = eager.length === enabledServices.length; + if (!allServicesEager) { const readiness: Array> = []; + yield* prepareServices(["postgres", ...eager]); if ( runtime.graph.startOrder.some((definition) => definition.name === "postgres-init") ) { @@ -732,7 +756,7 @@ export const localStackLayer = ( ), ); } - for (const service of eagerServices(enabledServices)) { + for (const service of eager) { const started = yield* beginStartTargets( service, new Set(lifecycleTargetsForService(enabledServices, service)), @@ -743,6 +767,7 @@ export const localStackLayer = ( (effect) => withReadinessPolicy(effect, "stack"), ); } else { + yield* prepareServices(enabledServices); yield* runtime.orchestrator.start(serviceStartOptions); yield* runtime.orchestrator .waitAllReady() @@ -775,6 +800,7 @@ export const localStackLayer = ( const started = yield* Effect.gen(function* () { yield* requireMutable(`start service ${name}`); const service = yield* requireKnownServiceName(name); + yield* prepareServices([service]); return yield* beginStartTargets( service, new Set(lifecycleTargetsForService(enabledServices, service)), @@ -840,7 +866,7 @@ export const localStackLayer = ( opts.functions === undefined ? currentBundle : yield* decodeFunctionsBundle(opts.functions); - const prepared = yield* ensurePrepared; + const prepared = yield* ensurePlanned; const runtime = yield* ensureRuntime; const buildResult = yield* builder.build(nextConfig, prepared); const edgeRuntimeDef = buildResult.graph.startOrder.find( diff --git a/packages/stack/src/Platform.ts b/packages/stack/src/Platform.ts index 5ed0e87e5e..cf819bf33c 100644 --- a/packages/stack/src/Platform.ts +++ b/packages/stack/src/Platform.ts @@ -5,40 +5,22 @@ export interface PlatformInfo { readonly arch: string; } +/** Native slim-service release targets. The release set intentionally has no + * windows or x64 macOS artifacts. */ +export type NativeTarget = "darwin-arm64" | "linux-amd64" | "linux-arm64"; + +export const nativeTargetForPlatform = (platform: PlatformInfo): NativeTarget | undefined => { + if (platform.os === "darwin" && platform.arch === "arm64") return "darwin-arm64"; + if (platform.os === "linux" && platform.arch === "x64") return "linux-amd64"; + if (platform.os === "linux" && platform.arch === "arm64") return "linux-arm64"; + return undefined; +}; + export const detectPlatform: Effect.Effect = Effect.sync(() => ({ os: process.platform, arch: process.arch, })); -export const postgresAssetName = (p: PlatformInfo): string | null => { - if (p.os === "darwin" && p.arch === "arm64") return "darwin-arm64"; - if (p.os === "linux" && p.arch === "x64") return "linux-x64"; - if (p.os === "linux" && p.arch === "arm64") return "linux-arm64"; - return null; -}; - -export const postgrestAssetName = (p: PlatformInfo): string | null => { - if (p.os === "darwin" && p.arch === "arm64") return "macos-aarch64"; - if (p.os === "linux" && p.arch === "x64") return "linux-static-x86-64"; - if (p.os === "linux" && p.arch === "arm64") return "ubuntu-aarch64"; - if (p.os === "win32" && p.arch === "x64") return "windows-x86-64"; - return null; -}; - -export const authAssetName = (p: PlatformInfo): string | null => { - if (p.os === "darwin" && p.arch === "arm64") return "darwin-arm64"; - if (p.os === "linux" && p.arch === "x64") return "x86"; - if (p.os === "linux" && p.arch === "arm64") return "arm64"; - return null; -}; - -export const edgeRuntimeAssetName = (p: PlatformInfo): string | null => { - if (p.os === "darwin" && p.arch === "arm64") return "aarch64-darwin"; - if (p.os === "linux" && p.arch === "x64") return "x86_64-linux"; - if (p.os === "linux" && p.arch === "arm64") return "aarch64-linux"; - return null; -}; - /** Host address that Docker containers should use to reach services on the host machine. */ export const dockerHostAddress = (_os: string): string => "host.docker.internal"; diff --git a/packages/stack/src/Platform.unit.test.ts b/packages/stack/src/Platform.unit.test.ts index 07a10d2049..67a8095557 100644 --- a/packages/stack/src/Platform.unit.test.ts +++ b/packages/stack/src/Platform.unit.test.ts @@ -4,10 +4,7 @@ import { detectPlatform, dockerHostAddress, dockerNetworkArgs, - postgresAssetName, - postgrestAssetName, - authAssetName, - edgeRuntimeAssetName, + nativeTargetForPlatform, } from "./Platform.ts"; describe("detectPlatform", () => { @@ -22,79 +19,21 @@ describe("detectPlatform", () => { ); }); -describe("postgresAssetName", () => { +describe("nativeTargetForPlatform", () => { it("maps darwin-arm64", () => { - expect(postgresAssetName({ os: "darwin", arch: "arm64" })).toBe("darwin-arm64"); + expect(nativeTargetForPlatform({ os: "darwin", arch: "arm64" })).toBe("darwin-arm64"); }); it("maps linux-x64", () => { - expect(postgresAssetName({ os: "linux", arch: "x64" })).toBe("linux-x64"); + expect(nativeTargetForPlatform({ os: "linux", arch: "x64" })).toBe("linux-amd64"); }); it("maps linux-arm64", () => { - expect(postgresAssetName({ os: "linux", arch: "arm64" })).toBe("linux-arm64"); + expect(nativeTargetForPlatform({ os: "linux", arch: "arm64" })).toBe("linux-arm64"); }); it("returns null for unsupported", () => { - expect(postgresAssetName({ os: "win32", arch: "x64" })).toBeNull(); - }); -}); - -describe("postgrestAssetName", () => { - it("maps darwin-arm64 to macos-aarch64", () => { - expect(postgrestAssetName({ os: "darwin", arch: "arm64" })).toBe("macos-aarch64"); - }); - - it("maps linux-x64 to linux-static-x86-64", () => { - expect(postgrestAssetName({ os: "linux", arch: "x64" })).toBe("linux-static-x86-64"); - }); - - it("maps linux-arm64 to ubuntu-aarch64", () => { - expect(postgrestAssetName({ os: "linux", arch: "arm64" })).toBe("ubuntu-aarch64"); - }); - - it("maps win32-x64 to windows-x86-64", () => { - expect(postgrestAssetName({ os: "win32", arch: "x64" })).toBe("windows-x86-64"); - }); - - it("returns null for unsupported", () => { - expect(postgrestAssetName({ os: "win32", arch: "arm64" })).toBeNull(); - }); -}); - -describe("authAssetName", () => { - it("maps darwin-arm64 to darwin-arm64", () => { - expect(authAssetName({ os: "darwin", arch: "arm64" })).toBe("darwin-arm64"); - }); - - it("maps linux-x64 to x86", () => { - expect(authAssetName({ os: "linux", arch: "x64" })).toBe("x86"); - }); - - it("maps linux-arm64 to arm64", () => { - expect(authAssetName({ os: "linux", arch: "arm64" })).toBe("arm64"); - }); - - it("returns null for unsupported", () => { - expect(authAssetName({ os: "darwin", arch: "x64" })).toBeNull(); - }); -}); - -describe("edgeRuntimeAssetName", () => { - it("maps darwin-arm64 to aarch64-darwin", () => { - expect(edgeRuntimeAssetName({ os: "darwin", arch: "arm64" })).toBe("aarch64-darwin"); - }); - - it("maps linux-x64 to x86_64-linux", () => { - expect(edgeRuntimeAssetName({ os: "linux", arch: "x64" })).toBe("x86_64-linux"); - }); - - it("maps linux-arm64 to aarch64-linux", () => { - expect(edgeRuntimeAssetName({ os: "linux", arch: "arm64" })).toBe("aarch64-linux"); - }); - - it("returns null for unsupported", () => { - expect(edgeRuntimeAssetName({ os: "win32", arch: "x64" })).toBeNull(); + expect(nativeTargetForPlatform({ os: "win32", arch: "x64" })).toBeUndefined(); }); }); diff --git a/packages/stack/src/ServiceActivation.ts b/packages/stack/src/ServiceActivation.ts index f61281abf4..78b45c0a2f 100644 --- a/packages/stack/src/ServiceActivation.ts +++ b/packages/stack/src/ServiceActivation.ts @@ -6,9 +6,12 @@ import { stackServiceStartupBudgetSeconds } from "./services/health-budgets.ts"; import { SERVICE_NAMES, serviceMetadata } from "./ServiceCatalog.ts"; import type { ServiceName } from "./ServiceName.ts"; import type { ReadinessPolicy } from "./StackConfig.ts"; +import type { ServicePolicyManifest } from "./StackConfig.ts"; -export const eagerServices = (enabled: ReadonlyArray): ReadonlyArray => - enabled.filter((service) => serviceMetadata(service).activation.startup === "eager"); +export const eagerServices = ( + enabled: ReadonlyArray, + policies: ServicePolicyManifest, +): ReadonlyArray => enabled.filter((service) => policies[service] === "eager"); export const activationTargetsForService = ( enabledServices: ReadonlyArray, diff --git a/packages/stack/src/ServiceActivation.unit.test.ts b/packages/stack/src/ServiceActivation.unit.test.ts index 0f28d4d6f2..531e155d26 100644 --- a/packages/stack/src/ServiceActivation.unit.test.ts +++ b/packages/stack/src/ServiceActivation.unit.test.ts @@ -7,6 +7,7 @@ import { lifecycleTargetsForService, } from "./ServiceActivation.ts"; import { SERVICE_CATALOG, SERVICE_NAMES } from "./ServiceCatalog.ts"; +import { DEFAULT_SERVICE_POLICIES } from "./ServiceCatalog.ts"; describe("service activation", () => { it("defines an access policy for every stack service", () => { @@ -14,7 +15,7 @@ describe("service activation", () => { }); it("starts direct endpoints eagerly", () => { - expect(eagerServices(SERVICE_NAMES)).toEqual([ + expect(eagerServices(SERVICE_NAMES, DEFAULT_SERVICE_POLICIES)).toEqual([ "postgres", "realtime", "mailpit", diff --git a/packages/stack/src/ServiceCatalog.ts b/packages/stack/src/ServiceCatalog.ts index ef6f7844bd..f1244d02e4 100644 --- a/packages/stack/src/ServiceCatalog.ts +++ b/packages/stack/src/ServiceCatalog.ts @@ -1,25 +1,24 @@ import { Record } from "effect"; -import { - authAssetName, - edgeRuntimeAssetName, - postgresAssetName, - postgrestAssetName, - type PlatformInfo, -} from "./Platform.ts"; +import { nativeTargetForPlatform, type NativeTarget, type PlatformInfo } from "./Platform.ts"; import type { PortField } from "./PortCatalog.ts"; import type { ServiceName } from "./ServiceName.ts"; -type ArtifactOwnership = "supabase" | "upstream"; type ServiceRuntimeSupport = "native-preferred" | "docker-only"; -export type ArchiveFormat = "tar.gz" | "tar.xz" | "zip"; +type ArchiveFormat = "tar.zst"; +export type ServicePreparationPolicy = "off" | "lazy" | "eager"; export interface NativeReleaseArtifact { + readonly service: ServiceName; + readonly version: string; readonly provider: string; readonly assetName: string; + readonly releaseTag: string; + readonly target: NativeTarget; readonly archive: ArchiveFormat; readonly downloadUrl: string; - readonly checksumUrl: string | null; - readonly stripComponents: boolean; + readonly manifestUrl: string; + readonly checksumUrl: string; + readonly requiredRuntimePaths: ReadonlyArray; } interface NativeReleaseSource { @@ -28,7 +27,6 @@ interface NativeReleaseSource { } interface DockerImageSource { - readonly ownership: ArtifactOwnership; readonly repository: string; readonly tagPrefix?: string; } @@ -39,14 +37,20 @@ interface ServiceArtifactDefinition { } interface ServiceActivationPolicy { - /** Whether the public service must already be running when lazy startup completes. */ - readonly startup: "eager" | "lazy"; /** Other public services required when this service is activated. */ readonly activates: ReadonlyArray; /** Private companions whose lifecycle is exclusively owned by this service. */ readonly owns: ReadonlyArray; } +export interface ServicePreparationMetadata { + /** Policies supported by the service's runtime/resource implementation. */ + readonly supported: ReadonlyArray>; + readonly default: Exclude; + /** Services whose resources must be materialized before this service can start. */ + readonly dependencies: ReadonlyArray; +} + type ServiceConfigKey = | "postgres" | "postgrest" @@ -69,36 +73,61 @@ export interface ServiceCatalogEntry { readonly runtimeSupport: ServiceRuntimeSupport; readonly artifact: ServiceArtifactDefinition; readonly activation: ServiceActivationPolicy; + readonly preparation: ServicePreparationMetadata; readonly portFields: ReadonlyArray; } -const SUPABASE_ECR_REGISTRY = "public.ecr.aws/supabase"; -const SUPABASE_DOCKER_HUB_REGISTRY = "supabase"; -const SUPABASE_GHCR_REGISTRY = "ghcr.io/supabase"; +const SUPABASE_GHCR_REGISTRY = "ghcr.io/supabase/cli"; +const SLIM_RELEASE_BASE = "https://github.com/supabase/slim-services/releases/download"; const nativeRelease = ( - provider: string, - assetName: string | null, - archive: ArchiveFormat, - downloadUrl: string, - options?: { - readonly checksumUrl?: string; - readonly stripComponents?: boolean; + service: ServiceName, + version: string, + platform: PlatformInfo, + options: { + readonly requiredRuntimePaths: ReadonlyArray; }, -): NativeReleaseArtifact | undefined => - assetName === null - ? undefined - : { - provider, - assetName, - archive, - downloadUrl, - checksumUrl: options?.checksumUrl ?? null, - stripComponents: options?.stripComponents ?? false, - }; +): NativeReleaseArtifact | undefined => { + const target = nativeTargetForPlatform(platform); + if (target === undefined) return undefined; + const releaseTag = `${service}-${version}`; + const base = `${SLIM_RELEASE_BASE}/${releaseTag}`; + const assetName = `${releaseTag}-${target}`; + return { + service, + version, + provider: "github.com/supabase/slim-services", + assetName, + releaseTag, + target, + archive: "tar.zst", + downloadUrl: `${base}/${assetName}.tar.zst`, + manifestUrl: `${base}/${assetName}.manifest.json`, + checksumUrl: `${base}/SHA256SUMS`, + requiredRuntimePaths: options.requiredRuntimePaths, + }; +}; -const authReleaseTag = (version: string): string => - version.includes("-rc.") ? `rc${version}` : `v${version}`; +const preparation = ( + supported: ReadonlyArray>, + defaultPolicy: Exclude, + dependencies: ReadonlyArray = [], +): ServicePreparationMetadata => ({ + supported, + default: defaultPolicy, + dependencies, +}); + +const genericNativeRelease = (service: ServiceName) => ({ + provider: "github.com/supabase/slim-services", + resolve: (version: string, platform: PlatformInfo) => + nativeRelease(service, version, platform, { + // These services are prepared generically before native process wiring + // exists. Their manifest-declared paths are authoritative; do not guess + // a consumer path that could make a valid archive fail installation. + requiredRuntimePaths: [], + }), +}); /** * Exhaustive static identity and capability metadata for public stack services. @@ -108,112 +137,103 @@ export const SERVICE_CATALOG = { postgres: { name: "postgres", configKey: "postgres", - defaultVersion: "17.6.1.159", + defaultVersion: "17.6.1.163", runtimeSupport: "native-preferred", artifact: { - docker: { ownership: "supabase", repository: "postgres" }, + docker: { repository: "postgres" }, native: { - provider: "github.com/supabase/postgres", - resolve: (version, platform) => { - const assetName = postgresAssetName(platform); - const cliVersion = `${version}-cli`; - const url = `https://github.com/supabase/postgres/releases/download/v${cliVersion}/supabase-postgres-v${cliVersion}-${assetName}.tar.gz`; - return nativeRelease("github.com/supabase/postgres", assetName, "tar.gz", url, { - checksumUrl: `${url}.sha256`, - stripComponents: true, - }); - }, + provider: "github.com/supabase/slim-services", + resolve: (version, platform) => + nativeRelease("postgres", version, platform, { + requiredRuntimePaths: [ + "bin/postgres", + "bin/pg_isready", + "bin/psql", + "share/supabase-cli/bin/supabase-postgres-init.sh", + "lib", + ], + }), }, }, - activation: { startup: "eager", activates: [], owns: [] }, + activation: { activates: [], owns: [] }, + preparation: preparation(["eager"], "eager"), portFields: ["dbPort"], }, postgrest: { name: "postgrest", configKey: "postgrest", - defaultVersion: "16.1", + defaultVersion: "v16.1", runtimeSupport: "native-preferred", artifact: { - docker: { ownership: "supabase", repository: "postgrest", tagPrefix: "v" }, + docker: { repository: "postgrest" }, native: { - provider: "github.com/PostgREST/postgrest", - resolve: (version, platform) => { - const assetName = postgrestAssetName(platform); - const archive = assetName?.startsWith("windows") === true ? "zip" : "tar.xz"; - return nativeRelease( - "github.com/PostgREST/postgrest", - assetName, - archive, - `https://github.com/PostgREST/postgrest/releases/download/v${version}/postgrest-v${version}-${assetName}.${archive}`, - ); - }, + provider: "github.com/supabase/slim-services", + resolve: (version, platform) => + nativeRelease("postgrest", version, platform, { + requiredRuntimePaths: ["bin/postgrest"], + }), }, }, - activation: { startup: "lazy", activates: [], owns: [] }, + activation: { activates: [], owns: [] }, + preparation: preparation(["lazy", "eager"], "lazy", ["postgres"]), portFields: ["postgrestPort", "postgrestAdminPort"], }, auth: { name: "auth", configKey: "auth", - defaultVersion: "2.195.0", + defaultVersion: "v2.195.0", runtimeSupport: "native-preferred", artifact: { - docker: { ownership: "supabase", repository: "gotrue", tagPrefix: "v" }, + docker: { repository: "auth" }, native: { - provider: "github.com/supabase/auth", - resolve: (version, platform) => { - const assetName = authAssetName(platform); - return nativeRelease( - "github.com/supabase/auth", - assetName, - "tar.gz", - `https://github.com/supabase/auth/releases/download/${authReleaseTag(version)}/auth-v${version}-${assetName}.tar.gz`, - ); - }, + provider: "github.com/supabase/slim-services", + resolve: (version, platform) => + nativeRelease("auth", version, platform, { + requiredRuntimePaths: ["bin/auth"], + }), }, }, - activation: { startup: "lazy", activates: [], owns: [] }, + activation: { activates: [], owns: [] }, + preparation: preparation(["lazy", "eager"], "lazy", ["postgres"]), portFields: ["authPort"], }, "edge-runtime": { name: "edge-runtime", configKey: "edgeRuntime", - defaultVersion: "1.74.3", + defaultVersion: "v1.74.3", runtimeSupport: "docker-only", artifact: { - docker: { ownership: "supabase", repository: "edge-runtime", tagPrefix: "v" }, - native: { - provider: "github.com/supabase/edge-runtime", - resolve: (version, platform) => { - const assetName = edgeRuntimeAssetName(platform); - return nativeRelease( - "github.com/supabase/edge-runtime", - assetName, - "tar.gz", - `https://github.com/supabase/edge-runtime/releases/download/v${version}/edge-runtime-v${version}-${assetName}.tar.gz`, - ); - }, - }, + docker: { repository: "edge-runtime" }, + native: genericNativeRelease("edge-runtime"), }, - activation: { startup: "lazy", activates: [], owns: [] }, + activation: { activates: [], owns: [] }, + preparation: preparation(["lazy", "eager"], "lazy", ["postgres"]), portFields: ["edgeRuntimePort", "edgeRuntimeInspectorPort"], }, realtime: { name: "realtime", configKey: "realtime", - defaultVersion: "2.129.0", + defaultVersion: "v2.129.1", runtimeSupport: "docker-only", - artifact: { docker: { ownership: "supabase", repository: "realtime", tagPrefix: "v" } }, - activation: { startup: "eager", activates: [], owns: [] }, + artifact: { + docker: { repository: "realtime" }, + native: genericNativeRelease("realtime"), + }, + activation: { activates: [], owns: [] }, + preparation: preparation(["eager"], "eager", ["postgres"]), portFields: ["realtimePort"], }, storage: { name: "storage", configKey: "storage", - defaultVersion: "1.69.11", + defaultVersion: "v1.70.1", runtimeSupport: "docker-only", - artifact: { docker: { ownership: "supabase", repository: "storage-api", tagPrefix: "v" } }, - activation: { startup: "lazy", activates: ["imgproxy"], owns: ["imgproxy"] }, + artifact: { + docker: { repository: "storage" }, + native: genericNativeRelease("storage"), + }, + activation: { activates: ["imgproxy"], owns: ["imgproxy"] }, + preparation: preparation(["lazy", "eager"], "lazy", ["postgres"]), portFields: ["storagePort"], }, imgproxy: { @@ -221,8 +241,12 @@ export const SERVICE_CATALOG = { configKey: "imgproxy", defaultVersion: "v3.8.0", runtimeSupport: "docker-only", - artifact: { docker: { ownership: "upstream", repository: "darthsim/imgproxy" } }, - activation: { startup: "lazy", activates: [], owns: [] }, + artifact: { + docker: { repository: "imgproxy" }, + native: genericNativeRelease("imgproxy"), + }, + activation: { activates: [], owns: [] }, + preparation: preparation(["lazy", "eager"], "lazy", ["storage"]), portFields: ["imgproxyPort"], }, mailpit: { @@ -230,8 +254,12 @@ export const SERVICE_CATALOG = { configKey: "mailpit", defaultVersion: "v1.30.2", runtimeSupport: "docker-only", - artifact: { docker: { ownership: "upstream", repository: "axllent/mailpit" } }, - activation: { startup: "eager", activates: [], owns: [] }, + artifact: { + docker: { repository: "mailpit" }, + native: genericNativeRelease("mailpit"), + }, + activation: { activates: [], owns: [] }, + preparation: preparation(["eager"], "eager"), portFields: ["mailpitPort", "mailpitSmtpPort", "mailpitPop3Port"], }, pgmeta: { @@ -240,9 +268,11 @@ export const SERVICE_CATALOG = { defaultVersion: "0.98.0", runtimeSupport: "docker-only", artifact: { - docker: { ownership: "supabase", repository: "postgres-meta", tagPrefix: "v" }, + docker: { repository: "pgmeta", tagPrefix: "v" }, + native: genericNativeRelease("pgmeta"), }, - activation: { startup: "lazy", activates: [], owns: [] }, + activation: { activates: [], owns: [] }, + preparation: preparation(["lazy", "eager"], "lazy", ["postgres"]), portFields: ["pgmetaPort"], }, studio: { @@ -250,35 +280,51 @@ export const SERVICE_CATALOG = { configKey: "studio", defaultVersion: "2026.08.17-sha-0c1da8f", runtimeSupport: "docker-only", - artifact: { docker: { ownership: "supabase", repository: "studio" } }, - activation: { startup: "eager", activates: ["analytics"], owns: [] }, + artifact: { + docker: { repository: "studio" }, + native: genericNativeRelease("studio"), + }, + activation: { activates: ["analytics"], owns: [] }, + preparation: preparation(["eager"], "eager", ["pgmeta", "analytics"]), portFields: ["studioPort"], }, analytics: { name: "analytics", configKey: "analytics", - defaultVersion: "1.50.2", + defaultVersion: "v1.50.3", runtimeSupport: "docker-only", - artifact: { docker: { ownership: "supabase", repository: "logflare" } }, - activation: { startup: "lazy", activates: ["vector"], owns: ["vector"] }, + artifact: { + docker: { repository: "analytics" }, + native: genericNativeRelease("analytics"), + }, + activation: { activates: ["vector"], owns: ["vector"] }, + preparation: preparation(["lazy", "eager"], "lazy", ["postgres"]), portFields: ["analyticsPort"], }, vector: { name: "vector", configKey: "vector", - defaultVersion: "0.53.0-alpine", + defaultVersion: "0.53.0", runtimeSupport: "docker-only", - artifact: { docker: { ownership: "upstream", repository: "timberio/vector" } }, - activation: { startup: "lazy", activates: [], owns: [] }, + artifact: { + docker: { repository: "vector" }, + native: genericNativeRelease("vector"), + }, + activation: { activates: [], owns: [] }, + preparation: preparation(["lazy", "eager"], "lazy", ["analytics"]), portFields: [], }, pooler: { name: "pooler", configKey: "pooler", - defaultVersion: "2.9.7", + defaultVersion: "v2.9.10", runtimeSupport: "docker-only", - artifact: { docker: { ownership: "supabase", repository: "supavisor" } }, - activation: { startup: "eager", activates: [], owns: [] }, + artifact: { + docker: { repository: "pooler" }, + native: genericNativeRelease("pooler"), + }, + activation: { activates: [], owns: [] }, + preparation: preparation(["eager"], "eager", ["postgres"]), portFields: ["poolerPort", "poolerApiPort"], }, } satisfies { readonly [Name in ServiceName]: ServiceCatalogEntry }; @@ -303,35 +349,28 @@ export const nativeReleaseForService = ( export const isDockerOnlyService = (service: ServiceName): boolean => SERVICE_CATALOG[service].runtimeSupport === "docker-only"; -const dockerTag = (service: ServiceName, version: string): string => { - const source = serviceMetadata(service).artifact.docker; - return `${source.tagPrefix ?? ""}${version}`; +export const DEFAULT_SERVICE_POLICIES: Readonly< + Record> +> = { + postgres: SERVICE_CATALOG.postgres.preparation.default, + postgrest: SERVICE_CATALOG.postgrest.preparation.default, + auth: SERVICE_CATALOG.auth.preparation.default, + "edge-runtime": SERVICE_CATALOG["edge-runtime"].preparation.default, + realtime: SERVICE_CATALOG.realtime.preparation.default, + storage: SERVICE_CATALOG.storage.preparation.default, + imgproxy: SERVICE_CATALOG.imgproxy.preparation.default, + mailpit: SERVICE_CATALOG.mailpit.preparation.default, + pgmeta: SERVICE_CATALOG.pgmeta.preparation.default, + studio: SERVICE_CATALOG.studio.preparation.default, + analytics: SERVICE_CATALOG.analytics.preparation.default, + vector: SERVICE_CATALOG.vector.preparation.default, + pooler: SERVICE_CATALOG.pooler.preparation.default, }; -export const dockerImageForArtifact = (service: ServiceName, version: string): string => { - const source = SERVICE_CATALOG[service].artifact.docker; - const repository = - source.ownership === "supabase" - ? `${SUPABASE_ECR_REGISTRY}/${source.repository}` - : source.repository; - return `${repository}:${dockerTag(service, version)}`; -}; +export const requiredPreparationDependencies = (service: ServiceName): ReadonlyArray => + serviceMetadata(service).preparation.dependencies; -export const dockerImageCandidatesForArtifact = ( - service: ServiceName, - version: string, -): ReadonlyArray => { - const source = SERVICE_CATALOG[service].artifact.docker; - const tag = dockerTag(service, version); - if (source.ownership === "upstream") { - return [`${source.repository}:${tag}`]; - } - return [ - `${SUPABASE_ECR_REGISTRY}/${source.repository}:${tag}`, - `${SUPABASE_DOCKER_HUB_REGISTRY}/${source.repository}:${tag}`, - `${SUPABASE_GHCR_REGISTRY}/${source.repository}:${tag}`, - ]; +export const dockerImageForArtifact = (service: ServiceName, version: string): string => { + const source = serviceMetadata(service).artifact.docker; + return `${SUPABASE_GHCR_REGISTRY}/${source.repository}:${source.tagPrefix ?? ""}${version}`; }; - -export const imageTagPrefixForService = (service: ServiceName): string | undefined => - serviceMetadata(service).artifact.docker.tagPrefix; diff --git a/packages/stack/src/ServicePorts.ts b/packages/stack/src/ServicePorts.ts index 6c1a7944e1..d5a7be4ece 100644 --- a/packages/stack/src/ServicePorts.ts +++ b/packages/stack/src/ServicePorts.ts @@ -2,10 +2,8 @@ import { PORT_CATALOG, PORT_FIELDS, type PortField } from "./PortCatalog.ts"; import { SERVICE_CATALOG, SERVICE_NAMES } from "./ServiceCatalog.ts"; import type { StackConfig } from "./StackConfig.ts"; -export const serviceEnabledForConfig = ( - config: StackConfig, - service: keyof typeof SERVICE_CATALOG, -) => { +const serviceEnabledForConfig = (config: StackConfig, service: keyof typeof SERVICE_CATALOG) => { + if (config.servicePolicies?.[service] === "off") return false; if (service === "postgres" || service === "postgrest" || service === "auth") { return config[service === "postgres" ? "postgres" : service] !== false; } diff --git a/packages/stack/src/Stack.unit.test.ts b/packages/stack/src/Stack.unit.test.ts index 44196e5dea..6847135891 100644 --- a/packages/stack/src/Stack.unit.test.ts +++ b/packages/stack/src/Stack.unit.test.ts @@ -1,11 +1,12 @@ import { describe, expect, it } from "@effect/vitest"; -import { BunServices } from "@effect/platform-bun"; +import { NodeServices } from "@effect/platform-node"; import { buildGraph } from "@supabase/process-compose"; import { createHmac } from "node:crypto"; import { mkdtempSync } from "node:fs"; import { chmod, readFile, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { createServer, type Server } from "node:http"; import { Deferred, Effect, Exit, Fiber, Layer, Stream } from "effect"; import { mockChildProcessSpawner } from "../../process-compose/tests/helpers/mocks.ts"; import { mockBinaryResolver } from "../tests/helpers/mocks.ts"; @@ -51,7 +52,21 @@ const defaultConfig: ResolvedStackConfig = { runtimeRoot: "/tmp/supabase-runtime", projectDir: "/tmp/supabase-project", mode: "native", - startupMode: "eager", + servicePolicies: { + postgres: "eager", + postgrest: "eager", + auth: "eager", + "edge-runtime": "off", + realtime: "off", + storage: "off", + imgproxy: "off", + mailpit: "off", + pgmeta: "off", + studio: "off", + analytics: "off", + vector: "off", + pooler: "off", + }, readiness: DEFAULT_STACK_READINESS_POLICY, readinessSource: "default", jwtSecret: testJwtSecret, @@ -100,6 +115,7 @@ const defaultConfig: ResolvedStackConfig = { const edgeRuntimeConfig: ResolvedStackConfig = { ...defaultConfig, mode: "auto", + servicePolicies: { ...defaultConfig.servicePolicies, "edge-runtime": "eager" }, edgeRuntime: { enabled: true, port: defaultPorts.edgeRuntimePort, @@ -142,7 +158,7 @@ function setupLayer( Layer.provide(StackBuilder.layer), Layer.provide(stackPreparationLayer), Layer.provide(spawner.layer), - Layer.provide(BunServices.layer), + Layer.provide(NodeServices.layer), ); return { layer, resolver, spawner }; @@ -211,7 +227,7 @@ describe("Stack", () => { Layer.provide(builderLayer), Layer.provide(StackPreparation.layer.pipe(Layer.provide(resolver.layer))), Layer.provide(mockChildProcessSpawner().layer), - Layer.provide(BunServices.layer), + Layer.provide(NodeServices.layer), ); const readRuntimeConfig = Effect.promise(() => readFile(functionsRuntimeConfigPath(runtimeRoot), "utf8").then((contents) => @@ -422,39 +438,6 @@ describe("Stack", () => { }).pipe(Effect.provide(layer)); }); - it.effect("emits Downloading when a service fetches assets before startup", () => { - const resolver = mockBinaryResolver({ - downloadedServices: ["postgres"], - downloadDelayMs: 20, - }); - const spawner = mockChildProcessSpawner(); - const stackPreparationLayer = StackPreparation.layer.pipe(Layer.provide(resolver.layer)); - const layer = localStackLayer(defaultConfig, noopPortLease(defaultConfig.ports)).pipe( - Layer.provide(StackBuilder.layer), - Layer.provide(stackPreparationLayer), - ); - const providedLayer = layer.pipe( - Layer.provide(spawner.layer), - Layer.provide(BunServices.layer), - ); - - return Effect.gen(function* () { - const stack = yield* Stack; - const statesFiber = yield* stack.allStateChanges().pipe( - Stream.filter((state) => state.name === "postgres"), - Stream.take(2), - Stream.runCollect, - Effect.forkChild({ startImmediately: true }), - ); - - const startFiber = yield* stack.start().pipe(Effect.forkChild({ startImmediately: true })); - const states = yield* Fiber.join(statesFiber); - yield* Fiber.interrupt(startFiber); - - expect(states.map((state) => state.status)).toContain("Downloading"); - }).pipe(Effect.provide(providedLayer)); - }); - it.live("starts the readiness deadline after artifact preparation", () => { const resolver = mockBinaryResolver({ downloadedServices: ["postgres"], @@ -473,7 +456,7 @@ describe("Stack", () => { Layer.provide(StackBuilder.layer), Layer.provide(stackPreparationLayer), Layer.provide(spawner.layer), - Layer.provide(BunServices.layer), + Layer.provide(NodeServices.layer), ); return Effect.gen(function* () { @@ -546,7 +529,7 @@ describe("Stack", () => { ); const providedLayer = layer.pipe( Layer.provide(spawner.layer), - Layer.provide(BunServices.layer), + Layer.provide(NodeServices.layer), ); return Effect.gen(function* () { @@ -587,7 +570,7 @@ describe("Stack", () => { Layer.provide(builderLayer), Layer.provide(stackPreparationLayer), Layer.provide(spawner.layer), - Layer.provide(BunServices.layer), + Layer.provide(NodeServices.layer), ); return Effect.gen(function* () { @@ -648,7 +631,7 @@ describe("Stack", () => { Layer.provide(builderLayer), Layer.provide(stackPreparationLayer), Layer.provide(spawner.layer), - Layer.provide(BunServices.layer), + Layer.provide(NodeServices.layer), ); return Effect.gen(function* () { @@ -662,7 +645,16 @@ describe("Stack", () => { }); it.live("lazy startup starts direct services without starting HTTP backends", () => { - const { layer, spawner } = setupLayer({ ...defaultConfig, startupMode: "lazy" }); + const { layer, spawner } = setupLayer({ + ...defaultConfig, + servicePolicies: { + ...defaultConfig.servicePolicies, + postgrest: "lazy", + auth: "lazy", + storage: "lazy", + imgproxy: "lazy", + }, + }); return Effect.gen(function* () { const stack = yield* Stack; @@ -687,7 +679,13 @@ describe("Stack", () => { const config: ResolvedStackConfig = { ...defaultConfig, mode: "auto", - startupMode: "lazy", + servicePolicies: { + ...defaultConfig.servicePolicies, + postgrest: "lazy", + auth: "lazy", + storage: "lazy", + imgproxy: "lazy", + }, storage: { port: defaultPorts.storagePort, dataDir: "/tmp/supabase/storage", @@ -729,7 +727,15 @@ describe("Stack", () => { ? Deferred.succeed(spawnStarted, undefined).pipe(Effect.andThen(Effect.never)) : Effect.void, }); - const config = { ...defaultConfig, startupMode: "lazy" } satisfies ResolvedStackConfig; + const config = { + ...defaultConfig, + servicePolicies: { + ...defaultConfig.servicePolicies, + postgrest: "lazy", + auth: "lazy", + mailpit: "eager", + }, + } satisfies ResolvedStackConfig; const { layer } = setupLayer(config, noopPortLease(config.ports), spawner); yield* Effect.gen(function* () { @@ -775,7 +781,10 @@ describe("Stack", () => { ? Deferred.succeed(authSpawnStarted, undefined).pipe(Effect.andThen(Effect.never)) : Effect.void, }); - const config = { ...defaultConfig, startupMode: "lazy" } satisfies ResolvedStackConfig; + const config = { + ...defaultConfig, + servicePolicies: { ...defaultConfig.servicePolicies, postgrest: "lazy", auth: "lazy" }, + } satisfies ResolvedStackConfig; const { layer } = setupLayer(config, noopPortLease(config.ports), spawner); yield* Effect.gen(function* () { @@ -803,7 +812,12 @@ describe("Stack", () => { const config = { ...defaultConfig, mode: "auto", - startupMode: "lazy", + servicePolicies: { + ...defaultConfig.servicePolicies, + postgrest: "lazy", + auth: "lazy", + mailpit: "eager", + }, mailpit: { port: defaultPorts.mailpitPort, smtpPort: defaultPorts.mailpitSmtpPort, @@ -856,7 +870,10 @@ describe("Stack", () => { ) : Effect.void, }); - const config = { ...defaultConfig, startupMode: "lazy" } satisfies ResolvedStackConfig; + const config = { + ...defaultConfig, + servicePolicies: { ...defaultConfig.servicePolicies, postgrest: "lazy", auth: "lazy" }, + } satisfies ResolvedStackConfig; const { layer } = setupLayer(config, noopPortLease(config.ports), spawner); yield* Effect.gen(function* () { @@ -883,57 +900,6 @@ describe("Stack", () => { }).pipe(Effect.scoped, Effect.timeout("5 seconds")), ); - it.live("uses the stack readiness deadline for explicit lazy activation and cleans up", () => - Effect.gen(function* () { - const spawner = mockChildProcessSpawner(); - let releasedAll = false; - const config = { - ...defaultConfig, - startupMode: "lazy", - readiness: { mode: "finite", timeoutMs: 100 }, - readinessSource: "configured", - } satisfies ResolvedStackConfig; - const lease: PortLease = { - ...noopPortLease(config.ports), - releaseAll: Effect.sync(() => { - releasedAll = true; - }), - }; - const { layer } = setupLayer(config, lease, spawner); - - yield* Effect.gen(function* () { - const stack = yield* Stack; - const activator = yield* StackServiceActivator; - yield* stack.start(); - - const error = yield* activator.activate("auth").pipe(Effect.flip); - - expect(error._tag).toBe("StackReadinessError"); - if (error._tag === "StackReadinessError") { - expect(error.target).toBe("auth"); - expect(error.timeoutMs).toBe(100); - } - expect(releasedAll).toBe(true); - const spawnCountAfterDisposal = spawner.spawned.length; - expect((yield* activator.activate("postgres").pipe(Effect.flip))._tag).toBe( - "StackNotRunningError", - ); - for (const operation of [ - stack.start(), - stack.startService("postgres"), - stack.stopService("postgres"), - stack.restartService("postgres"), - stack.reloadFunctions(), - stack.reloadEdgeRuntime({ edgeRuntime: {} }), - ]) { - expect((yield* operation.pipe(Effect.flip))._tag).toBe("StackBuildError"); - } - yield* stack.stop(); - expect(spawner.spawned).toHaveLength(spawnCountAfterDisposal); - }).pipe(Effect.provide(layer)); - }).pipe(Effect.scoped, Effect.timeout("5 seconds")), - ); - it.live("allows a finite wait override against an infinite stack policy", () => Effect.gen(function* () { const spawnStarted = yield* Deferred.make(); @@ -948,7 +914,7 @@ describe("Stack", () => { let releasedAll = false; const config = { ...defaultConfig, - startupMode: "lazy", + servicePolicies: { ...defaultConfig.servicePolicies, postgrest: "lazy", auth: "lazy" }, readiness: { mode: "infinite" }, readinessSource: "configured", } satisfies ResolvedStackConfig; @@ -987,15 +953,22 @@ describe("Stack", () => { it.live("does not revive stopped lazy dependents when restarting a dependency", () => { return Effect.gen(function* () { const authHealthServer = yield* Effect.acquireRelease( - Effect.sync(() => - Bun.serve({ - port: 0, - fetch: () => new Response("ok"), - }), + Effect.tryPromise( + () => + new Promise((resolve, reject) => { + const server = createServer((_request, response) => { + response.writeHead(200, { "content-type": "text/plain" }); + response.end("ok"); + }); + server.once("error", reject); + server.listen(0, "127.0.0.1", () => resolve(server)); + }), ), - (server) => Effect.sync(() => server.stop(true)), + (server) => + Effect.tryPromise(() => new Promise((resolve) => server.close(() => resolve()))), ); - const authPort = authHealthServer.port; + const address = authHealthServer.address(); + const authPort = typeof address === "object" && address !== null ? address.port : undefined; if (authPort === undefined) { throw new Error("Expected the auth health test server to bind a TCP port"); } @@ -1005,7 +978,7 @@ describe("Stack", () => { } const { layer, spawner } = setupLayer({ ...defaultConfig, - startupMode: "lazy", + servicePolicies: { ...defaultConfig.servicePolicies, postgrest: "lazy", auth: "lazy" }, ports: { ...defaultPorts, authPort }, auth: { ...authConfig, port: authPort }, }); @@ -1033,7 +1006,10 @@ describe("Stack", () => { }); it.live("lazy readiness fails fast before a service is activated", () => { - const { layer } = setupLayer({ ...defaultConfig, startupMode: "lazy" }); + const { layer } = setupLayer({ + ...defaultConfig, + servicePolicies: { ...defaultConfig.servicePolicies, postgrest: "lazy", auth: "lazy" }, + }); return Effect.gen(function* () { const stack = yield* Stack; @@ -1048,8 +1024,88 @@ describe("Stack", () => { }).pipe(Effect.provide(layer), Effect.timeout("5 seconds")); }); + it.live( + "restores dormant lazy services after preparation failure and retries successfully", + () => { + return Effect.gen(function* () { + const healthServer = yield* Effect.acquireRelease( + Effect.tryPromise( + () => + new Promise((resolve, reject) => { + const server = createServer((_request, response) => { + response.writeHead(200, { "content-type": "text/plain" }); + response.end("ok"); + }); + server.once("error", reject); + server.listen(0, "127.0.0.1", () => resolve(server)); + }), + ), + (server) => + Effect.tryPromise(() => new Promise((resolve) => server.close(() => resolve()))), + ); + const address = healthServer.address(); + const postgrestPort = typeof address === "object" && address !== null ? address.port : 0; + if (postgrestPort === 0) throw new Error("Expected a PostgREST health port"); + const basePostgrest = defaultConfig.postgrest; + if (basePostgrest === false) throw new Error("Expected PostgREST in the default config"); + const config = { + ...defaultConfig, + servicePolicies: { + ...defaultConfig.servicePolicies, + postgrest: "lazy", + auth: "off", + }, + auth: false, + ports: { + ...defaultConfig.ports, + postgrestPort, + postgrestAdminPort: postgrestPort + 1, + }, + postgrest: { + ...basePostgrest, + port: postgrestPort, + adminPort: postgrestPort + 1, + }, + } satisfies ResolvedStackConfig; + const failingResolver = mockBinaryResolver({ failOnceServices: ["postgrest"] }); + const stackPreparationLayer = StackPreparation.layer.pipe( + Layer.provide(failingResolver.layer), + ); + const testLayer = localStackLayer(config, noopPortLease(config.ports)).pipe( + Layer.provide(StackBuilder.layer), + Layer.provide(stackPreparationLayer), + Layer.provide(mockChildProcessSpawner().layer), + Layer.provide(NodeServices.layer), + ); + + yield* Effect.gen(function* () { + const stack = yield* Stack; + const activator = yield* StackServiceActivator; + yield* stack.start(); + expect(["Running", "Healthy", "Initializing"]).toContain( + (yield* stack.getState("postgres")).status, + ); + expect((yield* stack.getState("postgrest")).status).toBe("Dormant"); + const first = yield* activator.activate("postgrest").pipe(Effect.flip); + expect(first._tag).toBe("StackBuildError"); + expect(["Running", "Healthy", "Initializing"]).toContain( + (yield* stack.getState("postgres")).status, + ); + expect((yield* stack.getState("postgrest")).status).toBe("Dormant"); + + yield* activator.activate("postgrest"); + expect(["Running", "Healthy"]).toContain((yield* stack.getState("postgrest")).status); + yield* stack.stop(); + }).pipe(Effect.provide(testLayer)); + }).pipe(Effect.scoped, Effect.timeout("5 seconds")); + }, + ); + it.live("keeps unactivated services dormant after a stop and start cycle", () => { - const config = { ...defaultConfig, startupMode: "lazy" } satisfies ResolvedStackConfig; + const config = { + ...defaultConfig, + servicePolicies: { ...defaultConfig.servicePolicies, postgrest: "lazy", auth: "lazy" }, + } satisfies ResolvedStackConfig; const { layer } = setupLayer(config); return Effect.gen(function* () { @@ -1066,7 +1122,10 @@ describe("Stack", () => { }); it.live("rejects a cached activation after the stack has stopped", () => { - const config = { ...defaultConfig, startupMode: "lazy" } satisfies ResolvedStackConfig; + const config = { + ...defaultConfig, + servicePolicies: { ...defaultConfig.servicePolicies, postgrest: "lazy", auth: "lazy" }, + } satisfies ResolvedStackConfig; const { layer } = setupLayer(config); return Effect.gen(function* () { @@ -1081,7 +1140,10 @@ describe("Stack", () => { }); it.live("preserves an explicitly stopped service across a stack restart", () => { - const config = { ...defaultConfig, startupMode: "lazy" } satisfies ResolvedStackConfig; + const config = { + ...defaultConfig, + servicePolicies: { ...defaultConfig.servicePolicies, postgrest: "lazy", auth: "lazy" }, + } satisfies ResolvedStackConfig; const { layer } = setupLayer(config); return Effect.gen(function* () { @@ -1119,7 +1181,13 @@ describe("Stack", () => { }), releaseAll: Effect.void, }; - const { layer } = setupLayer({ ...defaultConfig, startupMode: "lazy" }, lease); + const { layer } = setupLayer( + { + ...defaultConfig, + servicePolicies: { ...defaultConfig.servicePolicies, postgrest: "lazy", auth: "lazy" }, + }, + lease, + ); return Effect.gen(function* () { const stack = yield* Stack; diff --git a/packages/stack/src/StackBuilder.ts b/packages/stack/src/StackBuilder.ts index b45e351f1b..e089cded05 100644 --- a/packages/stack/src/StackBuilder.ts +++ b/packages/stack/src/StackBuilder.ts @@ -153,8 +153,11 @@ export const validateResolvedConfig = ( }); export const enabledServicesForConfig = (config: ResolvedStackConfig): ReadonlyArray => - SERVICE_NAMES.filter( - (service) => service === "postgres" || resolvedConfigForService(config, service) !== false, + SERVICE_NAMES.filter((service) => + service === "postgres" && config.servicePolicies?.postgres !== "off" + ? true + : config.servicePolicies?.[service] !== "off" && + resolvedConfigForService(config, service) !== false, ); export const versionsForConfig = (config: ResolvedStackConfig): Partial => { diff --git a/packages/stack/src/StackBuilder.unit.test.ts b/packages/stack/src/StackBuilder.unit.test.ts index 84cbe83e3d..074f82b60f 100644 --- a/packages/stack/src/StackBuilder.unit.test.ts +++ b/packages/stack/src/StackBuilder.unit.test.ts @@ -48,7 +48,21 @@ const baseConfig: ResolvedStackConfig = { runtimeRoot: "/tmp/supabase-runtime", projectDir: "/tmp/supabase-project", mode: "auto", - startupMode: "eager", + servicePolicies: { + postgres: "eager", + postgrest: "eager", + auth: "eager", + "edge-runtime": "off", + realtime: "off", + storage: "off", + imgproxy: "off", + mailpit: "off", + pgmeta: "off", + studio: "off", + analytics: "off", + vector: "off", + pooler: "off", + }, readiness: DEFAULT_STACK_READINESS_POLICY, readinessSource: "default", jwtSecret: testJwtSecret, @@ -120,6 +134,7 @@ const siblingManagedConfig: ResolvedStackConfig = { const edgeRuntimeConfig: ResolvedStackConfig = { ...baseConfig, mode: "auto", + servicePolicies: { ...baseConfig.servicePolicies, "edge-runtime": "eager" }, edgeRuntime: { enabled: true, port: basePorts.edgeRuntimePort, @@ -255,70 +270,6 @@ describe("StackBuilder", () => { }).pipe(Effect.provide(layer)); }); - it.effect("uses docker fallback when auth binary not found", () => { - const resolver = mockBinaryResolver({ failServices: ["auth"] }); - const layer = builderLayer(resolver); - - return Effect.gen(function* () { - const builder = yield* StackBuilder; - const preparation = yield* StackPreparation; - const { graph } = yield* prepareAndBuild(builder, preparation, baseConfig); - - expect(graph.startOrder.length).toBe(4); - - const authDef = graph.startOrder.find((s) => s.name === "auth"); - expect(authDef).toBeDefined(); - expect(authDef?.command).toBe("docker"); - expect(authDef?.dependencies).toEqual([{ service: "postgres-init", condition: "completed" }]); - expect(authDef?.supervision).toBeDefined(); - }).pipe(Effect.provide(layer)); - }); - - it.effect("uses docker fallback when postgres binary not found", () => { - const resolver = mockBinaryResolver({ failServices: ["postgres"] }); - const layer = builderLayer(resolver); - - return Effect.gen(function* () { - const builder = yield* StackBuilder; - const preparation = yield* StackPreparation; - const { graph } = yield* prepareAndBuild(builder, preparation, baseConfig); - - // No postgres-init when postgres falls back to Docker. - expect(graph.startOrder.length).toBe(3); - - const postgresDef = graph.startOrder.find((s) => s.name === "postgres"); - expect(postgresDef).toBeDefined(); - expect(postgresDef?.command).toBe("docker"); - expect(postgresDef?.supervision).toBeDefined(); - - // postgrest falls back to postgres(healthy) dependency - const postgrestDef = graph.startOrder.find((s) => s.name === "postgrest"); - expect(postgrestDef?.dependencies).toEqual([{ service: "postgres", condition: "healthy" }]); - - const names = graph.startOrder.map((s) => s.name); - expect(names).not.toContain("postgres-init"); - }).pipe(Effect.provide(layer)); - }); - - it.effect("uses docker fallback when postgrest binary not found", () => { - const resolver = mockBinaryResolver({ failServices: ["postgrest"] }); - const layer = builderLayer(resolver); - - return Effect.gen(function* () { - const builder = yield* StackBuilder; - const preparation = yield* StackPreparation; - const { graph } = yield* prepareAndBuild(builder, preparation, baseConfig); - - // All 4 services still present (postgrest falls back to Docker, not removed) - expect(graph.startOrder.length).toBe(4); - - const postgrestDef = graph.startOrder.find((s) => s.name === "postgrest"); - expect(postgrestDef).toBeDefined(); - expect(postgrestDef?.command).toBe("docker"); - expect(postgrestDef?.supervision).toBeDefined(); - }).pipe(Effect.provide(layer)); - }); - it.effect("excludes disabled services", () => { const resolver = mockBinaryResolver(); const layer = builderLayer(resolver); @@ -562,38 +513,4 @@ describe("StackBuilder", () => { ); }).pipe(Effect.provide(layer)); }); - - it.effect("falls back to the next registry for docker-only services", () => { - const resolver = mockBinaryResolver(); - const spawnerLayer = mockSequenceSpawner([ - { exitCode: 0 }, - { exitCode: 0 }, - { exitCode: 0 }, - { exitCode: 1, stderr: ["manifest unknown"] }, - { exitCode: 0 }, - ]); - const layer = builderLayer(resolver, spawnerLayer); - - return Effect.gen(function* () { - const builder = yield* StackBuilder; - const preparation = yield* StackPreparation; - const { graph } = yield* prepareAndBuild(builder, preparation, { - ...dockerConfig, - realtime: { - port: 3010, - version: DEFAULT_VERSIONS.realtime, - tenantId: "realtime-dev", - encryptionKey: "supabaserealtime", - secretKeyBase: "EAx3IQ/wRG1v47ZD4NE4/9RzBI8Jmil3x0yhcW4V2NHBP6c2iPIzwjofi2Ep4HIG", - maxHeaderLength: 4096, - }, - }); - - const realtimeDef = graph.startOrder.find((service) => service.name === "realtime"); - expect(realtimeDef?.args).toContain(`supabase/realtime:v${DEFAULT_VERSIONS.realtime}`); - expect(realtimeDef?.args).not.toContain( - `public.ecr.aws/supabase/realtime:v${DEFAULT_VERSIONS.realtime}`, - ); - }).pipe(Effect.provide(layer)); - }); }); diff --git a/packages/stack/src/StackConfig.ts b/packages/stack/src/StackConfig.ts index 25ba352931..d82d906f8e 100644 --- a/packages/stack/src/StackConfig.ts +++ b/packages/stack/src/StackConfig.ts @@ -1,9 +1,11 @@ import { Schema } from "effect"; import type { ResolvedFunctionsBundle } from "./functions.ts"; import type { ResolvedPorts } from "./PortCatalog.ts"; +import type { ServiceName } from "./ServiceName.ts"; type StackMode = "native" | "auto" | "docker"; -type StackStartupMode = "eager" | "lazy"; +export type ServicePolicy = "off" | "lazy" | "eager"; +export type ServicePolicyManifest = Readonly>; export type ReadinessPolicy = | { readonly mode: "finite"; readonly timeoutMs: number } @@ -174,8 +176,8 @@ export interface StackConfig { readonly runtimeRoot?: string; readonly projectDir?: string; readonly mode?: StackMode; - /** Start all services immediately, or defer proxied services until first use. */ - readonly startupMode?: StackStartupMode; + /** Per-service resource policy. `off` excludes a service from the graph. */ + readonly servicePolicies?: Partial>; /** Stack-wide readiness policy. Per-call ReadyOptions take precedence. */ readonly readiness?: ReadinessPolicy; readonly jwtSecret?: string; @@ -304,7 +306,7 @@ export interface ResolvedStackConfig { readonly runtimeRoot: string; readonly projectDir: string; readonly mode: StackMode; - readonly startupMode: StackStartupMode; + readonly servicePolicies: ServicePolicyManifest; readonly readiness: ReadinessPolicy; /** Whether readiness came from the package default or an explicit stack policy. */ readonly readinessSource: "default" | "configured"; diff --git a/packages/stack/src/StackConfigResolver.policy.unit.test.ts b/packages/stack/src/StackConfigResolver.policy.unit.test.ts new file mode 100644 index 0000000000..b6bd0d08f6 --- /dev/null +++ b/packages/stack/src/StackConfigResolver.policy.unit.test.ts @@ -0,0 +1,41 @@ +import { describe, expect, it } from "vitest"; +import { resolveConfig } from "./StackConfigResolver.ts"; +import { StackBuildError } from "./errors.ts"; + +describe("resolved service preparation policies", () => { + it("applies explicit policies and catalog defaults while keeping Postgres eager", async () => { + const config = await resolveConfig({ + servicePolicies: { postgrest: "eager", mailpit: "eager" }, + mailpit: {}, + stackRoot: "/tmp/stack-policy-test", + runtimeRoot: "/tmp/runtime-policy-test", + }); + + expect(config.servicePolicies.postgres).toBe("eager"); + expect(config.servicePolicies.postgrest).toBe("eager"); + expect(config.servicePolicies.auth).toBe("lazy"); + expect(config.servicePolicies.mailpit).toBe("eager"); + }); + + it("rejects an unsupported lazy policy before port allocation", async () => { + let allocated = false; + await expect( + resolveConfig( + { servicePolicies: { postgres: "lazy" } }, + { + portAllocator: () => { + allocated = true; + throw new Error("must not allocate"); + }, + }, + ), + ).rejects.toBeInstanceOf(StackBuildError); + expect(allocated).toBe(false); + }); + + it("resolves explicitly disabled core services to false without reserving ports", async () => { + const config = await resolveConfig({ servicePolicies: { postgrest: "off" } }); + expect(config.postgrest).toBe(false); + expect(config.servicePolicies.postgrest).toBe("off"); + }); +}); diff --git a/packages/stack/src/StackConfigResolver.ts b/packages/stack/src/StackConfigResolver.ts index 9aa29921a9..dccc80c3d8 100644 --- a/packages/stack/src/StackConfigResolver.ts +++ b/packages/stack/src/StackConfigResolver.ts @@ -17,7 +17,7 @@ import { type PortSelectionOptions, } from "./PortAllocator.ts"; import { PORT_CATALOG, type PortField, type PortSet, type ResolvedPorts } from "./PortCatalog.ts"; -import { portFieldsForConfigInput, serviceEnabledForConfig } from "./ServicePorts.ts"; +import { portFieldsForConfigInput } from "./ServicePorts.ts"; import { INSTANCE_ID_PATTERN, InstanceIdSchema, resolveReadinessPolicy } from "./StackConfig.ts"; import type { AnalyticsConfig, @@ -42,12 +42,20 @@ import type { ResolvedStorageConfig, ResolvedStudioConfig, ResolvedVectorConfig, + ServicePolicy, + ServicePolicyManifest, StackConfig, StorageConfig, StudioConfig, VectorConfig, } from "./StackConfig.ts"; -import { DEFAULT_VERSIONS } from "./ServiceCatalog.ts"; +import { + DEFAULT_SERVICE_POLICIES, + DEFAULT_VERSIONS, + SERVICE_NAMES, + serviceMetadata, +} from "./ServiceCatalog.ts"; +import type { ServiceName } from "./ServiceName.ts"; export interface ResolveConfigOptions { readonly stackRoot?: string; @@ -344,29 +352,120 @@ const enabledServiceConfig = ( config: Config | false | undefined, ): Config | undefined => (enabled && config !== false ? config : undefined); +const rawServiceEnabled = (config: StackConfig, service: ServiceName): boolean => { + switch (service) { + case "postgres": + return true; + case "postgrest": + return config.postgrest !== false; + case "auth": + return config.auth !== false; + case "edge-runtime": + return ( + ((config.mode ?? "auto") !== "native" || config.edgeRuntime !== undefined) && + config.edgeRuntime !== false && + (config.edgeRuntime?.enabled ?? true) !== false + ); + case "realtime": + return config.realtime !== undefined && config.realtime !== false; + case "storage": + return config.storage !== undefined && config.storage !== false; + case "imgproxy": + return config.imgproxy !== undefined && config.imgproxy !== false; + case "mailpit": + return config.mailpit !== undefined && config.mailpit !== false; + case "pgmeta": + return config.pgmeta !== undefined && config.pgmeta !== false; + case "studio": + return config.studio !== undefined && config.studio !== false; + case "analytics": + return config.analytics !== undefined && config.analytics !== false; + case "vector": + return config.vector !== undefined && config.vector !== false; + case "pooler": + return config.pooler !== undefined && config.pooler !== false; + } +}; + +/** + * Resolve policy declarations before roots, ports, or config-dependent effects + * are acquired. This keeps unsupported policies a pure user/configuration error. + */ +const resolveServicePolicies = (config: StackConfig): ServicePolicyManifest => { + const policies: Record = { + postgres: "off", + postgrest: "off", + auth: "off", + "edge-runtime": "off", + realtime: "off", + storage: "off", + imgproxy: "off", + mailpit: "off", + pgmeta: "off", + studio: "off", + analytics: "off", + vector: "off", + pooler: "off", + }; + for (const service of SERVICE_NAMES) { + const requested = config.servicePolicies?.[service]; + if (service === "postgres" && requested !== undefined && requested !== "eager") { + throw new StackBuildError({ + detail: "postgres supports only the eager service preparation policy", + reason: "invalid_config", + }); + } + + const enabled = rawServiceEnabled(config, service); + if (!enabled || requested === "off") { + policies[service] = "off"; + continue; + } + + const policy: Exclude = + requested === undefined ? DEFAULT_SERVICE_POLICIES[service] : requested; + if (!serviceMetadata(service).preparation.supported.includes(policy)) { + throw new StackBuildError({ + detail: `${service} does not support the ${policy} service preparation policy`, + reason: "invalid_config", + }); + } + policies[service] = policy; + } + return policies; +}; + export async function resolveConfig( input?: StackConfig, opts: ResolveConfigOptions = {}, ): Promise { const config = input ?? {}; + // Deliberately first: unsupported policies must not create roots or reserve ports. + const servicePolicies = resolveServicePolicies(config); const projectDir = config.projectDir ?? process.cwd(); const instanceId = resolveInstanceId(config.instanceId); const functions = await resolveFunctionsConfig(config, projectDir); const resolvedMode = config.mode ?? "auto"; const roots = resolveRoots(config, opts); const postgresInput = config.postgres ?? {}; - const postgrestInput = config.postgrest !== false ? (config.postgrest ?? undefined) : undefined; - const authInput = config.auth !== false ? (config.auth ?? undefined) : undefined; - const edgeRuntimeEnabled = serviceEnabledForConfig(config, "edge-runtime"); - const realtimeEnabled = serviceEnabledForConfig(config, "realtime"); - const storageEnabled = serviceEnabledForConfig(config, "storage"); - const imgproxyEnabled = serviceEnabledForConfig(config, "imgproxy"); - const mailpitEnabled = serviceEnabledForConfig(config, "mailpit"); - const pgmetaEnabled = serviceEnabledForConfig(config, "pgmeta"); - const studioEnabled = serviceEnabledForConfig(config, "studio"); - const analyticsEnabled = serviceEnabledForConfig(config, "analytics"); - const vectorEnabled = serviceEnabledForConfig(config, "vector"); - const poolerEnabled = serviceEnabledForConfig(config, "pooler"); + const postgrestInput = + servicePolicies.postgrest !== "off" && config.postgrest !== false + ? (config.postgrest ?? undefined) + : undefined; + const authInput = + servicePolicies.auth !== "off" && config.auth !== false + ? (config.auth ?? undefined) + : undefined; + const edgeRuntimeEnabled = servicePolicies["edge-runtime"] !== "off"; + const realtimeEnabled = servicePolicies.realtime !== "off"; + const storageEnabled = servicePolicies.storage !== "off"; + const imgproxyEnabled = servicePolicies.imgproxy !== "off"; + const mailpitEnabled = servicePolicies.mailpit !== "off"; + const pgmetaEnabled = servicePolicies.pgmeta !== "off"; + const studioEnabled = servicePolicies.studio !== "off"; + const analyticsEnabled = servicePolicies.analytics !== "off"; + const vectorEnabled = servicePolicies.vector !== "off"; + const poolerEnabled = servicePolicies.pooler !== "off"; const edgeRuntimeInput = enabledServiceConfig(edgeRuntimeEnabled, config.edgeRuntime); const realtimeInput = enabledServiceConfig(realtimeEnabled, config.realtime); const storageInput = enabledServiceConfig(storageEnabled, config.storage); @@ -457,7 +556,7 @@ export async function resolveConfig( runtimeRoot: roots.runtimeRoot, projectDir, mode: resolvedMode, - startupMode: config.startupMode ?? "eager", + servicePolicies, readiness: resolveReadinessPolicy({ stackPolicy: config.readiness }), readinessSource: config.readiness === undefined ? "default" : "configured", jwtSecret, @@ -476,8 +575,17 @@ export async function resolveConfig( version: postgresInput.version ?? DEFAULT_VERSIONS.postgres, autoExposeNewTables: postgresInput.autoExposeNewTables ?? true, }, - postgrest: resolvePostgrestConfig(postgrestInput, config.postgrest, ports), - auth: resolveAuthConfig(authInput, config.auth, ports, apiPort), + postgrest: resolvePostgrestConfig( + postgrestInput, + servicePolicies.postgrest === "off" ? false : config.postgrest, + ports, + ), + auth: resolveAuthConfig( + authInput, + servicePolicies.auth === "off" ? false : config.auth, + ports, + apiPort, + ), edgeRuntime: edgeRuntimeEnabled ? resolveEdgeRuntimeConfig(edgeRuntimeInput, config.edgeRuntime, ports) : false, diff --git a/packages/stack/src/StackPreparation.ts b/packages/stack/src/StackPreparation.ts index 50761ed59e..50d0ee3322 100644 --- a/packages/stack/src/StackPreparation.ts +++ b/packages/stack/src/StackPreparation.ts @@ -1,13 +1,31 @@ -import { Cause, Data, Effect, Exit, Layer, Queue, Context, Stream } from "effect"; +import { + Cause, + Context, + Data, + Deferred, + Effect, + Exit, + Layer, + Queue, + Schedule, + Stream, +} from "effect"; import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; import { BinaryResolver } from "./BinaryResolver.ts"; -import type { ChecksumMismatchError } from "./errors.ts"; +import type { + BinaryHostCompatibilityError, + BinaryManifestError, + BinaryNotFoundError, + BinaryRuntimeError, + ChecksumMismatchError, + DownloadError, +} from "./errors.ts"; import { DockerPullError, isDockerDaemonDownMessage } from "./errors.ts"; -import { isDockerOnlyService } from "./ServiceCatalog.ts"; +import { isDockerOnlyService, requiredPreparationDependencies } from "./ServiceCatalog.ts"; import { DEFAULT_VERSIONS, SERVICE_NAMES, - dockerImageCandidatesForService, + dockerImageForService, type ServiceName, type VersionManifest, } from "./versions.ts"; @@ -16,6 +34,8 @@ export interface PreparedStackArtifacts { readonly resolutions: Partial>; } +export type PlannedStackArtifacts = PreparedStackArtifacts; + export type ServiceResolution = | { readonly type: "binary"; readonly path: string } | { readonly type: "docker"; readonly image: string }; @@ -26,6 +46,15 @@ export interface StackPreparationInput { readonly mode?: "native" | "auto" | "docker"; } +export type StackPreparationError = + | BinaryNotFoundError + | DownloadError + | ChecksumMismatchError + | BinaryManifestError + | BinaryRuntimeError + | BinaryHostCompatibilityError + | DockerPullError; + export class ServiceDownloadStarted extends Data.TaggedClass("ServiceDownloadStarted")<{ readonly service: ServiceName; }> {} @@ -34,16 +63,15 @@ export class ServiceDownloadFinished extends Data.TaggedClass("ServiceDownloadFi readonly service: ServiceName; }> {} -class PreparationCompleted extends Data.TaggedClass("PreparationCompleted")<{ +export class PreparationCompleted extends Data.TaggedClass("PreparationCompleted")<{ readonly artifacts: PreparedStackArtifacts; }> {} -type StackPreparationEvent = +export type StackPreparationEvent = | ServiceDownloadStarted | ServiceDownloadFinished | PreparationCompleted; -const DOCKER_PULL_RETRY_DELAYS_MS = [500] as const; const RETRYABLE_PULL_PATTERNS = [ /toomanyrequests/i, /rate exceeded/i, @@ -56,10 +84,14 @@ const RETRYABLE_PULL_PATTERNS = [ /i\/o timeout/i, ] as const; -interface PullAttemptFailure { - readonly image: string; - readonly attempt: number; - readonly message: string; +class PullAttemptError extends Error { + constructor( + readonly detail: string, + readonly daemonDown: boolean, + ) { + super(detail); + this.name = "PullAttemptError"; + } } const resolveDockerImageForService = ( @@ -70,90 +102,70 @@ const resolveDockerImageForService = ( readonly onDownloadStart?: Effect.Effect; }, ): Effect.Effect => - pullImage(spawner, dockerImageCandidatesForService(service, version), callbacks); + pullImage(spawner, dockerImageForService(service, version), callbacks); + +const preparationClosure = (services: ReadonlyArray): ReadonlyArray => { + const closure = new Set(); + const add = (service: ServiceName): void => { + if (closure.has(service)) return; + closure.add(service); + for (const dependency of requiredPreparationDependencies(service)) add(dependency); + }; + for (const service of services) add(service); + return [...closure]; +}; + +const selectedServices = (input?: StackPreparationInput): ReadonlyArray => + preparationClosure(input?.services ?? SERVICE_NAMES); + +const plannedResolution = ( + resolver: BinaryResolver["Service"], + service: ServiceName, + version: string, + mode: "native" | "auto" | "docker", +): Effect.Effect => { + if (mode === "docker" || (mode === "auto" && isDockerOnlyService(service))) { + return Effect.succeed({ + type: "docker", + image: dockerImageForService(service, version), + }); + } + return resolver.plan({ service, version }).pipe(Effect.map((path) => ({ type: "binary", path }))); +}; -export const prepareAssetsWithDependencies = ( +const planAssetsWithDependencies = ( resolver: BinaryResolver["Service"], - spawner: ChildProcessSpawner.ChildProcessSpawner["Service"], input?: StackPreparationInput, - publishEvent?: (event: StackPreparationEvent) => Effect.Effect, -): Effect.Effect => +): Effect.Effect => Effect.gen(function* () { const versions = { ...DEFAULT_VERSIONS, ...input?.versions }; - const services: ReadonlyArray = input?.services ?? SERVICE_NAMES; const mode = input?.mode ?? "auto"; - - type Entry = readonly [ServiceName, ServiceResolution]; - - const resolveService = ( - service: ServiceName, - ): Effect.Effect => { - let isDownloading = false; - const markDownloadStart = () => - Effect.sync(() => { - isDownloading = true; - }).pipe( - Effect.andThen(publishEvent?.(new ServiceDownloadStarted({ service })) ?? Effect.void), - ); - const markDownloadFinished = () => - Effect.suspend(() => - isDownloading - ? (publishEvent?.(new ServiceDownloadFinished({ service })) ?? Effect.void) - : Effect.void, - ); - - if (mode === "docker") { - return resolveDockerImageForService(spawner, service, versions[service], { - onDownloadStart: markDownloadStart(), - }).pipe( - Effect.map((image): Entry => [service, { type: "docker", image }]), - Effect.ensuring(markDownloadFinished()), - ); - } - - if (isDockerOnlyService(service)) { - return resolveDockerImageForService(spawner, service, versions[service], { - onDownloadStart: markDownloadStart(), - }).pipe( - Effect.map((image): Entry => [service, { type: "docker", image }]), - Effect.ensuring(markDownloadFinished()), - ); - } - - return resolveServiceWithMetadata( - resolver, - spawner, - service, - versions[service], - markDownloadStart(), - ).pipe( - Effect.map((resolution): Entry => [service, resolution]), - Effect.ensuring(markDownloadFinished()), - ); - }; - - const results = yield* Effect.all(services.map(resolveService), { - concurrency: "unbounded", - }); - - const resolutions: Partial> = {}; - for (const [service, resolution] of results) { - resolutions[service] = resolution; - } - const artifacts = { resolutions } satisfies PreparedStackArtifacts; - yield* publishEvent?.(new PreparationCompleted({ artifacts })) ?? Effect.void; - return artifacts; + const services = selectedServices(input); + const results = yield* Effect.all( + services.map((service) => + plannedResolution(resolver, service, versions[service], mode).pipe( + Effect.map((resolution) => [service, resolution] as const), + ), + ), + { concurrency: "unbounded" }, + ); + return { + resolutions: Object.fromEntries(results), + } satisfies PlannedStackArtifacts; }); export class StackPreparation extends Context.Service< StackPreparation, { + readonly plan: ( + input?: StackPreparationInput, + ) => Effect.Effect; readonly prepare: ( input?: StackPreparationInput, - ) => Effect.Effect; + ) => Effect.Effect; readonly prepareEvents: ( input?: StackPreparationInput, - ) => Stream.Stream; + ) => Stream.Stream; } >()("stack/StackPreparation") { static layer: Layer.Layer< @@ -165,15 +177,97 @@ export class StackPreparation extends Context.Service< Effect.gen(function* () { const resolver = yield* BinaryResolver; const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const scope = yield* Effect.scope; + const inFlight = new Map< + string, + Deferred.Deferred + >(); + + const materialize = ( + service: ServiceName, + resolution: ServiceResolution, + input: StackPreparationInput, + publishEvent?: (event: StackPreparationEvent) => Effect.Effect, + ): Effect.Effect => + Effect.suspend(() => { + let downloadStarted = false; + const markDownloadStart = () => + Effect.sync(() => { + downloadStarted = true; + }).pipe( + Effect.andThen( + publishEvent?.(new ServiceDownloadStarted({ service })) ?? Effect.void, + ), + ); + const markDownloadFinished = () => + Effect.suspend(() => + downloadStarted + ? (publishEvent?.(new ServiceDownloadFinished({ service })) ?? Effect.void) + : Effect.void, + ); + const key = `${service}\0${JSON.stringify(resolution)}`; + const existing = inFlight.get(key); + if (existing !== undefined) return Deferred.await(existing); + const deferred = Deferred.makeUnsafe(); + inFlight.set(key, deferred); + const version = input.versions?.[service] ?? DEFAULT_VERSIONS[service]; + const effect: Effect.Effect = + resolution.type === "docker" + ? resolveDockerImageForService(spawner, service, version, { + onDownloadStart: markDownloadStart(), + }).pipe(Effect.map((image): ServiceResolution => ({ type: "docker", image }))) + : resolver + .resolveWithMetadata( + { service, version }, + { + onDownloadStart: markDownloadStart(), + }, + ) + .pipe(Effect.map(({ path }): ServiceResolution => ({ type: "binary", path }))); + const coordinated = effect.pipe( + Effect.matchCauseEffect({ + onSuccess: (value) => + Effect.andThen(markDownloadFinished(), Deferred.succeed(deferred, value)), + onFailure: (cause) => + Deferred.failCause(deferred, cause).pipe(Effect.andThen(Effect.failCause(cause))), + }), + Effect.ensuring(Effect.sync(() => inFlight.delete(key))), + ); + return Effect.gen(function* () { + yield* Effect.forkIn(coordinated, scope, { startImmediately: true }); + return yield* Deferred.await(deferred); + }); + }); + + const prepareWithEvents = ( + input: StackPreparationInput | undefined, + publishEvent?: (event: StackPreparationEvent) => Effect.Effect, + ): Effect.Effect => + Effect.gen(function* () { + const planned = yield* planAssetsWithDependencies(resolver, input); + const entries = yield* Effect.all( + selectedServices(input).map((service) => { + const resolution = planned.resolutions[service]; + if (resolution === undefined) return Effect.die(`Missing plan for ${service}`); + return materialize(service, resolution, input ?? {}, publishEvent).pipe( + Effect.map((resolved) => [service, resolved] as const), + ); + }), + { concurrency: "unbounded" }, + ); + const artifacts = { + resolutions: Object.fromEntries(entries), + } satisfies PreparedStackArtifacts; + yield* publishEvent?.(new PreparationCompleted({ artifacts })) ?? Effect.void; + return artifacts; + }); return { - prepare: (input?: StackPreparationInput) => - prepareAssetsWithDependencies(resolver, spawner, input), + plan: (input?: StackPreparationInput) => planAssetsWithDependencies(resolver, input), + prepare: (input?: StackPreparationInput) => prepareWithEvents(input), prepareEvents: (input?: StackPreparationInput) => - Stream.callback((queue) => - prepareAssetsWithDependencies(resolver, spawner, input, (event) => - Queue.offer(queue, event), - ).pipe( + Stream.callback((queue) => + prepareWithEvents(input, (event) => Queue.offer(queue, event)).pipe( Effect.matchCauseEffect({ onFailure: (cause) => Queue.failCause(queue, cause), onSuccess: () => Queue.end(queue), @@ -188,131 +282,68 @@ export class StackPreparation extends Context.Service< const pullImage = ( spawner: ChildProcessSpawner.ChildProcessSpawner["Service"], - images: ReadonlyArray, + image: string, callbacks?: { readonly onDownloadStart?: Effect.Effect; }, ): Effect.Effect => Effect.gen(function* () { - const cachedImage = yield* findLocalDockerImage(spawner, images); - if (cachedImage !== undefined) { - return cachedImage; + if (yield* hasLocalDockerImage(spawner, image)) { + return image; } yield* callbacks?.onDownloadStart ?? Effect.void; - const failures: PullAttemptFailure[] = []; - let spawnFailed = false; - - for (const image of images) { - for ( - let attemptIndex = 0; - attemptIndex <= DOCKER_PULL_RETRY_DELAYS_MS.length; - attemptIndex += 1 - ) { - const attempt = attemptIndex + 1; - const result = yield* Effect.exit(runPullCommand(spawner, image)); - if (Exit.isSuccess(result)) { - // A successful spawn proves the runtime is usable; an earlier - // transient spawn failure must not taint the final classification. - spawnFailed = false; - if (result.value.exitCode === 0) { - return image; - } - - const message = - result.value.stderr.length > 0 - ? result.value.stderr - : `docker pull exited with code ${result.value.exitCode}`; - failures.push({ image, attempt, message }); - - if (!shouldRetryPull(message) || attemptIndex === DOCKER_PULL_RETRY_DELAYS_MS.length) { - break; - } - } else { - // A failed effect (rather than a non-zero exit) means the container - // runtime could not be spawned at all — a local Docker setup - // problem, not a registry failure. - spawnFailed = true; - const cause = Cause.squash(result.cause); - const message = cause instanceof Error ? cause.message : String(cause); - failures.push({ image, attempt, message }); - if (!shouldRetryPull(message) || attemptIndex === DOCKER_PULL_RETRY_DELAYS_MS.length) { - break; - } - } - - const retryDelay = DOCKER_PULL_RETRY_DELAYS_MS[attemptIndex]; - if (retryDelay === undefined) { - break; - } - yield* Effect.sleep(`${retryDelay} millis`); - } - } + const attempt = runPullCommand(spawner, image).pipe( + Effect.retry({ + while: (error) => shouldRetryPull(error.detail), + schedule: Schedule.recurs(1), + }), + ); + const result = yield* Effect.exit(attempt); + if (Exit.isSuccess(result)) return image; - const detail = failures - .map((failure) => `${failure.image} attempt ${failure.attempt}: ${failure.message}`) - .join("; "); + const failure = Cause.squash(result.cause); + const detail = failure instanceof PullAttemptError ? failure.detail : String(failure); + const daemonDown = failure instanceof PullAttemptError && failure.daemonDown; return yield* Effect.fail( new DockerPullError({ - image: images[0] ?? "unknown", - detail: `Failed to pull Docker image from all registries. ${detail}`, + image, + detail: `Failed to pull canonical Docker image. ${detail}`, cause: new Error(detail), - daemonDown: - spawnFailed || failures.some((failure) => isDockerDaemonDownMessage(failure.message)), + daemonDown, }), ); }); -const resolveServiceWithMetadata = ( - resolver: BinaryResolver["Service"], - spawner: ChildProcessSpawner.ChildProcessSpawner["Service"], - service: ServiceName, - version: string, - onDownloadStart: Effect.Effect, -): Effect.Effect => - resolver.resolveWithMetadata({ service, version }, { onDownloadStart }).pipe( - Effect.map(({ path }): ServiceResolution => ({ type: "binary", path })), - Effect.catchTag("BinaryNotFoundError", () => - resolveDockerImageForService(spawner, service, version, { - onDownloadStart, - }).pipe( - Effect.map((image): ServiceResolution => ({ - type: "docker", - image, - })), - ), - ), - Effect.catchTag("DownloadError", () => - resolveDockerImageForService(spawner, service, version, { - onDownloadStart, - }).pipe( - Effect.map((image): ServiceResolution => ({ - type: "docker", - image, - })), - ), - ), - ); - const runPullCommand = ( spawner: ChildProcessSpawner.ChildProcessSpawner["Service"], image: string, -): Effect.Effect<{ readonly exitCode: number; readonly stderr: string }, Error> => +): Effect.Effect<{ readonly exitCode: number; readonly stderr: string }, PullAttemptError> => Effect.gen(function* () { const child = yield* spawner.spawn(ChildProcess.make("docker", ["pull", image])); const [stderr, exitCode] = yield* Effect.all( [collectStreamAsString(child.stderr), child.exitCode.pipe(Effect.map(Number))], { concurrency: "unbounded" }, ); - return { + const result = { exitCode, stderr: stderr.trim(), }; + if (result.exitCode !== 0) { + const detail = + result.stderr.length > 0 + ? result.stderr + : `docker pull exited with code ${result.exitCode}`; + return yield* Effect.fail(new PullAttemptError(detail, isDockerDaemonDownMessage(detail))); + } + return result; }).pipe( Effect.scoped, - Effect.catchTag("PlatformError", (error) => Effect.fail(new Error(String(error)))), + Effect.catchTag("PlatformError", (error) => + Effect.fail(new PullAttemptError(String(error), true)), + ), ); const hasLocalDockerImage = ( @@ -324,19 +355,6 @@ const hasLocalDockerImage = ( Effect.catchTag("PlatformError", () => Effect.succeed(false)), ); -const findLocalDockerImage = ( - spawner: ChildProcessSpawner.ChildProcessSpawner["Service"], - images: ReadonlyArray, -): Effect.Effect => - Effect.gen(function* () { - for (const image of images) { - if (yield* hasLocalDockerImage(spawner, image)) { - return image; - } - } - return undefined; - }); - const collectStreamAsString = (stream: Stream.Stream): Effect.Effect => Stream.runFold( stream, diff --git a/packages/stack/src/createStack.integration.test.ts b/packages/stack/src/createStack.integration.test.ts index 59110eeadd..286e19463f 100644 --- a/packages/stack/src/createStack.integration.test.ts +++ b/packages/stack/src/createStack.integration.test.ts @@ -70,7 +70,6 @@ describe("direct createStack port ownership", () => { const stack = await createStackWithFreshPorts( { mode: "native", - startupMode: "lazy", postgrest: false, auth: false, edgeRuntime: false, diff --git a/packages/stack/src/createStack.unit.test.ts b/packages/stack/src/createStack.unit.test.ts index d0962a7191..5bad2ed488 100644 --- a/packages/stack/src/createStack.unit.test.ts +++ b/packages/stack/src/createStack.unit.test.ts @@ -211,21 +211,18 @@ describe("resolveConfig instanceId validation", () => { }); }); -describe("resolveConfig startup mode", () => { - it("keeps eager startup as the package default", async () => { - const config = await resolveConfig(); - expect(config.startupMode).toBe("eager"); - }); - - it("preserves an explicit lazy startup mode", async () => { - const config = await resolveConfig({ startupMode: "lazy" }); - expect(config.startupMode).toBe("lazy"); +describe("resolveConfig service policies", () => { + it("uses catalog defaults and resolves explicit policies", async () => { + const config = await resolveConfig({ servicePolicies: { postgrest: "eager" } }); + expect(config.servicePolicies.postgres).toBe("eager"); + expect(config.servicePolicies.postgrest).toBe("eager"); + expect(config.servicePolicies.auth).toBe("lazy"); }); }); describe("resolveConfig state roots", () => { it("uses disposable temporary roots when direct callers omit them", async () => { - const config = await resolveConfig({ startupMode: "lazy" }); + const config = await resolveConfig(); try { expect(config.autoManagedPaths).toEqual([config.stackRoot, config.runtimeRoot]); diff --git a/packages/stack/src/effect.ts b/packages/stack/src/effect.ts index 56ca2752cf..5184b47f8f 100644 --- a/packages/stack/src/effect.ts +++ b/packages/stack/src/effect.ts @@ -5,7 +5,10 @@ export type { StackServiceStatus } from "./StackServiceState.ts"; export { StackServiceState, fromRawServiceState } from "./StackServiceState.ts"; export { + BinaryHostCompatibilityError, + BinaryManifestError, BinaryNotFoundError, + BinaryRuntimeError, ChecksumMismatchError, DockerPullError, DownloadError, @@ -17,15 +20,10 @@ export { toStackError, } from "./errors.ts"; -export type { PlatformInfo } from "./Platform.ts"; -export { - authAssetName, - detectPlatform, - postgresAssetName, - postgrestAssetName, -} from "./Platform.ts"; +export type { NativeTarget, PlatformInfo } from "./Platform.ts"; +export { detectPlatform, nativeTargetForPlatform } from "./Platform.ts"; -export type { ServiceResolution } from "./StackPreparation.ts"; +export type { ServiceResolution, StackPreparationError } from "./StackPreparation.ts"; export type { PrefetchOptions, PrefetchResult } from "./prefetch.ts"; export { prefetch } from "./prefetch.ts"; @@ -93,6 +91,8 @@ export type { ResolvedVectorConfig, ReadinessPolicy, ReadyOptions, + ServicePolicy, + ServicePolicyManifest, StackConfig, StorageConfig, StudioConfig, @@ -126,7 +126,6 @@ export { dockerImageForService, fillServiceVersionManifest, fullVersionManifest, - IMAGE_TAG_PREFIX, normalizeServiceVersion, normalizeServiceVersions, SERVICE_NAMES, diff --git a/packages/stack/src/errors.ts b/packages/stack/src/errors.ts index 4d30ce2c47..168a9fe47f 100644 --- a/packages/stack/src/errors.ts +++ b/packages/stack/src/errors.ts @@ -16,6 +16,21 @@ export class ChecksumMismatchError extends Data.TaggedError("ChecksumMismatchErr readonly actual: string; }> {} +export class BinaryManifestError extends Data.TaggedError("BinaryManifestError")<{ + readonly url: string; + readonly detail: string; +}> {} + +export class BinaryRuntimeError extends Data.TaggedError("BinaryRuntimeError")<{ + readonly path: string; + readonly detail: string; +}> {} + +export class BinaryHostCompatibilityError extends Data.TaggedError("BinaryHostCompatibilityError")<{ + readonly target: string; + readonly detail: string; +}> {} + export class DockerPullError extends Data.TaggedError("DockerPullError")<{ readonly image: string; readonly detail: string; @@ -126,6 +141,12 @@ export function toStackError(err: unknown): StackError { message: taggedMessage, cause: err, }); + case "BinaryManifestError": + return new StackError({ code: "BINARY_MANIFEST", message: taggedMessage, cause: err }); + case "BinaryRuntimeError": + return new StackError({ code: "BINARY_RUNTIME", message: taggedMessage, cause: err }); + case "BinaryHostCompatibilityError": + return new StackError({ code: "BINARY_HOST", message: taggedMessage, cause: err }); case "DownloadError": return new StackError({ code: "DOWNLOAD_ERROR", diff --git a/packages/stack/src/index.ts b/packages/stack/src/index.ts index 3969233f6f..1e8e871dd4 100644 --- a/packages/stack/src/index.ts +++ b/packages/stack/src/index.ts @@ -16,6 +16,8 @@ export type { RealtimeConfig, ReadinessPolicy, ReadyOptions, + ServicePolicy, + ServicePolicyManifest, StackConfig, StorageConfig, StudioConfig, @@ -23,7 +25,7 @@ export type { } from "./StackConfig.ts"; export type { ServiceName, VersionManifest } from "./versions.ts"; -export type { ServiceResolution } from "./StackPreparation.ts"; +export type { ServiceResolution, StackPreparationError } from "./StackPreparation.ts"; export type { PrefetchOptions, PrefetchResult } from "./prefetch.ts"; export type { StackHandle } from "./createStack.ts"; export type { diff --git a/packages/stack/src/prefetch.ts b/packages/stack/src/prefetch.ts index b68d4b5f4d..06e6f8806d 100644 --- a/packages/stack/src/prefetch.ts +++ b/packages/stack/src/prefetch.ts @@ -1,6 +1,5 @@ import { Effect } from "effect"; -import type { ChecksumMismatchError } from "./errors.ts"; -import type { DockerPullError } from "./errors.ts"; +import type { StackPreparationError } from "./StackPreparation.ts"; import { type PreparedStackArtifacts, type ServiceResolution, @@ -18,7 +17,7 @@ const toPrefetchResult = (artifacts: PreparedStackArtifacts): PrefetchResult => export const prefetch = ( options?: PrefetchOptions, -): Effect.Effect => +): Effect.Effect => Effect.gen(function* () { const preparation = yield* StackPreparation; return yield* preparation.prepare(options).pipe(Effect.map(toPrefetchResult)); diff --git a/packages/stack/src/prefetch.unit.test.ts b/packages/stack/src/prefetch.unit.test.ts index 1cf509d074..30c2316a0c 100644 --- a/packages/stack/src/prefetch.unit.test.ts +++ b/packages/stack/src/prefetch.unit.test.ts @@ -2,21 +2,18 @@ import { describe, expect, test } from "vitest"; import { Deferred, Effect, Layer, Sink, Stream } from "effect"; import { ChildProcessSpawner } from "effect/unstable/process"; import { mockBinaryResolver } from "../tests/helpers/mocks.ts"; -import { BinaryResolver } from "./BinaryResolver.ts"; import { DockerPullError } from "./errors.ts"; import { prefetch } from "./prefetch.ts"; import { ServiceDownloadFinished, ServiceDownloadStarted, + PreparationCompleted, StackPreparation, } from "./StackPreparation.ts"; -import { prepareAssetsWithDependencies } from "./StackPreparation.ts"; import { DEFAULT_VERSIONS, SERVICE_NAMES } from "./versions.ts"; const encoder = new TextEncoder(); -const defaultAuthEcrImage = `public.ecr.aws/supabase/gotrue:v${DEFAULT_VERSIONS.auth}`; -const defaultAuthDockerHubImage = `supabase/gotrue:v${DEFAULT_VERSIONS.auth}`; -const defaultAuthGhcrImage = `ghcr.io/supabase/gotrue:v${DEFAULT_VERSIONS.auth}`; +const defaultAuthGhcrImage = `ghcr.io/supabase/cli/auth:${DEFAULT_VERSIONS.auth}`; interface SpawnResult { readonly exitCode: number; @@ -40,12 +37,7 @@ function mockSequenceSpawner(results: ReadonlyArray) { index += 1; const exitDeferred = yield* Deferred.make(); - yield* Effect.forkDetach( - Effect.andThen( - Effect.sleep("1 millis"), - Deferred.succeed(exitDeferred, ChildProcessSpawner.ExitCode(result.exitCode)), - ), - ); + yield* Deferred.succeed(exitDeferred, ChildProcessSpawner.ExitCode(result.exitCode)); return ChildProcessSpawner.makeHandle({ pid: ChildProcessSpawner.ProcessId(2000 + index), @@ -90,15 +82,15 @@ describe("prefetch", () => { expect(Object.keys(result).sort()).toEqual([...SERVICE_NAMES].sort()); }); - test("falls back to Docker Hub after ECR rate limiting", async () => { + test("preparation fails with DockerPullError when the canonical image fails", async () => { const resolver = mockBinaryResolver({ failServices: ["auth"] }); + // One image inspect followed by one canonical pull. Preparation must fail + // rather than defer the pull to startup. const spawner = mockSequenceSpawner([ - { exitCode: 1 }, - { exitCode: 1 }, - { exitCode: 1 }, - { exitCode: 1, stderr: ["toomanyrequests: Rate exceeded"] }, - { exitCode: 1, stderr: ["toomanyrequests: Rate exceeded"] }, - { exitCode: 0 }, + { exitCode: 1, stderr: ["manifest unknown"] }, + { exitCode: 1, stderr: ["manifest unknown"] }, + { exitCode: 1, stderr: ["manifest unknown"] }, + { exitCode: 1, stderr: ["manifest unknown"] }, ]); const layer = StackPreparation.layer.pipe( @@ -106,199 +98,99 @@ describe("prefetch", () => { Layer.provide(spawner.layer), ); - const result = await Effect.runPromise( - prefetch({ - mode: "docker", - services: ["auth"], - }).pipe(Effect.provide(layer)), + const error = await Effect.runPromise( + prefetch({ mode: "docker", services: ["auth"] }).pipe(Effect.provide(layer), Effect.flip), ); - expect(result.auth).toEqual({ - type: "docker", - image: defaultAuthDockerHubImage, - }); - expect( - spawner.spawned.filter((record) => record.args[0] === "pull").map((record) => record.args[1]), - ).toEqual([defaultAuthEcrImage, defaultAuthEcrImage, defaultAuthDockerHubImage]); + expect(error).toBeInstanceOf(DockerPullError); + expect(spawner.spawned).toHaveLength(2); }); - test("falls back to GHCR after ECR and Docker Hub fail", async () => { - const resolver = mockBinaryResolver({ failServices: ["auth"] }); - const spawner = mockSequenceSpawner([ - { exitCode: 1 }, - { exitCode: 1 }, - { exitCode: 1 }, - { exitCode: 1, stderr: ["manifest unknown"] }, - { exitCode: 1, stderr: ["toomanyrequests: Rate exceeded"] }, - { exitCode: 1, stderr: ["toomanyrequests: Rate exceeded"] }, - { exitCode: 0 }, - ]); - + test("prefetching one service includes its required preparation dependencies", async () => { + const resolver = mockBinaryResolver(); + const spawner = mockSequenceSpawner([{ exitCode: 0 }, { exitCode: 0 }]); const layer = StackPreparation.layer.pipe( Layer.provide(resolver.layer), Layer.provide(spawner.layer), ); const result = await Effect.runPromise( - prefetch({ - mode: "docker", - services: ["auth"], - }).pipe(Effect.provide(layer)), + prefetch({ mode: "docker", services: ["postgrest"] }).pipe(Effect.provide(layer)), ); - expect(result.auth).toEqual({ - type: "docker", - image: defaultAuthGhcrImage, - }); - expect( - spawner.spawned.filter((record) => record.args[0] === "pull").map((record) => record.args[1]), - ).toEqual([ - defaultAuthEcrImage, - defaultAuthDockerHubImage, - defaultAuthDockerHubImage, - defaultAuthGhcrImage, - ]); + expect(Object.keys(result).sort()).toEqual(["postgres", "postgrest"]); }); - test("preparation fails with DockerPullError when all registry candidates fail", async () => { - const resolver = mockBinaryResolver({ failServices: ["auth"] }); - // 3 image inspects (not cached locally) followed by a non-retryable pull for - // each registry candidate (ECR, Docker Hub, GHCR). "manifest unknown" is not a - // retryable pattern, so each candidate gets exactly one pull attempt: 3 + 3 = 6 - // spawns. With the whole fallback chain failing, preparation must fail rather - // than defer the pull to startup. - const spawner = mockSequenceSpawner([ - { exitCode: 1 }, - { exitCode: 1 }, - { exitCode: 1 }, - { exitCode: 1, stderr: ["manifest unknown"] }, - { exitCode: 1, stderr: ["manifest unknown"] }, - { exitCode: 1, stderr: ["manifest unknown"] }, - ]); - + test("prefetches pgmeta using its published container tag", async () => { + const resolver = mockBinaryResolver(); + const spawner = mockSequenceSpawner([{ exitCode: 0 }, { exitCode: 0 }]); const layer = StackPreparation.layer.pipe( Layer.provide(resolver.layer), Layer.provide(spawner.layer), ); - const error = await Effect.runPromise( - prefetch({ mode: "docker", services: ["auth"] }).pipe(Effect.provide(layer), Effect.flip), + const result = await Effect.runPromise( + prefetch({ mode: "docker", services: ["pgmeta"] }).pipe(Effect.provide(layer)), ); - expect(error).toBeInstanceOf(DockerPullError); - // Guard the spawn-count assumption above: if the retry/candidate logic changes - // so more spawns occur, the mock would default the extras to success and mask - // the failure. Assert the exact count so that regresses loudly instead. - expect(spawner.spawned).toHaveLength(6); + expect(result.pgmeta).toEqual({ + type: "docker", + image: "ghcr.io/supabase/cli/pgmeta:v0.98.0", + }); }); test("does not report downloading when the docker image is already cached locally", async () => { const resolver = mockBinaryResolver({ failServices: ["auth"] }); const spawner = mockSequenceSpawner([{ exitCode: 0 }]); - const events: string[] = []; + const layer = StackPreparation.layer.pipe( + Layer.provide(resolver.layer), + Layer.provide(spawner.layer), + ); const result = await Effect.runPromise( Effect.gen(function* () { - const resolverService = yield* BinaryResolver; - const spawnerService = yield* ChildProcessSpawner.ChildProcessSpawner; - const artifacts = yield* prepareAssetsWithDependencies( - resolverService, - spawnerService, - { - mode: "docker", - services: ["auth"], - }, - (event) => - Effect.sync(() => { - if ( - event instanceof ServiceDownloadStarted || - event instanceof ServiceDownloadFinished - ) { - events.push(event._tag); - } - }), + const preparation = yield* StackPreparation; + const streamEvents = yield* preparation + .prepareEvents({ mode: "docker", services: ["auth"] }) + .pipe(Stream.runCollect); + const downloadEvents = streamEvents.flatMap((event) => + event instanceof ServiceDownloadStarted || event instanceof ServiceDownloadFinished + ? [event._tag] + : [], ); - return artifacts.resolutions; - }).pipe(Effect.provide(resolver.layer), Effect.provide(spawner.layer)), + const completed = streamEvents.find((event) => event instanceof PreparationCompleted); + expect(downloadEvents).toEqual([]); + return completed instanceof PreparationCompleted ? completed.artifacts.resolutions : {}; + }).pipe(Effect.provide(layer)), ); expect(result.auth).toEqual({ type: "docker", - image: defaultAuthEcrImage, - }); - expect(events).toEqual([]); - }); - - test("reports per-service download finished events as each service completes", async () => { - const resolver = mockBinaryResolver({ - downloadedServices: ["postgres", "postgrest", "auth"], - downloadDelaysMs: { - postgres: 10, - auth: 30, - postgrest: 50, - }, + image: defaultAuthGhcrImage, }); - const events: string[] = []; - - await Effect.runPromise( - Effect.gen(function* () { - const resolverService = yield* BinaryResolver; - const artifacts = yield* prepareAssetsWithDependencies( - resolverService, - {} as ChildProcessSpawner.ChildProcessSpawner["Service"], - { - mode: "native", - services: ["postgres", "postgrest", "auth"], - }, - (event) => - Effect.sync(() => { - switch (event._tag) { - case "ServiceDownloadStarted": - case "ServiceDownloadFinished": - events.push(`${event._tag}:${event.service}`); - break; - case "PreparationCompleted": - events.push("PreparationCompleted"); - break; - } - }), - ); - expect(Object.keys(artifacts.resolutions)).toEqual(["postgres", "postgrest", "auth"]); - }).pipe(Effect.provide(resolver.layer)), - ); - - expect(events.slice(0, 3)).toEqual([ - "ServiceDownloadStarted:postgres", - "ServiceDownloadStarted:postgrest", - "ServiceDownloadStarted:auth", - ]); - expect(events.slice(3, 6).sort()).toEqual([ - "ServiceDownloadFinished:auth", - "ServiceDownloadFinished:postgres", - "ServiceDownloadFinished:postgrest", - ]); - expect(events.at(-1)).toBe("PreparationCompleted"); }); test("uses docker for edge-runtime in auto mode even when a native binary exists", async () => { const resolver = mockBinaryResolver(); const spawner = mockSequenceSpawner([{ exitCode: 0 }]); + const layer = StackPreparation.layer.pipe( + Layer.provide(resolver.layer), + Layer.provide(spawner.layer), + ); const result = await Effect.runPromise( Effect.gen(function* () { - const resolverService = yield* BinaryResolver; - const spawnerService = yield* ChildProcessSpawner.ChildProcessSpawner; - const artifacts = yield* prepareAssetsWithDependencies(resolverService, spawnerService, { - mode: "auto", - services: ["edge-runtime"], - }); + const preparation = yield* StackPreparation; + const artifacts = yield* preparation.prepare({ mode: "auto", services: ["edge-runtime"] }); return artifacts.resolutions; - }).pipe(Effect.provide(resolver.layer), Effect.provide(spawner.layer)), + }).pipe(Effect.provide(layer)), ); expect(result["edge-runtime"]).toEqual({ type: "docker", - image: `public.ecr.aws/supabase/edge-runtime:v${DEFAULT_VERSIONS["edge-runtime"]}`, + image: `ghcr.io/supabase/cli/edge-runtime:${DEFAULT_VERSIONS["edge-runtime"]}`, }); - expect(resolver.resolved).toEqual([]); + expect(resolver.resolved).toEqual([ + { service: "postgres", version: DEFAULT_VERSIONS.postgres }, + ]); }); }); diff --git a/packages/stack/src/services/auth.ts b/packages/stack/src/services/auth.ts index 28fa6ddf98..fce6322761 100644 --- a/packages/stack/src/services/auth.ts +++ b/packages/stack/src/services/auth.ts @@ -71,7 +71,7 @@ const authHealthCheck = (port: number) => ({ export const makeAuthServiceNative = (opts: NativeAuthOptions): ServiceDef => ({ name: "auth", - command: `${opts.binPath}/auth`, + command: `${opts.binPath}/bin/auth`, env: authEnv(opts), dependencies: opts.dependencies, healthCheck: authHealthCheck(opts.authPort), diff --git a/packages/stack/src/services/postgrest.ts b/packages/stack/src/services/postgrest.ts index e9aea4ce05..0573e8b4e7 100644 --- a/packages/stack/src/services/postgrest.ts +++ b/packages/stack/src/services/postgrest.ts @@ -52,7 +52,7 @@ const postgrestHealthCheck = (port: number) => ({ export const makePostgrestService = (opts: NativePostgrestOptions): ServiceDef => ({ name: "postgrest", - command: `${opts.binPath}/postgrest`, + command: `${opts.binPath}/bin/postgrest`, env: postgrestEnv(opts), dependencies: opts.dependencies, healthCheck: postgrestHealthCheck(opts.port), diff --git a/packages/stack/src/services/services.unit.test.ts b/packages/stack/src/services/services.unit.test.ts index 6259462de5..c5d0c814d6 100644 --- a/packages/stack/src/services/services.unit.test.ts +++ b/packages/stack/src/services/services.unit.test.ts @@ -259,7 +259,7 @@ describe("makePostgrestService", () => { }); expect(def.name).toBe("postgrest"); - expect(def.command).toBe(`${POSTGREST_BIN_PATH}/postgrest`); + expect(def.command).toBe(`${POSTGREST_BIN_PATH}/bin/postgrest`); expect(def.env?.PGRST_DB_URI).toBe( `postgresql://authenticator:postgres@127.0.0.1:${DB_PORT}/postgres`, ); @@ -324,7 +324,7 @@ describe("makeAuthServiceNative", () => { }); expect(def.name).toBe("auth"); - expect(def.command).toBe(`${AUTH_BIN_PATH}/auth`); + expect(def.command).toBe(`${AUTH_BIN_PATH}/bin/auth`); expect(def.env?.GOTRUE_DB_DATABASE_URL).toContain(`127.0.0.1:${DB_PORT}`); expect(def.env?.GOTRUE_SITE_URL).toBe("http://localhost:3000"); expect(def.env?.GOTRUE_JWT_SECRET).toBe(JWT_SECRET); diff --git a/packages/stack/src/version-plan.unit.test.ts b/packages/stack/src/version-plan.unit.test.ts index afc6e400c1..0f74bd061b 100644 --- a/packages/stack/src/version-plan.unit.test.ts +++ b/packages/stack/src/version-plan.unit.test.ts @@ -16,14 +16,14 @@ describe("planStackVersions", () => { candidateBaseline: { ...DEFAULT_VERSIONS, postgres: "17.6.1.090", - postgrest: "14.5", - auth: "2.187.0", + postgrest: "v14.5", + auth: "v2.187.0", }, pinnedBaseline: { ...DEFAULT_VERSIONS, postgres: "17.6.1.090", - postgrest: "14.5", - auth: "2.187.0", + postgrest: "v14.5", + auth: "v2.187.0", }, }); }); @@ -49,13 +49,13 @@ describe("planStackVersions", () => { runtimeVersions: { ...DEFAULT_VERSIONS, postgres: "17.4.1.045", - postgrest: "14.5", - auth: "2.170.0", + postgrest: "v14.5", + auth: "v2.170.0", storage: "1.40.0", }, activeOverrides: [ { service: "postgres", version: "17.4.1.045", source: "flag" }, - { service: "auth", version: "2.170.0", source: "flag" }, + { service: "auth", version: "v2.170.0", source: "flag" }, { service: "storage", version: "1.40.0", source: "local" }, ], }); @@ -79,15 +79,15 @@ describe("planStackVersions", () => { { service: "auth", pinnedVersion: "2.188.0-rc.15", - availableVersion: "2.188.1", + availableVersion: "v2.188.1", }, { service: "storage", pinnedVersion: "1.41.8", - availableVersion: "1.43.3", + availableVersion: "v1.43.3", }, ], - updateFingerprint: "auth:2.188.0-rc.15->2.188.1|storage:1.41.8->1.43.3", + updateFingerprint: "auth:2.188.0-rc.15->v2.188.1|storage:1.41.8->v1.43.3", }); }); }); diff --git a/packages/stack/src/versions.ts b/packages/stack/src/versions.ts index 4828de5064..ac33e86b87 100644 --- a/packages/stack/src/versions.ts +++ b/packages/stack/src/versions.ts @@ -1,11 +1,6 @@ -import { - DEFAULT_VERSIONS, - SERVICE_NAMES, - dockerImageCandidatesForArtifact, - dockerImageForArtifact, - imageTagPrefixForService, -} from "./ServiceCatalog.ts"; +import { DEFAULT_VERSIONS, SERVICE_NAMES, dockerImageForArtifact } from "./ServiceCatalog.ts"; import type { ServiceName } from "./ServiceName.ts"; +import { Schema } from "effect"; export { DEFAULT_VERSIONS, SERVICE_NAMES } from "./ServiceCatalog.ts"; export type { ServiceName } from "./ServiceName.ts"; @@ -30,30 +25,13 @@ export const PartialVersionManifestSchema = Schema.Struct({ export type PartialVersionManifest = Schema.Schema.Type; -export const IMAGE_TAG_PREFIX: Partial> = Object.fromEntries( - SERVICE_NAMES.flatMap((service) => { - const prefix = imageTagPrefixForService(service); - return prefix === undefined ? [] : [[service, prefix]]; - }), -); - /** * Returns the full Docker image URL for a service. - * - * Uses the same registry resolution as the Go CLI: images are pulled from - * `public.ecr.aws/supabase/` by default (faster than Docker Hub). */ export function dockerImageForService(service: ServiceName, version: string): string { return dockerImageForArtifact(service, version); } -export function dockerImageCandidatesForService( - service: ServiceName, - version: string, -): ReadonlyArray { - return dockerImageCandidatesForArtifact(service, version); -} - function assertFullVersions( versions: Partial>, ): asserts versions is Record { @@ -71,27 +49,10 @@ export function fullVersionManifest( } /** - * Normalizes a version string for a service based on its image tag prefix. - * - * Services with a "v" prefix in IMAGE_TAG_PREFIX (e.g. postgrest, auth) store - * versions without the "v" prefix (it gets prepended at image-pull time). - * Services without a prefix entry but whose DEFAULT_VERSIONS start with "v" - * (e.g. imgproxy, mailpit) store versions with the "v" prefix. - * All other services pass through trimmed. + * Normalizes a version string without rewriting the frozen release tag. */ -export function normalizeServiceVersion(service: ServiceName, version: string): string { - const trimmed = version.trim(); - const prefix = IMAGE_TAG_PREFIX[service]; - - if (prefix === "v") { - return trimmed.replace(/^v/i, ""); - } - - if (prefix === undefined && DEFAULT_VERSIONS[service].startsWith("v")) { - return /^v/i.test(trimmed) ? `v${trimmed.slice(1)}` : `v${trimmed}`; - } - - return trimmed; +export function normalizeServiceVersion(_service: ServiceName, version: string): string { + return version.trim(); } export function normalizeServiceVersions( @@ -136,4 +97,3 @@ export function diffPinnedAndAvailableVersions( return [{ service, pinnedVersion, availableVersion }]; }); } -import { Schema } from "effect"; diff --git a/packages/stack/src/versions.unit.test.ts b/packages/stack/src/versions.unit.test.ts index bd5e584c3f..6a80fe7a15 100644 --- a/packages/stack/src/versions.unit.test.ts +++ b/packages/stack/src/versions.unit.test.ts @@ -6,7 +6,6 @@ import { import { DEFAULT_VERSIONS, diffPinnedAndAvailableVersions, - dockerImageCandidatesForService, dockerImageForService, fillServiceVersionManifest, normalizeServiceVersion, @@ -52,7 +51,7 @@ describe("syncDefaultVersionsSource", () => { 'name: "postgres",\n configKey: "example",\n defaultVersion: "17.0.0.1"', ); expect(updated).toContain( - 'name: "edge-runtime",\n configKey: "example",\n defaultVersion: "1.70.0"', + 'name: "edge-runtime",\n configKey: "example",\n defaultVersion: "v1.70.0"', ); expect(updated).toContain( 'name: "mailpit",\n configKey: "example",\n defaultVersion: "v1.2.3"', @@ -82,68 +81,60 @@ describe("dockerImageForService", () => { it("returns correct image for postgres", () => { expect(dockerImageForService("postgres", DEFAULT_VERSIONS.postgres)).toBe( - `public.ecr.aws/supabase/postgres:${DEFAULT_VERSIONS.postgres}`, + `ghcr.io/supabase/cli/postgres:${DEFAULT_VERSIONS.postgres}`, ); }); it("returns correct image for postgrest (with v prefix)", () => { expect(dockerImageForService("postgrest", DEFAULT_VERSIONS.postgrest)).toBe( - `public.ecr.aws/supabase/postgrest:v${DEFAULT_VERSIONS.postgrest}`, + `ghcr.io/supabase/cli/postgrest:${DEFAULT_VERSIONS.postgrest}`, ); }); it("returns correct image for auth (with v prefix)", () => { expect(dockerImageForService("auth", DEFAULT_VERSIONS.auth)).toBe( - `public.ecr.aws/supabase/gotrue:v${DEFAULT_VERSIONS.auth}`, + `ghcr.io/supabase/cli/auth:${DEFAULT_VERSIONS.auth}`, ); }); it("returns correct image for edge-runtime (with v prefix)", () => { expect(dockerImageForService("edge-runtime", DEFAULT_VERSIONS["edge-runtime"])).toBe( - `public.ecr.aws/supabase/edge-runtime:v${DEFAULT_VERSIONS["edge-runtime"]}`, + `ghcr.io/supabase/cli/edge-runtime:${DEFAULT_VERSIONS["edge-runtime"]}`, ); }); - it("returns ECR, Docker Hub, and GHCR candidates for Supabase-owned images", () => { - expect(dockerImageCandidatesForService("auth", DEFAULT_VERSIONS.auth)).toEqual([ - `public.ecr.aws/supabase/gotrue:v${DEFAULT_VERSIONS.auth}`, - `supabase/gotrue:v${DEFAULT_VERSIONS.auth}`, - `ghcr.io/supabase/gotrue:v${DEFAULT_VERSIONS.auth}`, - ]); - }); - - it("does not add fallback registries for third-party images", () => { - expect(dockerImageCandidatesForService("imgproxy", DEFAULT_VERSIONS.imgproxy)).toEqual([ - `darthsim/imgproxy:${DEFAULT_VERSIONS.imgproxy}`, - ]); + it("uses canonical GHCR for every service", () => { + expect(dockerImageForService("imgproxy", DEFAULT_VERSIONS.imgproxy)).toBe( + `ghcr.io/supabase/cli/imgproxy:${DEFAULT_VERSIONS.imgproxy}`, + ); }); it("keeps non-managed services Docker-only", () => { expect(SERVICE_CATALOG.imgproxy).toMatchObject({ runtimeSupport: "docker-only", - artifact: { docker: { ownership: "upstream", repository: "darthsim/imgproxy" } }, + artifact: { docker: { repository: "imgproxy" } }, }); expect(SERVICE_CATALOG.mailpit).toMatchObject({ runtimeSupport: "docker-only", - artifact: { docker: { ownership: "upstream", repository: "axllent/mailpit" } }, + artifact: { docker: { repository: "mailpit" } }, }); expect(SERVICE_CATALOG.vector).toMatchObject({ runtimeSupport: "docker-only", - artifact: { docker: { ownership: "upstream", repository: "timberio/vector" } }, + artifact: { docker: { repository: "vector" } }, }); }); }); describe("normalizeServiceVersion", () => { - it("strips v prefix for services with IMAGE_TAG_PREFIX 'v'", () => { - expect(normalizeServiceVersion("postgrest", "v14.5")).toBe("14.5"); - expect(normalizeServiceVersion("auth", "v2.188.0")).toBe("2.188.0"); - expect(normalizeServiceVersion("edge-runtime", "v1.73.0")).toBe("1.73.0"); + it("preserves frozen leading v tags", () => { + expect(normalizeServiceVersion("postgrest", "v14.5")).toBe("v14.5"); + expect(normalizeServiceVersion("auth", "v2.188.0")).toBe("v2.188.0"); + expect(normalizeServiceVersion("edge-runtime", "v1.73.0")).toBe("v1.73.0"); }); - it("ensures v prefix for services whose defaults start with v", () => { - expect(normalizeServiceVersion("mailpit", "1.30.2")).toBe("v1.30.2"); - expect(normalizeServiceVersion("imgproxy", "3.8.0")).toBe("v3.8.0"); + it("preserves explicit release tags", () => { + expect(normalizeServiceVersion("mailpit", "1.30.2")).toBe("1.30.2"); + expect(normalizeServiceVersion("imgproxy", "3.8.0")).toBe("3.8.0"); }); it("passes through other services unchanged", () => { diff --git a/packages/stack/tests/createStack-docker.e2e.test.ts b/packages/stack/tests/createStack-docker.e2e.test.ts index 8f81ad820e..8da2ae9f9f 100644 --- a/packages/stack/tests/createStack-docker.e2e.test.ts +++ b/packages/stack/tests/createStack-docker.e2e.test.ts @@ -7,6 +7,7 @@ import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { activationTimeoutSecondsForService } from "../src/ServiceActivation.ts"; import { createStack, type StackHandle } from "../src/node.ts"; import { dependencyTimeoutSecondsForServices } from "../src/services/health-budgets.ts"; +import { DEFAULT_VERSIONS } from "../src/versions.ts"; import { setupTestTable } from "./helpers/e2e.ts"; const STACK_DOCKER_E2E_TEST_TIMEOUT_MS = 180_000; @@ -37,7 +38,6 @@ dockerDescribe("createStack e2e (docker mode)", () => { stack = await createStack({ mode: "docker", - startupMode: "lazy", jwtSecret: "super-secret-jwt-token-with-at-least-32-characters-long", postgres: { dataDir }, analytics: {}, @@ -78,9 +78,11 @@ dockerDescribe("createStack e2e (docker mode)", () => { await Promise.all([stack.startService("postgrest"), stack.startService("auth")]); const runningImages = execSync("docker ps --format '{{.Image}}'").toString(); - expect(runningImages).toContain("supabase/postgrest"); - expect(runningImages).toContain("supabase/postgres"); - expect(runningImages).toContain("supabase/gotrue"); + expect(runningImages).toContain( + `ghcr.io/supabase/cli/postgrest:${DEFAULT_VERSIONS.postgrest}`, + ); + expect(runningImages).toContain(`ghcr.io/supabase/cli/postgres:${DEFAULT_VERSIONS.postgres}`); + expect(runningImages).toContain(`ghcr.io/supabase/cli/auth:${DEFAULT_VERSIONS.auth}`); const [proxyRes, authRes] = await Promise.all([ fetch(`${stack.url}/health`), @@ -104,7 +106,9 @@ dockerDescribe("createStack e2e (docker mode)", () => { const runningImages = execSync("docker ps --format '{{.Image}}'").toString(); const states = await stack.getStatus(); - expect(runningImages).toContain("supabase/edge-runtime"); + expect(runningImages).toContain( + `ghcr.io/supabase/cli/edge-runtime:${DEFAULT_VERSIONS["edge-runtime"]}`, + ); expect(states).toEqual( expect.arrayContaining([ expect.objectContaining({ name: "edge-runtime", status: "Healthy" }), @@ -133,7 +137,9 @@ dockerDescribe("createStack e2e (docker mode)", () => { stack.getStatus(), ]); - expect(runningImages).toContain("supabase/logflare"); + expect(runningImages).toContain( + `ghcr.io/supabase/cli/analytics:${DEFAULT_VERSIONS.analytics}`, + ); expect(states).toEqual( expect.arrayContaining([expect.objectContaining({ name: "analytics", status: "Healthy" })]), ); diff --git a/packages/stack/tests/createStack.e2e.test.ts b/packages/stack/tests/createStack.e2e.test.ts index 6511c750ad..f5947aafa8 100644 --- a/packages/stack/tests/createStack.e2e.test.ts +++ b/packages/stack/tests/createStack.e2e.test.ts @@ -126,38 +126,34 @@ describe("createStack e2e", () => { }, ); - test( - "supports a full PostgREST CRUD golden path", - { timeout: STACK_E2E_TEST_TIMEOUT_MS }, - async () => { - const seeded = await supabase.from("todos").select("*").order("id"); - expect(seeded.error).toBeNull(); - expect(seeded.data).toHaveLength(2); - - const inserted = await supabase - .from("todos") - .insert({ title: "E2E test todo" }) - .select() - .single(); - expect(inserted.error).toBeNull(); - expect(inserted.data?.title).toBe("E2E test todo"); - - const updated = await supabase - .from("todos") - .update({ completed: true }) - .eq("title", "E2E test todo") - .select() - .single(); - expect(updated.error).toBeNull(); - expect(updated.data?.completed).toBe(true); - - const deleted = await supabase.from("todos").delete().eq("title", "E2E test todo"); - expect(deleted.error).toBeNull(); - - const remaining = await supabase.from("todos").select("*").eq("title", "E2E test todo"); - expect(remaining.data).toHaveLength(0); - }, - ); + test("supports a full PostgREST CRUD golden path", { timeout: 30_000 }, async () => { + const seeded = await supabase.from("todos").select("*").order("id"); + expect(seeded.error).toBeNull(); + expect(seeded.data).toHaveLength(2); + + const inserted = await supabase + .from("todos") + .insert({ title: "E2E test todo" }) + .select() + .single(); + expect(inserted.error).toBeNull(); + expect(inserted.data?.title).toBe("E2E test todo"); + + const updated = await supabase + .from("todos") + .update({ completed: true }) + .eq("title", "E2E test todo") + .select() + .single(); + expect(updated.error).toBeNull(); + expect(updated.data?.completed).toBe(true); + + const deleted = await supabase.from("todos").delete().eq("title", "E2E test todo"); + expect(deleted.error).toBeNull(); + + const remaining = await supabase.from("todos").select("*").eq("title", "E2E test todo"); + expect(remaining.data).toHaveLength(0); + }); }); function writeFunction(projectDir: string, slug: string, body: string) { diff --git a/packages/stack/tests/helpers/mocks.ts b/packages/stack/tests/helpers/mocks.ts index 6017124333..d300b991a5 100644 --- a/packages/stack/tests/helpers/mocks.ts +++ b/packages/stack/tests/helpers/mocks.ts @@ -14,6 +14,7 @@ export function mockBinaryResolver( downloadDelayMs?: number; downloadDelaysMs?: Partial>; failServices?: string[]; + failOnceServices?: string[]; } = {}, ) { const resolved: Array<{ service: string; version: string }> = []; @@ -23,9 +24,10 @@ export function mockBinaryResolver( auth: `/cache/auth/${DEFAULT_VERSIONS.auth}/arm64`, "edge-runtime": `/cache/edge-runtime/${DEFAULT_VERSIONS["edge-runtime"]}/aarch64-darwin`, }; + const failOnceServices = new Set(opts.failOnceServices ?? []); const resolveWithMetadata = (spec: BinarySpec, options?: ResolveBinaryOptions) => Effect.gen(function* () { - if (opts.failServices?.includes(spec.service)) { + if (opts.failServices?.includes(spec.service) || failOnceServices.delete(spec.service)) { return yield* new BinaryNotFoundError({ service: spec.service, platform: "darwin-arm64", @@ -52,6 +54,14 @@ export function mockBinaryResolver( return { layer: Layer.succeed(BinaryResolver, { + plan: (spec) => { + const path = binaries[spec.service]; + return path + ? Effect.succeed(path) + : Effect.fail( + new BinaryNotFoundError({ service: spec.service, platform: "darwin-arm64" }), + ); + }, resolveWithMetadata, resolve: (spec) => Effect.map(resolveWithMetadata(spec), ({ path }) => path), }), From db8411aeac7175b5082207a0a7e097acb14422a3 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 18 Aug 2026 19:06:27 +0200 Subject: [PATCH 02/20] fix(stack): harden resource preparation seams --- .../src/BinaryResolver.integration.test.ts | 117 +++++++++++++-- packages/stack/src/BinaryResolver.ts | 36 ++--- packages/stack/src/LocalStack.ts | 21 ++- packages/stack/src/ServiceCatalog.ts | 29 +--- packages/stack/src/Stack.unit.test.ts | 142 +++++++++++++++++- .../StackConfigResolver.policy.unit.test.ts | 7 + packages/stack/src/StackConfigResolver.ts | 6 + packages/stack/src/StackPreparation.ts | 29 +++- packages/stack/src/prefetch.unit.test.ts | 92 +++++++++++- packages/stack/tests/helpers/mocks.ts | 2 + 10 files changed, 410 insertions(+), 71 deletions(-) diff --git a/packages/stack/src/BinaryResolver.integration.test.ts b/packages/stack/src/BinaryResolver.integration.test.ts index 3c1e5149f2..baa62c2027 100644 --- a/packages/stack/src/BinaryResolver.integration.test.ts +++ b/packages/stack/src/BinaryResolver.integration.test.ts @@ -1,5 +1,6 @@ import { createHash } from "node:crypto"; import { execFileSync } from "node:child_process"; +import { zstdCompressSync } from "node:zlib"; import { existsSync, mkdirSync, @@ -16,6 +17,7 @@ import { NodeServices } from "@effect/platform-node"; import { describe, expect, it } from "@effect/vitest"; import { Effect, Layer } from "effect"; import { HttpClient, HttpClientResponse } from "effect/unstable/http"; +import { ChildProcessSpawner } from "effect/unstable/process"; import { BinaryResolver } from "./BinaryResolver.ts"; import { BinaryManifestError, @@ -35,40 +37,50 @@ const makeFixture = ( includePostgrest = true, ) => { const source = join(root, "source"); + const tar = join(root, "postgrest.tar"); const archive = join(root, "postgrest.tar.zst"); rmSync(source, { recursive: true, force: true }); mkdirSync(join(source, "bin"), { recursive: true }); if (includePostgrest) { writeFileSync(join(source, "bin", "postgrest"), "#!/bin/sh\necho postgrest\n"); } - execFileSync("tar", ["--zstd", "-cf", archive, "-C", source, "."]); + execFileSync("tar", ["-cf", tar, "-C", source, "."]); + writeFileSync(archive, zstdCompressSync(readFileSync(tar))); return { archive: readFileSync(archive), manifestOverride }; }; const makeTraversalFixture = (root: string) => { const source = join(root, "traversal-source"); + const tar = join(root, "postgrest-traversal.tar"); const archive = join(root, "postgrest-traversal.tar.zst"); rmSync(source, { recursive: true, force: true }); mkdirSync(source, { recursive: true }); writeFileSync(join(root, "outside.txt"), "must not extract\n"); - execFileSync("tar", ["--zstd", "-cf", archive, "-C", source, "../outside.txt"]); + execFileSync("tar", ["-cf", tar, "-C", source, "../outside.txt"]); + writeFileSync(archive, zstdCompressSync(readFileSync(tar))); return { archive: readFileSync(archive), manifestOverride: {} }; }; const makeEscapingSymlinkFixture = (root: string) => { const source = join(root, "symlink-source"); + const tar = join(root, "postgrest-symlink.tar"); const archive = join(root, "postgrest-symlink.tar.zst"); rmSync(source, { recursive: true, force: true }); mkdirSync(join(source, "bin"), { recursive: true }); symlinkSync("/bin/sh", join(source, "bin/postgrest")); - execFileSync("tar", ["--zstd", "-cf", archive, "-C", source, "."]); + execFileSync("tar", ["-cf", tar, "-C", source, "."]); + writeFileSync(archive, zstdCompressSync(readFileSync(tar))); return { archive: readFileSync(archive), manifestOverride: {} }; }; const makeResolverLayer = ( cacheRoot: string, fixture: ReturnType, - options: { readonly checksum?: string } = {}, + options: { + readonly checksum?: string; + readonly checksumText?: string; + readonly spawnedCommands?: Array<{ command: string; args: ReadonlyArray }>; + } = {}, ) => { const client = HttpClient.make((request) => Effect.sync(() => { @@ -96,19 +108,35 @@ const makeResolverLayer = ( } if (request.url.endsWith("SHA256SUMS")) { const hash = options.checksum ?? createHash("sha256").update(fixture.archive).digest("hex"); - return HttpClientResponse.fromWeb( - request, - new Response( - `${hash} postgrest-${DEFAULT_VERSIONS.postgrest}-${process.platform === "darwin" ? "darwin-arm64" : "linux-amd64"}.tar.zst\n`, - { status: 200 }, - ), - ); + const checksumText = + options.checksumText ?? + `${hash} postgrest-${DEFAULT_VERSIONS.postgrest}-${process.platform === "darwin" ? "darwin-arm64" : "linux-amd64"}.tar.zst\n`; + return HttpClientResponse.fromWeb(request, new Response(checksumText, { status: 200 })); } return HttpClientResponse.fromWeb(request, new Response(fixture.archive, { status: 200 })); }), ); + const spawnerLayer = + options.spawnedCommands === undefined + ? NodeServices.layer + : Layer.effect( + ChildProcessSpawner.ChildProcessSpawner, + Effect.gen(function* () { + const delegate = yield* ChildProcessSpawner.ChildProcessSpawner; + return ChildProcessSpawner.make((command) => { + if (command._tag === "StandardCommand") { + options.spawnedCommands?.push({ + command: command.command, + args: command.args, + }); + } + return delegate.spawn(command); + }); + }), + ).pipe(Layer.provide(NodeServices.layer)); return BinaryResolver.make(cacheRoot).pipe( Layer.provide(Layer.succeed(HttpClient.HttpClient, client)), + Layer.provide(spawnerLayer), Layer.provide(NodeServices.layer), ); }; @@ -150,6 +178,27 @@ describe("BinaryResolver slim-services installer", () => { }), ); + it.live("installs slim archives without requiring an external zstd executable", () => + Effect.gen(function* () { + const root = makeRoot(); + try { + const spawnedCommands: Array<{ command: string; args: ReadonlyArray }> = []; + const resolverLayer = makeResolverLayer(root, makeFixture(root), { spawnedCommands }); + yield* Effect.gen(function* () { + const resolver = yield* BinaryResolver; + yield* resolver.resolve({ + service: "postgrest", + version: DEFAULT_VERSIONS.postgrest, + }); + }).pipe(Effect.provide(resolverLayer)); + + expect(spawnedCommands.some(({ args }) => args.includes("--zstd"))).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }), + ); + it.live("accepts the Mailpit-style glibc runtime requirement", () => Effect.gen(function* () { if (process.platform !== "linux") return; @@ -386,4 +435,50 @@ describe("BinaryResolver slim-services installer", () => { } }), ); + + it.live("fails closed when SHA256SUMS has no entry for the requested archive", () => + Effect.gen(function* () { + const root = makeRoot(); + try { + const fixture = makeFixture(root); + const unrelated = createHash("sha256").update(fixture.archive).digest("hex"); + const resolverLayer = makeResolverLayer(root, fixture, { + checksumText: `${unrelated} unrelated.sbom.spdx.json\n`, + }); + const failure = yield* Effect.gen(function* () { + const resolver = yield* BinaryResolver; + return yield* resolver.resolve({ + service: "postgrest", + version: DEFAULT_VERSIONS.postgrest, + }); + }).pipe(Effect.provide(resolverLayer), Effect.flip); + + expect(failure).toBeInstanceOf(BinaryManifestError); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }), + ); + + it.live("accepts uppercase SHA256SUMS digests", () => + Effect.gen(function* () { + const root = makeRoot(); + try { + const fixture = makeFixture(root); + const hash = createHash("sha256").update(fixture.archive).digest("hex").toUpperCase(); + const resolverLayer = makeResolverLayer(root, fixture, { checksum: hash }); + const resolved = yield* Effect.gen(function* () { + const resolver = yield* BinaryResolver; + return yield* resolver.resolve({ + service: "postgrest", + version: DEFAULT_VERSIONS.postgrest, + }); + }).pipe(Effect.provide(resolverLayer)); + + expect(existsSync(join(resolved, "bin/postgrest"))).toBe(true); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }), + ); }); diff --git a/packages/stack/src/BinaryResolver.ts b/packages/stack/src/BinaryResolver.ts index 5ba3b59887..521e240dc8 100644 --- a/packages/stack/src/BinaryResolver.ts +++ b/packages/stack/src/BinaryResolver.ts @@ -1,4 +1,5 @@ import { createHash } from "node:crypto"; +import { zstdDecompressSync } from "node:zlib"; import { Context, Effect, FileSystem, Layer, Option, Path, Result } from "effect"; import { HttpClient } from "effect/unstable/http"; import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; @@ -71,11 +72,6 @@ const cachePath = (baseDir: string, info: AssetInfo): string => const CACHE_COMPLETE_MARKER = ".complete"; const STALE_STAGING_AGE_MS = 24 * 60 * 60 * 1_000; -const extractCommand = (archivePath: string, destDir: string): string[] => { - const args = ["tar", "--zstd", "-xf", archivePath, "-C", destDir]; - return args; -}; - const hasTraversalSegment = (value: string): boolean => value.split(/[\\/]/).some((segment) => segment === ".."); @@ -104,7 +100,7 @@ const verifyChecksum = ( Effect.sync(() => { const actual = createHash("sha256").update(new Uint8Array(data)).digest("hex"); // The .sha256 file typically contains "hex filename" or just "hex" - const expectedHex = expected.trim().split(/\s+/)[0] ?? ""; + const expectedHex = (expected.trim().split(/\s+/)[0] ?? "").toLowerCase(); return { actual, expectedHex }; }).pipe( Effect.flatMap(({ actual, expectedHex }) => { @@ -118,10 +114,11 @@ const verifyChecksum = ( const checksumForArchive = (contents: string, archiveName: string): string | undefined => { for (const line of contents.split(/\r?\n/)) { const match = line.trim().match(/^([a-f0-9]{64})\s+[* ]?(.+)$/i); - if (match?.[2] === archiveName || match?.[2]?.endsWith(`/${archiveName}`)) return match[1]; + if (match?.[2] === archiveName || match?.[2]?.endsWith(`/${archiveName}`)) { + return match[1]?.toLowerCase(); + } } - const single = contents.trim().match(/^([a-f0-9]{64})(?:\s|$)/i); - return single?.[1]; + return undefined; }; const manifestError = (url: string, detail: string): BinaryManifestError => @@ -584,11 +581,15 @@ export class BinaryResolver extends Context.Service< } yield* verifyChecksum(tarball, expected, release.checksumUrl); - const archivePath = path.join(destination, `_download.${release.archive}`); - yield* fs.writeFile(archivePath, new Uint8Array(tarball)); + const archivePath = path.join(destination, "_download.tar"); + const archive = yield* Effect.try({ + try: () => zstdDecompressSync(new Uint8Array(tarball)), + catch: (cause) => new DownloadError({ url: release.downloadUrl, cause }), + }); + yield* fs.writeFile(archivePath, archive); const members = yield* spawner - .string(ChildProcess.make("tar", ["--zstd", "-tf", archivePath])) + .string(ChildProcess.make("tar", ["-tf", archivePath])) .pipe( Effect.catch((cause) => Effect.fail( @@ -612,17 +613,8 @@ export class BinaryResolver extends Context.Service< ); } - const [command, ...args] = extractCommand(archivePath, destination); - if (command === undefined) { - return yield* Effect.fail( - new DownloadError({ - url: release.downloadUrl, - cause: new Error("No extraction command was configured"), - }), - ); - } const exitCode = yield* spawner - .exitCode(ChildProcess.make(command, args)) + .exitCode(ChildProcess.make("tar", ["-xf", archivePath, "-C", destination])) .pipe( Effect.catchTag("PlatformError", (cause) => Effect.fail(new DownloadError({ url: release.downloadUrl, cause })), diff --git a/packages/stack/src/LocalStack.ts b/packages/stack/src/LocalStack.ts index f212a457d1..fef4fe4f03 100644 --- a/packages/stack/src/LocalStack.ts +++ b/packages/stack/src/LocalStack.ts @@ -260,6 +260,7 @@ export const localStackLayer = ( .plan({ mode: config.mode, services: enabledServicesForConfig(config), + enabledServices, versions: versionsForConfig(config), }) .pipe( @@ -314,6 +315,7 @@ export const localStackLayer = ( preparation.prepareEvents({ mode: config.mode, services: pending, + enabledServices, versions: versionsForConfig(config), }), () => ({ resolutions: {} }) satisfies PreparedStackArtifacts, @@ -704,11 +706,11 @@ export const localStackLayer = ( ); return; } + yield* prepareServices([service]); const started = yield* Effect.gen(function* () { yield* requireRunningPhase; const concurrentlyStarted = yield* inspectStartedTargets(service); if (concurrentlyStarted !== undefined) return concurrentlyStarted; - yield* prepareServices([service]); return yield* beginStartTargets(service, new Set()); }).pipe(withLifecycleLock); yield* waitForTargets(started).pipe((effect) => @@ -797,10 +799,11 @@ export const localStackLayer = ( dispose: disposeOnce, startService: (name) => Effect.gen(function* () { + yield* requireMutable(`start service ${name}`); + const service = yield* requireKnownServiceName(name); + yield* prepareServices([service]); const started = yield* Effect.gen(function* () { yield* requireMutable(`start service ${name}`); - const service = yield* requireKnownServiceName(name); - yield* prepareServices([service]); return yield* beginStartTargets( service, new Set(lifecycleTargetsForService(enabledServices, service)), @@ -822,9 +825,11 @@ export const localStackLayer = ( }).pipe(withLifecycleLock), restartService: (name) => Effect.gen(function* () { + yield* requireMutable(`restart service ${name}`); + const service = yield* requireKnownServiceName(name); + yield* prepareServices([service]); const started = yield* Effect.gen(function* () { yield* requireMutable(`restart service ${name}`); - const service = yield* requireKnownServiceName(name); const runtime = yield* ensureRuntime; yield* runtime.orchestrator.restartService(service, serviceStartOptions); return { runtime, targets: [service] }; @@ -833,9 +838,11 @@ export const localStackLayer = ( }).pipe(cleanupOnReadinessFailure), reloadFunctions: (opts) => Effect.gen(function* () { + yield* requireMutable("reload functions"); + yield* requireKnownService("edge-runtime"); + yield* prepareServices(["edge-runtime"]); const started = yield* Effect.gen(function* () { yield* requireMutable("reload functions"); - yield* requireKnownService("edge-runtime"); const currentBundle = yield* Ref.get(functionsBundleRef); const nextBundle = opts?.functions === undefined @@ -857,9 +864,11 @@ export const localStackLayer = ( }).pipe(cleanupOnReadinessFailure), reloadEdgeRuntime: (opts) => Effect.gen(function* () { + yield* requireMutable("reload Edge Runtime"); + yield* requireKnownService("edge-runtime"); + yield* prepareServices(["edge-runtime"]); const started = yield* Effect.gen(function* () { yield* requireMutable("reload Edge Runtime"); - yield* requireKnownService("edge-runtime"); const nextConfig = yield* configWithEdgeRuntimeOptions(opts); const currentBundle = yield* Ref.get(functionsBundleRef); const nextBundle = diff --git a/packages/stack/src/ServiceCatalog.ts b/packages/stack/src/ServiceCatalog.ts index f1244d02e4..be4e29c25f 100644 --- a/packages/stack/src/ServiceCatalog.ts +++ b/packages/stack/src/ServiceCatalog.ts @@ -118,17 +118,6 @@ const preparation = ( dependencies, }); -const genericNativeRelease = (service: ServiceName) => ({ - provider: "github.com/supabase/slim-services", - resolve: (version: string, platform: PlatformInfo) => - nativeRelease(service, version, platform, { - // These services are prepared generically before native process wiring - // exists. Their manifest-declared paths are authoritative; do not guess - // a consumer path that could make a valid archive fail installation. - requiredRuntimePaths: [], - }), -}); - /** * Exhaustive static identity and capability metadata for public stack services. * Cross-service topology and process definitions deliberately remain in StackBuilder. @@ -204,7 +193,6 @@ export const SERVICE_CATALOG = { runtimeSupport: "docker-only", artifact: { docker: { repository: "edge-runtime" }, - native: genericNativeRelease("edge-runtime"), }, activation: { activates: [], owns: [] }, preparation: preparation(["lazy", "eager"], "lazy", ["postgres"]), @@ -217,7 +205,6 @@ export const SERVICE_CATALOG = { runtimeSupport: "docker-only", artifact: { docker: { repository: "realtime" }, - native: genericNativeRelease("realtime"), }, activation: { activates: [], owns: [] }, preparation: preparation(["eager"], "eager", ["postgres"]), @@ -230,10 +217,9 @@ export const SERVICE_CATALOG = { runtimeSupport: "docker-only", artifact: { docker: { repository: "storage" }, - native: genericNativeRelease("storage"), }, activation: { activates: ["imgproxy"], owns: ["imgproxy"] }, - preparation: preparation(["lazy", "eager"], "lazy", ["postgres"]), + preparation: preparation(["lazy", "eager"], "lazy", ["postgres", "imgproxy"]), portFields: ["storagePort"], }, imgproxy: { @@ -243,10 +229,9 @@ export const SERVICE_CATALOG = { runtimeSupport: "docker-only", artifact: { docker: { repository: "imgproxy" }, - native: genericNativeRelease("imgproxy"), }, activation: { activates: [], owns: [] }, - preparation: preparation(["lazy", "eager"], "lazy", ["storage"]), + preparation: preparation(["lazy", "eager"], "lazy"), portFields: ["imgproxyPort"], }, mailpit: { @@ -256,7 +241,6 @@ export const SERVICE_CATALOG = { runtimeSupport: "docker-only", artifact: { docker: { repository: "mailpit" }, - native: genericNativeRelease("mailpit"), }, activation: { activates: [], owns: [] }, preparation: preparation(["eager"], "eager"), @@ -269,7 +253,6 @@ export const SERVICE_CATALOG = { runtimeSupport: "docker-only", artifact: { docker: { repository: "pgmeta", tagPrefix: "v" }, - native: genericNativeRelease("pgmeta"), }, activation: { activates: [], owns: [] }, preparation: preparation(["lazy", "eager"], "lazy", ["postgres"]), @@ -282,7 +265,6 @@ export const SERVICE_CATALOG = { runtimeSupport: "docker-only", artifact: { docker: { repository: "studio" }, - native: genericNativeRelease("studio"), }, activation: { activates: ["analytics"], owns: [] }, preparation: preparation(["eager"], "eager", ["pgmeta", "analytics"]), @@ -295,10 +277,9 @@ export const SERVICE_CATALOG = { runtimeSupport: "docker-only", artifact: { docker: { repository: "analytics" }, - native: genericNativeRelease("analytics"), }, activation: { activates: ["vector"], owns: ["vector"] }, - preparation: preparation(["lazy", "eager"], "lazy", ["postgres"]), + preparation: preparation(["lazy", "eager"], "lazy", ["postgres", "vector"]), portFields: ["analyticsPort"], }, vector: { @@ -308,10 +289,9 @@ export const SERVICE_CATALOG = { runtimeSupport: "docker-only", artifact: { docker: { repository: "vector" }, - native: genericNativeRelease("vector"), }, activation: { activates: [], owns: [] }, - preparation: preparation(["lazy", "eager"], "lazy", ["analytics"]), + preparation: preparation(["lazy", "eager"], "lazy"), portFields: [], }, pooler: { @@ -321,7 +301,6 @@ export const SERVICE_CATALOG = { runtimeSupport: "docker-only", artifact: { docker: { repository: "pooler" }, - native: genericNativeRelease("pooler"), }, activation: { activates: [], owns: [] }, preparation: preparation(["eager"], "eager", ["postgres"]), diff --git a/packages/stack/src/Stack.unit.test.ts b/packages/stack/src/Stack.unit.test.ts index 6847135891..ffdd2ef9f0 100644 --- a/packages/stack/src/Stack.unit.test.ts +++ b/packages/stack/src/Stack.unit.test.ts @@ -151,8 +151,8 @@ function setupLayer( config: ResolvedStackConfig = defaultConfig, portLease: PortLease = noopPortLease(config.ports), spawner = mockChildProcessSpawner(), + resolver = mockBinaryResolver(), ) { - const resolver = mockBinaryResolver(); const stackPreparationLayer = StackPreparation.layer.pipe(Layer.provide(resolver.layer)); const layer = localStackLayer(config, portLease).pipe( Layer.provide(StackBuilder.layer), @@ -294,6 +294,59 @@ describe("Stack", () => { ); }); + const coldEdgeRuntimeReload = (operation: "reloadFunctions" | "reloadEdgeRuntime") => + Effect.gen(function* () { + const runtimeRoot = mkdtempSync(join(tmpdir(), `supabase-${operation}-`)); + const edgeRuntimeSpawnStarted = yield* Deferred.make(); + const image = `ghcr.io/supabase/cli/edge-runtime:${DEFAULT_VERSIONS["edge-runtime"]}`; + const spawner = mockChildProcessSpawner({ + beforeSpawn: ({ command, args }) => + command === process.execPath && + args.some((arg) => Buffer.from(arg, "base64url").toString().includes(image)) + ? Deferred.succeed(edgeRuntimeSpawnStarted, undefined).pipe(Effect.asVoid) + : Effect.void, + }); + const config = { + ...edgeRuntimeConfig, + runtimeRoot, + projectDir: runtimeRoot, + servicePolicies: { + ...edgeRuntimeConfig.servicePolicies, + "edge-runtime": "lazy", + }, + } satisfies ResolvedStackConfig; + const { layer } = setupLayer(config, noopPortLease(config.ports), spawner); + + yield* Effect.gen(function* () { + const stack = yield* Stack; + const reloading = yield* ( + operation === "reloadFunctions" + ? stack.reloadFunctions() + : stack.reloadEdgeRuntime({ edgeRuntime: {} }) + ).pipe(Effect.forkChild({ startImmediately: true })); + yield* Deferred.await(edgeRuntimeSpawnStarted); + + expect(spawner.spawned).toContainEqual({ + command: "docker", + args: ["image", "inspect", image], + }); + + yield* Fiber.interrupt(reloading); + yield* stack.dispose(); + }).pipe( + Effect.provide(layer), + Effect.ensuring(Effect.promise(() => rm(runtimeRoot, { recursive: true, force: true }))), + ); + }); + + it.live("prepares dormant Edge Runtime before reloading Functions", () => + coldEdgeRuntimeReload("reloadFunctions").pipe(Effect.scoped, Effect.timeout("5 seconds")), + ); + + it.live("prepares dormant Edge Runtime before reloading its configuration", () => + coldEdgeRuntimeReload("reloadEdgeRuntime").pipe(Effect.scoped, Effect.timeout("5 seconds")), + ); + it.effect("getInfo returns valid JWT tokens", () => { const { layer } = setupLayer(); @@ -675,6 +728,50 @@ describe("Stack", () => { }).pipe(Effect.provide(layer), Effect.timeout("5 seconds")); }); + it.live("prepares a dormant service before restarting it", () => + Effect.gen(function* () { + const postgrestSpawnStarted = yield* Deferred.make(); + const spawner = mockChildProcessSpawner({ + beforeSpawn: ({ args }) => + args.some((arg) => + Buffer.from(arg, "base64url").toString().includes('"command":"/cache/postgrest/'), + ) + ? Deferred.succeed(postgrestSpawnStarted, undefined).pipe(Effect.asVoid) + : Effect.void, + }); + const resolver = mockBinaryResolver({ downloadedServices: ["postgrest"] }); + const config = { + ...defaultConfig, + servicePolicies: { + ...defaultConfig.servicePolicies, + postgrest: "lazy", + auth: "lazy", + }, + } satisfies ResolvedStackConfig; + const { layer } = setupLayer(config, noopPortLease(config.ports), spawner, resolver); + + yield* Effect.gen(function* () { + const stack = yield* Stack; + yield* stack.start(); + const stateChanges = yield* stack.stateChanges("postgrest"); + const downloading = yield* stateChanges.pipe( + Stream.filter((state) => state.status === "Downloading"), + Stream.runHead, + Effect.forkChild({ startImmediately: true }), + ); + const restarting = yield* stack + .restartService("postgrest") + .pipe(Effect.forkChild({ startImmediately: true })); + yield* Deferred.await(postgrestSpawnStarted); + + expect((yield* Fiber.join(downloading))._tag).toBe("Some"); + + yield* Fiber.interrupt(restarting); + yield* stack.stop(); + }).pipe(Effect.provide(layer)); + }).pipe(Effect.scoped, Effect.timeout("5 seconds")), + ); + it.live("lazy activation honors explicitly stopped transitive dependencies", () => { const config: ResolvedStackConfig = { ...defaultConfig, @@ -900,6 +997,49 @@ describe("Stack", () => { }).pipe(Effect.scoped, Effect.timeout("5 seconds")), ); + it.live("dispose is not blocked by in-flight lazy preparation", () => + Effect.gen(function* () { + const preparationStarted = yield* Deferred.make(); + const allowPreparation = yield* Deferred.make(); + const disposed = yield* Deferred.make(); + const resolver = mockBinaryResolver({ + downloadedServices: ["auth"], + beforeResolve: ({ service }) => + service === "auth" + ? Deferred.succeed(preparationStarted, undefined).pipe( + Effect.andThen(Deferred.await(allowPreparation)), + ) + : Effect.void, + }); + const config = { + ...defaultConfig, + servicePolicies: { ...defaultConfig.servicePolicies, postgrest: "lazy", auth: "lazy" }, + } satisfies ResolvedStackConfig; + const lease = { + ...noopPortLease(config.ports), + releaseAll: Deferred.succeed(disposed, undefined).pipe(Effect.asVoid), + } satisfies PortLease; + const { layer } = setupLayer(config, lease, mockChildProcessSpawner(), resolver); + + yield* Effect.gen(function* () { + const stack = yield* Stack; + const activator = yield* StackServiceActivator; + yield* stack.start(); + const activation = yield* activator + .activate("auth") + .pipe(Effect.forkChild({ startImmediately: true })); + yield* Deferred.await(preparationStarted); + + const disposing = yield* stack.dispose().pipe(Effect.forkChild({ startImmediately: true })); + yield* Deferred.await(disposed); + yield* Fiber.join(disposing); + + yield* Deferred.succeed(allowPreparation, undefined); + yield* Fiber.interrupt(activation); + }).pipe(Effect.provide(layer)); + }).pipe(Effect.scoped, Effect.timeout("5 seconds")), + ); + it.live("allows a finite wait override against an infinite stack policy", () => Effect.gen(function* () { const spawnStarted = yield* Deferred.make(); diff --git a/packages/stack/src/StackConfigResolver.policy.unit.test.ts b/packages/stack/src/StackConfigResolver.policy.unit.test.ts index b6bd0d08f6..890689885d 100644 --- a/packages/stack/src/StackConfigResolver.policy.unit.test.ts +++ b/packages/stack/src/StackConfigResolver.policy.unit.test.ts @@ -38,4 +38,11 @@ describe("resolved service preparation policies", () => { expect(config.postgrest).toBe(false); expect(config.servicePolicies.postgrest).toBe("off"); }); + + it("rejects a preparation policy for a service that is not configured", async () => { + await expect(resolveConfig({ servicePolicies: { realtime: "eager" } })).rejects.toMatchObject({ + _tag: "StackBuildError", + reason: "invalid_config", + }); + }); }); diff --git a/packages/stack/src/StackConfigResolver.ts b/packages/stack/src/StackConfigResolver.ts index dccc80c3d8..85d1d6459c 100644 --- a/packages/stack/src/StackConfigResolver.ts +++ b/packages/stack/src/StackConfigResolver.ts @@ -417,6 +417,12 @@ const resolveServicePolicies = (config: StackConfig): ServicePolicyManifest => { } const enabled = rawServiceEnabled(config, service); + if (!enabled && requested !== undefined && requested !== "off") { + throw new StackBuildError({ + detail: `${service} cannot use the ${requested} service preparation policy because the service is not configured`, + reason: "invalid_config", + }); + } if (!enabled || requested === "off") { policies[service] = "off"; continue; diff --git a/packages/stack/src/StackPreparation.ts b/packages/stack/src/StackPreparation.ts index 50d0ee3322..8087276085 100644 --- a/packages/stack/src/StackPreparation.ts +++ b/packages/stack/src/StackPreparation.ts @@ -15,12 +15,11 @@ import { BinaryResolver } from "./BinaryResolver.ts"; import type { BinaryHostCompatibilityError, BinaryManifestError, - BinaryNotFoundError, BinaryRuntimeError, ChecksumMismatchError, DownloadError, } from "./errors.ts"; -import { DockerPullError, isDockerDaemonDownMessage } from "./errors.ts"; +import { BinaryNotFoundError, DockerPullError, isDockerDaemonDownMessage } from "./errors.ts"; import { isDockerOnlyService, requiredPreparationDependencies } from "./ServiceCatalog.ts"; import { DEFAULT_VERSIONS, @@ -43,6 +42,7 @@ export type ServiceResolution = export interface StackPreparationInput { readonly versions?: Partial; readonly services?: ReadonlyArray; + readonly enabledServices?: ReadonlyArray; readonly mode?: "native" | "auto" | "docker"; } @@ -104,9 +104,14 @@ const resolveDockerImageForService = ( ): Effect.Effect => pullImage(spawner, dockerImageForService(service, version), callbacks); -const preparationClosure = (services: ReadonlyArray): ReadonlyArray => { +const preparationClosure = ( + services: ReadonlyArray, + enabledServices?: ReadonlyArray, +): ReadonlyArray => { + const enabled = enabledServices === undefined ? undefined : new Set(enabledServices); const closure = new Set(); const add = (service: ServiceName): void => { + if (enabled !== undefined && !enabled.has(service)) return; if (closure.has(service)) return; closure.add(service); for (const dependency of requiredPreparationDependencies(service)) add(dependency); @@ -116,7 +121,7 @@ const preparationClosure = (services: ReadonlyArray): ReadonlyArray }; const selectedServices = (input?: StackPreparationInput): ReadonlyArray => - preparationClosure(input?.services ?? SERVICE_NAMES); + preparationClosure(input?.services ?? SERVICE_NAMES, input?.enabledServices); const plannedResolution = ( resolver: BinaryResolver["Service"], @@ -130,7 +135,21 @@ const plannedResolution = ( image: dockerImageForService(service, version), }); } - return resolver.plan({ service, version }).pipe(Effect.map((path) => ({ type: "binary", path }))); + if (isDockerOnlyService(service)) { + return Effect.fail(new BinaryNotFoundError({ service, platform: "native" })); + } + const native = resolver + .plan({ service, version }) + .pipe(Effect.map((path): ServiceResolution => ({ type: "binary", path }))); + if (mode === "native") return native; + return native.pipe( + Effect.catchTag("BinaryNotFoundError", () => + Effect.succeed({ + type: "docker" as const, + image: dockerImageForService(service, version), + }), + ), + ); }; const planAssetsWithDependencies = ( diff --git a/packages/stack/src/prefetch.unit.test.ts b/packages/stack/src/prefetch.unit.test.ts index 30c2316a0c..8b4e015c2f 100644 --- a/packages/stack/src/prefetch.unit.test.ts +++ b/packages/stack/src/prefetch.unit.test.ts @@ -2,7 +2,7 @@ import { describe, expect, test } from "vitest"; import { Deferred, Effect, Layer, Sink, Stream } from "effect"; import { ChildProcessSpawner } from "effect/unstable/process"; import { mockBinaryResolver } from "../tests/helpers/mocks.ts"; -import { DockerPullError } from "./errors.ts"; +import { BinaryNotFoundError, DockerPullError } from "./errors.ts"; import { prefetch } from "./prefetch.ts"; import { ServiceDownloadFinished, @@ -121,6 +121,96 @@ describe("prefetch", () => { expect(Object.keys(result).sort()).toEqual(["postgres", "postgrest"]); }); + test("prefetching storage includes the companion it starts", async () => { + const resolver = mockBinaryResolver(); + const spawner = mockSequenceSpawner([{ exitCode: 0 }, { exitCode: 0 }, { exitCode: 0 }]); + const layer = StackPreparation.layer.pipe( + Layer.provide(resolver.layer), + Layer.provide(spawner.layer), + ); + + const result = await Effect.runPromise( + prefetch({ mode: "docker", services: ["storage"] }).pipe(Effect.provide(layer)), + ); + + expect(Object.keys(result).sort()).toEqual(["imgproxy", "postgres", "storage"]); + }); + + test("prefetching a companion does not pull its owner", async () => { + const resolver = mockBinaryResolver(); + const spawner = mockSequenceSpawner([{ exitCode: 0 }]); + const layer = StackPreparation.layer.pipe( + Layer.provide(resolver.layer), + Layer.provide(spawner.layer), + ); + + const result = await Effect.runPromise( + prefetch({ mode: "docker", services: ["imgproxy"] }).pipe(Effect.provide(layer)), + ); + + expect(Object.keys(result)).toEqual(["imgproxy"]); + }); + + test("does not prepare dependencies that are disabled in the stack", async () => { + const resolver = mockBinaryResolver(); + const spawner = mockSequenceSpawner([{ exitCode: 0 }, { exitCode: 0 }, { exitCode: 0 }]); + const layer = StackPreparation.layer.pipe( + Layer.provide(resolver.layer), + Layer.provide(spawner.layer), + ); + + const result = await Effect.runPromise( + Effect.gen(function* () { + const preparation = yield* StackPreparation; + return yield* preparation.prepare({ + mode: "docker", + services: ["studio"], + enabledServices: ["postgres", "pgmeta", "studio"], + }); + }).pipe(Effect.provide(layer)), + ); + + expect(Object.keys(result.resolutions).sort()).toEqual(["pgmeta", "postgres", "studio"]); + }); + + test("auto mode falls back to Docker when a service has no native target", async () => { + const resolver = mockBinaryResolver({ + binaries: { postgres: "/cache/postgres/native" }, + }); + const spawner = mockSequenceSpawner([{ exitCode: 0 }]); + const layer = StackPreparation.layer.pipe( + Layer.provide(resolver.layer), + Layer.provide(spawner.layer), + ); + + const result = await Effect.runPromise( + prefetch({ mode: "auto", services: ["postgrest"] }).pipe(Effect.provide(layer)), + ); + + expect(result.postgrest).toEqual({ + type: "docker", + image: `ghcr.io/supabase/cli/postgrest:${DEFAULT_VERSIONS.postgrest}`, + }); + }); + + test("native mode rejects services that have no native runtime", async () => { + const resolver = mockBinaryResolver(); + const spawner = mockSequenceSpawner([]); + const layer = StackPreparation.layer.pipe( + Layer.provide(resolver.layer), + Layer.provide(spawner.layer), + ); + + const error = await Effect.runPromise( + prefetch({ mode: "native", services: ["edge-runtime"] }).pipe( + Effect.provide(layer), + Effect.flip, + ), + ); + + expect(error).toBeInstanceOf(BinaryNotFoundError); + }); + test("prefetches pgmeta using its published container tag", async () => { const resolver = mockBinaryResolver(); const spawner = mockSequenceSpawner([{ exitCode: 0 }, { exitCode: 0 }]); diff --git a/packages/stack/tests/helpers/mocks.ts b/packages/stack/tests/helpers/mocks.ts index d300b991a5..efce61a07f 100644 --- a/packages/stack/tests/helpers/mocks.ts +++ b/packages/stack/tests/helpers/mocks.ts @@ -15,6 +15,7 @@ export function mockBinaryResolver( downloadDelaysMs?: Partial>; failServices?: string[]; failOnceServices?: string[]; + beforeResolve?: (spec: BinarySpec) => Effect.Effect; } = {}, ) { const resolved: Array<{ service: string; version: string }> = []; @@ -44,6 +45,7 @@ export function mockBinaryResolver( const downloaded = opts.downloadedServices?.includes(spec.service) ?? false; if (downloaded) { yield* options?.onDownloadStart ?? Effect.void; + yield* opts.beforeResolve?.(spec) ?? Effect.void; const delayMs = opts.downloadDelaysMs?.[spec.service] ?? opts.downloadDelayMs ?? 0; if (delayMs > 0) { yield* Effect.sleep(`${delayMs} millis`); From 044024b711561128296b3e0e84f560afaa0682b9 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 18 Aug 2026 20:14:07 +0200 Subject: [PATCH 03/20] fix(stack): select one execution mode --- .../branches/switch/switch.handler.ts | 2 +- .../functions/dev/functions-dev-runtime.ts | 3 +- .../commands/link/link.integration.test.ts | 4 +- .../src/next/commands/start/start.command.ts | 16 ++-- .../src/next/commands/start/start.handler.ts | 2 +- .../commands/start/start.integration.test.ts | 2 +- .../next/commands/status/status.handler.ts | 2 +- .../next/commands/update/update.handler.ts | 2 +- apps/cli/src/next/config/stack-config.ts | 6 +- .../src/next/config/stack-config.unit.test.ts | 20 ++-- apps/cli/tests/helpers/running-stack.ts | 6 +- packages/stack/README.md | 6 +- packages/stack/docs/architecture.md | 18 ++-- .../src/ContainerRuntime.integration.test.ts | 94 +++++++++++++++++++ packages/stack/src/ContainerRuntime.ts | 41 ++++++++ packages/stack/src/LocalStack.ts | 6 ++ packages/stack/src/ServicePorts.ts | 2 +- packages/stack/src/Stack.unit.test.ts | 10 +- packages/stack/src/StackBuilder.ts | 44 ++++++++- packages/stack/src/StackBuilder.unit.test.ts | 54 ++++------- packages/stack/src/StackConfig.ts | 6 +- packages/stack/src/StackConfigResolver.ts | 12 ++- packages/stack/src/StackPreparation.ts | 58 +++++++----- packages/stack/src/bun.ts | 18 +++- packages/stack/src/cleanup.ts | 15 ++- .../stack/src/createStack.integration.test.ts | 1 + packages/stack/src/createStack.ts | 10 +- packages/stack/src/createStack.unit.test.ts | 21 ++++- packages/stack/src/effect.ts | 1 + packages/stack/src/functions.unit.test.ts | 8 +- packages/stack/src/index.ts | 1 + packages/stack/src/layers.ts | 4 +- ...aged-manager-lifecycle.integration.test.ts | 4 +- packages/stack/src/managed/document.ts | 17 +++- packages/stack/src/managed/lifecycle.ts | 20 ++-- packages/stack/src/managed/manager.ts | 9 +- packages/stack/src/node.ts | 18 +++- packages/stack/src/prefetch.ts | 14 ++- packages/stack/src/prefetch.unit.test.ts | 50 ++++++++-- packages/stack/src/services/analytics.ts | 9 +- packages/stack/src/services/auth.ts | 9 +- packages/stack/src/services/docker-cleanup.ts | 11 ++- packages/stack/src/services/edge-runtime.ts | 10 +- packages/stack/src/services/imgproxy.ts | 10 +- packages/stack/src/services/mailpit.ts | 10 +- packages/stack/src/services/pgmeta.ts | 9 +- packages/stack/src/services/pooler.ts | 9 +- packages/stack/src/services/postgres.ts | 24 +++-- packages/stack/src/services/postgrest.ts | 9 +- packages/stack/src/services/realtime.ts | 9 +- packages/stack/src/services/service-utils.ts | 19 +++- .../stack/src/services/services.unit.test.ts | 15 +++ packages/stack/src/services/storage.ts | 9 +- packages/stack/src/services/studio.ts | 9 +- packages/stack/src/services/vector.ts | 5 +- .../stack/src/supervisor.integration.test.ts | 6 +- packages/stack/src/supervisor.ts | 39 ++++++-- .../tests/createStack-native.e2e.test.ts | 45 +++++++++ 58 files changed, 685 insertions(+), 208 deletions(-) create mode 100644 packages/stack/src/ContainerRuntime.integration.test.ts create mode 100644 packages/stack/src/ContainerRuntime.ts create mode 100644 packages/stack/tests/createStack-native.e2e.test.ts diff --git a/apps/cli/src/next/commands/branches/switch/switch.handler.ts b/apps/cli/src/next/commands/branches/switch/switch.handler.ts index fbf53b6bc0..843e812ea4 100644 --- a/apps/cli/src/next/commands/branches/switch/switch.handler.ts +++ b/apps/cli/src/next/commands/branches/switch/switch.handler.ts @@ -147,7 +147,7 @@ export const switchBranch = Effect.fn("branches.switch")(function* (opts: { // `pull` does not exist yet. const launchConfig = stackCheck.value.launch === undefined - ? toStartStackConfig([], "auto") + ? toStartStackConfig([], undefined) : withServiceVersions( toStartStackConfig( stackCheck.value.launch.excludedServices?.filter( diff --git a/apps/cli/src/next/commands/functions/dev/functions-dev-runtime.ts b/apps/cli/src/next/commands/functions/dev/functions-dev-runtime.ts index 680c67417e..815e406985 100644 --- a/apps/cli/src/next/commands/functions/dev/functions-dev-runtime.ts +++ b/apps/cli/src/next/commands/functions/dev/functions-dev-runtime.ts @@ -55,7 +55,7 @@ const startFullStack = Effect.fnUntraced(function* (opts: FunctionsDevStackOptio const serviceVersionContext = yield* resolveServiceVersionContext([], undefined); const loadedProjectConfig = yield* loadProjectConfig(projectHome.projectRoot); const stackConfig = withServiceVersions( - toStartStackConfig([], "auto"), + toStartStackConfig([], undefined), serviceVersionContext.runtimeVersions, ); const stackLayer = yield* daemonLayer({ @@ -65,7 +65,6 @@ const startFullStack = Effect.fnUntraced(function* (opts: FunctionsDevStackOptio name: opts.stack, edgeRuntime: opts.edgeRuntime, launch: { - mode: "auto", versions: serviceVersionContext.pinnedBaseline, excludedServices: [], }, diff --git a/apps/cli/src/next/commands/link/link.integration.test.ts b/apps/cli/src/next/commands/link/link.integration.test.ts index 5af28a05f9..943cae7945 100644 --- a/apps/cli/src/next/commands/link/link.integration.test.ts +++ b/apps/cli/src/next/commands/link/link.integration.test.ts @@ -346,7 +346,7 @@ describe("link handler", () => { }, ], behavior: { - mode: "auto", + mode: "docker", placeholder: "Search projects...", maxItems: 10, }, @@ -515,7 +515,7 @@ describe("link handler", () => { }, ], behavior: { - mode: "auto", + mode: "docker", placeholder: "Search projects...", maxItems: 10, }, diff --git a/apps/cli/src/next/commands/start/start.command.ts b/apps/cli/src/next/commands/start/start.command.ts index fbcd3043b6..38262c1814 100644 --- a/apps/cli/src/next/commands/start/start.command.ts +++ b/apps/cli/src/next/commands/start/start.command.ts @@ -1,4 +1,4 @@ -import { Effect, Layer, Context } from "effect"; +import { Effect, Layer, Context, Option } from "effect"; import { loadProjectConfig } from "@supabase/config"; import { DEFAULT_MANAGED_STACK_NAME, @@ -79,9 +79,10 @@ export const serviceVersionFlag = Flag.string("service-version").pipe( const modeFlag = Flag.choice("mode", startModes).pipe( Flag.withDescription( - 'Stack startup mode. "auto" prefers native binaries and falls back to Docker, "native" requires native-compatible services, and "docker" forces Docker for all services.', + 'Stack startup mode. "native" requires native-compatible services and "docker" requires a usable Docker or Podman runtime.', ), - Flag.withDefault("auto" as StartMode), + Flag.optional, + Flag.map(Option.getOrUndefined), ); interface StartVersionStateShape { @@ -124,7 +125,7 @@ export type StartFlags = CliCommand.Command.Config.Infer; export const startCommand = Command.make("start", flags).pipe( Command.withDescription( "Start the local Supabase development stack.\n\n" + - "Starts the full local Supabase stack. Use --mode auto (default) to prefer native binaries and fall back to Docker, --mode native to require native-compatible services, or --mode docker to force Docker-backed startup.\n\n" + + "Starts the full local Supabase stack. By default, a usable Docker or Podman runtime selects Docker mode; otherwise the stack uses native mode. Use --mode to require one explicitly.\n\n" + "Named CLI stacks persist managed runtime state under the Supabase home directory. Use --exclude to skip optional services. Use --detach to run in the background.", ), Command.withShortDescription("Start local Supabase stack"), @@ -219,7 +220,7 @@ export const startCommand = Command.make("start", flags).pipe( portDocument: portIntents, }); const launch = { - mode: flags.mode, + ...(flags.mode === undefined ? {} : { mode: flags.mode }), versions: serviceVersionContext.pinnedBaseline, excludedServices: flags.exclude, ...(existingSummary?.lastNotifiedUpdateFingerprint === undefined @@ -242,12 +243,15 @@ export const startCommand = Command.make("start", flags).pipe( cwd: runtimeInfo.cwd, name: flags.stack, }); + if (summary.launch === undefined) { + return yield* Effect.die("Managed stack started without persisted launch settings"); + } return { stackLayer, startVersionState: StartVersionState.of({ launch: { - mode: flags.mode, + mode: summary.launch.mode, versions: serviceVersionContext.pinnedBaseline, excludedServices: flags.exclude, }, diff --git a/apps/cli/src/next/commands/start/start.handler.ts b/apps/cli/src/next/commands/start/start.handler.ts index deafa672bf..5b6c5303e0 100644 --- a/apps/cli/src/next/commands/start/start.handler.ts +++ b/apps/cli/src/next/commands/start/start.handler.ts @@ -68,7 +68,7 @@ export const start = Effect.fnUntraced(function* (flags: StartFlags) { } yield* analytics.capture("cli_stack_started", { - mode: flags.mode, + mode: launch.mode, detach: flags.detach, stack: flags.stack, }); diff --git a/apps/cli/src/next/commands/start/start.integration.test.ts b/apps/cli/src/next/commands/start/start.integration.test.ts index e98b31af2a..920bfba263 100644 --- a/apps/cli/src/next/commands/start/start.integration.test.ts +++ b/apps/cli/src/next/commands/start/start.integration.test.ts @@ -80,7 +80,7 @@ describe("start handler", () => { ); return start({ stack: fixture.stackName, - mode: "auto", + mode: "docker", exclude: [], serviceVersion: [], detach: false, diff --git a/apps/cli/src/next/commands/status/status.handler.ts b/apps/cli/src/next/commands/status/status.handler.ts index 546c89b549..f11f2a1e54 100644 --- a/apps/cli/src/next/commands/status/status.handler.ts +++ b/apps/cli/src/next/commands/status/status.handler.ts @@ -43,7 +43,7 @@ const resolveConfiguredSummary = Effect.fnUntraced(function* (input: { return yield* resolveStackSummary({ ...input, portDocument: managedPortIntents( - toStartStackConfig(excluded, current.launch?.mode ?? "auto"), + toStartStackConfig(excluded, current.launch?.mode), loaded ?? undefined, ), }); diff --git a/apps/cli/src/next/commands/update/update.handler.ts b/apps/cli/src/next/commands/update/update.handler.ts index c5ad206314..4aed1af93a 100644 --- a/apps/cli/src/next/commands/update/update.handler.ts +++ b/apps/cli/src/next/commands/update/update.handler.ts @@ -104,7 +104,7 @@ export const update = Effect.fnUntraced(function* (flags: UpdateFlags) { if (Option.isSome(existingSummary)) { const persistedLaunch = existingSummary.value.launch ?? { - mode: "auto" as const, + mode: "docker" as const, excludedServices: [] as const, }; yield* updateManagedLaunch({ diff --git a/apps/cli/src/next/config/stack-config.ts b/apps/cli/src/next/config/stack-config.ts index 0469f96b51..e68b5873db 100644 --- a/apps/cli/src/next/config/stack-config.ts +++ b/apps/cli/src/next/config/stack-config.ts @@ -17,16 +17,16 @@ export const excludedStackServices = [ export type ExcludedStackService = (typeof excludedStackServices)[number]; export const isExcludedStackService = (value: string): value is ExcludedStackService => excludedStackServices.some((candidate) => candidate === value); -export const startModes = ["native", "auto", "docker"] as const; +export const startModes = ["native", "docker"] as const; export type StartMode = (typeof startModes)[number]; export function toStartStackConfig( exclude: ReadonlyArray, - mode: StartMode, + mode?: StartMode, ): StackConfig { const excluded = new Set(exclude); return { - mode, + ...(mode === undefined ? {} : { mode }), realtime: excluded.has("realtime") ? false : {}, storage: excluded.has("storage") ? false : {}, imgproxy: excluded.has("imgproxy") || excluded.has("storage") ? false : {}, diff --git a/apps/cli/src/next/config/stack-config.unit.test.ts b/apps/cli/src/next/config/stack-config.unit.test.ts index f11f460a1a..0ebc50d531 100644 --- a/apps/cli/src/next/config/stack-config.unit.test.ts +++ b/apps/cli/src/next/config/stack-config.unit.test.ts @@ -2,9 +2,13 @@ import { describe, expect, it } from "vitest"; import { toStartStackConfig, withServiceVersions } from "./stack-config.ts"; describe("toStartStackConfig", () => { + it("leaves mode unset so the stack package can select the usable runtime", () => { + expect(toStartStackConfig([], undefined)).not.toHaveProperty("mode"); + }); + it("uses the requested runtime mode and catalog service defaults", () => { - expect(toStartStackConfig([], "auto")).toMatchObject({ - mode: "auto", + expect(toStartStackConfig([], "docker")).toMatchObject({ + mode: "docker", }); expect(toStartStackConfig([], "docker")).toMatchObject({ mode: "docker", @@ -15,12 +19,12 @@ describe("toStartStackConfig", () => { }); it("dedupes excluded services when building stack config", () => { - expect(toStartStackConfig(["auth", "auth"], "auto")).toMatchObject({ - mode: "auto", + expect(toStartStackConfig(["auth", "auth"], "docker")).toMatchObject({ + mode: "docker", auth: false, }); - expect(toStartStackConfig(["auth", "postgrest"], "auto")).toMatchObject({ - mode: "auto", + expect(toStartStackConfig(["auth", "postgrest"], "docker")).toMatchObject({ + mode: "docker", auth: false, postgrest: false, }); @@ -30,7 +34,7 @@ describe("toStartStackConfig", () => { describe("withServiceVersions", () => { it("injects linked service versions without re-enabling excluded services", () => { expect( - withServiceVersions(toStartStackConfig([], "auto"), { + withServiceVersions(toStartStackConfig([], "docker"), { postgres: "17.6.1.090", postgrest: "14.5", auth: "2.187.0", @@ -46,7 +50,7 @@ describe("withServiceVersions", () => { }); expect( - withServiceVersions(toStartStackConfig(["auth", "storage"], "auto"), { + withServiceVersions(toStartStackConfig(["auth", "storage"], "docker"), { postgres: "17.6.1.090", auth: "2.187.0", storage: "1.39.2", diff --git a/apps/cli/tests/helpers/running-stack.ts b/apps/cli/tests/helpers/running-stack.ts index 0ac4e85d51..45e58edf4d 100644 --- a/apps/cli/tests/helpers/running-stack.ts +++ b/apps/cli/tests/helpers/running-stack.ts @@ -24,7 +24,11 @@ import { CliConfig } from "../../src/next/config/cli-config.service.ts"; import { ProjectHome } from "../../src/next/config/project-home.service.ts"; import { RuntimeInfo } from "../../src/shared/runtime/runtime-info.service.ts"; -const launch = { mode: "auto" as const, versions: { postgres: "17.6.1" }, excludedServices: [] }; +const launch = { + mode: "docker" as const, + versions: { postgres: "17.6.1" }, + excludedServices: [], +}; const portDocument: ManagedPortIntentDocument = { activeFields: ["apiPort", "dbPort"], document: {}, diff --git a/packages/stack/README.md b/packages/stack/README.md index b324a54241..88e4db6183 100644 --- a/packages/stack/README.md +++ b/packages/stack/README.md @@ -15,7 +15,9 @@ console.log((await stack.getInfo()).url); `createStack` resolves configuration, reserves ports, and builds a scoped handle. `stack.start()` starts services; disposing the handle stops them and -releases its lease. +releases its lease. When `mode` is omitted, creation uses Docker mode with a +usable Docker or Podman service and otherwise selects native mode. An explicit +mode never falls back to the other one. ## Managed stack @@ -47,7 +49,7 @@ const runtime = projectDir: projectRoot, name: "default", portIntents, - launch: { mode: "auto", versions: {}, excludedServices: [] }, + launch: { mode: "docker", versions: {}, excludedServices: [] }, }); ``` diff --git a/packages/stack/docs/architecture.md b/packages/stack/docs/architecture.md index 3ff0b4e3bd..14d1c0bc2b 100644 --- a/packages/stack/docs/architecture.md +++ b/packages/stack/docs/architecture.md @@ -226,17 +226,17 @@ requests for the same resource share one installation or pull. Native assets come from the pinned slim-services release contract and are checksum- and manifest-verified before atomic publication. Docker assets use one canonical `ghcr.io/supabase/cli/:` reference; a locally cached image is -reused and a missing image is pulled without registry fallback. `mode: -"native"` rejects Docker-only services, `mode: "docker"` resolves every service -to Docker, and `mode: "auto"` selects the catalog-supported runtime without -falling back after a preparation failure. +reused and a missing image is pulled without registry fallback. Stack creation +selects exactly one execution mode: a usable Docker daemon is preferred, then a +usable Podman service; if neither responds, the stack uses native mode. An +explicit `mode: "native"` rejects Docker-only services, while explicit `mode: +"docker"` requires a usable container runtime. Preparation never falls back to +the other mode after that choice. The `StackBuilder` turns planned resolutions into one process-compose graph, -so a stack can run native and Docker-backed services together without eagerly -materializing every graph resource. Docker resources are namespaced with the -managed stack id; when native Postgres is combined with Docker services, the -graph supplies the platform-specific host address so the containers can reach -it. +without eagerly materializing every graph resource. Container resources are +namespaced with the managed stack id and every pull, launch, health check, and +cleanup uses the selected Docker or Podman executable. `ApiProxy` listens on the configured public `apiPort` and routes Supabase API paths (`/auth`, `/rest`, `/functions`, `/realtime`, `/storage`, `/pg`, diff --git a/packages/stack/src/ContainerRuntime.integration.test.ts b/packages/stack/src/ContainerRuntime.integration.test.ts new file mode 100644 index 0000000000..b2ccc51852 --- /dev/null +++ b/packages/stack/src/ContainerRuntime.integration.test.ts @@ -0,0 +1,94 @@ +import { describe, expect, it } from "@effect/vitest"; +import { Deferred, Effect, Layer, Sink, Stream } from "effect"; +import { ChildProcessSpawner } from "effect/unstable/process"; +import { selectStackRuntime } from "./ContainerRuntime.ts"; + +const runtimeSpawner = (availability: Readonly>) => { + const commands: string[] = []; + return { + commands, + layer: Layer.succeed( + ChildProcessSpawner.ChildProcessSpawner, + ChildProcessSpawner.make((command) => + Effect.gen(function* () { + const executable = command._tag === "StandardCommand" ? command.command : ""; + commands.push(executable); + const exitCode = yield* Deferred.make(); + yield* Deferred.succeed( + exitCode, + ChildProcessSpawner.ExitCode(availability[executable] === true ? 0 : 1), + ); + return ChildProcessSpawner.makeHandle({ + pid: ChildProcessSpawner.ProcessId(1), + stdout: Stream.empty, + stderr: Stream.empty, + all: Stream.empty, + exitCode: Deferred.await(exitCode), + isRunning: Effect.succeed(false), + stdin: Sink.drain, + kill: () => Effect.void, + unref: Effect.succeed(Effect.void), + getInputFd: () => Sink.drain, + getOutputFd: () => Stream.empty, + }); + }), + ), + ), + }; +}; + +describe("stack runtime selection", () => { + it.effect("uses Docker mode when the Docker daemon is usable", () => { + const spawner = runtimeSpawner({ docker: true }); + return Effect.gen(function* () { + expect(yield* selectStackRuntime()).toEqual({ + mode: "docker", + containerRuntime: "docker", + }); + expect(spawner.commands).toEqual(["docker"]); + }).pipe(Effect.provide(spawner.layer)); + }); + + it.effect("uses Podman for Docker mode when Docker is unavailable", () => { + const spawner = runtimeSpawner({ podman: true }); + return Effect.gen(function* () { + expect(yield* selectStackRuntime()).toEqual({ + mode: "docker", + containerRuntime: "podman", + }); + expect(spawner.commands).toEqual(["docker", "podman"]); + }).pipe(Effect.provide(spawner.layer)); + }); + + it.effect("uses native mode when no container runtime is usable", () => { + const spawner = runtimeSpawner({}); + return Effect.gen(function* () { + expect(yield* selectStackRuntime()).toEqual({ + mode: "native", + containerRuntime: null, + }); + }).pipe(Effect.provide(spawner.layer)); + }); + + it.effect("does not probe container runtimes when native mode is explicit", () => { + const spawner = runtimeSpawner({ docker: true }); + return Effect.gen(function* () { + expect(yield* selectStackRuntime("native")).toEqual({ + mode: "native", + containerRuntime: null, + }); + expect(spawner.commands).toEqual([]); + }).pipe(Effect.provide(spawner.layer)); + }); + + it.effect("rejects explicit Docker mode when neither runtime is usable", () => { + const spawner = runtimeSpawner({}); + return Effect.gen(function* () { + const error = yield* selectStackRuntime("docker").pipe(Effect.flip); + expect(error).toMatchObject({ + _tag: "StackBuildError", + reason: "docker_not_running", + }); + }).pipe(Effect.provide(spawner.layer)); + }); +}); diff --git a/packages/stack/src/ContainerRuntime.ts b/packages/stack/src/ContainerRuntime.ts new file mode 100644 index 0000000000..e4a897a21d --- /dev/null +++ b/packages/stack/src/ContainerRuntime.ts @@ -0,0 +1,41 @@ +import { Effect, Exit } from "effect"; +import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; +import { StackBuildError } from "./errors.ts"; +import type { StackMode } from "./StackConfig.ts"; + +export type ContainerRuntime = "docker" | "podman"; + +export type StackRuntimeSelection = + | { readonly mode: "native"; readonly containerRuntime: null } + | { readonly mode: "docker"; readonly containerRuntime: ContainerRuntime }; + +export const selectStackRuntime = ( + requestedMode?: StackMode, +): Effect.Effect => + Effect.gen(function* () { + if (requestedMode === "native") { + return { mode: "native", containerRuntime: null }; + } + + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const runtimes: ReadonlyArray = ["docker", "podman"]; + for (const runtime of runtimes) { + const result = yield* Effect.exit( + spawner.exitCode(ChildProcess.make(runtime, ["info"])).pipe(Effect.timeout("3 seconds")), + ); + if (Exit.isSuccess(result) && result.value === 0) { + return { mode: "docker", containerRuntime: runtime }; + } + } + + if (requestedMode === "docker") { + return yield* Effect.fail( + new StackBuildError({ + detail: "Docker mode requires a usable Docker or Podman runtime", + reason: "docker_not_running", + }), + ); + } + + return { mode: "native", containerRuntime: null }; + }); diff --git a/packages/stack/src/LocalStack.ts b/packages/stack/src/LocalStack.ts index fef4fe4f03..6b6da842c2 100644 --- a/packages/stack/src/LocalStack.ts +++ b/packages/stack/src/LocalStack.ts @@ -259,6 +259,9 @@ export const localStackLayer = ( const effect = preparation .plan({ mode: config.mode, + ...(config.containerRuntime === null + ? {} + : { containerRuntime: config.containerRuntime }), services: enabledServicesForConfig(config), enabledServices, versions: versionsForConfig(config), @@ -314,6 +317,9 @@ export const localStackLayer = ( const effect = Stream.runFoldEffect( preparation.prepareEvents({ mode: config.mode, + ...(config.containerRuntime === null + ? {} + : { containerRuntime: config.containerRuntime }), services: pending, enabledServices, versions: versionsForConfig(config), diff --git a/packages/stack/src/ServicePorts.ts b/packages/stack/src/ServicePorts.ts index d5a7be4ece..5674f87ce9 100644 --- a/packages/stack/src/ServicePorts.ts +++ b/packages/stack/src/ServicePorts.ts @@ -8,7 +8,7 @@ const serviceEnabledForConfig = (config: StackConfig, service: keyof typeof SERV return config[service === "postgres" ? "postgres" : service] !== false; } if (service === "edge-runtime") { - const mode = config.mode ?? "auto"; + const mode = config.mode ?? "native"; return ( !(mode === "native" && config.edgeRuntime === undefined) && config.edgeRuntime !== false && diff --git a/packages/stack/src/Stack.unit.test.ts b/packages/stack/src/Stack.unit.test.ts index ffdd2ef9f0..c090d285e6 100644 --- a/packages/stack/src/Stack.unit.test.ts +++ b/packages/stack/src/Stack.unit.test.ts @@ -52,6 +52,7 @@ const defaultConfig: ResolvedStackConfig = { runtimeRoot: "/tmp/supabase-runtime", projectDir: "/tmp/supabase-project", mode: "native", + containerRuntime: null, servicePolicies: { postgres: "eager", postgrest: "eager", @@ -114,7 +115,8 @@ const defaultConfig: ResolvedStackConfig = { const edgeRuntimeConfig: ResolvedStackConfig = { ...defaultConfig, - mode: "auto", + mode: "docker", + containerRuntime: "docker", servicePolicies: { ...defaultConfig.servicePolicies, "edge-runtime": "eager" }, edgeRuntime: { enabled: true, @@ -775,7 +777,8 @@ describe("Stack", () => { it.live("lazy activation honors explicitly stopped transitive dependencies", () => { const config: ResolvedStackConfig = { ...defaultConfig, - mode: "auto", + mode: "docker", + containerRuntime: "docker", servicePolicies: { ...defaultConfig.servicePolicies, postgrest: "lazy", @@ -908,7 +911,8 @@ describe("Stack", () => { const mailpitReleaseStarted = yield* Deferred.make(); const config = { ...defaultConfig, - mode: "auto", + mode: "docker", + containerRuntime: "docker", servicePolicies: { ...defaultConfig.servicePolicies, postgrest: "lazy", diff --git a/packages/stack/src/StackBuilder.ts b/packages/stack/src/StackBuilder.ts index e089cded05..4959d7bb6e 100644 --- a/packages/stack/src/StackBuilder.ts +++ b/packages/stack/src/StackBuilder.ts @@ -101,6 +101,17 @@ export const validateResolvedConfig = ( config: ResolvedStackConfig, ): Effect.Effect => Effect.gen(function* () { + if ( + (config.mode === "native" && config.containerRuntime !== null) || + (config.mode === "docker" && config.containerRuntime === null) + ) { + return yield* Effect.fail( + new StackBuildError({ + detail: `Resolved ${config.mode} mode has an inconsistent container runtime`, + reason: "invalid_config", + }), + ); + } if (config.instanceId !== undefined && !INSTANCE_ID_PATTERN.test(config.instanceId)) { return yield* Effect.fail( new StackBuildError({ @@ -117,7 +128,7 @@ export const validateResolvedConfig = ( if (enabledDockerOnly.length > 0) { return yield* Effect.fail( new StackBuildError({ - detail: `mode "native" only supports postgres, auth, and postgrest. Disable ${enabledDockerOnly.join(", ")} or switch to "auto" or "docker".`, + detail: `mode "native" only supports postgres, auth, and postgrest. Disable ${enabledDockerOnly.join(", ")} or use "docker" mode.`, reason: "invalid_config", }), ); @@ -220,6 +231,17 @@ export class StackBuilder extends Context.Service< Effect.gen(function* () { yield* validateResolvedConfig(config); + const requireContainerRuntime = Effect.suspend(() => + config.containerRuntime === null + ? Effect.fail( + new StackBuildError({ + detail: "A Docker service requires a selected container runtime", + reason: "invalid_config", + }), + ) + : Effect.succeed(config.containerRuntime), + ); + const platform = yield* detectPlatform; const serviceHost = dockerHostAddress(platform.os); const projectDir = config.projectDir; @@ -285,6 +307,7 @@ export class StackBuilder extends Context.Service< dependencies: [], }) : makePostgresServiceDocker({ + runtime: yield* requireContainerRuntime, image: postgresResolution.image, dataDir: config.postgres.dataDir, port: config.dbPort, @@ -326,6 +349,7 @@ export class StackBuilder extends Context.Service< dependencies: postgresDeps, }) : makePostgrestServiceDocker({ + runtime: yield* requireContainerRuntime, image: postgrestResolution.image, dbHost: serviceHost, dbPort: config.dbPort, @@ -362,6 +386,7 @@ export class StackBuilder extends Context.Service< dependencies: postgresDeps, }) : makeAuthServiceDocker({ + runtime: yield* requireContainerRuntime, image: authResolution.image, dbHost: serviceHost, dbPort: config.dbPort, @@ -396,6 +421,7 @@ export class StackBuilder extends Context.Service< dependencies: postgresDeps, }) : makeEdgeRuntimeServiceDocker({ + runtime: yield* requireContainerRuntime, image: edgeRuntimeResolution.image, identity, runtimeRoot: config.runtimeRoot, @@ -416,6 +442,7 @@ export class StackBuilder extends Context.Service< const mailpitImage = yield* requirePreparedDockerImage(prepared, "mailpit"); defs.push({ ...makeMailpitServiceDocker({ + runtime: yield* requireContainerRuntime, image: mailpitImage, identity, webPort: config.mailpit.port, @@ -432,6 +459,7 @@ export class StackBuilder extends Context.Service< const realtimeImage = yield* requirePreparedDockerImage(prepared, "realtime"); defs.push({ ...makeRealtimeServiceDocker({ + runtime: yield* requireContainerRuntime, image: realtimeImage, port: config.realtime.port, identity, @@ -455,6 +483,7 @@ export class StackBuilder extends Context.Service< const storageImage = yield* requirePreparedDockerImage(prepared, "storage"); defs.push({ ...makeStorageServiceDocker({ + runtime: yield* requireContainerRuntime, image: storageImage, port: config.storage.port, identity, @@ -484,6 +513,7 @@ export class StackBuilder extends Context.Service< const imgproxyImage = yield* requirePreparedDockerImage(prepared, "imgproxy"); defs.push({ ...makeImgproxyServiceDocker({ + runtime: yield* requireContainerRuntime, image: imgproxyImage, port: config.imgproxy.port, identity, @@ -500,6 +530,7 @@ export class StackBuilder extends Context.Service< const pgmetaImage = yield* requirePreparedDockerImage(prepared, "pgmeta"); defs.push({ ...makePgmetaServiceDocker({ + runtime: yield* requireContainerRuntime, image: pgmetaImage, identity, port: config.pgmeta.port, @@ -517,6 +548,7 @@ export class StackBuilder extends Context.Service< const analyticsImage = yield* requirePreparedDockerImage(prepared, "analytics"); defs.push({ ...makeAnalyticsServiceDocker({ + runtime: yield* requireContainerRuntime, image: analyticsImage, identity, hostPort: config.analytics.port, @@ -537,6 +569,7 @@ export class StackBuilder extends Context.Service< const vectorImage = yield* requirePreparedDockerImage(prepared, "vector"); defs.push({ ...makeVectorServiceDocker({ + runtime: yield* requireContainerRuntime, image: vectorImage, identity, serviceHost, @@ -554,6 +587,7 @@ export class StackBuilder extends Context.Service< const poolerImage = yield* requirePreparedDockerImage(prepared, "pooler"); defs.push({ ...makePoolerServiceDocker({ + runtime: yield* requireContainerRuntime, image: poolerImage, identity, hostAdminPort: config.pooler.apiPort, @@ -580,6 +614,7 @@ export class StackBuilder extends Context.Service< const studioImage = yield* requirePreparedDockerImage(prepared, "studio"); defs.push({ ...makeStudioServiceDocker({ + runtime: yield* requireContainerRuntime, image: studioImage, identity, port: config.studio.port, @@ -611,7 +646,12 @@ export class StackBuilder extends Context.Service< } const dockerContainerNames = SERVICE_NAMES.filter((service) => - defs.some((def) => def.name === service && def.command === "docker"), + defs.some( + (def) => + def.name === service && + config.containerRuntime !== null && + def.command === config.containerRuntime, + ), ).map((service) => dockerContainerName(service, identity.key)); const graph = yield* buildGraph(defs).pipe( diff --git a/packages/stack/src/StackBuilder.unit.test.ts b/packages/stack/src/StackBuilder.unit.test.ts index 074f82b60f..6d86be764a 100644 --- a/packages/stack/src/StackBuilder.unit.test.ts +++ b/packages/stack/src/StackBuilder.unit.test.ts @@ -47,7 +47,8 @@ const baseConfig: ResolvedStackConfig = { stackRoot: "/tmp/supabase-stack", runtimeRoot: "/tmp/supabase-runtime", projectDir: "/tmp/supabase-project", - mode: "auto", + mode: "native", + containerRuntime: null, servicePolicies: { postgres: "eager", postgrest: "eager", @@ -111,6 +112,7 @@ const baseConfig: ResolvedStackConfig = { const dockerConfig: ResolvedStackConfig = { ...baseConfig, mode: "docker", + containerRuntime: "docker", }; /** @@ -133,7 +135,8 @@ const siblingManagedConfig: ResolvedStackConfig = { const edgeRuntimeConfig: ResolvedStackConfig = { ...baseConfig, - mode: "auto", + mode: "docker", + containerRuntime: "docker", servicePolicies: { ...baseConfig.servicePolicies, "edge-runtime": "eager" }, edgeRuntime: { enabled: true, @@ -292,14 +295,15 @@ describe("StackBuilder", () => { }).pipe(Effect.provide(layer)); }); - it.effect("docker mode produces Docker service defs for all services", () => { + it.effect("Docker mode consistently uses the selected container runtime", () => { const resolver = mockBinaryResolver(); const layer = builderLayer(resolver); return Effect.gen(function* () { const builder = yield* StackBuilder; const preparation = yield* StackPreparation; - const { graph, cleanupTargets } = yield* prepareAndBuild(builder, preparation, dockerConfig); + const config = { ...dockerConfig, containerRuntime: "podman" } satisfies ResolvedStackConfig; + const { graph, cleanupTargets } = yield* prepareAndBuild(builder, preparation, config); expect(graph.startOrder.length).toBe(3); @@ -314,15 +318,18 @@ describe("StackBuilder", () => { for (const name of ["postgres", "postgrest", "auth"]) { const def = graph.startOrder.find((s) => s.name === name); expect(def).toBeDefined(); - expect(def?.command).toBe("docker"); + expect(def?.command).toBe("podman"); expect(def?.supervision).toBeDefined(); + expect(def?.supervision?.orphanCleanup).toContainEqual( + expect.objectContaining({ executable: "podman" }), + ); } // Docker container names are collected for cleanup expect(cleanupTargets.dockerContainerNames).toEqual([ - `supabase-postgres-${dockerConfig.apiPort}`, - `supabase-postgrest-${dockerConfig.apiPort}`, - `supabase-auth-${dockerConfig.apiPort}`, + `supabase-postgres-${config.apiPort}`, + `supabase-postgrest-${config.apiPort}`, + `supabase-auth-${config.apiPort}`, ]); }).pipe(Effect.provide(layer)); }); @@ -464,7 +471,7 @@ describe("StackBuilder", () => { }).pipe(Effect.provide(layer)); }); - it.effect("uses docker-backed edge-runtime even when a native binary is available", () => { + it.effect("uses Docker for edge-runtime and its dependencies in Docker mode", () => { const resolver = mockBinaryResolver(); const layer = builderLayer(resolver); @@ -480,34 +487,7 @@ describe("StackBuilder", () => { const edgeRuntimeDef = graph.startOrder.find((service) => service.name === "edge-runtime"); expect(edgeRuntimeDef).toBeDefined(); expect(edgeRuntimeDef?.command).toBe("docker"); - expect(edgeRuntimeDef?.dependencies).toEqual([ - { service: "postgres-init", condition: "completed" }, - ]); - expect(cleanupTargets.dockerContainerNames).toContain( - `supabase-edge-runtime-${edgeRuntimeConfig.apiPort}`, - ); - }).pipe(Effect.provide(layer)); - }); - - it.effect("uses docker-backed edge-runtime when the binary is unavailable", () => { - const resolver = mockBinaryResolver({ failServices: ["edge-runtime"] }); - const layer = builderLayer(resolver); - - return Effect.gen(function* () { - const builder = yield* StackBuilder; - const preparation = yield* StackPreparation; - const { graph, cleanupTargets } = yield* prepareAndBuild( - builder, - preparation, - edgeRuntimeConfig, - ); - - const edgeRuntimeDef = graph.startOrder.find((service) => service.name === "edge-runtime"); - expect(edgeRuntimeDef).toBeDefined(); - expect(edgeRuntimeDef?.command).toBe("docker"); - expect(edgeRuntimeDef?.dependencies).toEqual([ - { service: "postgres-init", condition: "completed" }, - ]); + expect(edgeRuntimeDef?.dependencies).toEqual([{ service: "postgres", condition: "healthy" }]); expect(cleanupTargets.dockerContainerNames).toContain( `supabase-edge-runtime-${edgeRuntimeConfig.apiPort}`, ); diff --git a/packages/stack/src/StackConfig.ts b/packages/stack/src/StackConfig.ts index d82d906f8e..13694f9747 100644 --- a/packages/stack/src/StackConfig.ts +++ b/packages/stack/src/StackConfig.ts @@ -3,7 +3,9 @@ import type { ResolvedFunctionsBundle } from "./functions.ts"; import type { ResolvedPorts } from "./PortCatalog.ts"; import type { ServiceName } from "./ServiceName.ts"; -type StackMode = "native" | "auto" | "docker"; +import type { ContainerRuntime } from "./ContainerRuntime.ts"; + +export type StackMode = "native" | "docker"; export type ServicePolicy = "off" | "lazy" | "eager"; export type ServicePolicyManifest = Readonly>; @@ -306,6 +308,8 @@ export interface ResolvedStackConfig { readonly runtimeRoot: string; readonly projectDir: string; readonly mode: StackMode; + /** Concrete container executable selected once when the stack was created. */ + readonly containerRuntime: ContainerRuntime | null; readonly servicePolicies: ServicePolicyManifest; readonly readiness: ReadinessPolicy; /** Whether readiness came from the package default or an explicit stack policy. */ diff --git a/packages/stack/src/StackConfigResolver.ts b/packages/stack/src/StackConfigResolver.ts index 85d1d6459c..3ad70bb228 100644 --- a/packages/stack/src/StackConfigResolver.ts +++ b/packages/stack/src/StackConfigResolver.ts @@ -49,6 +49,7 @@ import type { StudioConfig, VectorConfig, } from "./StackConfig.ts"; +import type { StackRuntimeSelection } from "./ContainerRuntime.ts"; import { DEFAULT_SERVICE_POLICIES, DEFAULT_VERSIONS, @@ -66,6 +67,7 @@ export interface ResolveConfigOptions { requests: ReadonlyArray, options: PortSelectionOptions, ) => Effect.Effect; + readonly runtime?: StackRuntimeSelection; } interface ResolvedRoots { @@ -362,7 +364,7 @@ const rawServiceEnabled = (config: StackConfig, service: ServiceName): boolean = return config.auth !== false; case "edge-runtime": return ( - ((config.mode ?? "auto") !== "native" || config.edgeRuntime !== undefined) && + ((config.mode ?? "native") !== "native" || config.edgeRuntime !== undefined) && config.edgeRuntime !== false && (config.edgeRuntime?.enabled ?? true) !== false ); @@ -445,13 +447,16 @@ export async function resolveConfig( input?: StackConfig, opts: ResolveConfigOptions = {}, ): Promise { - const config = input ?? {}; + const inputConfig = input ?? {}; + const resolvedMode = opts.runtime?.mode ?? inputConfig.mode ?? "native"; + const containerRuntime = + resolvedMode === "docker" ? (opts.runtime?.containerRuntime ?? "docker") : null; + const config: StackConfig = { ...inputConfig, mode: resolvedMode }; // Deliberately first: unsupported policies must not create roots or reserve ports. const servicePolicies = resolveServicePolicies(config); const projectDir = config.projectDir ?? process.cwd(); const instanceId = resolveInstanceId(config.instanceId); const functions = await resolveFunctionsConfig(config, projectDir); - const resolvedMode = config.mode ?? "auto"; const roots = resolveRoots(config, opts); const postgresInput = config.postgres ?? {}; const postgrestInput = @@ -562,6 +567,7 @@ export async function resolveConfig( runtimeRoot: roots.runtimeRoot, projectDir, mode: resolvedMode, + containerRuntime, servicePolicies, readiness: resolveReadinessPolicy({ stackPolicy: config.readiness }), readinessSource: config.readiness === undefined ? "default" : "configured", diff --git a/packages/stack/src/StackPreparation.ts b/packages/stack/src/StackPreparation.ts index 8087276085..025cda26a9 100644 --- a/packages/stack/src/StackPreparation.ts +++ b/packages/stack/src/StackPreparation.ts @@ -12,6 +12,7 @@ import { } from "effect"; import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; import { BinaryResolver } from "./BinaryResolver.ts"; +import type { ContainerRuntime } from "./ContainerRuntime.ts"; import type { BinaryHostCompatibilityError, BinaryManifestError, @@ -43,7 +44,8 @@ export interface StackPreparationInput { readonly versions?: Partial; readonly services?: ReadonlyArray; readonly enabledServices?: ReadonlyArray; - readonly mode?: "native" | "auto" | "docker"; + readonly mode: "native" | "docker"; + readonly containerRuntime?: ContainerRuntime; } export type StackPreparationError = @@ -96,13 +98,14 @@ class PullAttemptError extends Error { const resolveDockerImageForService = ( spawner: ChildProcessSpawner.ChildProcessSpawner["Service"], + runtime: ContainerRuntime, service: ServiceName, version: string, callbacks?: { readonly onDownloadStart?: Effect.Effect; }, ): Effect.Effect => - pullImage(spawner, dockerImageForService(service, version), callbacks); + pullImage(spawner, runtime, dockerImageForService(service, version), callbacks); const preparationClosure = ( services: ReadonlyArray, @@ -127,9 +130,9 @@ const plannedResolution = ( resolver: BinaryResolver["Service"], service: ServiceName, version: string, - mode: "native" | "auto" | "docker", + mode: "native" | "docker", ): Effect.Effect => { - if (mode === "docker" || (mode === "auto" && isDockerOnlyService(service))) { + if (mode === "docker") { return Effect.succeed({ type: "docker", image: dockerImageForService(service, version), @@ -138,18 +141,9 @@ const plannedResolution = ( if (isDockerOnlyService(service)) { return Effect.fail(new BinaryNotFoundError({ service, platform: "native" })); } - const native = resolver + return resolver .plan({ service, version }) .pipe(Effect.map((path): ServiceResolution => ({ type: "binary", path }))); - if (mode === "native") return native; - return native.pipe( - Effect.catchTag("BinaryNotFoundError", () => - Effect.succeed({ - type: "docker" as const, - image: dockerImageForService(service, version), - }), - ), - ); }; const planAssetsWithDependencies = ( @@ -158,7 +152,7 @@ const planAssetsWithDependencies = ( ): Effect.Effect => Effect.gen(function* () { const versions = { ...DEFAULT_VERSIONS, ...input?.versions }; - const mode = input?.mode ?? "auto"; + const mode = input?.mode ?? "native"; const services = selectedServices(input); const results = yield* Effect.all( services.map((service) => @@ -232,9 +226,15 @@ export class StackPreparation extends Context.Service< const version = input.versions?.[service] ?? DEFAULT_VERSIONS[service]; const effect: Effect.Effect = resolution.type === "docker" - ? resolveDockerImageForService(spawner, service, version, { - onDownloadStart: markDownloadStart(), - }).pipe(Effect.map((image): ServiceResolution => ({ type: "docker", image }))) + ? resolveDockerImageForService( + spawner, + input.containerRuntime ?? "docker", + service, + version, + { + onDownloadStart: markDownloadStart(), + }, + ).pipe(Effect.map((image): ServiceResolution => ({ type: "docker", image }))) : resolver .resolveWithMetadata( { service, version }, @@ -268,9 +268,12 @@ export class StackPreparation extends Context.Service< selectedServices(input).map((service) => { const resolution = planned.resolutions[service]; if (resolution === undefined) return Effect.die(`Missing plan for ${service}`); - return materialize(service, resolution, input ?? {}, publishEvent).pipe( - Effect.map((resolved) => [service, resolved] as const), - ); + return materialize( + service, + resolution, + { mode: input?.mode ?? "native", ...input }, + publishEvent, + ).pipe(Effect.map((resolved) => [service, resolved] as const)); }), { concurrency: "unbounded" }, ); @@ -301,19 +304,20 @@ export class StackPreparation extends Context.Service< const pullImage = ( spawner: ChildProcessSpawner.ChildProcessSpawner["Service"], + runtime: ContainerRuntime, image: string, callbacks?: { readonly onDownloadStart?: Effect.Effect; }, ): Effect.Effect => Effect.gen(function* () { - if (yield* hasLocalDockerImage(spawner, image)) { + if (yield* hasLocalDockerImage(spawner, runtime, image)) { return image; } yield* callbacks?.onDownloadStart ?? Effect.void; - const attempt = runPullCommand(spawner, image).pipe( + const attempt = runPullCommand(spawner, runtime, image).pipe( Effect.retry({ while: (error) => shouldRetryPull(error.detail), schedule: Schedule.recurs(1), @@ -338,10 +342,11 @@ const pullImage = ( const runPullCommand = ( spawner: ChildProcessSpawner.ChildProcessSpawner["Service"], + runtime: ContainerRuntime, image: string, ): Effect.Effect<{ readonly exitCode: number; readonly stderr: string }, PullAttemptError> => Effect.gen(function* () { - const child = yield* spawner.spawn(ChildProcess.make("docker", ["pull", image])); + const child = yield* spawner.spawn(ChildProcess.make(runtime, ["pull", image])); const [stderr, exitCode] = yield* Effect.all( [collectStreamAsString(child.stderr), child.exitCode.pipe(Effect.map(Number))], { concurrency: "unbounded" }, @@ -354,7 +359,7 @@ const runPullCommand = ( const detail = result.stderr.length > 0 ? result.stderr - : `docker pull exited with code ${result.exitCode}`; + : `${runtime} pull exited with code ${result.exitCode}`; return yield* Effect.fail(new PullAttemptError(detail, isDockerDaemonDownMessage(detail))); } return result; @@ -367,9 +372,10 @@ const runPullCommand = ( const hasLocalDockerImage = ( spawner: ChildProcessSpawner.ChildProcessSpawner["Service"], + runtime: ContainerRuntime, image: string, ): Effect.Effect => - spawner.exitCode(ChildProcess.make("docker", ["image", "inspect", image])).pipe( + spawner.exitCode(ChildProcess.make(runtime, ["image", "inspect", image])).pipe( Effect.map((exitCode) => exitCode === 0), Effect.catchTag("PlatformError", () => Effect.succeed(false)), ); diff --git a/packages/stack/src/bun.ts b/packages/stack/src/bun.ts index 2d05640881..ca22284037 100644 --- a/packages/stack/src/bun.ts +++ b/packages/stack/src/bun.ts @@ -2,6 +2,7 @@ import { BunServices } from "@effect/platform-bun"; import { Effect, Layer } from "effect"; import { FetchHttpClient } from "effect/unstable/http"; import { BinaryResolver } from "./BinaryResolver.ts"; +import { selectStackRuntime } from "./ContainerRuntime.ts"; import { createStack as createStackCore, type StackHandle } from "./createStack.ts"; import { prefetch as prefetchEffect, @@ -14,19 +15,26 @@ import { StackPreparation } from "./StackPreparation.ts"; import type { StackConfig } from "./StackConfig.ts"; export async function createStack(config?: StackConfig): Promise { - return createStackCore(config, platformFactory); + const runtime = await Effect.runPromise( + selectStackRuntime(config?.mode).pipe(Effect.provide(BunServices.layer)), + ); + return createStackCore(config, platformFactory, runtime); } export async function prefetch(options?: PrefetchOptions): Promise { + const runtime = await Effect.runPromise( + selectStackRuntime(options?.mode).pipe(Effect.provide(BunServices.layer)), + ); const resolverLayer = BinaryResolver.make(defaultCacheRoot()).pipe( Layer.provide(FetchHttpClient.layer), ); const preparationLayer = StackPreparation.layer.pipe(Layer.provide(resolverLayer)); return Effect.runPromise( - prefetchEffect(options).pipe( - Effect.provide(preparationLayer), - Effect.provide(BunServices.layer), - ), + prefetchEffect({ + ...options, + mode: runtime.mode, + ...(runtime.containerRuntime === null ? {} : { containerRuntime: runtime.containerRuntime }), + }).pipe(Effect.provide(preparationLayer), Effect.provide(BunServices.layer)), ); } diff --git a/packages/stack/src/cleanup.ts b/packages/stack/src/cleanup.ts index db5db4e596..af00a8c93e 100644 --- a/packages/stack/src/cleanup.ts +++ b/packages/stack/src/cleanup.ts @@ -1,6 +1,7 @@ import { execFile } from "node:child_process"; import { existsSync, rmSync } from "node:fs"; import { Duration, Effect } from "effect"; +import type { ContainerRuntime } from "./ContainerRuntime.ts"; import type { CleanupTargets } from "./CleanupTargets.ts"; import { SERVICE_NAMES, serviceMetadata } from "./ServiceCatalog.ts"; import type { ResolvedStackConfig } from "./StackConfig.ts"; @@ -20,12 +21,15 @@ export const candidateCleanupTargets = (config: ResolvedStackConfig): CleanupTar * Force-remove Docker containers by name. Best-effort safety net — * silently ignores containers that don't exist or are already removed. */ -export const dockerForceRemove = (containerNames: ReadonlyArray): Effect.Effect => +export const dockerForceRemove = ( + runtime: ContainerRuntime, + containerNames: ReadonlyArray, +): Effect.Effect => Effect.forEach( containerNames, (name) => Effect.callback((resume) => { - const child = execFile("docker", ["rm", "-f", name], { timeout: 5_000 }, () => + const child = execFile(runtime, ["rm", "-f", name], { timeout: 5_000 }, () => resume(Effect.void), ); return Effect.sync(() => child.kill()); @@ -94,6 +98,11 @@ export const cleanupLocalStackResources = (opts: { // Safety net: force-remove any Docker containers that survived // signal-based shutdown. On macOS, killing the `docker run` client // may not stop the container. - yield* dockerForceRemove(opts.cleanupTargets.dockerContainerNames); + if (opts.config.containerRuntime !== null) { + yield* dockerForceRemove( + opts.config.containerRuntime, + opts.cleanupTargets.dockerContainerNames, + ); + } yield* cleanupAutoManagedPathsWithRetry(opts.config); }); diff --git a/packages/stack/src/createStack.integration.test.ts b/packages/stack/src/createStack.integration.test.ts index 286e19463f..6945a90132 100644 --- a/packages/stack/src/createStack.integration.test.ts +++ b/packages/stack/src/createStack.integration.test.ts @@ -53,6 +53,7 @@ const createStackWithFreshPorts = async ( postgres: { ...config?.postgres, port: dbPort }, }, platform, + { mode: "native", containerRuntime: null }, ); } catch (error) { if (!isAddressInUse(error) || attempt === 2) throw error; diff --git a/packages/stack/src/createStack.ts b/packages/stack/src/createStack.ts index a80dfc0b04..cef27c50c3 100644 --- a/packages/stack/src/createStack.ts +++ b/packages/stack/src/createStack.ts @@ -3,6 +3,7 @@ import { Context, Effect, FileSystem, type Layer, ManagedRuntime, Path, Stream } import { HttpServer } from "effect/unstable/http"; import { ChildProcessSpawner } from "effect/unstable/process"; import { ApiProxy } from "./ApiProxy.ts"; +import type { StackRuntimeSelection } from "./ContainerRuntime.ts"; import { candidateCleanupTargets, cleanupAutoManagedPaths, dockerForceRemove } from "./cleanup.ts"; import { toStackError } from "./errors.ts"; import type { FunctionsReloadConfig } from "./functions.ts"; @@ -73,11 +74,13 @@ export interface StackHandle extends AsyncDisposable { export async function createStack( config: StackConfig | undefined, platformFactory: PlatformFactory, + runtime: StackRuntimeSelection, ): Promise { let portLease: PortLease | undefined; let resolved: ResolvedStackConfig; try { resolved = await resolveConfig(config, { + runtime, portAllocator: (requests, options) => reservePortSet(requests, options).pipe( Effect.tap((lease) => @@ -162,7 +165,12 @@ export async function createStack( } catch (error: unknown) { await Effect.runPromise(portLease.releaseAll); await Effect.runPromise( - dockerForceRemove(candidateCleanupTargets(resolved).dockerContainerNames), + resolved.containerRuntime === null + ? Effect.void + : dockerForceRemove( + resolved.containerRuntime, + candidateCleanupTargets(resolved).dockerContainerNames, + ), ); cleanupAutoManagedPaths(resolved); throw toStackError(error); diff --git a/packages/stack/src/createStack.unit.test.ts b/packages/stack/src/createStack.unit.test.ts index 5bad2ed488..21fe435f04 100644 --- a/packages/stack/src/createStack.unit.test.ts +++ b/packages/stack/src/createStack.unit.test.ts @@ -83,10 +83,25 @@ describe("resolveConfig edge runtime defaults", () => { expect(config.edgeRuntime).toBe(false); }); - it("enables edge runtime when omitted in auto mode", async () => { - const config = await resolveConfig(); + it("enables edge runtime when omitted in Docker mode", async () => { + const config = await resolveConfig({ mode: "docker" }); + + expect(config.mode).toBe("docker"); + expect(config.edgeRuntime).toEqual( + expect.objectContaining({ + enabled: true, + version: DEFAULT_VERSIONS["edge-runtime"], + }), + ); + }); + + it("applies the detected Docker mode before resolving services and ports", async () => { + const config = await resolveConfig(undefined, { + runtime: { mode: "docker", containerRuntime: "podman" }, + }); - expect(config.mode).toBe("auto"); + expect(config.mode).toBe("docker"); + expect(config.containerRuntime).toBe("podman"); expect(config.edgeRuntime).toEqual( expect.objectContaining({ enabled: true, diff --git a/packages/stack/src/effect.ts b/packages/stack/src/effect.ts index 5184b47f8f..5b92833fcc 100644 --- a/packages/stack/src/effect.ts +++ b/packages/stack/src/effect.ts @@ -93,6 +93,7 @@ export type { ReadyOptions, ServicePolicy, ServicePolicyManifest, + StackMode, StackConfig, StorageConfig, StudioConfig, diff --git a/packages/stack/src/functions.unit.test.ts b/packages/stack/src/functions.unit.test.ts index 46f9298c83..321d26dbea 100644 --- a/packages/stack/src/functions.unit.test.ts +++ b/packages/stack/src/functions.unit.test.ts @@ -84,7 +84,11 @@ const authFailureCases = [ describe("stack Functions runtime config", () => { it("projects an explicit bundle without project discovery", async () => { const root = makeTempProject(); - const stackConfig = await resolveConfig({ projectDir: root, functions: makeBundle(root) }); + const stackConfig = await resolveConfig({ + mode: "docker", + projectDir: root, + functions: makeBundle(root), + }); const config = resolveFunctionsRuntimeConfig( stackConfig, { hostname: "127.0.0.1" }, @@ -212,7 +216,7 @@ describe("stack Functions runtime config", () => { return Effect.gen(function* () { const bundle = makeBundle(cwd); const stackConfig = yield* Effect.promise(() => - resolveConfig({ projectDir: cwd, runtimeRoot: cwd, functions: bundle }), + resolveConfig({ mode: "docker", projectDir: cwd, runtimeRoot: cwd, functions: bundle }), ); yield* configureFunctionsRuntime(stackConfig, { hostname: "127.0.0.1" }, bundle); const filePath = functionsRuntimeConfigPath(stackConfig.runtimeRoot); diff --git a/packages/stack/src/index.ts b/packages/stack/src/index.ts index 1e8e871dd4..43e077632a 100644 --- a/packages/stack/src/index.ts +++ b/packages/stack/src/index.ts @@ -18,6 +18,7 @@ export type { ReadyOptions, ServicePolicy, ServicePolicyManifest, + StackMode, StackConfig, StorageConfig, StudioConfig, diff --git a/packages/stack/src/layers.ts b/packages/stack/src/layers.ts index f6b8170b57..46204768de 100644 --- a/packages/stack/src/layers.ts +++ b/packages/stack/src/layers.ts @@ -15,7 +15,7 @@ import type { ResolvedStackConfig } from "./StackConfig.ts"; import { sanitizeDaemonConfigInput, type DaemonConfigInput } from "./StackConfigResolver.ts"; import { HttpTransportClient } from "./HttpTransportClient.ts"; import { DEFAULT_MANAGED_STACK_NAME, defaultCacheRoot } from "./paths.ts"; -import type { ManagedStackDocument } from "./managed/document.ts"; +import type { ManagedStackLaunchInput } from "./managed/document.ts"; import type { ManagedPortIntentDocument } from "./managed/model.ts"; import { deriveStackId, ensureEnvironment } from "./managed/environment.ts"; import { gitConfigStoreLayer } from "./managed/git.ts"; @@ -107,7 +107,7 @@ export class DaemonStartError extends Data.TaggedError("DaemonStartError")<{ /** Managed-only additions kept outside the generic daemon config resolver. */ export type ManagedDaemonConfigInput = DaemonConfigInput & { readonly portIntents: ManagedPortIntentDocument; - readonly launch?: ManagedStackDocument["launch"]; + readonly launch?: ManagedStackLaunchInput; }; // --------------------------------------------------------------------------- diff --git a/packages/stack/src/managed-manager-lifecycle.integration.test.ts b/packages/stack/src/managed-manager-lifecycle.integration.test.ts index 0954f2b1d7..34e32ef332 100644 --- a/packages/stack/src/managed-manager-lifecycle.integration.test.ts +++ b/packages/stack/src/managed-manager-lifecycle.integration.test.ts @@ -58,7 +58,7 @@ describe("managed stack lifecycle journeys", () => { workspacePath: workspace, stackName: "default", launch: { - mode: "auto" as const, + mode: "docker" as const, versions: { postgres: "17.6.1" }, excludedServices: [], }, @@ -194,7 +194,7 @@ describe("managed stack lifecycle journeys", () => { lifecycle: "running", }); yield* releaseLease(initial); - const launch = { mode: "auto" as const, versions: { postgres: "17.6.1" } }; + const launch = { mode: "docker" as const, versions: { postgres: "17.6.1" } }; gate.enabled = true; const launchFiber = yield* Effect.forkScoped( manager.updateLaunch(owner, { stackId, launch }), diff --git a/packages/stack/src/managed/document.ts b/packages/stack/src/managed/document.ts index 843d967e16..509ab7e44f 100644 --- a/packages/stack/src/managed/document.ts +++ b/packages/stack/src/managed/document.ts @@ -1,6 +1,7 @@ import { Data, Effect, Schema } from "effect"; import type { ManagedPortAssignment } from "./model.ts"; import { PartialVersionManifestSchema, type PartialVersionManifest } from "../versions.ts"; +import type { ContainerRuntime } from "../ContainerRuntime.ts"; export type ManagedStackDocumentLifecycle = | "stopped" @@ -34,7 +35,8 @@ export interface ManagedStackDocument { readonly protocolVersion: 1; }; readonly launch?: { - readonly mode: "native" | "auto" | "docker"; + readonly mode: "native" | "docker"; + readonly containerRuntime?: ContainerRuntime; readonly versions: PartialVersionManifest; readonly excludedServices?: ReadonlyArray; readonly lastNotifiedUpdateFingerprint?: string; @@ -44,7 +46,8 @@ export interface ManagedStackDocument { } export const managedStackLaunchSchema = Schema.Struct({ - mode: Schema.Literals(["native", "auto", "docker"] as const), + mode: Schema.Literals(["native", "docker"] as const), + containerRuntime: Schema.optionalKey(Schema.Literals(["docker", "podman"] as const)), versions: PartialVersionManifestSchema, excludedServices: Schema.optionalKey(Schema.Array(Schema.String)), lastNotifiedUpdateFingerprint: Schema.optionalKey(Schema.String), @@ -52,6 +55,16 @@ export const managedStackLaunchSchema = Schema.Struct({ export type ManagedStackLaunch = Schema.Schema.Type; +/** Launch request before the supervisor selects a concrete execution mode. */ +export const managedStackLaunchInputSchema = Schema.Struct({ + mode: Schema.optionalKey(Schema.Literals(["native", "docker"] as const)), + versions: PartialVersionManifestSchema, + excludedServices: Schema.optionalKey(Schema.Array(Schema.String)), + lastNotifiedUpdateFingerprint: Schema.optionalKey(Schema.String), +}); + +export type ManagedStackLaunchInput = Schema.Schema.Type; + const managedPortAssignmentSchema = Schema.Struct({ key: Schema.Literals([ "api.port", diff --git a/packages/stack/src/managed/lifecycle.ts b/packages/stack/src/managed/lifecycle.ts index 90cb704711..8085715279 100644 --- a/packages/stack/src/managed/lifecycle.ts +++ b/packages/stack/src/managed/lifecycle.ts @@ -126,6 +126,8 @@ export const stopManagedStack = ( const manager = yield* ManagedStackManager; const document = yield* resolveManagedDocument(input); const stackId = document.id; + const containerRuntime = + document.launch?.mode === "docker" ? (document.launch.containerRuntime ?? "docker") : null; const acquisition = yield* manager.acquireControl(stackId); if (acquisition._tag === "Owned") { if ( @@ -133,9 +135,12 @@ export const stopManagedStack = ( document.lifecycle === "starting" || document.lifecycle === "failed" ) { - yield* dockerForceRemove( - SERVICE_NAMES.map((service) => dockerContainerName(service, `id-${stackId}`)), - ); + if (containerRuntime !== null) { + yield* dockerForceRemove( + containerRuntime, + SERVICE_NAMES.map((service) => dockerContainerName(service, `id-${stackId}`)), + ); + } yield* manager.recordLifecycle(acquisition, { stackId, lifecycle: "stopped" }); } yield* acquisition.close; @@ -148,9 +153,12 @@ export const stopManagedStack = ( const cleanupOwned = (owned: import("./control.ts").ControlOwnership) => Effect.ensuring( Effect.gen(function* () { - yield* dockerForceRemove( - SERVICE_NAMES.map((service) => dockerContainerName(service, `id-${stackId}`)), - ); + if (containerRuntime !== null) { + yield* dockerForceRemove( + containerRuntime, + SERVICE_NAMES.map((service) => dockerContainerName(service, `id-${stackId}`)), + ); + } yield* manager.recordLifecycle(owned, { stackId, lifecycle: "stopped" }); }), owned.close, diff --git a/packages/stack/src/managed/manager.ts b/packages/stack/src/managed/manager.ts index 585ecaabfc..e22a633adb 100644 --- a/packages/stack/src/managed/manager.ts +++ b/packages/stack/src/managed/manager.ts @@ -897,9 +897,12 @@ const makeManager = ( if (current === undefined) return { outcome: "already-absent", stackId }; if ("outcome" in current) return current; yield* acquisition.setState("deleting", false); - yield* dockerForceRemove( - SERVICE_NAMES.map((service) => dockerContainerName(service, `id-${stackId}`)), - ); + if (current.launch?.mode === "docker") { + yield* dockerForceRemove( + current.launch.containerRuntime ?? "docker", + SERVICE_NAMES.map((service) => dockerContainerName(service, `id-${stackId}`)), + ); + } const { runtime: _runtime, ...withoutRuntime } = current; const deleting = { ...withoutRuntime, lifecycle: "deleting" as const, updatedAt: now() }; yield* store.write(deleting); diff --git a/packages/stack/src/node.ts b/packages/stack/src/node.ts index 8be38e7ce1..5c81d60f8d 100644 --- a/packages/stack/src/node.ts +++ b/packages/stack/src/node.ts @@ -2,6 +2,7 @@ import { NodeServices } from "@effect/platform-node"; import { Effect, Layer } from "effect"; import { FetchHttpClient } from "effect/unstable/http"; import { BinaryResolver } from "./BinaryResolver.ts"; +import { selectStackRuntime } from "./ContainerRuntime.ts"; import { createStack as createStackCore, type StackHandle } from "./createStack.ts"; import { prefetch as prefetchEffect, @@ -20,19 +21,26 @@ import type { StackConfig } from "./StackConfig.ts"; */ export async function createStack(config?: StackConfig): Promise { - return createStackCore(config, platformFactory); + const runtime = await Effect.runPromise( + selectStackRuntime(config?.mode).pipe(Effect.provide(NodeServices.layer)), + ); + return createStackCore(config, platformFactory, runtime); } export async function prefetch(options?: PrefetchOptions): Promise { + const runtime = await Effect.runPromise( + selectStackRuntime(options?.mode).pipe(Effect.provide(NodeServices.layer)), + ); const resolverLayer = BinaryResolver.make(defaultCacheRoot()).pipe( Layer.provide(FetchHttpClient.layer), ); const preparationLayer = StackPreparation.layer.pipe(Layer.provide(resolverLayer)); return Effect.runPromise( - prefetchEffect(options).pipe( - Effect.provide(preparationLayer), - Effect.provide(NodeServices.layer), - ), + prefetchEffect({ + ...options, + mode: runtime.mode, + ...(runtime.containerRuntime === null ? {} : { containerRuntime: runtime.containerRuntime }), + }).pipe(Effect.provide(preparationLayer), Effect.provide(NodeServices.layer)), ); } diff --git a/packages/stack/src/prefetch.ts b/packages/stack/src/prefetch.ts index 06e6f8806d..50a799452a 100644 --- a/packages/stack/src/prefetch.ts +++ b/packages/stack/src/prefetch.ts @@ -8,7 +8,13 @@ import { import { StackPreparation } from "./StackPreparation.ts"; import type { ServiceName } from "./ServiceName.ts"; -export interface PrefetchOptions extends StackPreparationInput {} +export interface PrefetchOptions extends Omit { + readonly mode?: StackPreparationInput["mode"]; +} + +interface ResolvedPrefetchOptions extends PrefetchOptions { + readonly containerRuntime?: import("./ContainerRuntime.ts").ContainerRuntime; +} export type PrefetchResult = Partial>; @@ -16,9 +22,11 @@ const toPrefetchResult = (artifacts: PreparedStackArtifacts): PrefetchResult => artifacts.resolutions; export const prefetch = ( - options?: PrefetchOptions, + options?: ResolvedPrefetchOptions, ): Effect.Effect => Effect.gen(function* () { const preparation = yield* StackPreparation; - return yield* preparation.prepare(options).pipe(Effect.map(toPrefetchResult)); + return yield* preparation + .prepare({ mode: options?.mode ?? "native", ...options }) + .pipe(Effect.map(toPrefetchResult)); }); diff --git a/packages/stack/src/prefetch.unit.test.ts b/packages/stack/src/prefetch.unit.test.ts index 8b4e015c2f..c418b4d418 100644 --- a/packages/stack/src/prefetch.unit.test.ts +++ b/packages/stack/src/prefetch.unit.test.ts @@ -77,7 +77,9 @@ describe("prefetch", () => { Layer.provide(spawner.layer), ); - const result = await Effect.runPromise(prefetch().pipe(Effect.provide(layer))); + const result = await Effect.runPromise( + prefetch({ mode: "docker" }).pipe(Effect.provide(layer)), + ); expect(Object.keys(result).sort()).toEqual([...SERVICE_NAMES].sort()); }); @@ -136,7 +138,7 @@ describe("prefetch", () => { expect(Object.keys(result).sort()).toEqual(["imgproxy", "postgres", "storage"]); }); - test("prefetching a companion does not pull its owner", async () => { + test("prefetching uses the selected container runtime without pulling an owner", async () => { const resolver = mockBinaryResolver(); const spawner = mockSequenceSpawner([{ exitCode: 0 }]); const layer = StackPreparation.layer.pipe( @@ -145,10 +147,18 @@ describe("prefetch", () => { ); const result = await Effect.runPromise( - prefetch({ mode: "docker", services: ["imgproxy"] }).pipe(Effect.provide(layer)), + prefetch({ mode: "docker", containerRuntime: "podman", services: ["imgproxy"] }).pipe( + Effect.provide(layer), + ), ); expect(Object.keys(result)).toEqual(["imgproxy"]); + expect(spawner.spawned).toEqual([ + { + command: "podman", + args: ["image", "inspect", `ghcr.io/supabase/cli/imgproxy:${DEFAULT_VERSIONS.imgproxy}`], + }, + ]); }); test("does not prepare dependencies that are disabled in the stack", async () => { @@ -173,7 +183,7 @@ describe("prefetch", () => { expect(Object.keys(result.resolutions).sort()).toEqual(["pgmeta", "postgres", "studio"]); }); - test("auto mode falls back to Docker when a service has no native target", async () => { + test("Docker mode uses Docker when a native artifact is unavailable", async () => { const resolver = mockBinaryResolver({ binaries: { postgres: "/cache/postgres/native" }, }); @@ -184,7 +194,7 @@ describe("prefetch", () => { ); const result = await Effect.runPromise( - prefetch({ mode: "auto", services: ["postgrest"] }).pipe(Effect.provide(layer)), + prefetch({ mode: "docker", services: ["postgrest"] }).pipe(Effect.provide(layer)), ); expect(result.postgrest).toEqual({ @@ -211,6 +221,25 @@ describe("prefetch", () => { expect(error).toBeInstanceOf(BinaryNotFoundError); }); + test("native mode does not fall back when a native artifact is unavailable", async () => { + const resolver = mockBinaryResolver({ failServices: ["postgrest"] }); + const spawner = mockSequenceSpawner([{ exitCode: 0 }]); + const layer = StackPreparation.layer.pipe( + Layer.provide(resolver.layer), + Layer.provide(spawner.layer), + ); + + const error = await Effect.runPromise( + prefetch({ mode: "native", services: ["postgrest"] }).pipe( + Effect.provide(layer), + Effect.flip, + ), + ); + + expect(error).toBeInstanceOf(BinaryNotFoundError); + expect(spawner.spawned).toEqual([]); + }); + test("prefetches pgmeta using its published container tag", async () => { const resolver = mockBinaryResolver(); const spawner = mockSequenceSpawner([{ exitCode: 0 }, { exitCode: 0 }]); @@ -259,7 +288,7 @@ describe("prefetch", () => { }); }); - test("uses docker for edge-runtime in auto mode even when a native binary exists", async () => { + test("uses Docker for every service in Docker mode", async () => { const resolver = mockBinaryResolver(); const spawner = mockSequenceSpawner([{ exitCode: 0 }]); const layer = StackPreparation.layer.pipe( @@ -270,7 +299,10 @@ describe("prefetch", () => { const result = await Effect.runPromise( Effect.gen(function* () { const preparation = yield* StackPreparation; - const artifacts = yield* preparation.prepare({ mode: "auto", services: ["edge-runtime"] }); + const artifacts = yield* preparation.prepare({ + mode: "docker", + services: ["edge-runtime"], + }); return artifacts.resolutions; }).pipe(Effect.provide(layer)), ); @@ -279,8 +311,6 @@ describe("prefetch", () => { type: "docker", image: `ghcr.io/supabase/cli/edge-runtime:${DEFAULT_VERSIONS["edge-runtime"]}`, }); - expect(resolver.resolved).toEqual([ - { service: "postgres", version: DEFAULT_VERSIONS.postgres }, - ]); + expect(resolver.resolved).toEqual([]); }); }); diff --git a/packages/stack/src/services/analytics.ts b/packages/stack/src/services/analytics.ts index e4039a989e..7eeba05c69 100644 --- a/packages/stack/src/services/analytics.ts +++ b/packages/stack/src/services/analytics.ts @@ -1,10 +1,14 @@ import type { ServiceDef } from "@supabase/process-compose"; import { dockerPortMapArgs } from "../Platform.ts"; import type { StackIdentity } from "../StackIdentity.ts"; -import { dockerRunService, type ServiceDependency } from "./service-utils.ts"; +import { + dockerRunService, + type ContainerRuntimeOptions, + type ServiceDependency, +} from "./service-utils.ts"; import { stackHealthBudgets } from "./health-budgets.ts"; -interface DockerAnalyticsOptions { +interface DockerAnalyticsOptions extends ContainerRuntimeOptions { readonly image: string; readonly identity: StackIdentity; readonly hostPort: number; @@ -69,6 +73,7 @@ export const makeAnalyticsServiceDocker = (opts: DockerAnalyticsOptions): Servic } return dockerRunService({ + runtime: opts.runtime, name: "analytics", identity: opts.identity, image: opts.image, diff --git a/packages/stack/src/services/auth.ts b/packages/stack/src/services/auth.ts index fce6322761..111c07335b 100644 --- a/packages/stack/src/services/auth.ts +++ b/packages/stack/src/services/auth.ts @@ -2,7 +2,11 @@ import type { ServiceDef } from "@supabase/process-compose"; import { dockerNetworkArgs } from "../Platform.ts"; import type { StackIdentity } from "../StackIdentity.ts"; import { stackHealthBudgets } from "./health-budgets.ts"; -import { dockerRunService, type ServiceDependency } from "./service-utils.ts"; +import { + dockerRunService, + type ContainerRuntimeOptions, + type ServiceDependency, +} from "./service-utils.ts"; interface AuthServiceOptions { readonly dbPort: number; @@ -22,7 +26,7 @@ interface NativeAuthOptions extends AuthServiceOptions { readonly binPath: string; } -interface DockerAuthOptions extends AuthServiceOptions { +interface DockerAuthOptions extends AuthServiceOptions, ContainerRuntimeOptions { readonly image: string; readonly dbHost: string; readonly platformOs: string; @@ -82,6 +86,7 @@ export const makeAuthServiceNative = (opts: NativeAuthOptions): ServiceDef => ({ export const makeAuthServiceDocker = (opts: DockerAuthOptions): ServiceDef => { const env = authEnv(opts, opts.dbHost); return dockerRunService({ + runtime: opts.runtime, name: "auth", identity: opts.identity, image: opts.image, diff --git a/packages/stack/src/services/docker-cleanup.ts b/packages/stack/src/services/docker-cleanup.ts index c96c7af6b9..0e9b5a2a8a 100644 --- a/packages/stack/src/services/docker-cleanup.ts +++ b/packages/stack/src/services/docker-cleanup.ts @@ -1,11 +1,15 @@ import type { ExternalCleanupAction } from "@supabase/process-compose"; import { execFileSync } from "node:child_process"; import { Effect } from "effect"; +import type { ContainerRuntime } from "../ContainerRuntime.ts"; -export const dockerServiceCleanup = (containerName: string): Effect.Effect => +export const dockerServiceCleanup = ( + runtime: ContainerRuntime, + containerName: string, +): Effect.Effect => Effect.sync(() => { try { - execFileSync("docker", ["rm", "-f", containerName], { + execFileSync(runtime, ["rm", "-f", containerName], { stdio: "ignore", timeout: 5_000, }); @@ -13,11 +17,12 @@ export const dockerServiceCleanup = (containerName: string): Effect.Effect }); export const dockerServiceOrphanCleanup = ( + runtime: ContainerRuntime, containerName: string, ): ReadonlyArray => [ { _tag: "RunCommand", - executable: "docker", + executable: runtime, args: ["rm", "-f", containerName], timeoutMs: 5_000, }, diff --git a/packages/stack/src/services/edge-runtime.ts b/packages/stack/src/services/edge-runtime.ts index 168a14cd26..a4dfdb89e5 100644 --- a/packages/stack/src/services/edge-runtime.ts +++ b/packages/stack/src/services/edge-runtime.ts @@ -3,7 +3,12 @@ import { join } from "node:path"; import type { ServiceDef } from "@supabase/process-compose"; import { dockerNetworkArgs } from "../Platform.ts"; import type { StackIdentity } from "../StackIdentity.ts"; -import { dockerRunService, hostHttpHealthCheck, type ServiceDependency } from "./service-utils.ts"; +import { + dockerRunService, + hostHttpHealthCheck, + type ContainerRuntimeOptions, + type ServiceDependency, +} from "./service-utils.ts"; import bootstrapSource from "./edge-runtime-main.ts" with { type: "text" }; import { stackHealthBudgets } from "./health-budgets.ts"; @@ -21,7 +26,7 @@ interface NativeEdgeRuntimeOptions extends EdgeRuntimeOptions { readonly binPath: string; } -interface DockerEdgeRuntimeOptions extends EdgeRuntimeOptions { +interface DockerEdgeRuntimeOptions extends EdgeRuntimeOptions, ContainerRuntimeOptions { readonly image: string; readonly identity: StackIdentity; readonly platformOs: string; @@ -86,6 +91,7 @@ export const makeEdgeRuntimeServiceDocker = (opts: DockerEdgeRuntimeOptions): Se const bootstrapDir = ensureBootstrapScript(opts.runtimeRoot); return dockerRunService({ + runtime: opts.runtime, name: "edge-runtime", identity: opts.identity, image: opts.image, diff --git a/packages/stack/src/services/imgproxy.ts b/packages/stack/src/services/imgproxy.ts index 06d1fcd64f..2c9acf0bc7 100644 --- a/packages/stack/src/services/imgproxy.ts +++ b/packages/stack/src/services/imgproxy.ts @@ -1,10 +1,15 @@ import type { ServiceDef } from "@supabase/process-compose"; import { dockerNetworkArgs } from "../Platform.ts"; import type { StackIdentity } from "../StackIdentity.ts"; -import { dockerRunService, hostHttpHealthCheck, type ServiceDependency } from "./service-utils.ts"; +import { + dockerRunService, + hostHttpHealthCheck, + type ContainerRuntimeOptions, + type ServiceDependency, +} from "./service-utils.ts"; import { stackHealthBudgets } from "./health-budgets.ts"; -interface DockerImgproxyOptions { +interface DockerImgproxyOptions extends ContainerRuntimeOptions { readonly image: string; readonly port: number; readonly identity: StackIdentity; @@ -22,6 +27,7 @@ const imgproxyHealthCheck = (port: number): ServiceDef["healthCheck"] => export const makeImgproxyServiceDocker = (opts: DockerImgproxyOptions): ServiceDef => dockerRunService({ + runtime: opts.runtime, name: "imgproxy", identity: opts.identity, image: opts.image, diff --git a/packages/stack/src/services/mailpit.ts b/packages/stack/src/services/mailpit.ts index a57d51b60c..193f408e02 100644 --- a/packages/stack/src/services/mailpit.ts +++ b/packages/stack/src/services/mailpit.ts @@ -1,10 +1,15 @@ import type { ServiceDef } from "@supabase/process-compose"; import { dockerNetworkArgs } from "../Platform.ts"; import type { StackIdentity } from "../StackIdentity.ts"; -import { dockerRunService, hostHttpHealthCheck, type ServiceDependency } from "./service-utils.ts"; +import { + dockerRunService, + hostHttpHealthCheck, + type ContainerRuntimeOptions, + type ServiceDependency, +} from "./service-utils.ts"; import { stackHealthBudgets } from "./health-budgets.ts"; -interface DockerMailpitOptions { +interface DockerMailpitOptions extends ContainerRuntimeOptions { readonly image: string; readonly identity: StackIdentity; readonly webPort: number; @@ -21,6 +26,7 @@ const mailpitHealthCheck = (port: number): ServiceDef["healthCheck"] => export const makeMailpitServiceDocker = (opts: DockerMailpitOptions): ServiceDef => dockerRunService({ + runtime: opts.runtime, name: "mailpit", identity: opts.identity, image: opts.image, diff --git a/packages/stack/src/services/pgmeta.ts b/packages/stack/src/services/pgmeta.ts index b4a95779c6..fe5d74631b 100644 --- a/packages/stack/src/services/pgmeta.ts +++ b/packages/stack/src/services/pgmeta.ts @@ -1,10 +1,14 @@ import type { ServiceDef } from "@supabase/process-compose"; import { dockerNetworkArgs } from "../Platform.ts"; import type { StackIdentity } from "../StackIdentity.ts"; -import { dockerRunService, type ServiceDependency } from "./service-utils.ts"; +import { + dockerRunService, + type ContainerRuntimeOptions, + type ServiceDependency, +} from "./service-utils.ts"; import { stackHealthBudgets } from "./health-budgets.ts"; -interface DockerPgmetaOptions { +interface DockerPgmetaOptions extends ContainerRuntimeOptions { readonly image: string; readonly identity: StackIdentity; readonly port: number; @@ -27,6 +31,7 @@ const pgmetaHealthCheck = (port: number): ServiceDef["healthCheck"] => ({ export const makePgmetaServiceDocker = (opts: DockerPgmetaOptions): ServiceDef => dockerRunService({ + runtime: opts.runtime, name: "pgmeta", identity: opts.identity, image: opts.image, diff --git a/packages/stack/src/services/pooler.ts b/packages/stack/src/services/pooler.ts index d638ed2029..10e53633bb 100644 --- a/packages/stack/src/services/pooler.ts +++ b/packages/stack/src/services/pooler.ts @@ -1,12 +1,16 @@ import type { ServiceDef } from "@supabase/process-compose"; import { dockerPortMapArgs } from "../Platform.ts"; import type { StackIdentity } from "../StackIdentity.ts"; -import { dockerRunService, type ServiceDependency } from "./service-utils.ts"; +import { + dockerRunService, + type ContainerRuntimeOptions, + type ServiceDependency, +} from "./service-utils.ts"; import { stackHealthBudgets } from "./health-budgets.ts"; type PoolMode = "transaction" | "session"; -interface DockerPoolerOptions { +interface DockerPoolerOptions extends ContainerRuntimeOptions { readonly image: string; readonly identity: StackIdentity; readonly hostAdminPort: number; @@ -70,6 +74,7 @@ end`; export const makePoolerServiceDocker = (opts: DockerPoolerOptions): ServiceDef => (() => { return dockerRunService({ + runtime: opts.runtime, name: "pooler", identity: opts.identity, image: opts.image, diff --git a/packages/stack/src/services/postgres.ts b/packages/stack/src/services/postgres.ts index 910b76d544..1df6f8f3bf 100644 --- a/packages/stack/src/services/postgres.ts +++ b/packages/stack/src/services/postgres.ts @@ -7,6 +7,7 @@ import { stackHealthBudgets } from "./health-budgets.ts"; import { dockerExecHealthCheck, dockerRunService, + type ContainerRuntimeOptions, type ServiceDependency, } from "./service-utils.ts"; @@ -23,7 +24,7 @@ interface NativePostgresOptions extends PostgresServiceOptions { readonly dockerAccessible?: boolean; } -interface DockerPostgresOptions extends PostgresServiceOptions { +interface DockerPostgresOptions extends PostgresServiceOptions, ContainerRuntimeOptions { readonly image: string; readonly platformOs: string; readonly jwtSecret: string; @@ -105,10 +106,20 @@ const postgresHealthCheck = (binPath: string, port: number) => ({ * queries with "unexpected EOF". We use `docker exec` to run pg_isready * inside the container, which verifies postgres is accepting commands. */ -const postgresDockerHealthCheck = (containerName: string, port: number) => - dockerExecHealthCheck(containerName, "pg_isready", ["-p", String(port), "-U", "postgres"], { - ...stackHealthBudgets.postgresDocker, - }); +const postgresDockerHealthCheck = ( + runtime: DockerPostgresOptions["runtime"], + containerName: string, + port: number, +) => + dockerExecHealthCheck( + runtime, + containerName, + "pg_isready", + ["-p", String(port), "-U", "postgres"], + { + ...stackHealthBudgets.postgresDocker, + }, + ); export const makePostgresService = (opts: NativePostgresOptions): ServiceDef => { const initScript = `${opts.binPath}/share/supabase-cli/bin/supabase-postgres-init.sh`; @@ -176,6 +187,7 @@ export const makePostgresServiceDocker = (opts: DockerPostgresOptions): ServiceD const env = postgresDockerEnv(opts); const containerName = dockerContainerName("postgres", opts.identity.key); return dockerRunService({ + runtime: opts.runtime, name: "postgres", identity: opts.identity, image: opts.image, @@ -185,7 +197,7 @@ export const makePostgresServiceDocker = (opts: DockerPostgresOptions): ServiceD entrypoint: "sh", cmd: ["-c", dockerPostgresEntrypoint(opts.port)], dependencies: opts.dependencies, - healthCheck: postgresDockerHealthCheck(containerName, opts.port), + healthCheck: postgresDockerHealthCheck(opts.runtime, containerName, opts.port), shutdown: { signal: "SIGTERM", timeoutSeconds: 10 }, orphanCleanup: orphanCleanup(opts), }); diff --git a/packages/stack/src/services/postgrest.ts b/packages/stack/src/services/postgrest.ts index 0573e8b4e7..bfb211b8c1 100644 --- a/packages/stack/src/services/postgrest.ts +++ b/packages/stack/src/services/postgrest.ts @@ -2,7 +2,11 @@ import type { ServiceDef } from "@supabase/process-compose"; import { dockerNetworkArgs } from "../Platform.ts"; import type { StackIdentity } from "../StackIdentity.ts"; import { stackHealthBudgets } from "./health-budgets.ts"; -import { dockerRunService, type ServiceDependency } from "./service-utils.ts"; +import { + dockerRunService, + type ContainerRuntimeOptions, + type ServiceDependency, +} from "./service-utils.ts"; interface PostgrestServiceOptions { readonly dbPort: number; @@ -18,7 +22,7 @@ interface NativePostgrestOptions extends PostgrestServiceOptions { readonly binPath: string; } -interface DockerPostgrestOptions extends PostgrestServiceOptions { +interface DockerPostgrestOptions extends PostgrestServiceOptions, ContainerRuntimeOptions { readonly image: string; readonly dbHost: string; readonly platformOs: string; @@ -66,6 +70,7 @@ export const makePostgrestServiceDocker = (opts: DockerPostgrestOptions): Servic PGRST_ADMIN_SERVER_PORT: String(opts.adminPort), }; return dockerRunService({ + runtime: opts.runtime, name: "postgrest", identity: opts.identity, image: opts.image, diff --git a/packages/stack/src/services/realtime.ts b/packages/stack/src/services/realtime.ts index 32ac4d53a0..0783ed830f 100644 --- a/packages/stack/src/services/realtime.ts +++ b/packages/stack/src/services/realtime.ts @@ -1,10 +1,14 @@ import type { ServiceDef } from "@supabase/process-compose"; import { dockerNetworkArgs } from "../Platform.ts"; import type { StackIdentity } from "../StackIdentity.ts"; -import { dockerRunService, type ServiceDependency } from "./service-utils.ts"; +import { + dockerRunService, + type ContainerRuntimeOptions, + type ServiceDependency, +} from "./service-utils.ts"; import { stackHealthBudgets } from "./health-budgets.ts"; -interface DockerRealtimeOptions { +interface DockerRealtimeOptions extends ContainerRuntimeOptions { readonly image: string; readonly port: number; readonly identity: StackIdentity; @@ -39,6 +43,7 @@ const realtimeHealthCheck = (port: number, tenantId: string): ServiceDef["health export const makeRealtimeServiceDocker = (opts: DockerRealtimeOptions): ServiceDef => dockerRunService({ + runtime: opts.runtime, name: "realtime", identity: opts.identity, image: opts.image, diff --git a/packages/stack/src/services/service-utils.ts b/packages/stack/src/services/service-utils.ts index 230741cfde..51a36a4858 100644 --- a/packages/stack/src/services/service-utils.ts +++ b/packages/stack/src/services/service-utils.ts @@ -1,5 +1,6 @@ import type { ExternalCleanupAction, ServiceDef } from "@supabase/process-compose"; import type { ServiceName } from "../ServiceName.ts"; +import type { ContainerRuntime } from "../ContainerRuntime.ts"; import { dockerContainerName, STACK_ID_LABEL, type StackIdentity } from "../StackIdentity.ts"; import { dockerServiceCleanup, dockerServiceOrphanCleanup } from "./docker-cleanup.ts"; @@ -8,7 +9,11 @@ export interface ServiceDependency { readonly condition: "healthy" | "completed"; } -interface DockerRunServiceOptions { +export interface ContainerRuntimeOptions { + readonly runtime: ContainerRuntime; +} + +interface DockerRunServiceOptions extends ContainerRuntimeOptions { readonly name: ServiceName; readonly identity: StackIdentity; readonly image: string; @@ -44,6 +49,7 @@ export const hostHttpHealthCheck = ( }); export const dockerExecHealthCheck = ( + runtime: ContainerRuntime, containerName: string, command: string, args: ReadonlyArray, @@ -51,7 +57,7 @@ export const dockerExecHealthCheck = ( ): ServiceDef["healthCheck"] => ({ probe: { _tag: "Exec", - command: "docker", + command: runtime, args: ["exec", containerName, command, ...args], }, ...opts, @@ -80,14 +86,17 @@ export const dockerRunService = (opts: DockerRunServiceOptions): ServiceDef => { return { name: opts.name, - command: "docker", + command: opts.runtime, args: dockerArgs, dependencies: opts.dependencies, healthCheck: opts.healthCheck, shutdown: opts.shutdown, - cleanup: dockerServiceCleanup(containerName), + cleanup: dockerServiceCleanup(opts.runtime, containerName), supervision: { - orphanCleanup: [...dockerServiceOrphanCleanup(containerName), ...(opts.orphanCleanup ?? [])], + orphanCleanup: [ + ...dockerServiceOrphanCleanup(opts.runtime, containerName), + ...(opts.orphanCleanup ?? []), + ], }, restart: opts.restart ?? "unless-stopped", }; diff --git a/packages/stack/src/services/services.unit.test.ts b/packages/stack/src/services/services.unit.test.ts index c5d0c814d6..72367fc302 100644 --- a/packages/stack/src/services/services.unit.test.ts +++ b/packages/stack/src/services/services.unit.test.ts @@ -95,6 +95,7 @@ describe("analyticsDockerRuntimeNetwork", () => { describe("makeStudioServiceDocker", () => { it("injects legacy keys, opaque keys, and S3 protocol credentials", () => { const def = makeStudioServiceDocker({ + runtime: "docker", image: dockerImageForService("studio", DEFAULT_VERSIONS.studio), identity: EPHEMERAL_IDENTITY, port: 54323, @@ -173,6 +174,7 @@ describe("makePostgresService (dockerAccessible)", () => { describe("makePostgresServiceDocker", () => { it("creates a docker-based postgres ServiceDef", () => { const def = makePostgresServiceDocker({ + runtime: "docker", image: dockerImageForService("postgres", DEFAULT_VERSIONS.postgres), dataDir: "/tmp/supabase/data", port: DB_PORT, @@ -224,6 +226,7 @@ describe("makePostgresServiceDocker", () => { it("bootstraps auxiliary databases and schemas used by docker-backed services", () => { const def = makePostgresServiceDocker({ + runtime: "docker", image: dockerImageForService("postgres", DEFAULT_VERSIONS.postgres), dataDir: "/tmp/supabase/data", port: DB_PORT, @@ -280,6 +283,7 @@ describe("makePostgrestService", () => { it("creates a docker definition with caller-supplied topology and derived identity", () => { const dependencies = [{ service: "postgres", condition: "healthy" }] as const; const def = makePostgrestServiceDocker({ + runtime: "docker", image: dockerImageForService("postgrest", DEFAULT_VERSIONS.postgrest), identity: EPHEMERAL_IDENTITY, dbHost: "host.docker.internal", @@ -343,6 +347,7 @@ describe("makeAuthServiceNative", () => { describe("makeAuthServiceDocker", () => { it("creates a docker-based auth ServiceDef", () => { const def = makeAuthServiceDocker({ + runtime: "docker", image: dockerImageForService("auth", DEFAULT_VERSIONS.auth), dbPort: DB_PORT, authPort: 9999, @@ -383,6 +388,7 @@ describe("makeEdgeRuntimeServiceDocker", () => { try { const def = makeEdgeRuntimeServiceDocker({ + runtime: "docker", image: dockerImageForService("edge-runtime", DEFAULT_VERSIONS["edge-runtime"]), identity: EPHEMERAL_IDENTITY, runtimeRoot: tempDir, @@ -570,6 +576,7 @@ describe("docker-backed auxiliary services", () => { it("defines realtime command, topology, environment, and readiness locally", () => { const dependencies = [{ service: "postgres", condition: "healthy" }] as const; const def = makeRealtimeServiceDocker({ + runtime: "docker", image: dockerImageForService("realtime", DEFAULT_VERSIONS.realtime), identity: EPHEMERAL_IDENTITY, port: 54330, @@ -597,6 +604,7 @@ describe("docker-backed auxiliary services", () => { it("defines storage mounts, cleanup, topology, and readiness locally", () => { const dependencies = [{ service: "postgres-init", condition: "completed" }] as const; const def = makeStorageServiceDocker({ + runtime: "docker", image: dockerImageForService("storage", DEFAULT_VERSIONS.storage), identity: EPHEMERAL_IDENTITY, port: 54331, @@ -633,6 +641,7 @@ describe("docker-backed auxiliary services", () => { it("defines postgres metadata command, topology, environment, and readiness locally", () => { const dependencies = [{ service: "postgres", condition: "healthy" }] as const; const def = makePgmetaServiceDocker({ + runtime: "docker", image: dockerImageForService("pgmeta", DEFAULT_VERSIONS.pgmeta), identity: EPHEMERAL_IDENTITY, port: 54336, @@ -653,6 +662,7 @@ describe("docker-backed auxiliary services", () => { it("uses a host HTTP readiness probe for mailpit", () => { const def = makeMailpitServiceDocker({ + runtime: "docker", image: dockerImageForService("mailpit", DEFAULT_VERSIONS.mailpit), identity: EPHEMERAL_IDENTITY, webPort: 54323, @@ -673,6 +683,7 @@ describe("docker-backed auxiliary services", () => { it("uses a host HTTP health probe for imgproxy", () => { const def = makeImgproxyServiceDocker({ + runtime: "docker", image: dockerImageForService("imgproxy", DEFAULT_VERSIONS.imgproxy), identity: EPHEMERAL_IDENTITY, port: 54326, @@ -693,6 +704,7 @@ describe("docker-backed auxiliary services", () => { it("uses docker exec for vector health because its admin port is not published", () => { const def = makeVectorServiceDocker({ + runtime: "docker", image: dockerImageForService("vector", DEFAULT_VERSIONS.vector), identity: EPHEMERAL_IDENTITY, serviceHost: "127.0.0.1", @@ -717,6 +729,7 @@ describe("docker-backed auxiliary services", () => { it("binds analytics on all interfaces so published ports and proxy health checks work", () => { const def = makeAnalyticsServiceDocker({ + runtime: "docker", image: dockerImageForService("analytics", DEFAULT_VERSIONS.analytics), identity: EPHEMERAL_IDENTITY, hostPort: 54328, @@ -748,6 +761,7 @@ describe("docker-backed auxiliary services", () => { it("keeps analytics on its container port when Linux uses bridge networking", () => { const def = makeAnalyticsServiceDocker({ + runtime: "docker", image: dockerImageForService("analytics", DEFAULT_VERSIONS.analytics), identity: EPHEMERAL_IDENTITY, hostPort: 54328, @@ -768,6 +782,7 @@ describe("docker-backed auxiliary services", () => { it("keeps pooler container ports fixed and maps only the selected proxy port outward", () => { const def = makePoolerServiceDocker({ + runtime: "docker", image: dockerImageForService("pooler", DEFAULT_VERSIONS.pooler), identity: EPHEMERAL_IDENTITY, hostAdminPort: 54329, diff --git a/packages/stack/src/services/storage.ts b/packages/stack/src/services/storage.ts index b0da6a5997..e739aa962e 100644 --- a/packages/stack/src/services/storage.ts +++ b/packages/stack/src/services/storage.ts @@ -2,10 +2,14 @@ import type { ServiceDef } from "@supabase/process-compose"; import { dockerNetworkArgs } from "../Platform.ts"; import type { StackIdentity } from "../StackIdentity.ts"; import { removePathOnOrphanCleanup } from "./docker-cleanup.ts"; -import { dockerRunService, type ServiceDependency } from "./service-utils.ts"; +import { + dockerRunService, + type ContainerRuntimeOptions, + type ServiceDependency, +} from "./service-utils.ts"; import { stackHealthBudgets } from "./health-budgets.ts"; -interface DockerStorageOptions { +interface DockerStorageOptions extends ContainerRuntimeOptions { readonly image: string; readonly port: number; readonly identity: StackIdentity; @@ -46,6 +50,7 @@ const storageHealthCheck = (port: number): ServiceDef["healthCheck"] => ({ export const makeStorageServiceDocker = (opts: DockerStorageOptions): ServiceDef => dockerRunService({ + runtime: opts.runtime, name: "storage", identity: opts.identity, image: opts.image, diff --git a/packages/stack/src/services/studio.ts b/packages/stack/src/services/studio.ts index c1223a4882..bb5f1009d5 100644 --- a/packages/stack/src/services/studio.ts +++ b/packages/stack/src/services/studio.ts @@ -1,10 +1,14 @@ import type { ServiceDef } from "@supabase/process-compose"; import { dockerNetworkArgs } from "../Platform.ts"; import type { StackIdentity } from "../StackIdentity.ts"; -import { dockerRunService, type ServiceDependency } from "./service-utils.ts"; +import { + dockerRunService, + type ContainerRuntimeOptions, + type ServiceDependency, +} from "./service-utils.ts"; import { stackHealthBudgets } from "./health-budgets.ts"; -interface DockerStudioOptions { +interface DockerStudioOptions extends ContainerRuntimeOptions { readonly image: string; readonly identity: StackIdentity; readonly port: number; @@ -37,6 +41,7 @@ const studioHealthCheck = (port: number): ServiceDef["healthCheck"] => ({ export const makeStudioServiceDocker = (opts: DockerStudioOptions): ServiceDef => dockerRunService({ + runtime: opts.runtime, name: "studio", identity: opts.identity, image: opts.image, diff --git a/packages/stack/src/services/vector.ts b/packages/stack/src/services/vector.ts index b0c5841d5c..945fe80e22 100644 --- a/packages/stack/src/services/vector.ts +++ b/packages/stack/src/services/vector.ts @@ -4,11 +4,12 @@ import { dockerContainerName, type StackIdentity } from "../StackIdentity.ts"; import { dockerExecHealthCheck, dockerRunService, + type ContainerRuntimeOptions, type ServiceDependency, } from "./service-utils.ts"; import { stackHealthBudgets } from "./health-budgets.ts"; -interface DockerVectorOptions { +interface DockerVectorOptions extends ContainerRuntimeOptions { readonly image: string; readonly identity: StackIdentity; readonly serviceHost: string; @@ -49,6 +50,7 @@ export const makeVectorServiceDocker = (opts: DockerVectorOptions) => { const volumes = existsSync(dockerSocket) ? [`${dockerSocket}:/var/run/docker.sock:ro`] : []; return dockerRunService({ + runtime: opts.runtime, name: "vector", identity: opts.identity, image: opts.image, @@ -66,6 +68,7 @@ ${VECTOR_CONFIG(opts.serviceHost, opts.analyticsPort, opts.analyticsApiKey)}EOF ], dependencies: opts.dependencies, healthCheck: dockerExecHealthCheck( + opts.runtime, containerName, "sh", ["-ec", "wget -q -O /dev/null http://127.0.0.1:9001/health"], diff --git a/packages/stack/src/supervisor.integration.test.ts b/packages/stack/src/supervisor.integration.test.ts index da96e82da0..d209702242 100644 --- a/packages/stack/src/supervisor.integration.test.ts +++ b/packages/stack/src/supervisor.integration.test.ts @@ -256,7 +256,7 @@ const remoteInfo = (endpoint: ControlEndpoint): Promise<{ readonly url: string } const updateLaunch = async ( endpoint: ControlEndpoint, launch: { - readonly mode: "native" | "auto" | "docker"; + readonly mode: "native" | "docker"; readonly versions: Record; }, ): Promise => { @@ -905,9 +905,9 @@ describe("detached supervisor child journeys", () => { const attached = await later.started; expect(attached.attached).toBe(true); expect(await remoteInfo(attached.endpoint)).toMatchObject({ url: expect.any(String) }); - await updateLaunch(attached.endpoint, { mode: "auto", versions: { postgres: "17.6.1" } }); + await updateLaunch(attached.endpoint, { mode: "docker", versions: { postgres: "17.6.1" } }); expect(readStackDocument(roots)?.launch).toEqual({ - mode: "auto", + mode: "docker", versions: { postgres: "17.6.1" }, }); await remoteStop(attached.endpoint); diff --git a/packages/stack/src/supervisor.ts b/packages/stack/src/supervisor.ts index ca0aa68f0a..f8ba002c0d 100644 --- a/packages/stack/src/supervisor.ts +++ b/packages/stack/src/supervisor.ts @@ -12,6 +12,8 @@ import { Schema, } from "effect"; import { HttpServer } from "effect/unstable/http"; +import { ChildProcessSpawner } from "effect/unstable/process"; +import { selectStackRuntime } from "./ContainerRuntime.ts"; import type { PlatformFactory } from "./createStack.ts"; import { DaemonServer } from "./DaemonServer.ts"; import { Stack } from "./Stack.ts"; @@ -26,7 +28,7 @@ import { type ControlTransport, } from "./managed/control.ts"; import { ManagedStackManager, type ManagedStackStartResult } from "./managed/manager.ts"; -import { managedStackLaunchSchema } from "./managed/document.ts"; +import { managedStackLaunchInputSchema, type ManagedStackLaunchInput } from "./managed/document.ts"; import { deriveStackId, ensureEnvironment } from "./managed/environment.ts"; import { gitConfigStoreLayer } from "./managed/git.ts"; import { validateManagedStackName, type ManagedPortIntentDocument } from "./managed/model.ts"; @@ -51,7 +53,7 @@ export interface SupervisorStartMessage { readonly stateRoot: string; readonly config: Readonly>; readonly portIntents: ManagedPortIntentDocument; - readonly launch?: import("./managed/document.ts").ManagedStackDocument["launch"]; + readonly launch?: ManagedStackLaunchInput; } export interface SupervisorStartedMessage { @@ -73,7 +75,7 @@ export interface ManagedDaemonStartInput { readonly stateRoot: string; readonly config: Readonly>; readonly portIntents: ManagedPortIntentDocument; - readonly launch?: import("./managed/document.ts").ManagedStackDocument["launch"]; + readonly launch?: ManagedStackLaunchInput; } const supervisorPortIntentSchema = Schema.Struct({ @@ -90,7 +92,7 @@ const supervisorStartMessageSchema = Schema.Struct({ stateRoot: Schema.String, config: Schema.Record(Schema.String, Schema.Unknown), portIntents: supervisorPortIntentSchema, - launch: Schema.optionalKey(managedStackLaunchSchema), + launch: Schema.optionalKey(managedStackLaunchInputSchema), }); const isRecord = (value: unknown): value is Readonly> => @@ -310,6 +312,7 @@ const runManaged = ( | ControlTransport | import("effect").FileSystem.FileSystem | import("effect").Path.Path + | ChildProcessSpawner.ChildProcessSpawner | Scope.Scope > => { let owner: ControlOwnership | undefined; @@ -433,6 +436,7 @@ const runManaged = ( ); } } + const runtime = yield* selectStackRuntime(configInput.mode ?? input.launch?.mode); const startup = Effect.gen(function* () { const existing = yield* manager.inspectStack(stackId); if ( @@ -442,9 +446,12 @@ const runManaged = ( existing.lifecycle === "failed" || existing.lifecycle === "deleting") ) { - yield* dockerForceRemove( - SERVICE_NAMES.map((service) => dockerContainerName(service, `id-${stackId}`)), - ); + if (runtime.containerRuntime !== null) { + yield* dockerForceRemove( + runtime.containerRuntime, + SERVICE_NAMES.map((service) => dockerContainerName(service, `id-${stackId}`)), + ); + } } const started: ManagedStackStartResult = yield* manager.startStack({ workspacePath: input.workspacePath, @@ -452,7 +459,16 @@ const runManaged = ( portDocument: input.portIntents, ownership, lifecycle: "starting", - launch: input.launch, + launch: + input.launch === undefined + ? undefined + : { + ...input.launch, + mode: runtime.mode, + ...(runtime.containerRuntime === null + ? {} + : { containerRuntime: runtime.containerRuntime }), + }, }); claimedStack = true; const resolved = yield* Effect.tryPromise({ @@ -465,7 +481,7 @@ const runManaged = ( runtimeRoot: managedStackPaths(input.stateRoot, started.stack.id).runtime, instanceId: started.stack.id, }, - { portAllocator: () => Effect.succeed(started.lease.ports) }, + { runtime, portAllocator: () => Effect.succeed(started.lease.ports) }, ), catch: (cause) => cause, }); @@ -557,7 +573,10 @@ export const runSupervisor = ( ): Effect.Effect< void, SupervisorStartError | unknown, - ControlTransport | import("effect").FileSystem.FileSystem | import("effect").Path.Path + | ControlTransport + | import("effect").FileSystem.FileSystem + | import("effect").Path.Path + | ChildProcessSpawner.ChildProcessSpawner > => Effect.scoped( Effect.gen(function* () { diff --git a/packages/stack/tests/createStack-native.e2e.test.ts b/packages/stack/tests/createStack-native.e2e.test.ts new file mode 100644 index 0000000000..34a145b271 --- /dev/null +++ b/packages/stack/tests/createStack-native.e2e.test.ts @@ -0,0 +1,45 @@ +import { createClient } from "@supabase/supabase-js"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterAll, beforeAll, describe, expect, test } from "vitest"; +import { createStack, type StackHandle } from "../src/node.ts"; +import { setupTestTable } from "./helpers/e2e.ts"; + +describe("native PostgREST tracer bullet", () => { + let stack: StackHandle; + let dataDir: string; + + beforeAll(async () => { + dataDir = mkdtempSync(join(tmpdir(), "supabase-native-postgrest-e2e-")); + stack = await createStack({ + mode: "native", + functions: false, + edgeRuntime: false, + auth: false, + postgres: { dataDir }, + }); + await stack.start(); + await setupTestTable(parseInt(new URL(stack.dbUrl).port)); + }, 45_000); + + afterAll(async () => { + await stack?.dispose(); + rmSync(dataDir, { recursive: true, force: true }); + }, 30_000); + + test("serves a CRUD request through the native PostgREST resource", async () => { + const client = createClient(stack.url, stack.publishableKey); + const inserted = await client + .from("todos") + .insert({ title: "native tracer bullet" }) + .select() + .single(); + + expect(inserted.error).toBeNull(); + expect(inserted.data).toEqual(expect.objectContaining({ title: "native tracer bullet" })); + + const deleted = await client.from("todos").delete().eq("title", "native tracer bullet"); + expect(deleted.error).toBeNull(); + }, 30_000); +}); From f7501f4845250aa74c86b3daf9cea7214396483b Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 18 Aug 2026 22:11:26 +0200 Subject: [PATCH 04/20] fix(stack): own runtime selection lifecycle --- .../commands/link/link.integration.test.ts | 4 +- .../service-version-overrides.unit.test.ts | 14 +- .../src/next/commands/start/start.handler.ts | 1 - .../next/commands/update/update.handler.ts | 7 +- apps/cli/tests/helpers/running-stack.ts | 1 + .../process-compose/tests/helpers/mocks.ts | 4 +- packages/stack/src/ContainerRuntime.ts | 39 +++- packages/stack/src/DaemonServer.ts | 12 +- packages/stack/src/LocalStack.ts | 147 ++++++++------ packages/stack/src/Stack.ts | 26 ++- packages/stack/src/Stack.unit.test.ts | 189 ++++++++++-------- packages/stack/src/StackBuilder.ts | 118 +++++------ packages/stack/src/StackBuilder.unit.test.ts | 61 ++---- packages/stack/src/StackConfigResolver.ts | 19 +- packages/stack/src/StackPreparation.ts | 67 ++++--- packages/stack/src/bun.ts | 23 ++- packages/stack/src/createStack.unit.test.ts | 65 +++--- packages/stack/src/effect-bun.ts | 2 +- packages/stack/src/effect-node.ts | 2 +- packages/stack/src/effect.ts | 5 +- packages/stack/src/functions.unit.test.ts | 18 +- ...aged-manager-lifecycle.integration.test.ts | 11 +- .../src/managed-store.integration.test.ts | 2 + packages/stack/src/managed.ts | 4 +- packages/stack/src/managed/document.ts | 48 +++-- packages/stack/src/managed/lifecycle.ts | 11 +- packages/stack/src/managed/manager.ts | 47 ++++- packages/stack/src/node.ts | 23 ++- packages/stack/src/prefetch.ts | 24 ++- packages/stack/src/prefetch.unit.test.ts | 33 ++- packages/stack/src/services/postgres-init.ts | 59 +++++- packages/stack/src/services/postgres.ts | 91 +-------- .../stack/src/services/services.unit.test.ts | 75 +------ packages/stack/src/services/vector.ts | 38 ++-- .../stack/src/supervisor.integration.test.ts | 86 +++++++- packages/stack/src/supervisor.ts | 85 ++++++-- packages/stack/tests/helpers/warmup.ts | 6 +- .../stack/tests/helpers/warmup.unit.test.ts | 20 +- 38 files changed, 851 insertions(+), 636 deletions(-) diff --git a/apps/cli/src/next/commands/link/link.integration.test.ts b/apps/cli/src/next/commands/link/link.integration.test.ts index 943cae7945..5af28a05f9 100644 --- a/apps/cli/src/next/commands/link/link.integration.test.ts +++ b/apps/cli/src/next/commands/link/link.integration.test.ts @@ -346,7 +346,7 @@ describe("link handler", () => { }, ], behavior: { - mode: "docker", + mode: "auto", placeholder: "Search projects...", maxItems: 10, }, @@ -515,7 +515,7 @@ describe("link handler", () => { }, ], behavior: { - mode: "docker", + mode: "auto", placeholder: "Search projects...", maxItems: 10, }, diff --git a/apps/cli/src/next/commands/start/service-version-overrides.unit.test.ts b/apps/cli/src/next/commands/start/service-version-overrides.unit.test.ts index dc23a41bac..2316f1ec8b 100644 --- a/apps/cli/src/next/commands/start/service-version-overrides.unit.test.ts +++ b/apps/cli/src/next/commands/start/service-version-overrides.unit.test.ts @@ -11,14 +11,14 @@ import { } from "../../config/service-version-resolution.ts"; describe("service version overrides", () => { - test("parses and normalizes repeated flag overrides", async () => { + test("parses repeated flag overrides as exact image tags", async () => { await expect( Effect.runPromise( parseServiceVersionOverrides(["postgrest=v14.5", "mailpit=1.30.2", "auth=2.180.0"]), ), ).resolves.toEqual({ - postgrest: "14.5", - mailpit: "v1.30.2", + postgrest: "v14.5", + mailpit: "1.30.2", auth: "2.180.0", }); }); @@ -27,8 +27,8 @@ describe("service version overrides", () => { const candidateBaseline = { ...DEFAULT_VERSIONS, postgres: "17.6.1.090", - postgrest: "14.5", - auth: "2.187.0", + postgrest: "v14.5", + auth: "v2.187.0", }; const layer = Layer.mergeAll( @@ -68,12 +68,12 @@ describe("service version overrides", () => { runtimeVersions: { ...candidateBaseline, postgres: "17.4.1.045", - auth: "2.170.0", + auth: "v2.170.0", storage: "1.40.0", }, activeOverrides: [ { service: "postgres", version: "17.4.1.045", source: "flag" }, - { service: "auth", version: "2.170.0", source: "flag" }, + { service: "auth", version: "v2.170.0", source: "flag" }, { service: "storage", version: "1.40.0", source: "local" }, ], availableUpdates: [], diff --git a/apps/cli/src/next/commands/start/start.handler.ts b/apps/cli/src/next/commands/start/start.handler.ts index 5b6c5303e0..2a388237f7 100644 --- a/apps/cli/src/next/commands/start/start.handler.ts +++ b/apps/cli/src/next/commands/start/start.handler.ts @@ -50,7 +50,6 @@ export const start = Effect.fnUntraced(function* (flags: StartFlags) { yield* updateManagedLaunch({ ...lifecycleInput, launch: { - mode: launch.mode, versions: launch.versions, excludedServices: launch.excludedServices, lastNotifiedUpdateFingerprint: serviceVersionContext.updateFingerprint, diff --git a/apps/cli/src/next/commands/update/update.handler.ts b/apps/cli/src/next/commands/update/update.handler.ts index 4aed1af93a..165b4b6739 100644 --- a/apps/cli/src/next/commands/update/update.handler.ts +++ b/apps/cli/src/next/commands/update/update.handler.ts @@ -103,19 +103,14 @@ export const update = Effect.fnUntraced(function* (flags: UpdateFlags) { ); if (Option.isSome(existingSummary)) { - const persistedLaunch = existingSummary.value.launch ?? { - mode: "docker" as const, - excludedServices: [] as const, - }; yield* updateManagedLaunch({ cacheRoot: cliConfig.supabaseHome, cwd: runtimeInfo.cwd, workspacePath: projectHome.projectRoot, stackName: flags.stack, launch: { - mode: persistedLaunch.mode, versions: serviceVersionContext.candidateBaseline, - excludedServices: persistedLaunch.excludedServices, + excludedServices: existingSummary.value.launch?.excludedServices ?? [], ...(existingSummary.value.lastNotifiedUpdateFingerprint === undefined ? {} : { diff --git a/apps/cli/tests/helpers/running-stack.ts b/apps/cli/tests/helpers/running-stack.ts index 45e58edf4d..712dad8ea8 100644 --- a/apps/cli/tests/helpers/running-stack.ts +++ b/apps/cli/tests/helpers/running-stack.ts @@ -26,6 +26,7 @@ import { RuntimeInfo } from "../../src/shared/runtime/runtime-info.service.ts"; const launch = { mode: "docker" as const, + containerRuntime: "docker" as const, versions: { postgres: "17.6.1" }, excludedServices: [], }; diff --git a/packages/process-compose/tests/helpers/mocks.ts b/packages/process-compose/tests/helpers/mocks.ts index 2df118417b..849c85eef5 100644 --- a/packages/process-compose/tests/helpers/mocks.ts +++ b/packages/process-compose/tests/helpers/mocks.ts @@ -17,10 +17,10 @@ const isOneShotSupervisor = (args: ReadonlyArray): boolean => { typeof config === "object" && config !== null && "command" in config && - config.command === "bash" && "args" in config && Array.isArray(config.args) && - config.args[0] === "-c" + ((config.command === "bash" && config.args[0] === "-c") || + ((config.command === "docker" || config.command === "podman") && config.args[0] === "exec")) ); } catch { return false; diff --git a/packages/stack/src/ContainerRuntime.ts b/packages/stack/src/ContainerRuntime.ts index e4a897a21d..aeb3dc84f1 100644 --- a/packages/stack/src/ContainerRuntime.ts +++ b/packages/stack/src/ContainerRuntime.ts @@ -9,6 +9,39 @@ export type StackRuntimeSelection = | { readonly mode: "native"; readonly containerRuntime: null } | { readonly mode: "docker"; readonly containerRuntime: ContainerRuntime }; +const probeContainerRuntime = ( + runtime: ContainerRuntime, +): Effect.Effect => + Effect.gen(function* () { + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const result = yield* Effect.exit( + spawner.exitCode(ChildProcess.make(runtime, ["info"])).pipe(Effect.timeout("30 seconds")), + ); + return Exit.isSuccess(result) && result.value === 0; + }); + +export const validateStackRuntime = ( + selection: StackRuntimeSelection, +): Effect.Effect< + StackRuntimeSelection, + StackBuildError, + ChildProcessSpawner.ChildProcessSpawner +> => + selection.containerRuntime === null + ? Effect.succeed(selection) + : probeContainerRuntime(selection.containerRuntime).pipe( + Effect.flatMap((usable) => + usable + ? Effect.succeed(selection) + : Effect.fail( + new StackBuildError({ + detail: `Docker mode requires a usable ${selection.containerRuntime} runtime`, + reason: "docker_not_running", + }), + ), + ), + ); + export const selectStackRuntime = ( requestedMode?: StackMode, ): Effect.Effect => @@ -17,13 +50,9 @@ export const selectStackRuntime = ( return { mode: "native", containerRuntime: null }; } - const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const runtimes: ReadonlyArray = ["docker", "podman"]; for (const runtime of runtimes) { - const result = yield* Effect.exit( - spawner.exitCode(ChildProcess.make(runtime, ["info"])).pipe(Effect.timeout("3 seconds")), - ); - if (Exit.isSuccess(result) && result.value === 0) { + if (yield* probeContainerRuntime(runtime)) { return { mode: "docker", containerRuntime: runtime }; } } diff --git a/packages/stack/src/DaemonServer.ts b/packages/stack/src/DaemonServer.ts index 0c56371133..d2efe9f830 100644 --- a/packages/stack/src/DaemonServer.ts +++ b/packages/stack/src/DaemonServer.ts @@ -11,7 +11,10 @@ import type { ControlOwnerStatus, DaemonErrorResponse } from "./DaemonProtocol.t import { FunctionsReloadConfigSchema } from "./functions.ts"; import { EdgeRuntimeReloadConfigSchema, Stack } from "./Stack.ts"; import { ReadyOptionsSchema } from "./StackConfig.ts"; -import { managedStackLaunchSchema, type ManagedStackLaunch } from "./managed/document.ts"; +import { + managedStackLaunchUpdateSchema, + type ManagedStackLaunchUpdate, +} from "./managed/document.ts"; // --------------------------------------------------------------------------- // Service @@ -35,7 +38,7 @@ export class DaemonServer extends Context.Service< }), options: { readonly includeOwnerRoute?: boolean; - readonly launchUpdate?: (launch: ManagedStackLaunch) => Effect.Effect; + readonly launchUpdate?: (launch: ManagedStackLaunchUpdate) => Effect.Effect; /** Supervisor-owned shutdown callbacks already stop the local stack. */ readonly stopOnShutdown?: boolean; } = {}, @@ -178,8 +181,9 @@ export class DaemonServer extends Context.Service< "POST", "/managed/launch", Effect.gen(function* () { - const launch = - yield* HttpServerRequest.schemaBodyJson(managedStackLaunchSchema); + const launch = yield* HttpServerRequest.schemaBodyJson( + managedStackLaunchUpdateSchema, + ); yield* launchUpdate(launch); return HttpServerResponse.jsonUnsafe({ ok: true }); }), diff --git a/packages/stack/src/LocalStack.ts b/packages/stack/src/LocalStack.ts index 6b6da842c2..2a1dfbfb70 100644 --- a/packages/stack/src/LocalStack.ts +++ b/packages/stack/src/LocalStack.ts @@ -7,11 +7,13 @@ import { Duration, Effect, Equal, + Exit, FileSystem, Layer, Path, Ref, Schema, + Scope, Semaphore, Stream, SubscriptionRef, @@ -46,8 +48,13 @@ import { ServiceDownloadStarted, StackPreparation, } from "./StackPreparation.ts"; -import type { PreparedStackArtifacts } from "./StackPreparation.ts"; -import { enabledServicesForConfig, StackBuilder, versionsForConfig } from "./StackBuilder.ts"; +import type { PreparedStackArtifacts, StackPreparationInput } from "./StackPreparation.ts"; +import { + enabledServicesForConfig, + StackBuilder, + validateResolvedConfig, + versionsForConfig, +} from "./StackBuilder.ts"; import { resolveReadinessPolicy } from "./StackConfig.ts"; import type { ReadinessPolicy, ReadyOptions, ResolvedStackConfig } from "./StackConfig.ts"; import { projectStackStates, type StackServiceProjectionCatalog } from "./StackStateProjection.ts"; @@ -181,6 +188,7 @@ export const localStackLayer = ( const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; const scope = yield* Effect.scope; + const preparationScope = yield* Scope.fork(scope, "parallel"); const info = stackInfoFor(config); const enabledServices = enabledServicesForConfig(config); @@ -251,22 +259,38 @@ export const localStackLayer = ( let runtimeDeferred: Deferred.Deferred | undefined; let exactCleanupTargets: CleanupTargets | undefined; + const preparationInput = ( + services: ReadonlyArray, + ): Effect.Effect => { + const shared = { + services, + enabledServices, + versions: versionsForConfig(config), + }; + if (config.mode === "native") return Effect.succeed({ ...shared, mode: "native" }); + return config.containerRuntime === null + ? Effect.fail( + new StackBuildError({ + detail: "Docker mode requires a selected Docker or Podman runtime", + reason: "invalid_config", + }), + ) + : Effect.succeed({ + ...shared, + mode: "docker", + containerRuntime: config.containerRuntime, + }); + }; + const ensurePlanned = Effect.suspend(() => { if (plannedArtifacts !== undefined) return Effect.succeed(plannedArtifacts); if (planDeferred !== undefined) return Deferred.await(planDeferred); const deferred = Deferred.makeUnsafe(); planDeferred = deferred; - const effect = preparation - .plan({ - mode: config.mode, - ...(config.containerRuntime === null - ? {} - : { containerRuntime: config.containerRuntime }), - services: enabledServicesForConfig(config), - enabledServices, - versions: versionsForConfig(config), - }) - .pipe( + const effect = Effect.gen(function* () { + yield* validateResolvedConfig(config); + const input = yield* preparationInput(enabledServicesForConfig(config)); + return yield* preparation.plan(input).pipe( Effect.mapError( (cause) => new StackBuildError({ @@ -275,15 +299,17 @@ export const localStackLayer = ( reason: "asset_preparation", }), ), - Effect.tap((value) => - Effect.sync(() => { - plannedArtifacts = value; - }), - ), - Effect.ensuring(Effect.sync(() => (planDeferred = undefined))), ); + }).pipe( + Effect.tap((value) => + Effect.sync(() => { + plannedArtifacts = value; + }), + ), + Effect.ensuring(Effect.sync(() => (planDeferred = undefined))), + ); return Effect.gen(function* () { - yield* Effect.forkIn(effect.pipe(Deferred.into(deferred)), scope); + yield* Effect.forkIn(effect.pipe(Deferred.into(deferred)), preparationScope); return yield* Deferred.await(deferred); }); }); @@ -314,30 +340,27 @@ export const localStackLayer = ( ); const deferred = Deferred.makeUnsafe(); preparationInFlight.set(key, deferred); - const effect = Stream.runFoldEffect( - preparation.prepareEvents({ - mode: config.mode, - ...(config.containerRuntime === null - ? {} - : { containerRuntime: config.containerRuntime }), - services: pending, - enabledServices, - versions: versionsForConfig(config), - }), - () => ({ resolutions: {} }) satisfies PreparedStackArtifacts, - (current, event) => - Effect.gen(function* () { - if (event instanceof ServiceDownloadStarted) { - const state = SubscriptionRef.getUnsafe(stateRef).find( - (entry) => entry.name === event.service, - ); - if (state !== undefined && state.status !== "Downloading") { - yield* updateState(new StackServiceState({ ...state, status: "Downloading" })); - } - } - return event instanceof PreparationCompleted ? event.artifacts : current; - }), - ).pipe( + const effect = preparationInput(pending).pipe( + Effect.flatMap((input) => + Stream.runFoldEffect( + preparation.prepareEvents(input), + () => ({ resolutions: {} }) satisfies PreparedStackArtifacts, + (current, event) => + Effect.gen(function* () { + if (event instanceof ServiceDownloadStarted) { + const state = SubscriptionRef.getUnsafe(stateRef).find( + (entry) => entry.name === event.service, + ); + if (state !== undefined && state.status !== "Downloading") { + yield* updateState( + new StackServiceState({ ...state, status: "Downloading" }), + ); + } + } + return event instanceof PreparationCompleted ? event.artifacts : current; + }), + ), + ), Effect.mapError( (cause) => new StackBuildError({ @@ -370,7 +393,7 @@ export const localStackLayer = ( Effect.ensuring(Effect.sync(() => preparationInFlight.delete(key))), ); return Effect.gen(function* () { - yield* Effect.forkIn(effect.pipe(Deferred.into(deferred)), scope); + yield* Effect.forkIn(effect.pipe(Deferred.into(deferred)), preparationScope); return yield* Deferred.await(deferred); }); }); @@ -427,7 +450,7 @@ export const localStackLayer = ( ); return Effect.gen(function* () { - yield* Effect.forkIn(effect.pipe(Deferred.into(deferred)), scope); + yield* Effect.forkIn(effect.pipe(Deferred.into(deferred)), preparationScope); return yield* Deferred.await(deferred); }); }); @@ -639,6 +662,7 @@ export const localStackLayer = ( } disposed = true; yield* Ref.set(phaseRef, "stopping"); + yield* Scope.close(preparationScope, Exit.void); yield* cleanupLocalStackResources({ stop: () => runtimeState === undefined ? Effect.void : runtimeState.orchestrator.stop(), @@ -774,6 +798,7 @@ export const localStackLayer = ( yield* Effect.all(readiness, { concurrency: "unbounded", discard: true }).pipe( (effect) => withReadinessPolicy(effect, "stack"), ); + yield* syncRuntimeProjectedStates(runtime); } else { yield* prepareServices(enabledServices); yield* runtime.orchestrator.start(serviceStartOptions); @@ -806,10 +831,12 @@ export const localStackLayer = ( startService: (name) => Effect.gen(function* () { yield* requireMutable(`start service ${name}`); + yield* requireRunningPhase; const service = yield* requireKnownServiceName(name); yield* prepareServices([service]); const started = yield* Effect.gen(function* () { yield* requireMutable(`start service ${name}`); + yield* requireRunningPhase; return yield* beginStartTargets( service, new Set(lifecycleTargetsForService(enabledServices, service)), @@ -832,10 +859,12 @@ export const localStackLayer = ( restartService: (name) => Effect.gen(function* () { yield* requireMutable(`restart service ${name}`); + yield* requireRunningPhase; const service = yield* requireKnownServiceName(name); yield* prepareServices([service]); const started = yield* Effect.gen(function* () { yield* requireMutable(`restart service ${name}`); + yield* requireRunningPhase; const runtime = yield* ensureRuntime; yield* runtime.orchestrator.restartService(service, serviceStartOptions); return { runtime, targets: [service] }; @@ -845,15 +874,17 @@ export const localStackLayer = ( reloadFunctions: (opts) => Effect.gen(function* () { yield* requireMutable("reload functions"); + yield* requireRunningPhase; yield* requireKnownService("edge-runtime"); + const currentBundle = yield* Ref.get(functionsBundleRef); + const nextBundle = + opts?.functions === undefined + ? currentBundle + : yield* decodeFunctionsBundle(opts.functions); yield* prepareServices(["edge-runtime"]); const started = yield* Effect.gen(function* () { yield* requireMutable("reload functions"); - const currentBundle = yield* Ref.get(functionsBundleRef); - const nextBundle = - opts?.functions === undefined - ? currentBundle - : yield* decodeFunctionsBundle(opts.functions); + yield* requireRunningPhase; yield* configureFunctions(config, nextBundle); yield* Ref.set(functionsBundleRef, nextBundle); const runtime = yield* ensureRuntime; @@ -871,16 +902,18 @@ export const localStackLayer = ( reloadEdgeRuntime: (opts) => Effect.gen(function* () { yield* requireMutable("reload Edge Runtime"); + yield* requireRunningPhase; yield* requireKnownService("edge-runtime"); + const nextConfig = yield* configWithEdgeRuntimeOptions(opts); + const currentBundle = yield* Ref.get(functionsBundleRef); + const nextBundle = + opts.functions === undefined + ? currentBundle + : yield* decodeFunctionsBundle(opts.functions); yield* prepareServices(["edge-runtime"]); const started = yield* Effect.gen(function* () { yield* requireMutable("reload Edge Runtime"); - const nextConfig = yield* configWithEdgeRuntimeOptions(opts); - const currentBundle = yield* Ref.get(functionsBundleRef); - const nextBundle = - opts.functions === undefined - ? currentBundle - : yield* decodeFunctionsBundle(opts.functions); + yield* requireRunningPhase; const prepared = yield* ensurePlanned; const runtime = yield* ensureRuntime; const buildResult = yield* builder.build(nextConfig, prepared); diff --git a/packages/stack/src/Stack.ts b/packages/stack/src/Stack.ts index 9693b08d02..650ff69715 100644 --- a/packages/stack/src/Stack.ts +++ b/packages/stack/src/Stack.ts @@ -1,7 +1,7 @@ import { ServiceNotFoundError } from "@supabase/process-compose"; import type { LogEntry, ServiceReadyError } from "@supabase/process-compose"; import { Context, Effect, Schema, Stream } from "effect"; -import { StackBuildError, StackReadinessError } from "./errors.ts"; +import { StackBuildError, StackNotRunningError, StackReadinessError } from "./errors.ts"; import { ResolvedFunctionsBundleSchema, type FunctionsReloadConfig, @@ -61,7 +61,11 @@ export class Stack extends Context.Service< name: string, ) => Effect.Effect< void, - ServiceNotFoundError | ServiceReadyError | StackBuildError | StackReadinessError + | ServiceNotFoundError + | ServiceReadyError + | StackBuildError + | StackNotRunningError + | StackReadinessError >; readonly stopService: ( name: string, @@ -70,19 +74,31 @@ export class Stack extends Context.Service< name: string, ) => Effect.Effect< void, - ServiceNotFoundError | ServiceReadyError | StackBuildError | StackReadinessError + | ServiceNotFoundError + | ServiceReadyError + | StackBuildError + | StackNotRunningError + | StackReadinessError >; readonly reloadFunctions: ( opts?: FunctionsReloadConfig, ) => Effect.Effect< void, - ServiceNotFoundError | ServiceReadyError | StackBuildError | StackReadinessError + | ServiceNotFoundError + | ServiceReadyError + | StackBuildError + | StackNotRunningError + | StackReadinessError >; readonly reloadEdgeRuntime: ( opts: EdgeRuntimeReloadConfig, ) => Effect.Effect< void, - ServiceNotFoundError | ServiceReadyError | StackBuildError | StackReadinessError + | ServiceNotFoundError + | ServiceReadyError + | StackBuildError + | StackNotRunningError + | StackReadinessError >; readonly getState: (name: string) => Effect.Effect; readonly getAllStates: () => Effect.Effect>; diff --git a/packages/stack/src/Stack.unit.test.ts b/packages/stack/src/Stack.unit.test.ts index c090d285e6..50a090d3ed 100644 --- a/packages/stack/src/Stack.unit.test.ts +++ b/packages/stack/src/Stack.unit.test.ts @@ -202,9 +202,22 @@ describe("Stack", () => { projectDir: runtimeRoot, runtimeRoot, functions: initialBundle, + postgrest: false, + auth: false, + servicePolicies: { + ...edgeRuntimeConfig.servicePolicies, + postgrest: "off", + auth: "off", + "edge-runtime": "lazy", + }, } satisfies ResolvedStackConfig; const graph = Effect.runSync( buildGraph([ + { + name: "postgres", + command: process.execPath, + restart: "unless-stopped", + }, { name: "edge-runtime", command: process.execPath, @@ -220,7 +233,10 @@ describe("Stack", () => { return { graph, cleanupTargets: { dockerContainerNames: [] }, - serviceProjection: new Map([["edge-runtime", { visibility: "public" as const }]]), + serviceProjection: new Map([ + ["postgres", { visibility: "public" as const }], + ["edge-runtime", { visibility: "public" as const }], + ]), }; }), }); @@ -240,8 +256,10 @@ describe("Stack", () => { return Effect.gen(function* () { const stack = yield* Stack; yield* stack.start(); + expect((yield* stack.getState("edge-runtime")).status).toBe("Dormant"); yield* stack.reloadFunctions({ functions: replacementBundle }); + expect((yield* stack.getState("edge-runtime")).status).toBe("Healthy"); expect((yield* readRuntimeConfig).env.SHARED).toBe("replacement-secret"); yield* stack.reloadEdgeRuntime({ edgeRuntime: { policy: "oneshot" } }); @@ -296,59 +314,6 @@ describe("Stack", () => { ); }); - const coldEdgeRuntimeReload = (operation: "reloadFunctions" | "reloadEdgeRuntime") => - Effect.gen(function* () { - const runtimeRoot = mkdtempSync(join(tmpdir(), `supabase-${operation}-`)); - const edgeRuntimeSpawnStarted = yield* Deferred.make(); - const image = `ghcr.io/supabase/cli/edge-runtime:${DEFAULT_VERSIONS["edge-runtime"]}`; - const spawner = mockChildProcessSpawner({ - beforeSpawn: ({ command, args }) => - command === process.execPath && - args.some((arg) => Buffer.from(arg, "base64url").toString().includes(image)) - ? Deferred.succeed(edgeRuntimeSpawnStarted, undefined).pipe(Effect.asVoid) - : Effect.void, - }); - const config = { - ...edgeRuntimeConfig, - runtimeRoot, - projectDir: runtimeRoot, - servicePolicies: { - ...edgeRuntimeConfig.servicePolicies, - "edge-runtime": "lazy", - }, - } satisfies ResolvedStackConfig; - const { layer } = setupLayer(config, noopPortLease(config.ports), spawner); - - yield* Effect.gen(function* () { - const stack = yield* Stack; - const reloading = yield* ( - operation === "reloadFunctions" - ? stack.reloadFunctions() - : stack.reloadEdgeRuntime({ edgeRuntime: {} }) - ).pipe(Effect.forkChild({ startImmediately: true })); - yield* Deferred.await(edgeRuntimeSpawnStarted); - - expect(spawner.spawned).toContainEqual({ - command: "docker", - args: ["image", "inspect", image], - }); - - yield* Fiber.interrupt(reloading); - yield* stack.dispose(); - }).pipe( - Effect.provide(layer), - Effect.ensuring(Effect.promise(() => rm(runtimeRoot, { recursive: true, force: true }))), - ); - }); - - it.live("prepares dormant Edge Runtime before reloading Functions", () => - coldEdgeRuntimeReload("reloadFunctions").pipe(Effect.scoped, Effect.timeout("5 seconds")), - ); - - it.live("prepares dormant Edge Runtime before reloading its configuration", () => - coldEdgeRuntimeReload("reloadEdgeRuntime").pipe(Effect.scoped, Effect.timeout("5 seconds")), - ); - it.effect("getInfo returns valid JWT tokens", () => { const { layer } = setupLayer(); @@ -524,6 +489,26 @@ describe("Stack", () => { }).pipe(Effect.provide(layer), Effect.scoped, Effect.timeout("5 seconds")); }); + it.effect("rejects unsupported native services before resource planning", () => { + const config = { + ...edgeRuntimeConfig, + mode: "native", + containerRuntime: null, + } satisfies ResolvedStackConfig; + const { layer } = setupLayer(config, noopPortLease(config.ports)); + + return Effect.gen(function* () { + const stack = yield* Stack; + const error = yield* stack.start().pipe(Effect.flip); + + expect(error).toMatchObject({ + _tag: "StackBuildError", + reason: "invalid_config", + }); + if (error._tag === "StackBuildError") expect(error.detail).toContain("edge-runtime"); + }).pipe(Effect.provide(layer)); + }); + it.effect("getState fails for internal helper services", () => { const { layer } = setupLayer(); @@ -700,7 +685,7 @@ describe("Stack", () => { }); it.live("lazy startup starts direct services without starting HTTP backends", () => { - const { layer, spawner } = setupLayer({ + const { layer } = setupLayer({ ...defaultConfig, servicePolicies: { ...defaultConfig.servicePolicies, @@ -716,15 +701,9 @@ describe("Stack", () => { yield* stack.start(); yield* stack.waitAllReady(); - expect( - spawner.spawned.some((record) => - record.args.some((arg) => - Buffer.from(arg, "base64url").toString().includes('"command":"bash"'), - ), - ), - ).toBe(true); - expect(spawner.spawned.some((record) => record.command.endsWith("/auth"))).toBe(false); - expect(spawner.spawned.some((record) => record.command.endsWith("/postgrest"))).toBe(false); + expect((yield* stack.getState("postgres")).status).toBe("Healthy"); + expect((yield* stack.getState("auth")).status).toBe("Dormant"); + expect((yield* stack.getState("postgrest")).status).toBe("Dormant"); yield* stack.stop(); }).pipe(Effect.provide(layer), Effect.timeout("5 seconds")); @@ -732,15 +711,6 @@ describe("Stack", () => { it.live("prepares a dormant service before restarting it", () => Effect.gen(function* () { - const postgrestSpawnStarted = yield* Deferred.make(); - const spawner = mockChildProcessSpawner({ - beforeSpawn: ({ args }) => - args.some((arg) => - Buffer.from(arg, "base64url").toString().includes('"command":"/cache/postgrest/'), - ) - ? Deferred.succeed(postgrestSpawnStarted, undefined).pipe(Effect.asVoid) - : Effect.void, - }); const resolver = mockBinaryResolver({ downloadedServices: ["postgrest"] }); const config = { ...defaultConfig, @@ -750,7 +720,7 @@ describe("Stack", () => { auth: "lazy", }, } satisfies ResolvedStackConfig; - const { layer } = setupLayer(config, noopPortLease(config.ports), spawner, resolver); + const { layer } = setupLayer(config, noopPortLease(config.ports), undefined, resolver); yield* Effect.gen(function* () { const stack = yield* Stack; @@ -761,12 +731,17 @@ describe("Stack", () => { Stream.runHead, Effect.forkChild({ startImmediately: true }), ); + const running = yield* stateChanges.pipe( + Stream.filter((state) => state.status === "Running"), + Stream.runHead, + Effect.forkChild({ startImmediately: true }), + ); const restarting = yield* stack .restartService("postgrest") .pipe(Effect.forkChild({ startImmediately: true })); - yield* Deferred.await(postgrestSpawnStarted); expect((yield* Fiber.join(downloading))._tag).toBe("Some"); + expect((yield* Fiber.join(running))._tag).toBe("Some"); yield* Fiber.interrupt(restarting); yield* stack.stop(); @@ -774,6 +749,59 @@ describe("Stack", () => { }).pipe(Effect.scoped, Effect.timeout("5 seconds")), ); + it.live("does not restart a service after the stack stops during preparation", () => + Effect.gen(function* () { + const allowPreparation = yield* Deferred.make(); + const resolver = mockBinaryResolver({ + downloadedServices: ["postgrest"], + beforeResolve: ({ service }) => + service === "postgrest" ? Deferred.await(allowPreparation) : Effect.void, + }); + const config = { + ...defaultConfig, + servicePolicies: { + ...defaultConfig.servicePolicies, + postgrest: "lazy", + auth: "lazy", + }, + } satisfies ResolvedStackConfig; + const { layer } = setupLayer(config, noopPortLease(config.ports), undefined, resolver); + + yield* Effect.gen(function* () { + const stack = yield* Stack; + yield* stack.start(); + const downloading = yield* (yield* stack.stateChanges("postgrest")).pipe( + Stream.filter((state) => state.status === "Downloading"), + Stream.runHead, + Effect.forkChild({ startImmediately: true }), + ); + const restarting = yield* stack + .restartService("postgrest") + .pipe(Effect.forkChild({ startImmediately: true })); + yield* Fiber.join(downloading); + const running = yield* (yield* stack.stateChanges("postgrest")).pipe( + Stream.filter((state) => state.status === "Running"), + Stream.runHead, + Effect.forkChild({ startImmediately: true }), + ); + + yield* stack.stop(); + yield* Deferred.succeed(allowPreparation, undefined); + const outcome = yield* Effect.race( + Fiber.join(restarting).pipe( + Effect.exit, + Effect.map((exit) => ({ type: "restart" as const, exit })), + ), + Fiber.join(running).pipe(Effect.as({ type: "resurrected" as const })), + ); + + expect(outcome.type).toBe("restart"); + if (outcome.type === "restart") expect(Exit.isFailure(outcome.exit)).toBe(true); + expect((yield* stack.getState("postgrest")).status).not.toBe("Running"); + }).pipe(Effect.provide(layer)); + }).pipe(Effect.scoped, Effect.timeout("5 seconds")), + ); + it.live("lazy activation honors explicitly stopped transitive dependencies", () => { const config: ResolvedStackConfig = { ...defaultConfig, @@ -1001,18 +1029,15 @@ describe("Stack", () => { }).pipe(Effect.scoped, Effect.timeout("5 seconds")), ); - it.live("dispose is not blocked by in-flight lazy preparation", () => + it.live("dispose cancels in-flight lazy preparation", () => Effect.gen(function* () { const preparationStarted = yield* Deferred.make(); - const allowPreparation = yield* Deferred.make(); const disposed = yield* Deferred.make(); const resolver = mockBinaryResolver({ downloadedServices: ["auth"], beforeResolve: ({ service }) => service === "auth" - ? Deferred.succeed(preparationStarted, undefined).pipe( - Effect.andThen(Deferred.await(allowPreparation)), - ) + ? Deferred.succeed(preparationStarted, undefined).pipe(Effect.andThen(Effect.never)) : Effect.void, }); const config = { @@ -1038,8 +1063,8 @@ describe("Stack", () => { yield* Deferred.await(disposed); yield* Fiber.join(disposing); - yield* Deferred.succeed(allowPreparation, undefined); - yield* Fiber.interrupt(activation); + expect(Exit.isFailure(yield* Fiber.await(activation))).toBe(true); + expect((yield* stack.getState("auth")).status).not.toBe("Downloading"); }).pipe(Effect.provide(layer)); }).pipe(Effect.scoped, Effect.timeout("5 seconds")), ); diff --git a/packages/stack/src/StackBuilder.ts b/packages/stack/src/StackBuilder.ts index 4959d7bb6e..3006f0001f 100644 --- a/packages/stack/src/StackBuilder.ts +++ b/packages/stack/src/StackBuilder.ts @@ -15,7 +15,10 @@ import { makeImgproxyServiceDocker } from "./services/imgproxy.ts"; import { makeMailpitServiceDocker } from "./services/mailpit.ts"; import { makePgmetaServiceDocker } from "./services/pgmeta.ts"; import { makePoolerServiceDocker } from "./services/pooler.ts"; -import { makePostgresInitService } from "./services/postgres-init.ts"; +import { + makePostgresInitService, + makePostgresInitServiceDocker, +} from "./services/postgres-init.ts"; import { makePostgresService, makePostgresServiceDocker } from "./services/postgres.ts"; import { makePostgrestService, makePostgrestServiceDocker } from "./services/postgrest.ts"; import { makeRealtimeServiceDocker } from "./services/realtime.ts"; @@ -48,6 +51,9 @@ export interface BuildResult { const dockerOnlyServices = SERVICE_NAMES.filter( (service) => serviceMetadata(service).runtimeSupport === "docker-only", ); +const nativeServices = SERVICE_NAMES.filter( + (service) => serviceMetadata(service).runtimeSupport !== "docker-only", +); // Serial health-check paths used by dependency waits; keep each path aligned // with the corresponding service's transitive dependencies. @@ -57,14 +63,12 @@ const analyticsStartupPath: ReadonlyArray = ["postgres", "analytics const postgresDependencyTimeoutSeconds = dependencyTimeoutSecondsForServices(postgresStartupPath); -const dependsOnPostgres = (hasPostgresInit: boolean): ReadonlyArray => - hasPostgresInit - ? [{ service: "postgres-init", condition: "completed" }] - : [{ service: "postgres", condition: "healthy" }]; +const postgresDependencies: ReadonlyArray = [ + { service: "postgres-init", condition: "completed" }, +]; const publicServiceProjection = ( defs: ReadonlyArray, - hasPostgresInit: boolean, ): StackServiceProjectionCatalog => { const serviceProjection: Map< string, @@ -75,13 +79,11 @@ const publicServiceProjection = ( } > = new Map(defs.map((def) => [def.name, { visibility: "public" as const }] as const)); - if (hasPostgresInit) { - serviceProjection.set("postgres-init", { - visibility: "internal", - owner: "postgres", - ownerStatusWhileActive: "Initializing", - }); - } + serviceProjection.set("postgres-init", { + visibility: "internal", + owner: "postgres", + ownerStatusWhileActive: "Initializing", + }); return serviceProjection; }; @@ -128,7 +130,7 @@ export const validateResolvedConfig = ( if (enabledDockerOnly.length > 0) { return yield* Effect.fail( new StackBuildError({ - detail: `mode "native" only supports postgres, auth, and postgrest. Disable ${enabledDockerOnly.join(", ")} or use "docker" mode.`, + detail: `Native mode supports only ${nativeServices.join(", ")}. Disable ${enabledDockerOnly.join(", ")} or select Docker mode with a usable Docker or Podman runtime.`, reason: "invalid_config", }), ); @@ -212,11 +214,6 @@ const requirePreparedDockerImage = ( ), ); -export const nativePostgresNeedsDockerAccess = ( - postgresResolution: ServiceResolution, - dockerServicesEnabled: boolean, -): boolean => postgresResolution.type === "binary" && dockerServicesEnabled; - export class StackBuilder extends Context.Service< StackBuilder, { @@ -261,29 +258,7 @@ export class StackBuilder extends Context.Service< ? false : yield* requirePreparedResolution(prepared, "postgrest"); - const dockerServicesEnabled = - config.realtime !== false || - config.storage !== false || - config.imgproxy !== false || - config.mailpit !== false || - config.pgmeta !== false || - config.studio !== false || - config.analytics !== false || - config.vector !== false || - config.pooler !== false || - (edgeRuntimeResolution !== false && edgeRuntimeResolution.type === "docker") || - (authResolution !== false && authResolution.type === "docker") || - (postgrestResolution !== false && postgrestResolution.type === "docker"); - - const needsDockerAccess = nativePostgresNeedsDockerAccess( - postgresResolution, - dockerServicesEnabled, - ); - const hasPostgresInit = postgresResolution.type === "binary"; - const postgresDeps = dependsOnPostgres(hasPostgresInit); - const postgresInitCompletionBudgetSeconds = hasPostgresInit - ? POSTGRES_INIT_COMPLETION_BUDGET_SECONDS - : 0; + const postgresInitCompletionBudgetSeconds = POSTGRES_INIT_COMPLETION_BUDGET_SECONDS; const postgresConsumerDependencyTimeoutSeconds = postgresDependencyTimeoutSeconds + postgresInitCompletionBudgetSeconds; const storageDependencyTimeoutSeconds = @@ -302,7 +277,6 @@ export class StackBuilder extends Context.Service< binPath: postgresResolution.path, dataDir: config.postgres.dataDir, port: config.dbPort, - dockerAccessible: needsDockerAccess, cleanupDataDirOnExit: hasAutoManagedPath(config, config.postgres.dataDir), dependencies: [], }) @@ -312,8 +286,6 @@ export class StackBuilder extends Context.Service< dataDir: config.postgres.dataDir, port: config.dbPort, platformOs: platform.os, - jwtSecret: config.jwtSecret, - jwtExpiry: config.auth !== false ? config.auth.jwtExpiry : 3600, identity, cleanupDataDirOnExit: hasAutoManagedPath(config, config.postgres.dataDir), dependencies: [], @@ -322,18 +294,26 @@ export class StackBuilder extends Context.Service< }, ]; - if (hasPostgresInit) { - defs.push({ - ...makePostgresInitService({ - postgresDir: postgresResolution.path, - dbPort: config.dbPort, - autoExposeNewTables: config.postgres.autoExposeNewTables, - dependencies: [{ service: "postgres", condition: "healthy" }], - }), - dependencyTimeoutSeconds: postgresDependencyTimeoutSeconds, - enabled: true, - }); - } + defs.push({ + ...(postgresResolution.type === "binary" + ? makePostgresInitService({ + postgresDir: postgresResolution.path, + dbPort: config.dbPort, + autoExposeNewTables: config.postgres.autoExposeNewTables, + dependencies: [{ service: "postgres", condition: "healthy" }], + }) + : makePostgresInitServiceDocker({ + runtime: yield* requireContainerRuntime, + dbPort: config.dbPort, + jwtSecret: config.jwtSecret, + jwtExpiry: config.auth !== false ? config.auth.jwtExpiry : 3600, + autoExposeNewTables: config.postgres.autoExposeNewTables, + identity, + dependencies: [{ service: "postgres", condition: "healthy" }], + })), + dependencyTimeoutSeconds: postgresDependencyTimeoutSeconds, + enabled: true, + }); if (config.postgrest !== false && postgrestResolution !== false) { defs.push({ @@ -346,7 +326,7 @@ export class StackBuilder extends Context.Service< extraSearchPath: config.postgrest.extraSearchPath, maxRows: config.postgrest.maxRows, jwtSecret: config.jwtSecret, - dependencies: postgresDeps, + dependencies: postgresDependencies, }) : makePostgrestServiceDocker({ runtime: yield* requireContainerRuntime, @@ -361,7 +341,7 @@ export class StackBuilder extends Context.Service< jwtSecret: config.jwtSecret, platformOs: platform.os, identity, - dependencies: postgresDeps, + dependencies: postgresDependencies, })), dependencyTimeoutSeconds: postgresConsumerDependencyTimeoutSeconds, enabled: true, @@ -383,7 +363,7 @@ export class StackBuilder extends Context.Service< smtpPort: config.mailpit !== false ? config.mailpit.smtpPort : undefined, smtpAdminEmail: config.mailpit !== false ? config.mailpit.adminEmail : undefined, smtpSenderName: config.mailpit !== false ? config.mailpit.senderName : undefined, - dependencies: postgresDeps, + dependencies: postgresDependencies, }) : makeAuthServiceDocker({ runtime: yield* requireContainerRuntime, @@ -401,7 +381,7 @@ export class StackBuilder extends Context.Service< smtpSenderName: config.mailpit !== false ? config.mailpit.senderName : undefined, platformOs: platform.os, identity, - dependencies: postgresDeps, + dependencies: postgresDependencies, })), dependencyTimeoutSeconds: postgresConsumerDependencyTimeoutSeconds, enabled: true, @@ -418,7 +398,7 @@ export class StackBuilder extends Context.Service< inspectorPort: config.edgeRuntime.inspectorPort, policy: config.edgeRuntime.policy, env: config.edgeRuntime.env, - dependencies: postgresDeps, + dependencies: postgresDependencies, }) : makeEdgeRuntimeServiceDocker({ runtime: yield* requireContainerRuntime, @@ -431,7 +411,7 @@ export class StackBuilder extends Context.Service< policy: config.edgeRuntime.policy, env: config.edgeRuntime.env, platformOs: platform.os, - dependencies: postgresDeps, + dependencies: postgresDependencies, })), dependencyTimeoutSeconds: postgresConsumerDependencyTimeoutSeconds, enabled: true, @@ -472,7 +452,7 @@ export class StackBuilder extends Context.Service< secretKeyBase: config.realtime.secretKeyBase, maxHeaderLength: config.realtime.maxHeaderLength, platformOs: platform.os, - dependencies: postgresDeps, + dependencies: postgresDependencies, }), dependencyTimeoutSeconds: postgresConsumerDependencyTimeoutSeconds, enabled: true, @@ -500,7 +480,7 @@ export class StackBuilder extends Context.Service< config.imgproxy !== false ? `http://${serviceHost}:${config.imgproxy.port}` : "", s3ProtocolEnabled: config.storage.s3ProtocolEnabled, platformOs: platform.os, - dependencies: postgresDeps, + dependencies: postgresDependencies, cleanupDataDirOnExit: hasAutoManagedPath(config, config.storage.dataDir), }), dependencyTimeoutSeconds: postgresConsumerDependencyTimeoutSeconds, @@ -537,7 +517,7 @@ export class StackBuilder extends Context.Service< dbHost: serviceHost, dbPort: config.dbPort, platformOs: platform.os, - dependencies: postgresDeps, + dependencies: postgresDependencies, }), dependencyTimeoutSeconds: postgresConsumerDependencyTimeoutSeconds, enabled: true, @@ -557,7 +537,7 @@ export class StackBuilder extends Context.Service< dbPort: config.dbPort, apiKey: config.analytics.apiKey, backend: config.analytics.backend, - dependencies: postgresDeps, + dependencies: postgresDependencies, }), dependencyTimeoutSeconds: postgresConsumerDependencyTimeoutSeconds, enabled: true, @@ -602,7 +582,7 @@ export class StackBuilder extends Context.Service< tenantId: config.pooler.tenantId, encryptionKey: config.pooler.encryptionKey, secretKeyBase: config.pooler.secretKeyBase, - dependencies: postgresDeps, + dependencies: postgresDependencies, }), dependencyTimeoutSeconds: postgresConsumerDependencyTimeoutSeconds, enabled: true, @@ -669,7 +649,7 @@ export class StackBuilder extends Context.Service< cleanupTargets: { dockerContainerNames, }, - serviceProjection: publicServiceProjection(defs, hasPostgresInit), + serviceProjection: publicServiceProjection(defs), }; }), }); diff --git a/packages/stack/src/StackBuilder.unit.test.ts b/packages/stack/src/StackBuilder.unit.test.ts index 6d86be764a..285d27595a 100644 --- a/packages/stack/src/StackBuilder.unit.test.ts +++ b/packages/stack/src/StackBuilder.unit.test.ts @@ -9,7 +9,6 @@ import type { BuildResult } from "./StackBuilder.ts"; import { DEFAULT_STACK_READINESS_POLICY, type ResolvedStackConfig } from "./StackConfig.ts"; import { STACK_ID_LABEL } from "./StackIdentity.ts"; import { enabledServicesForConfig, versionsForConfig } from "./StackBuilder.ts"; -import { nativePostgresNeedsDockerAccess } from "./StackBuilder.ts"; import type { AllocatedPorts } from "./PortCatalog.ts"; import { StackPreparation } from "./StackPreparation.ts"; import type { StackPreparationInput } from "./StackPreparation.ts"; @@ -199,28 +198,21 @@ const prepareAndBuild = ( config: ResolvedStackConfig, ): Effect.Effect => Effect.gen(function* () { - const input: StackPreparationInput = { - mode: config.mode, + const shared = { services: enabledServicesForConfig(config), versions: versionsForConfig(config), }; + const input: StackPreparationInput = + config.mode === "native" + ? { ...shared, mode: "native" } + : config.containerRuntime === null + ? yield* Effect.die("Docker test config is missing its container runtime") + : { ...shared, mode: "docker", containerRuntime: config.containerRuntime }; const prepared = yield* preparation.prepare(input); return yield* builder.build(config, prepared); }); describe("StackBuilder", () => { - it("makes native postgres reachable by docker services on every platform", () => { - expect(nativePostgresNeedsDockerAccess({ type: "binary", path: "/cache/postgres" }, true)).toBe( - true, - ); - expect( - nativePostgresNeedsDockerAccess({ type: "binary", path: "/cache/postgres" }, false), - ).toBe(false); - expect( - nativePostgresNeedsDockerAccess({ type: "docker", image: "supabase/postgres" }, true), - ).toBe(false); - }); - it.effect("builds graph with all native binaries", () => { const resolver = mockBinaryResolver(); const layer = builderLayer(resolver); @@ -305,11 +297,11 @@ describe("StackBuilder", () => { const config = { ...dockerConfig, containerRuntime: "podman" } satisfies ResolvedStackConfig; const { graph, cleanupTargets } = yield* prepareAndBuild(builder, preparation, config); - expect(graph.startOrder.length).toBe(3); + expect(graph.startOrder.length).toBe(4); const names = graph.startOrder.map((s) => s.name); expect(names).toContain("postgres"); - expect(names).not.toContain("postgres-init"); + expect(names).toContain("postgres-init"); expect(names).toContain("postgrest"); expect(names).toContain("auth"); @@ -357,7 +349,7 @@ describe("StackBuilder", () => { expect(name).not.toContain(String(managedConfig.apiPort)); } - for (const def of graph.startOrder) { + for (const def of graph.startOrder.filter((service) => service.args?.[0] === "run")) { expect(def.args).toContain(`supabase-${def.name}-id-${firstManagedId}`); // The label carries the whole identity, so the containers stay findable // by it even if the names are ever built differently. @@ -415,14 +407,14 @@ describe("StackBuilder", () => { `supabase-postgrest-${dockerConfig.apiPort}`, `supabase-auth-${dockerConfig.apiPort}`, ]); - for (const def of graph.startOrder) { + for (const def of graph.startOrder.filter((service) => service.args?.[0] === "run")) { expect(def.args).toContain(`supabase-${def.name}-${dockerConfig.apiPort}`); expect(def.args?.join(" ")).not.toContain(STACK_ID_LABEL); } }).pipe(Effect.provide(layer)); }); - it.effect("docker mode wires auth directly to postgres readiness", () => { + it.effect("docker consumers wait for database initialization", () => { const resolver = mockBinaryResolver(); const layer = builderLayer(resolver); @@ -432,27 +424,13 @@ describe("StackBuilder", () => { const { graph } = yield* prepareAndBuild(builder, preparation, dockerConfig); const authDef = graph.startOrder.find((s) => s.name === "auth"); - expect(authDef?.dependencies).toEqual([{ service: "postgres", condition: "healthy" }]); + expect(authDef?.dependencies).toEqual([{ service: "postgres-init", condition: "completed" }]); expect(authDef?.dependencyTimeoutSeconds).toBe( - dependencyTimeoutSecondsForServices(["postgres"]), + dependencyTimeoutSecondsForServices(["postgres"]) + POSTGRES_INIT_COMPLETION_BUDGET_SECONDS, ); }).pipe(Effect.provide(layer)); }); - it.effect("docker mode has no postgres-init service for Docker postgres", () => { - const resolver = mockBinaryResolver(); - const layer = builderLayer(resolver); - - return Effect.gen(function* () { - const builder = yield* StackBuilder; - const preparation = yield* StackPreparation; - const { graph } = yield* prepareAndBuild(builder, preparation, dockerConfig); - - const names = graph.startOrder.map((s) => s.name); - expect(names).not.toContain("postgres-init"); - }).pipe(Effect.provide(layer)); - }); - it.effect("docker mode wires dependencies correctly", () => { const resolver = mockBinaryResolver(); const layer = builderLayer(resolver); @@ -463,11 +441,12 @@ describe("StackBuilder", () => { const { graph } = yield* prepareAndBuild(builder, preparation, dockerConfig); const authDef = graph.startOrder.find((s) => s.name === "auth"); - expect(authDef?.dependencies).toEqual([{ service: "postgres", condition: "healthy" }]); + expect(authDef?.dependencies).toEqual([{ service: "postgres-init", condition: "completed" }]); - // postgrest depends on postgres(healthy) — no postgres-init in Docker mode const postgrestDef = graph.startOrder.find((s) => s.name === "postgrest"); - expect(postgrestDef?.dependencies).toEqual([{ service: "postgres", condition: "healthy" }]); + expect(postgrestDef?.dependencies).toEqual([ + { service: "postgres-init", condition: "completed" }, + ]); }).pipe(Effect.provide(layer)); }); @@ -487,7 +466,9 @@ describe("StackBuilder", () => { const edgeRuntimeDef = graph.startOrder.find((service) => service.name === "edge-runtime"); expect(edgeRuntimeDef).toBeDefined(); expect(edgeRuntimeDef?.command).toBe("docker"); - expect(edgeRuntimeDef?.dependencies).toEqual([{ service: "postgres", condition: "healthy" }]); + expect(edgeRuntimeDef?.dependencies).toEqual([ + { service: "postgres-init", condition: "completed" }, + ]); expect(cleanupTargets.dockerContainerNames).toContain( `supabase-edge-runtime-${edgeRuntimeConfig.apiPort}`, ); diff --git a/packages/stack/src/StackConfigResolver.ts b/packages/stack/src/StackConfigResolver.ts index 3ad70bb228..2d06d12468 100644 --- a/packages/stack/src/StackConfigResolver.ts +++ b/packages/stack/src/StackConfigResolver.ts @@ -448,9 +448,24 @@ export async function resolveConfig( opts: ResolveConfigOptions = {}, ): Promise { const inputConfig = input ?? {}; + if ( + inputConfig.mode !== undefined && + opts.runtime !== undefined && + inputConfig.mode !== opts.runtime.mode + ) { + throw new StackBuildError({ + detail: `Selected ${opts.runtime.mode} runtime does not match requested ${inputConfig.mode} mode`, + reason: "invalid_config", + }); + } const resolvedMode = opts.runtime?.mode ?? inputConfig.mode ?? "native"; - const containerRuntime = - resolvedMode === "docker" ? (opts.runtime?.containerRuntime ?? "docker") : null; + if (resolvedMode === "docker" && opts.runtime?.containerRuntime == null) { + throw new StackBuildError({ + detail: "Docker mode requires a selected Docker or Podman runtime", + reason: "invalid_config", + }); + } + const containerRuntime = opts.runtime?.containerRuntime ?? null; const config: StackConfig = { ...inputConfig, mode: resolvedMode }; // Deliberately first: unsupported policies must not create roots or reserve ports. const servicePolicies = resolveServicePolicies(config); diff --git a/packages/stack/src/StackPreparation.ts b/packages/stack/src/StackPreparation.ts index 025cda26a9..4e567611df 100644 --- a/packages/stack/src/StackPreparation.ts +++ b/packages/stack/src/StackPreparation.ts @@ -40,14 +40,18 @@ export type ServiceResolution = | { readonly type: "binary"; readonly path: string } | { readonly type: "docker"; readonly image: string }; -export interface StackPreparationInput { +interface StackPreparationOptions { readonly versions?: Partial; readonly services?: ReadonlyArray; readonly enabledServices?: ReadonlyArray; - readonly mode: "native" | "docker"; - readonly containerRuntime?: ContainerRuntime; } +export type StackPreparationInput = StackPreparationOptions & + ( + | { readonly mode: "native"; readonly containerRuntime?: never } + | { readonly mode: "docker"; readonly containerRuntime: ContainerRuntime } + ); + export type StackPreparationError = | BinaryNotFoundError | DownloadError @@ -123,8 +127,13 @@ const preparationClosure = ( return [...closure]; }; -const selectedServices = (input?: StackPreparationInput): ReadonlyArray => - preparationClosure(input?.services ?? SERVICE_NAMES, input?.enabledServices); +const selectedServices = (input: StackPreparationInput): ReadonlyArray => { + const defaults = + input.mode === "docker" + ? SERVICE_NAMES + : SERVICE_NAMES.filter((service) => !isDockerOnlyService(service)); + return preparationClosure(input.services ?? defaults, input.enabledServices); +}; const plannedResolution = ( resolver: BinaryResolver["Service"], @@ -148,15 +157,14 @@ const plannedResolution = ( const planAssetsWithDependencies = ( resolver: BinaryResolver["Service"], - input?: StackPreparationInput, + input: StackPreparationInput, ): Effect.Effect => Effect.gen(function* () { - const versions = { ...DEFAULT_VERSIONS, ...input?.versions }; - const mode = input?.mode ?? "native"; + const versions = { ...DEFAULT_VERSIONS, ...input.versions }; const services = selectedServices(input); const results = yield* Effect.all( services.map((service) => - plannedResolution(resolver, service, versions[service], mode).pipe( + plannedResolution(resolver, service, versions[service], input.mode).pipe( Effect.map((resolution) => [service, resolution] as const), ), ), @@ -171,13 +179,13 @@ export class StackPreparation extends Context.Service< StackPreparation, { readonly plan: ( - input?: StackPreparationInput, + input: StackPreparationInput, ) => Effect.Effect; readonly prepare: ( - input?: StackPreparationInput, + input: StackPreparationInput, ) => Effect.Effect; readonly prepareEvents: ( - input?: StackPreparationInput, + input: StackPreparationInput, ) => Stream.Stream; } >()("stack/StackPreparation") { @@ -218,7 +226,11 @@ export class StackPreparation extends Context.Service< ? (publishEvent?.(new ServiceDownloadFinished({ service })) ?? Effect.void) : Effect.void, ); - const key = `${service}\0${JSON.stringify(resolution)}`; + const key = JSON.stringify({ + service, + resolution, + containerRuntime: input.mode === "docker" ? input.containerRuntime : null, + }); const existing = inFlight.get(key); if (existing !== undefined) return Deferred.await(existing); const deferred = Deferred.makeUnsafe(); @@ -226,15 +238,11 @@ export class StackPreparation extends Context.Service< const version = input.versions?.[service] ?? DEFAULT_VERSIONS[service]; const effect: Effect.Effect = resolution.type === "docker" - ? resolveDockerImageForService( - spawner, - input.containerRuntime ?? "docker", - service, - version, - { + ? input.mode === "docker" + ? resolveDockerImageForService(spawner, input.containerRuntime, service, version, { onDownloadStart: markDownloadStart(), - }, - ).pipe(Effect.map((image): ServiceResolution => ({ type: "docker", image }))) + }).pipe(Effect.map((image): ServiceResolution => ({ type: "docker", image }))) + : Effect.die("Native preparation planned a Docker resolution") : resolver .resolveWithMetadata( { service, version }, @@ -259,7 +267,7 @@ export class StackPreparation extends Context.Service< }); const prepareWithEvents = ( - input: StackPreparationInput | undefined, + input: StackPreparationInput, publishEvent?: (event: StackPreparationEvent) => Effect.Effect, ): Effect.Effect => Effect.gen(function* () { @@ -268,12 +276,9 @@ export class StackPreparation extends Context.Service< selectedServices(input).map((service) => { const resolution = planned.resolutions[service]; if (resolution === undefined) return Effect.die(`Missing plan for ${service}`); - return materialize( - service, - resolution, - { mode: input?.mode ?? "native", ...input }, - publishEvent, - ).pipe(Effect.map((resolved) => [service, resolved] as const)); + return materialize(service, resolution, input, publishEvent).pipe( + Effect.map((resolved) => [service, resolved] as const), + ); }), { concurrency: "unbounded" }, ); @@ -285,9 +290,9 @@ export class StackPreparation extends Context.Service< }); return { - plan: (input?: StackPreparationInput) => planAssetsWithDependencies(resolver, input), - prepare: (input?: StackPreparationInput) => prepareWithEvents(input), - prepareEvents: (input?: StackPreparationInput) => + plan: (input: StackPreparationInput) => planAssetsWithDependencies(resolver, input), + prepare: (input: StackPreparationInput) => prepareWithEvents(input), + prepareEvents: (input: StackPreparationInput) => Stream.callback((queue) => prepareWithEvents(input, (event) => Queue.offer(queue, event)).pipe( Effect.matchCauseEffect({ diff --git a/packages/stack/src/bun.ts b/packages/stack/src/bun.ts index ca22284037..e4bd2caa93 100644 --- a/packages/stack/src/bun.ts +++ b/packages/stack/src/bun.ts @@ -4,8 +4,10 @@ import { FetchHttpClient } from "effect/unstable/http"; import { BinaryResolver } from "./BinaryResolver.ts"; import { selectStackRuntime } from "./ContainerRuntime.ts"; import { createStack as createStackCore, type StackHandle } from "./createStack.ts"; +import { toStackError } from "./errors.ts"; import { prefetch as prefetchEffect, + type PrefetchEffectOptions, type PrefetchOptions, type PrefetchResult, } from "./prefetch.ts"; @@ -17,24 +19,31 @@ import type { StackConfig } from "./StackConfig.ts"; export async function createStack(config?: StackConfig): Promise { const runtime = await Effect.runPromise( selectStackRuntime(config?.mode).pipe(Effect.provide(BunServices.layer)), - ); + ).catch((error: unknown) => { + throw toStackError(error); + }); return createStackCore(config, platformFactory, runtime); } export async function prefetch(options?: PrefetchOptions): Promise { const runtime = await Effect.runPromise( selectStackRuntime(options?.mode).pipe(Effect.provide(BunServices.layer)), - ); + ).catch((error: unknown) => { + throw toStackError(error); + }); const resolverLayer = BinaryResolver.make(defaultCacheRoot()).pipe( Layer.provide(FetchHttpClient.layer), ); const preparationLayer = StackPreparation.layer.pipe(Layer.provide(resolverLayer)); + const resolvedOptions: PrefetchEffectOptions = + runtime.mode === "native" + ? { ...options, mode: "native" } + : { ...options, mode: "docker", containerRuntime: runtime.containerRuntime }; return Effect.runPromise( - prefetchEffect({ - ...options, - mode: runtime.mode, - ...(runtime.containerRuntime === null ? {} : { containerRuntime: runtime.containerRuntime }), - }).pipe(Effect.provide(preparationLayer), Effect.provide(BunServices.layer)), + prefetchEffect(resolvedOptions).pipe( + Effect.provide(preparationLayer), + Effect.provide(BunServices.layer), + ), ); } diff --git a/packages/stack/src/createStack.unit.test.ts b/packages/stack/src/createStack.unit.test.ts index 21fe435f04..459aec256d 100644 --- a/packages/stack/src/createStack.unit.test.ts +++ b/packages/stack/src/createStack.unit.test.ts @@ -84,7 +84,10 @@ describe("resolveConfig edge runtime defaults", () => { }); it("enables edge runtime when omitted in Docker mode", async () => { - const config = await resolveConfig({ mode: "docker" }); + const config = await resolveConfig( + { mode: "docker" }, + { runtime: { mode: "docker", containerRuntime: "docker" } }, + ); expect(config.mode).toBe("docker"); expect(config.edgeRuntime).toEqual( @@ -95,6 +98,13 @@ describe("resolveConfig edge runtime defaults", () => { ); }); + it("requires Effect consumers to provide the selected Docker runtime", async () => { + await expect(resolveConfig({ mode: "docker" })).rejects.toMatchObject({ + _tag: "StackBuildError", + reason: "invalid_config", + }); + }); + it("applies the detected Docker mode before resolving services and ports", async () => { const config = await resolveConfig(undefined, { runtime: { mode: "docker", containerRuntime: "podman" }, @@ -125,13 +135,16 @@ describe("resolveConfig edge runtime defaults", () => { describe("resolveConfig explicit keyless ports", () => { it("preserves an explicit pooler api port", async () => { - const config = await resolveConfig({ - mode: "docker", - edgeRuntime: false, - postgrest: false, - auth: false, - pooler: { port: 42423, apiPort: 42424 }, - }); + const config = await resolveConfig( + { + mode: "docker", + edgeRuntime: false, + postgrest: false, + auth: false, + pooler: { port: 42423, apiPort: 42424 }, + }, + { runtime: { mode: "docker", containerRuntime: "docker" } }, + ); expect(config.ports.poolerPort).toBe(42423); expect(config.ports.poolerApiPort).toBe(42424); @@ -170,20 +183,23 @@ describe("resolveConfig explicit keyless ports", () => { describe("candidateCleanupTargets", () => { it("derives fallback Docker identities from enabled catalog services", async () => { - const config = await resolveConfig({ - mode: "docker", - auth: false, - edgeRuntime: false, - realtime: false, - storage: false, - imgproxy: false, - mailpit: false, - pgmeta: false, - studio: false, - analytics: false, - vector: false, - pooler: false, - }); + const config = await resolveConfig( + { + mode: "docker", + auth: false, + edgeRuntime: false, + realtime: false, + storage: false, + imgproxy: false, + mailpit: false, + pgmeta: false, + studio: false, + analytics: false, + vector: false, + pooler: false, + }, + { runtime: { mode: "docker", containerRuntime: "docker" } }, + ); expect(candidateCleanupTargets(config)).toEqual({ dockerContainerNames: [ @@ -195,7 +211,10 @@ describe("candidateCleanupTargets", () => { it("keys fallback Docker identities by the stack's own identity when it has one", async () => { const instanceId = "0f9d2b3c-4a5e-4c7d-8e9f-1a2b3c4d5e6f"; - const config = await resolveConfig({ mode: "docker", instanceId }); + const config = await resolveConfig( + { mode: "docker", instanceId }, + { runtime: { mode: "docker", containerRuntime: "docker" } }, + ); expect(config.instanceId).toBe(instanceId); const { dockerContainerNames } = candidateCleanupTargets(config); diff --git a/packages/stack/src/effect-bun.ts b/packages/stack/src/effect-bun.ts index 490b6cedd8..409b895276 100644 --- a/packages/stack/src/effect-bun.ts +++ b/packages/stack/src/effect-bun.ts @@ -66,7 +66,7 @@ export const updateManagedLaunch = (opts: { readonly stackName?: string; readonly cwd?: string; readonly cacheRoot: string; - readonly launch: NonNullable; + readonly launch: import("./managed/document.ts").ManagedStackLaunchUpdate; }) => updateManagedLaunchCore(opts).pipe( Effect.provide(managedLayer(opts.cacheRoot)), diff --git a/packages/stack/src/effect-node.ts b/packages/stack/src/effect-node.ts index 97f5841af2..1959dbce15 100644 --- a/packages/stack/src/effect-node.ts +++ b/packages/stack/src/effect-node.ts @@ -66,7 +66,7 @@ export const updateManagedLaunch = (opts: { readonly stackName?: string; readonly cwd?: string; readonly cacheRoot: string; - readonly launch: NonNullable; + readonly launch: import("./managed/document.ts").ManagedStackLaunchUpdate; }) => updateManagedLaunchCore(opts).pipe( Effect.provide(managedLayer(opts.cacheRoot)), diff --git a/packages/stack/src/effect.ts b/packages/stack/src/effect.ts index 5b92833fcc..0c4c92113a 100644 --- a/packages/stack/src/effect.ts +++ b/packages/stack/src/effect.ts @@ -23,9 +23,12 @@ export { export type { NativeTarget, PlatformInfo } from "./Platform.ts"; export { detectPlatform, nativeTargetForPlatform } from "./Platform.ts"; +export type { ContainerRuntime, StackRuntimeSelection } from "./ContainerRuntime.ts"; +export { selectStackRuntime, validateStackRuntime } from "./ContainerRuntime.ts"; + export type { ServiceResolution, StackPreparationError } from "./StackPreparation.ts"; -export type { PrefetchOptions, PrefetchResult } from "./prefetch.ts"; +export type { PrefetchEffectOptions, PrefetchOptions, PrefetchResult } from "./prefetch.ts"; export { prefetch } from "./prefetch.ts"; export { diff --git a/packages/stack/src/functions.unit.test.ts b/packages/stack/src/functions.unit.test.ts index 321d26dbea..7aed30c261 100644 --- a/packages/stack/src/functions.unit.test.ts +++ b/packages/stack/src/functions.unit.test.ts @@ -84,11 +84,14 @@ const authFailureCases = [ describe("stack Functions runtime config", () => { it("projects an explicit bundle without project discovery", async () => { const root = makeTempProject(); - const stackConfig = await resolveConfig({ - mode: "docker", - projectDir: root, - functions: makeBundle(root), - }); + const stackConfig = await resolveConfig( + { + mode: "docker", + projectDir: root, + functions: makeBundle(root), + }, + { runtime: { mode: "docker", containerRuntime: "docker" } }, + ); const config = resolveFunctionsRuntimeConfig( stackConfig, { hostname: "127.0.0.1" }, @@ -216,7 +219,10 @@ describe("stack Functions runtime config", () => { return Effect.gen(function* () { const bundle = makeBundle(cwd); const stackConfig = yield* Effect.promise(() => - resolveConfig({ mode: "docker", projectDir: cwd, runtimeRoot: cwd, functions: bundle }), + resolveConfig( + { mode: "docker", projectDir: cwd, runtimeRoot: cwd, functions: bundle }, + { runtime: { mode: "docker", containerRuntime: "docker" } }, + ), ); yield* configureFunctionsRuntime(stackConfig, { hostname: "127.0.0.1" }, bundle); const filePath = functionsRuntimeConfigPath(stackConfig.runtimeRoot); diff --git a/packages/stack/src/managed-manager-lifecycle.integration.test.ts b/packages/stack/src/managed-manager-lifecycle.integration.test.ts index 34e32ef332..16cf28faec 100644 --- a/packages/stack/src/managed-manager-lifecycle.integration.test.ts +++ b/packages/stack/src/managed-manager-lifecycle.integration.test.ts @@ -50,6 +50,7 @@ describe("managed stack lifecycle journeys", () => { portDocument, ownership: owner, lifecycle: "running", + launch: { mode: "native", versions: {} }, }); yield* releaseLease(started); yield* owner.close; @@ -58,13 +59,12 @@ describe("managed stack lifecycle journeys", () => { workspacePath: workspace, stackName: "default", launch: { - mode: "docker" as const, versions: { postgres: "17.6.1" }, excludedServices: [], }, }; const updated = yield* updateManagedLaunch(input); - expect(updated.launch).toEqual(input.launch); + expect(updated.launch).toEqual({ mode: "native", ...input.launch }); yield* stopManagedStack(input); const stopped = yield* manager.inspectStack(stackId); @@ -192,9 +192,12 @@ describe("managed stack lifecycle journeys", () => { portDocument: automaticDocument(), ownership: owner, lifecycle: "running", + launch: { mode: "native", versions: {} }, }); yield* releaseLease(initial); - const launch = { mode: "docker" as const, versions: { postgres: "17.6.1" } }; + const launch = { + versions: { postgres: "17.6.1" }, + }; gate.enabled = true; const launchFiber = yield* Effect.forkScoped( manager.updateLaunch(owner, { stackId, launch }), @@ -220,7 +223,7 @@ describe("managed stack lifecycle journeys", () => { yield* Fiber.join(stopFiber); const final = yield* manager.inspectStack(stackId); expect(final?.lifecycle).toBe("stopped"); - expect(final?.launch).toEqual(launch); + expect(final?.launch).toEqual({ mode: "native", ...launch }); }), ).pipe( Effect.provide(managerLayer), diff --git a/packages/stack/src/managed-store.integration.test.ts b/packages/stack/src/managed-store.integration.test.ts index 3154555e76..a1baf516b1 100644 --- a/packages/stack/src/managed-store.integration.test.ts +++ b/packages/stack/src/managed-store.integration.test.ts @@ -139,6 +139,7 @@ describe("managed stack document store", () => { document({ launch: { mode: "docker", + containerRuntime: "docker", versions: { postgres: "17.6.1" }, excludedServices: ["studio", "analytics"], lastNotifiedUpdateFingerprint: "fingerprint", @@ -147,6 +148,7 @@ describe("managed stack document store", () => { ); expect((yield* store.read(STACK_ID))?.launch).toEqual({ mode: "docker", + containerRuntime: "docker", versions: { postgres: "17.6.1" }, excludedServices: ["studio", "analytics"], lastNotifiedUpdateFingerprint: "fingerprint", diff --git a/packages/stack/src/managed.ts b/packages/stack/src/managed.ts index 51b1a65411..4b7942ff9e 100644 --- a/packages/stack/src/managed.ts +++ b/packages/stack/src/managed.ts @@ -53,6 +53,7 @@ export { deriveRepairOwnershipId, ManagedStackControlRequiredError, ManagedStackAttachedError, + ManagedStackLaunchMissingError, ManagedWorkspaceRepairConflictError, } from "./managed/manager.ts"; export type { @@ -67,8 +68,9 @@ export type { AllocateManagedPortsRequest, ReadStackRequest, StartStackRequest, - ManagedStackLaunchUpdate, + ManagedStackLaunchUpdateRequest, } from "./managed/manager.ts"; +export type { ManagedStackLaunchUpdate } from "./managed/document.ts"; export { connectManagedStack, deleteManagedStack, diff --git a/packages/stack/src/managed/document.ts b/packages/stack/src/managed/document.ts index 509ab7e44f..8684294caa 100644 --- a/packages/stack/src/managed/document.ts +++ b/packages/stack/src/managed/document.ts @@ -1,7 +1,6 @@ import { Data, Effect, Schema } from "effect"; import type { ManagedPortAssignment } from "./model.ts"; -import { PartialVersionManifestSchema, type PartialVersionManifest } from "../versions.ts"; -import type { ContainerRuntime } from "../ContainerRuntime.ts"; +import { PartialVersionManifestSchema } from "../versions.ts"; export type ManagedStackDocumentLifecycle = | "stopped" @@ -10,6 +9,29 @@ export type ManagedStackDocumentLifecycle = | "deleting" | "failed"; +const managedStackLaunchFields = { + versions: PartialVersionManifestSchema, + excludedServices: Schema.optionalKey(Schema.Array(Schema.String)), + lastNotifiedUpdateFingerprint: Schema.optionalKey(Schema.String), +} as const; + +export const managedStackLaunchUpdateSchema = Schema.Struct(managedStackLaunchFields); +export type ManagedStackLaunchUpdate = Schema.Schema.Type; + +const managedStackLaunchSchema = Schema.Union([ + Schema.Struct({ + mode: Schema.Literal("native"), + ...managedStackLaunchFields, + }), + Schema.Struct({ + mode: Schema.Literal("docker"), + containerRuntime: Schema.Literals(["docker", "podman"] as const), + ...managedStackLaunchFields, + }), +]); + +export type ManagedStackLaunch = Schema.Schema.Type; + export interface ManagedStackDocument { readonly format: "supabase-stack"; readonly formatVersion: 1; @@ -34,33 +56,15 @@ export interface ManagedStackDocument { readonly controlEndpoint: string; readonly protocolVersion: 1; }; - readonly launch?: { - readonly mode: "native" | "docker"; - readonly containerRuntime?: ContainerRuntime; - readonly versions: PartialVersionManifest; - readonly excludedServices?: ReadonlyArray; - readonly lastNotifiedUpdateFingerprint?: string; - }; + readonly launch?: ManagedStackLaunch; readonly createdAt: string; readonly updatedAt: string; } -export const managedStackLaunchSchema = Schema.Struct({ - mode: Schema.Literals(["native", "docker"] as const), - containerRuntime: Schema.optionalKey(Schema.Literals(["docker", "podman"] as const)), - versions: PartialVersionManifestSchema, - excludedServices: Schema.optionalKey(Schema.Array(Schema.String)), - lastNotifiedUpdateFingerprint: Schema.optionalKey(Schema.String), -}); - -export type ManagedStackLaunch = Schema.Schema.Type; - /** Launch request before the supervisor selects a concrete execution mode. */ export const managedStackLaunchInputSchema = Schema.Struct({ mode: Schema.optionalKey(Schema.Literals(["native", "docker"] as const)), - versions: PartialVersionManifestSchema, - excludedServices: Schema.optionalKey(Schema.Array(Schema.String)), - lastNotifiedUpdateFingerprint: Schema.optionalKey(Schema.String), + ...managedStackLaunchFields, }); export type ManagedStackLaunchInput = Schema.Schema.Type; diff --git a/packages/stack/src/managed/lifecycle.ts b/packages/stack/src/managed/lifecycle.ts index 8085715279..f3097bb6f9 100644 --- a/packages/stack/src/managed/lifecycle.ts +++ b/packages/stack/src/managed/lifecycle.ts @@ -6,14 +6,14 @@ import { dockerForceRemove } from "../cleanup.ts"; import { dockerContainerName } from "../StackIdentity.ts"; import { SERVICE_NAMES } from "../ServiceCatalog.ts"; import { HttpTransportClient, HttpTransportClientError } from "../HttpTransportClient.ts"; -import type { ManagedStackDocument } from "./document.ts"; +import type { ManagedStackDocument, ManagedStackLaunchUpdate } from "./document.ts"; import { ManagedStackAttachedError, ManagedStackControlRequiredError, ManagedStackManager, workspaceRepairConflict, type ManagedStackManagerError, - type ManagedStackLaunchUpdate, + type ManagedStackLaunchUpdateRequest, } from "./manager.ts"; import { ControlTransportError, controlEndpoint, type ControlEndpoint } from "./control.ts"; import { @@ -289,7 +289,7 @@ export const deleteManagedStack = ( /** Persist launch selections in the managed document, owner-gated. */ export const updateManagedLaunch = ( - input: ManagedLifecycleInput & { readonly launch: NonNullable }, + input: ManagedLifecycleInput & { readonly launch: ManagedStackLaunchUpdate }, ): Effect.Effect< ManagedStackDocument, NoRunningStackError | ManagedStackManagerError | HttpTransportClientError, @@ -317,7 +317,10 @@ export const updateManagedLaunch = ( if (next === undefined) return yield* Effect.fail(noRunningStack(input)); return next; } - const update: ManagedStackLaunchUpdate = { stackId: document.id, launch: input.launch }; + const update: ManagedStackLaunchUpdateRequest = { + stackId: document.id, + launch: input.launch, + }; return yield* Effect.ensuring(manager.updateLaunch(acquisition, update), acquisition.close); }), ); diff --git a/packages/stack/src/managed/manager.ts b/packages/stack/src/managed/manager.ts index e22a633adb..b529e94425 100644 --- a/packages/stack/src/managed/manager.ts +++ b/packages/stack/src/managed/manager.ts @@ -60,7 +60,7 @@ import { } from "./port-plan.ts"; import { resolvePortIntents } from "./port-intent.ts"; import { makeStackStore, type ManagedStackListing } from "./store.ts"; -import type { ManagedStackDocument } from "./document.ts"; +import type { ManagedStackDocument, ManagedStackLaunchUpdate } from "./document.ts"; import { dockerForceRemove } from "../cleanup.ts"; import { SERVICE_NAMES } from "../ServiceCatalog.ts"; import { dockerContainerName } from "../StackIdentity.ts"; @@ -114,9 +114,9 @@ export interface ManagedStackLifecycleUpdate { readonly runtime?: ManagedStackDocument["runtime"] | null; } -export interface ManagedStackLaunchUpdate { +export interface ManagedStackLaunchUpdateRequest { readonly stackId: string; - readonly launch: NonNullable; + readonly launch: ManagedStackLaunchUpdate; } export interface ManagedPortLease { @@ -148,6 +148,16 @@ export class ManagedStackAttachedError extends Data.TaggedError("ManagedStackAtt } } +export class ManagedStackLaunchMissingError extends Data.TaggedError( + "ManagedStackLaunchMissingError", +)<{ + readonly stackId: string; +}> { + override get message(): string { + return `Managed stack ${this.stackId} has no selected runtime`; + } +} + export class ManagedWorkspaceRepairConflictError extends Data.TaggedError( "ManagedWorkspaceRepairConflictError", )<{ @@ -174,6 +184,7 @@ export const workspaceRepairConflict = ( export type ManagedStackManagerError = | ManagedStackControlRequiredError | ManagedStackAttachedError + | ManagedStackLaunchMissingError | ManagedWorkspaceRepairConflictError | ManagedStackNotFoundError | ManagedStackNotStoppedError @@ -235,7 +246,7 @@ export interface ManagedStackManagerShape { /** Persist launch selections under the stack's control ownership. */ readonly updateLaunch: ( ownership: ControlOwnership, - update: ManagedStackLaunchUpdate, + update: ManagedStackLaunchUpdateRequest, ) => Effect.Effect; readonly repairWorkspace: ( request: RepairRequest, @@ -764,7 +775,7 @@ const makeManager = ( const updateLaunch = ( ownership: ControlOwnership, - update: ManagedStackLaunchUpdate, + update: ManagedStackLaunchUpdateRequest, ): Effect.Effect => lifecycleLock.withPermit( Effect.gen(function* () { @@ -773,9 +784,33 @@ const makeManager = ( if (current === undefined) { return yield* Effect.fail(new ManagedStackNotFoundError({ stackId: update.stackId })); } + if (current.launch === undefined) { + return yield* Effect.fail( + new ManagedStackLaunchMissingError({ stackId: update.stackId }), + ); + } + const metadata = { + versions: update.launch.versions, + ...(update.launch.excludedServices === undefined + ? {} + : { excludedServices: update.launch.excludedServices }), + ...(update.launch.lastNotifiedUpdateFingerprint === undefined + ? {} + : { + lastNotifiedUpdateFingerprint: update.launch.lastNotifiedUpdateFingerprint, + }), + }; + const launch: NonNullable = + current.launch.mode === "native" + ? { ...metadata, mode: "native" } + : { + ...metadata, + mode: "docker", + containerRuntime: current.launch.containerRuntime, + }; const next: ManagedStackDocument = { ...current, - launch: update.launch, + launch, updatedAt: now(), }; yield* store.write(next); diff --git a/packages/stack/src/node.ts b/packages/stack/src/node.ts index 5c81d60f8d..49c37accad 100644 --- a/packages/stack/src/node.ts +++ b/packages/stack/src/node.ts @@ -4,8 +4,10 @@ import { FetchHttpClient } from "effect/unstable/http"; import { BinaryResolver } from "./BinaryResolver.ts"; import { selectStackRuntime } from "./ContainerRuntime.ts"; import { createStack as createStackCore, type StackHandle } from "./createStack.ts"; +import { toStackError } from "./errors.ts"; import { prefetch as prefetchEffect, + type PrefetchEffectOptions, type PrefetchOptions, type PrefetchResult, } from "./prefetch.ts"; @@ -23,24 +25,31 @@ import type { StackConfig } from "./StackConfig.ts"; export async function createStack(config?: StackConfig): Promise { const runtime = await Effect.runPromise( selectStackRuntime(config?.mode).pipe(Effect.provide(NodeServices.layer)), - ); + ).catch((error: unknown) => { + throw toStackError(error); + }); return createStackCore(config, platformFactory, runtime); } export async function prefetch(options?: PrefetchOptions): Promise { const runtime = await Effect.runPromise( selectStackRuntime(options?.mode).pipe(Effect.provide(NodeServices.layer)), - ); + ).catch((error: unknown) => { + throw toStackError(error); + }); const resolverLayer = BinaryResolver.make(defaultCacheRoot()).pipe( Layer.provide(FetchHttpClient.layer), ); const preparationLayer = StackPreparation.layer.pipe(Layer.provide(resolverLayer)); + const resolvedOptions: PrefetchEffectOptions = + runtime.mode === "native" + ? { ...options, mode: "native" } + : { ...options, mode: "docker", containerRuntime: runtime.containerRuntime }; return Effect.runPromise( - prefetchEffect({ - ...options, - mode: runtime.mode, - ...(runtime.containerRuntime === null ? {} : { containerRuntime: runtime.containerRuntime }), - }).pipe(Effect.provide(preparationLayer), Effect.provide(NodeServices.layer)), + prefetchEffect(resolvedOptions).pipe( + Effect.provide(preparationLayer), + Effect.provide(NodeServices.layer), + ), ); } diff --git a/packages/stack/src/prefetch.ts b/packages/stack/src/prefetch.ts index 50a799452a..c4a26af9fc 100644 --- a/packages/stack/src/prefetch.ts +++ b/packages/stack/src/prefetch.ts @@ -1,4 +1,5 @@ import { Effect } from "effect"; +import type { ContainerRuntime } from "./ContainerRuntime.ts"; import type { StackPreparationError } from "./StackPreparation.ts"; import { type PreparedStackArtifacts, @@ -8,13 +9,18 @@ import { import { StackPreparation } from "./StackPreparation.ts"; import type { ServiceName } from "./ServiceName.ts"; -export interface PrefetchOptions extends Omit { - readonly mode?: StackPreparationInput["mode"]; +export interface PrefetchOptions { + readonly versions?: StackPreparationInput["versions"]; + readonly services?: StackPreparationInput["services"]; + readonly enabledServices?: StackPreparationInput["enabledServices"]; + readonly mode?: "native" | "docker"; } -interface ResolvedPrefetchOptions extends PrefetchOptions { - readonly containerRuntime?: import("./ContainerRuntime.ts").ContainerRuntime; -} +export type PrefetchEffectOptions = Omit & + ( + | { readonly mode?: "native"; readonly containerRuntime?: never } + | { readonly mode: "docker"; readonly containerRuntime: ContainerRuntime } + ); export type PrefetchResult = Partial>; @@ -22,11 +28,11 @@ const toPrefetchResult = (artifacts: PreparedStackArtifacts): PrefetchResult => artifacts.resolutions; export const prefetch = ( - options?: ResolvedPrefetchOptions, + options?: PrefetchEffectOptions, ): Effect.Effect => Effect.gen(function* () { const preparation = yield* StackPreparation; - return yield* preparation - .prepare({ mode: options?.mode ?? "native", ...options }) - .pipe(Effect.map(toPrefetchResult)); + const input: StackPreparationInput = + options?.mode === "docker" ? options : { ...options, mode: "native" }; + return yield* preparation.prepare(input).pipe(Effect.map(toPrefetchResult)); }); diff --git a/packages/stack/src/prefetch.unit.test.ts b/packages/stack/src/prefetch.unit.test.ts index c418b4d418..c21582eeae 100644 --- a/packages/stack/src/prefetch.unit.test.ts +++ b/packages/stack/src/prefetch.unit.test.ts @@ -64,7 +64,7 @@ function mockSequenceSpawner(results: ReadonlyArray) { } describe("prefetch", () => { - test("prefetches all services by default", async () => { + test("prefetches every native-capable service by default in native mode", async () => { const resolver = mockBinaryResolver(); const spawner = mockSequenceSpawner( Array.from({ length: SERVICE_NAMES.length }, () => ({ @@ -77,11 +77,9 @@ describe("prefetch", () => { Layer.provide(spawner.layer), ); - const result = await Effect.runPromise( - prefetch({ mode: "docker" }).pipe(Effect.provide(layer)), - ); + const result = await Effect.runPromise(prefetch().pipe(Effect.provide(layer))); - expect(Object.keys(result).sort()).toEqual([...SERVICE_NAMES].sort()); + expect(Object.keys(result).sort()).toEqual(["auth", "postgres", "postgrest"]); }); test("preparation fails with DockerPullError when the canonical image fails", async () => { @@ -101,7 +99,10 @@ describe("prefetch", () => { ); const error = await Effect.runPromise( - prefetch({ mode: "docker", services: ["auth"] }).pipe(Effect.provide(layer), Effect.flip), + prefetch({ mode: "docker", containerRuntime: "docker", services: ["auth"] }).pipe( + Effect.provide(layer), + Effect.flip, + ), ); expect(error).toBeInstanceOf(DockerPullError); @@ -117,7 +118,9 @@ describe("prefetch", () => { ); const result = await Effect.runPromise( - prefetch({ mode: "docker", services: ["postgrest"] }).pipe(Effect.provide(layer)), + prefetch({ mode: "docker", containerRuntime: "docker", services: ["postgrest"] }).pipe( + Effect.provide(layer), + ), ); expect(Object.keys(result).sort()).toEqual(["postgres", "postgrest"]); @@ -132,7 +135,9 @@ describe("prefetch", () => { ); const result = await Effect.runPromise( - prefetch({ mode: "docker", services: ["storage"] }).pipe(Effect.provide(layer)), + prefetch({ mode: "docker", containerRuntime: "docker", services: ["storage"] }).pipe( + Effect.provide(layer), + ), ); expect(Object.keys(result).sort()).toEqual(["imgproxy", "postgres", "storage"]); @@ -174,6 +179,7 @@ describe("prefetch", () => { const preparation = yield* StackPreparation; return yield* preparation.prepare({ mode: "docker", + containerRuntime: "docker", services: ["studio"], enabledServices: ["postgres", "pgmeta", "studio"], }); @@ -194,7 +200,9 @@ describe("prefetch", () => { ); const result = await Effect.runPromise( - prefetch({ mode: "docker", services: ["postgrest"] }).pipe(Effect.provide(layer)), + prefetch({ mode: "docker", containerRuntime: "docker", services: ["postgrest"] }).pipe( + Effect.provide(layer), + ), ); expect(result.postgrest).toEqual({ @@ -249,7 +257,9 @@ describe("prefetch", () => { ); const result = await Effect.runPromise( - prefetch({ mode: "docker", services: ["pgmeta"] }).pipe(Effect.provide(layer)), + prefetch({ mode: "docker", containerRuntime: "docker", services: ["pgmeta"] }).pipe( + Effect.provide(layer), + ), ); expect(result.pgmeta).toEqual({ @@ -269,7 +279,7 @@ describe("prefetch", () => { Effect.gen(function* () { const preparation = yield* StackPreparation; const streamEvents = yield* preparation - .prepareEvents({ mode: "docker", services: ["auth"] }) + .prepareEvents({ mode: "docker", containerRuntime: "docker", services: ["auth"] }) .pipe(Stream.runCollect); const downloadEvents = streamEvents.flatMap((event) => event instanceof ServiceDownloadStarted || event instanceof ServiceDownloadFinished @@ -301,6 +311,7 @@ describe("prefetch", () => { const preparation = yield* StackPreparation; const artifacts = yield* preparation.prepare({ mode: "docker", + containerRuntime: "docker", services: ["edge-runtime"], }); return artifacts.resolutions; diff --git a/packages/stack/src/services/postgres-init.ts b/packages/stack/src/services/postgres-init.ts index 63917352b1..4447226169 100644 --- a/packages/stack/src/services/postgres-init.ts +++ b/packages/stack/src/services/postgres-init.ts @@ -1,5 +1,6 @@ import type { ServiceDef } from "@supabase/process-compose"; -import type { ServiceDependency } from "./service-utils.ts"; +import { dockerContainerName, type StackIdentity } from "../StackIdentity.ts"; +import type { ContainerRuntimeOptions, ServiceDependency } from "./service-utils.ts"; interface PostgresInitOptions { readonly postgresDir: string; @@ -12,6 +13,15 @@ interface PostgresInitOptions { readonly dependencies: ReadonlyArray; } +interface DockerPostgresInitOptions extends ContainerRuntimeOptions { + readonly dbPort: number; + readonly jwtSecret: string; + readonly jwtExpiry: number; + readonly autoExposeNewTables: boolean; + readonly identity: StackIdentity; + readonly dependencies: ReadonlyArray; +} + /** * SQL that matches what Studio runs at cloud project creation when "Default privileges for new * entities" is off. Revokes the default GRANTs installed by the bundled initial schema so new @@ -27,6 +37,53 @@ alter default privileges for role postgres in schema public revoke execute on functions from anon, authenticated, service_role; `.trim(); +const dockerPostgresSchemaSql = (opts: DockerPostgresInitOptions) => + ` +\\set jwt_secret \`echo "$JWT_SECRET"\` +\\set jwt_exp \`echo "$JWT_EXP"\` +ALTER DATABASE postgres SET "app.settings.jwt_secret" TO :'jwt_secret'; +ALTER DATABASE postgres SET "app.settings.jwt_exp" TO :'jwt_exp'; +ALTER USER postgres WITH PASSWORD 'postgres'; +ALTER USER authenticator WITH PASSWORD 'postgres'; +ALTER USER supabase_auth_admin WITH PASSWORD 'postgres'; +ALTER USER supabase_storage_admin WITH PASSWORD 'postgres'; +ALTER USER supabase_replication_admin WITH PASSWORD 'postgres'; +ALTER USER supabase_read_only_user WITH PASSWORD 'postgres'; +CREATE SCHEMA IF NOT EXISTS _realtime; +ALTER SCHEMA _realtime OWNER TO postgres; +${opts.autoExposeNewTables ? "" : REVOKE_DEFAULT_DATA_API_PRIVILEGES_SQL} +SELECT 'CREATE DATABASE _supabase WITH OWNER postgres' +WHERE NOT EXISTS (SELECT 1 FROM pg_database WHERE datname = '_supabase')\\gexec +\\connect _supabase +CREATE SCHEMA IF NOT EXISTS _analytics; +ALTER SCHEMA _analytics OWNER TO postgres; +CREATE SCHEMA IF NOT EXISTS _supavisor; +ALTER SCHEMA _supavisor OWNER TO postgres; +`.trim(); + +export const makePostgresInitServiceDocker = (opts: DockerPostgresInitOptions): ServiceDef => ({ + name: "postgres-init", + command: opts.runtime, + args: [ + "exec", + "-e", + "PGPASSWORD=postgres", + "-e", + `JWT_SECRET=${opts.jwtSecret}`, + "-e", + `JWT_EXP=${opts.jwtExpiry}`, + dockerContainerName("postgres", opts.identity.key), + "sh", + "-c", + `/opt/postgres/bin/psql -h 127.0.0.1 -p ${opts.dbPort} -v ON_ERROR_STOP=1 --no-password --no-psqlrc -U supabase_admin -d postgres <<'EOSQL' +${dockerPostgresSchemaSql(opts)} +EOSQL`, + ], + dependencies: opts.dependencies, + supervision: {}, + restart: "no", +}); + export const makePostgresInitService = (opts: PostgresInitOptions): ServiceDef => { const pgBinDir = `${opts.postgresDir}/bin`; const pgLibDir = `${opts.postgresDir}/lib`; diff --git a/packages/stack/src/services/postgres.ts b/packages/stack/src/services/postgres.ts index 1df6f8f3bf..39b99abb3b 100644 --- a/packages/stack/src/services/postgres.ts +++ b/packages/stack/src/services/postgres.ts @@ -1,4 +1,3 @@ -import { mkdirSync, writeFileSync } from "node:fs"; import type { ServiceDef } from "@supabase/process-compose"; import { dockerNetworkArgs } from "../Platform.ts"; import { dockerContainerName, type StackIdentity } from "../StackIdentity.ts"; @@ -20,15 +19,11 @@ interface PostgresServiceOptions { interface NativePostgresOptions extends PostgresServiceOptions { readonly binPath: string; - /** When true, patches postgres to listen on all interfaces so Docker containers can connect. */ - readonly dockerAccessible?: boolean; } interface DockerPostgresOptions extends PostgresServiceOptions, ContainerRuntimeOptions { readonly image: string; readonly platformOs: string; - readonly jwtSecret: string; - readonly jwtExpiry: number; readonly identity: StackIdentity; readonly cleanupDataDirOnExit?: boolean; } @@ -41,12 +36,6 @@ const postgresEnv = (opts: NativePostgresOptions): Record => ({ TZDIR: "/var/db/timezone/zoneinfo", }); -const postgresDockerEnv = (opts: DockerPostgresOptions): Record => ({ - POSTGRES_PASSWORD: "postgres", - JWT_SECRET: opts.jwtSecret, - JWT_EXP: String(opts.jwtExpiry), -}); - const NATIVE_POSTGRES_RUNTIME_ARGS = [ "-c", "wal_level=logical", @@ -59,32 +48,6 @@ const NATIVE_POSTGRES_RUNTIME_ARGS = [ const orphanCleanup = (opts: PostgresServiceOptions) => opts.cleanupDataDirOnExit ? removePathOnOrphanCleanup(opts.dataDir) : []; -const DOCKER_POSTGRES_SCHEMA_SQL = `\\set pgpass \`echo "$PGPASSWORD"\` -\\set jwt_secret \`echo "$JWT_SECRET"\` -\\set jwt_exp \`echo "$JWT_EXP"\` -ALTER DATABASE postgres SET "app.settings.jwt_secret" TO :'jwt_secret'; -ALTER DATABASE postgres SET "app.settings.jwt_exp" TO :'jwt_exp'; -ALTER USER postgres WITH PASSWORD :'pgpass'; -ALTER USER authenticator WITH PASSWORD :'pgpass'; -ALTER USER supabase_auth_admin WITH PASSWORD :'pgpass'; -ALTER USER supabase_storage_admin WITH PASSWORD :'pgpass'; -ALTER USER supabase_replication_admin WITH PASSWORD :'pgpass'; -ALTER USER supabase_read_only_user WITH PASSWORD :'pgpass'; -create schema if not exists _realtime; -alter schema _realtime owner to postgres; -SELECT 'CREATE DATABASE _supabase WITH OWNER postgres' -WHERE NOT EXISTS (SELECT 1 FROM pg_database WHERE datname = '_supabase')\\gexec -\\connect _supabase -create schema if not exists _analytics; -alter schema _analytics owner to postgres; -create schema if not exists _supavisor; -alter schema _supavisor owner to postgres;`; - -const dockerPostgresEntrypoint = (port: number) => - `cat <<'EOF' > /etc/postgresql.schema.sql && exec docker-entrypoint.sh postgres -D /etc/postgresql -p ${port} -${DOCKER_POSTGRES_SCHEMA_SQL} -EOF`; - const postgresHealthCheck = (binPath: string, port: number) => ({ probe: { _tag: "Exec" as const, @@ -115,7 +78,7 @@ const postgresDockerHealthCheck = ( runtime, containerName, "pg_isready", - ["-p", String(port), "-U", "postgres"], + ["-h", "127.0.0.1", "-p", String(port), "-U", "postgres"], { ...stackHealthBudgets.postgresDocker, }, @@ -124,52 +87,6 @@ const postgresDockerHealthCheck = ( export const makePostgresService = (opts: NativePostgresOptions): ServiceDef => { const initScript = `${opts.binPath}/share/supabase-cli/bin/supabase-postgres-init.sh`; - if (opts.dockerAccessible) { - // Docker containers connect via host.docker.internal, which resolves to a gateway IP - // rather than 127.0.0.1. We create a per-run pg_hba.conf that allows those - // connections, and use postgres -c flags to override listen_addresses and hba_file. - // This avoids mutating the shared binary cache. - const customHbaPath = `${opts.dataDir}_pg_hba_docker.conf`; - mkdirSync(opts.dataDir, { recursive: true }); - writeFileSync( - customHbaPath, - [ - "local all all scram-sha-256", - "host all all 127.0.0.1/32 scram-sha-256", - "host all all ::1/128 scram-sha-256", - "host all all 0.0.0.0/0 scram-sha-256", - "", - ].join("\n"), - "utf8", - ); - - return { - name: "postgres", - command: "bash", - args: [ - initScript, - "-p", - String(opts.port), - ...NATIVE_POSTGRES_RUNTIME_ARGS, - "-c", - "listen_addresses=*", - "-c", - `hba_file=${customHbaPath}`, - ], - env: postgresEnv(opts), - dependencies: opts.dependencies, - healthCheck: postgresHealthCheck(opts.binPath, opts.port), - shutdown: { signal: "SIGTERM", timeoutSeconds: 10 }, - supervision: { - orphanCleanup: [ - ...orphanCleanup(opts), - ...removePathOnOrphanCleanup(customHbaPath, { recursive: false }), - ], - }, - restart: "unless-stopped", - }; - } - return { name: "postgres", command: "bash", @@ -184,7 +101,6 @@ export const makePostgresService = (opts: NativePostgresOptions): ServiceDef => }; export const makePostgresServiceDocker = (opts: DockerPostgresOptions): ServiceDef => { - const env = postgresDockerEnv(opts); const containerName = dockerContainerName("postgres", opts.identity.key); return dockerRunService({ runtime: opts.runtime, @@ -193,9 +109,8 @@ export const makePostgresServiceDocker = (opts: DockerPostgresOptions): ServiceD image: opts.image, networkArgs: dockerNetworkArgs(opts.platformOs, [opts.port]), volumes: [`${opts.dataDir}:/var/lib/postgresql/data`], - env, - entrypoint: "sh", - cmd: ["-c", dockerPostgresEntrypoint(opts.port)], + env: { POSTGRES_PASSWORD: "postgres" }, + cmd: ["-p", String(opts.port)], dependencies: opts.dependencies, healthCheck: postgresDockerHealthCheck(opts.runtime, containerName, opts.port), shutdown: { signal: "SIGTERM", timeoutSeconds: 10 }, diff --git a/packages/stack/src/services/services.unit.test.ts b/packages/stack/src/services/services.unit.test.ts index 72367fc302..bea77ea535 100644 --- a/packages/stack/src/services/services.unit.test.ts +++ b/packages/stack/src/services/services.unit.test.ts @@ -126,51 +126,6 @@ describe("makeStudioServiceDocker", () => { }); }); -describe("makePostgresService (dockerAccessible)", () => { - it("creates per-run pg_hba.conf instead of mutating shared cache", () => { - const tempDir = mkdtempSync(path.join(tmpdir(), "stack-postgres-service-")); - const def = makePostgresService({ - binPath: POSTGRES_BIN_PATH, - dataDir: path.join(tempDir, "data"), - port: DB_PORT, - dockerAccessible: true, - cleanupDataDirOnExit: true, - dependencies: [], - }); - const customHbaPath = `${path.join(tempDir, "data")}_pg_hba_docker.conf`; - - try { - expect(def.name).toBe("postgres"); - expect(def.command).toBe("bash"); - expect(def.args).toEqual([ - `${POSTGRES_BIN_PATH}/share/supabase-cli/bin/supabase-postgres-init.sh`, - "-p", - "54322", - "-c", - "wal_level=logical", - "-c", - "max_wal_senders=5", - "-c", - "max_replication_slots=5", - "-c", - "listen_addresses=*", - "-c", - `hba_file=${customHbaPath}`, - ]); - expect(readFileSync(customHbaPath, "utf8")).toContain("0.0.0.0/0"); - expect(def.supervision).toEqual({ - orphanCleanup: [ - { _tag: "RemovePath", path: path.join(tempDir, "data") }, - { _tag: "RemovePath", path: customHbaPath, recursive: false }, - ], - }); - } finally { - rmSync(tempDir, { recursive: true, force: true }); - rmSync(customHbaPath, { force: true }); - } - }); -}); - describe("makePostgresServiceDocker", () => { it("creates a docker-based postgres ServiceDef", () => { const def = makePostgresServiceDocker({ @@ -179,8 +134,6 @@ describe("makePostgresServiceDocker", () => { dataDir: "/tmp/supabase/data", port: DB_PORT, platformOs: "linux", - jwtSecret: "test-jwt-secret-with-at-least-32-characters", - jwtExpiry: 3600, identity: EPHEMERAL_IDENTITY, dependencies: [], }); @@ -194,8 +147,7 @@ describe("makePostgresServiceDocker", () => { expect(def.args).toContain(`${DB_PORT}:${DB_PORT}`); expect(def.args).toContain(dockerImageForService("postgres", DEFAULT_VERSIONS.postgres)); expect(def.args).toContain("/tmp/supabase/data:/var/lib/postgresql/data"); - // Verify port is passed to postgres inside the container - expect(def.args?.[def.args.length - 1]).toContain(`-p ${DB_PORT}`); + expect(def.args?.slice(-2)).toEqual(["-p", String(DB_PORT)]); // Health check uses docker exec + pg_isready inside the container (host has no postgres tools) expect(def.healthCheck?.probe).toEqual({ _tag: "Exec", @@ -204,6 +156,8 @@ describe("makePostgresServiceDocker", () => { "exec", `supabase-postgres-${API_PORT}`, "pg_isready", + "-h", + "127.0.0.1", "-p", "54322", "-U", @@ -223,29 +177,6 @@ describe("makePostgresServiceDocker", () => { ], }); }); - - it("bootstraps auxiliary databases and schemas used by docker-backed services", () => { - const def = makePostgresServiceDocker({ - runtime: "docker", - image: dockerImageForService("postgres", DEFAULT_VERSIONS.postgres), - dataDir: "/tmp/supabase/data", - port: DB_PORT, - platformOs: "linux", - jwtSecret: "test-jwt-secret-with-at-least-32-characters", - jwtExpiry: 3600, - identity: EPHEMERAL_IDENTITY, - dependencies: [], - }); - - const script = def.args?.[def.args.length - 1] as string; - expect(script).toContain("CREATE DATABASE _supabase WITH OWNER postgres"); - expect(script).toContain( - "WHERE NOT EXISTS (SELECT 1 FROM pg_database WHERE datname = '_supabase')", - ); - expect(script).toContain("\\connect _supabase"); - expect(script).toContain("create schema if not exists _analytics;"); - expect(script).toContain("create schema if not exists _supavisor;"); - }); }); describe("makePostgrestService", () => { diff --git a/packages/stack/src/services/vector.ts b/packages/stack/src/services/vector.ts index 945fe80e22..f81d66fa0e 100644 --- a/packages/stack/src/services/vector.ts +++ b/packages/stack/src/services/vector.ts @@ -19,19 +19,24 @@ interface DockerVectorOptions extends ContainerRuntimeOptions { readonly dependencies: ReadonlyArray; } -const VECTOR_CONFIG = (host: string, port: number, apiKey: string) => `api: +const vectorConfig = ( + host: string, + port: number, + apiKey: string, + sourceType: "docker_logs" | "internal_logs", +) => `api: enabled: true address: 0.0.0.0:9001 sources: - docker: - type: docker_logs + runtime: + type: ${sourceType} sinks: logflare: type: http inputs: - - docker + - runtime encoding: codec: json method: post @@ -44,10 +49,16 @@ sinks: export const makeVectorServiceDocker = (opts: DockerVectorOptions) => { const containerName = dockerContainerName("vector", opts.identity.key); - const dockerSocket = process.env.DOCKER_HOST?.startsWith("unix://") - ? process.env.DOCKER_HOST.slice("unix://".length) - : "/var/run/docker.sock"; - const volumes = existsSync(dockerSocket) ? [`${dockerSocket}:/var/run/docker.sock:ro`] : []; + const dockerSocket = + opts.runtime === "docker" + ? process.env.DOCKER_HOST?.startsWith("unix://") + ? process.env.DOCKER_HOST.slice("unix://".length) + : "/var/run/docker.sock" + : undefined; + const volumes = + dockerSocket !== undefined && existsSync(dockerSocket) + ? [`${dockerSocket}:/var/run/docker.sock:ro`] + : []; return dockerRunService({ runtime: opts.runtime, @@ -56,14 +67,17 @@ export const makeVectorServiceDocker = (opts: DockerVectorOptions) => { image: opts.image, networkArgs: dockerNetworkArgs(opts.platformOs, []), volumes, - env: { - DOCKER_HOST: "unix:///var/run/docker.sock", - }, + env: opts.runtime === "docker" ? { DOCKER_HOST: "unix:///var/run/docker.sock" } : {}, entrypoint: "sh", cmd: [ "-c", `cat <<'EOF' > /etc/vector/vector.yaml && vector --config /etc/vector/vector.yaml -${VECTOR_CONFIG(opts.serviceHost, opts.analyticsPort, opts.analyticsApiKey)}EOF +${vectorConfig( + opts.serviceHost, + opts.analyticsPort, + opts.analyticsApiKey, + opts.runtime === "docker" ? "docker_logs" : "internal_logs", +)}EOF `, ], dependencies: opts.dependencies, diff --git a/packages/stack/src/supervisor.integration.test.ts b/packages/stack/src/supervisor.integration.test.ts index d209702242..4ea908a215 100644 --- a/packages/stack/src/supervisor.integration.test.ts +++ b/packages/stack/src/supervisor.integration.test.ts @@ -5,6 +5,7 @@ import { createServer as createHttpServer } from "node:http"; import { createConnection, createServer } from "node:net"; import { existsSync, + chmodSync, mkdtempSync, mkdirSync, readFileSync, @@ -256,7 +257,6 @@ const remoteInfo = (endpoint: ControlEndpoint): Promise<{ readonly url: string } const updateLaunch = async ( endpoint: ControlEndpoint, launch: { - readonly mode: "native" | "docker"; readonly versions: Record; }, ): Promise => { @@ -390,8 +390,12 @@ const readStackDocument = (roots: { | { readonly id: string; readonly lifecycle: string; - readonly ports: ReadonlyArray<{ port: number }>; - readonly launch?: { readonly mode: string; readonly versions: Record }; + readonly ports: ReadonlyArray<{ key: string; port: number }>; + readonly launch?: { + readonly mode: string; + readonly containerRuntime?: string; + readonly versions: Record; + }; } | undefined => { const stacksRoot = join(roots.stateRoot, "stacks"); @@ -402,7 +406,7 @@ const readStackDocument = (roots: { return JSON.parse(readFileSync(path, "utf8")) as { readonly id: string; readonly lifecycle: string; - readonly ports: ReadonlyArray<{ port: number }>; + readonly ports: ReadonlyArray<{ key: string; port: number }>; }; } return undefined; @@ -529,6 +533,76 @@ describe("detached supervisor child journeys", () => { } }); + test("starts an omitted-mode stack from one detected runtime selection", async () => { + const roots = await workspace(); + const binDir = mkdtempSync(join(tmpdir(), "sup-stack-runtime-")); + const docker = join(binDir, "docker"); + writeFileSync(docker, "#!/bin/sh\nexit 0\n"); + chmodSync(docker, 0o755); + const base = messageFor(roots); + const { mode: _mode, ...config } = base.config; + const child = spawnChild( + messageFor(roots, { + config: { ...config, edgeRuntime: {} }, + }), + { environment: { PATH: `${binDir}:${process.env["PATH"] ?? ""}` } }, + ); + try { + const started = await child.started; + const document = readStackDocument(roots); + expect(document?.launch).toMatchObject({ + mode: "docker", + containerRuntime: "docker", + }); + expect(document?.ports.map(({ key }) => key)).toContain("edge_runtime.inspector_port"); + await remoteStop(started.endpoint); + await waitForExit(child.child); + } finally { + if (child.child.exitCode === null) await kill(child.child); + cleanupRoots(roots); + rmSync(binDir, { recursive: true, force: true }); + } + }); + + test("reuses the persisted runtime instead of selecting a different one on restart", async () => { + const roots = await workspace(); + const binDir = mkdtempSync(join(tmpdir(), "sup-stack-sticky-runtime-")); + const docker = join(binDir, "docker"); + const podman = join(binDir, "podman"); + writeFileSync(docker, "#!/bin/sh\nexit 0\n"); + writeFileSync(podman, "#!/bin/sh\nexit 1\n"); + chmodSync(docker, 0o755); + chmodSync(podman, 0o755); + const base = messageFor(roots); + const { mode: _mode, ...config } = base.config; + const input = messageFor(roots, { config }); + const environment = { PATH: `${binDir}:${process.env["PATH"] ?? ""}` }; + const initial = spawnChild(input, { environment }); + let restarted: ChildHandle | undefined; + try { + const started = await initial.started; + await remoteStop(started.endpoint); + await waitForExit(initial.child); + + writeFileSync(docker, "#!/bin/sh\nexit 1\n"); + writeFileSync(podman, "#!/bin/sh\nexit 0\n"); + restarted = spawnChild(input, { environment }); + + await expect(restarted.started).rejects.toThrow( + "Docker mode requires a usable docker runtime", + ); + expect(readStackDocument(roots)?.launch).toMatchObject({ + mode: "docker", + containerRuntime: "docker", + }); + } finally { + if (initial.child.exitCode === null) await kill(initial.child); + if (restarted?.child.exitCode === null) await kill(restarted.child); + cleanupRoots(roots); + rmSync(binDir, { recursive: true, force: true }); + } + }); + test("publishes stopping before a slow owner shutdown can finish", async () => { const roots = await workspace(); const child = spawnChild(messageFor(roots), { testMode: "hold-stop" }); @@ -905,9 +979,9 @@ describe("detached supervisor child journeys", () => { const attached = await later.started; expect(attached.attached).toBe(true); expect(await remoteInfo(attached.endpoint)).toMatchObject({ url: expect.any(String) }); - await updateLaunch(attached.endpoint, { mode: "docker", versions: { postgres: "17.6.1" } }); + await updateLaunch(attached.endpoint, { versions: { postgres: "17.6.1" } }); expect(readStackDocument(roots)?.launch).toEqual({ - mode: "docker", + mode: "native", versions: { postgres: "17.6.1" }, }); await remoteStop(attached.endpoint); diff --git a/packages/stack/src/supervisor.ts b/packages/stack/src/supervisor.ts index f8ba002c0d..5eb0d8616f 100644 --- a/packages/stack/src/supervisor.ts +++ b/packages/stack/src/supervisor.ts @@ -13,7 +13,11 @@ import { } from "effect"; import { HttpServer } from "effect/unstable/http"; import { ChildProcessSpawner } from "effect/unstable/process"; -import { selectStackRuntime } from "./ContainerRuntime.ts"; +import { + selectStackRuntime, + validateStackRuntime, + type StackRuntimeSelection, +} from "./ContainerRuntime.ts"; import type { PlatformFactory } from "./createStack.ts"; import { DaemonServer } from "./DaemonServer.ts"; import { Stack } from "./Stack.ts"; @@ -28,12 +32,17 @@ import { type ControlTransport, } from "./managed/control.ts"; import { ManagedStackManager, type ManagedStackStartResult } from "./managed/manager.ts"; -import { managedStackLaunchInputSchema, type ManagedStackLaunchInput } from "./managed/document.ts"; +import { + managedStackLaunchInputSchema, + type ManagedStackLaunch, + type ManagedStackLaunchInput, +} from "./managed/document.ts"; import { deriveStackId, ensureEnvironment } from "./managed/environment.ts"; import { gitConfigStoreLayer } from "./managed/git.ts"; import { validateManagedStackName, type ManagedPortIntentDocument } from "./managed/model.ts"; import { managedStackPaths } from "./managed/paths.ts"; -import { PORT_FIELDS, type PortField, type PortSet } from "./PortCatalog.ts"; +import { PORT_CATALOG, PORT_FIELDS, type PortField, type PortSet } from "./PortCatalog.ts"; +import { portFieldsForConfigInput } from "./ServicePorts.ts"; import { SERVICE_NAMES } from "./ServiceCatalog.ts"; import { dockerContainerName } from "./StackIdentity.ts"; import type { PortAllocationError, PortLease } from "./PortAllocator.ts"; @@ -108,8 +117,18 @@ const decodeSupervisorStartMessage = (value: unknown): SupervisorStartMessage => return Schema.decodeUnknownSync(supervisorStartMessageSchema)(value); }; -const causeMessage = (cause: unknown): string => - cause instanceof Error ? cause.message : typeof cause === "string" ? cause : String(cause); +const causeMessage = (cause: unknown): string => { + if (cause instanceof Error && cause.message.length > 0) return cause.message; + if ( + typeof cause === "object" && + cause !== null && + "detail" in cause && + typeof cause.detail === "string" + ) { + return cause.detail; + } + return typeof cause === "string" ? cause : String(cause); +}; const toDaemonConfig = (value: Readonly>): DaemonConfigInput | undefined => typeof value.cwd === "string" ? { ...value, cwd: value.cwd } : undefined; @@ -268,7 +287,7 @@ const startDaemon = (input: { readonly platform: SupervisorPlatform; readonly scope: Scope.Scope; readonly launchUpdate?: ( - launch: NonNullable, + launch: import("./managed/document.ts").ManagedStackLaunchUpdate, ) => Effect.Effect; }): Effect.Effect< { readonly daemon: DaemonServer["Service"] }, @@ -436,9 +455,44 @@ const runManaged = ( ); } } - const runtime = yield* selectStackRuntime(configInput.mode ?? input.launch?.mode); + const existing = yield* manager.inspectStack(stackId); + const requestedMode = configInput.mode ?? input.launch?.mode; + const persistedRuntime: StackRuntimeSelection | undefined = + existing?.launch?.mode === "native" + ? { mode: "native", containerRuntime: null } + : existing?.launch?.mode === "docker" + ? { mode: "docker", containerRuntime: existing.launch.containerRuntime } + : undefined; + if ( + persistedRuntime !== undefined && + requestedMode !== undefined && + persistedRuntime.mode !== requestedMode + ) { + return yield* Effect.fail( + new SupervisorStartError({ + message: `Stack runtime is already ${persistedRuntime.mode}; requested ${requestedMode}`, + }), + ); + } + const runtime = + persistedRuntime === undefined + ? yield* selectStackRuntime(requestedMode) + : yield* validateStackRuntime(persistedRuntime); + const activeFields = portFieldsForConfigInput({ ...configInput, mode: runtime.mode }); + const activeFieldSet = new Set(activeFields); + const portIntents: ManagedPortIntentDocument = { + ...input.portIntents, + activeFields, + disabledFields: PORT_FIELDS.filter( + (field) => PORT_CATALOG[field].persistence === "sticky" && !activeFieldSet.has(field), + ), + }; + const launchInput = input.launch ?? { versions: {} }; + const launch: ManagedStackLaunch = + runtime.containerRuntime === null + ? { ...launchInput, mode: "native" } + : { ...launchInput, mode: "docker", containerRuntime: runtime.containerRuntime }; const startup = Effect.gen(function* () { - const existing = yield* manager.inspectStack(stackId); if ( existing !== undefined && (existing.lifecycle === "starting" || @@ -456,19 +510,10 @@ const runManaged = ( const started: ManagedStackStartResult = yield* manager.startStack({ workspacePath: input.workspacePath, stackName: input.stackName, - portDocument: input.portIntents, + portDocument: portIntents, ownership, lifecycle: "starting", - launch: - input.launch === undefined - ? undefined - : { - ...input.launch, - mode: runtime.mode, - ...(runtime.containerRuntime === null - ? {} - : { containerRuntime: runtime.containerRuntime }), - }, + launch, }); claimedStack = true; const resolved = yield* Effect.tryPromise({ @@ -584,7 +629,7 @@ export const runSupervisor = ( const input = yield* receiveStartMessage(); yield* Effect.matchCauseEffect(runManaged(input, platform, scope), { onFailure: (cause) => - sendMessage({ type: "error", message: causeMessage(cause) }).pipe( + sendMessage({ type: "error", message: causeMessage(Cause.squash(cause)) }).pipe( Effect.andThen(Effect.failCause(cause)), ), onSuccess: Effect.succeed, diff --git a/packages/stack/tests/helpers/warmup.ts b/packages/stack/tests/helpers/warmup.ts index 7660bdc578..157ce74076 100644 --- a/packages/stack/tests/helpers/warmup.ts +++ b/packages/stack/tests/helpers/warmup.ts @@ -30,11 +30,7 @@ export async function warmStackE2eDependencies( const dockerAvailable = (options.hasDockerDaemon ?? hasDockerDaemon)(); try { - const warmups = [prefetchDeps()]; - if (dockerAvailable) { - warmups.push(prefetchDeps({ mode: "docker" })); - } - await Promise.all(warmups); + await prefetchDeps(dockerAvailable ? { mode: "docker" } : undefined); } catch (error) { logger.warn( `[stack-e2e] Warmup failed: ${error instanceof Error ? error.message : String(error)}`, diff --git a/packages/stack/tests/helpers/warmup.unit.test.ts b/packages/stack/tests/helpers/warmup.unit.test.ts index 5e805ffb0f..aed75f906a 100644 --- a/packages/stack/tests/helpers/warmup.unit.test.ts +++ b/packages/stack/tests/helpers/warmup.unit.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, test, vi } from "vitest"; +import { describe, expect, test } from "vitest"; import type { PrefetchOptions, PrefetchResult } from "../../src/node.ts"; import { warmStackE2eDependencies } from "./warmup.ts"; @@ -29,32 +29,20 @@ function makeResult(type: "binary" | "docker"): PrefetchResult { } describe("stack e2e warmup", () => { - test("runs auto prefetch and docker image warmup when Docker is available", async () => { + test("warms Docker resources once when Docker is available", async () => { const calls: Array = []; const { logger } = makeLogger(); - let finishAutoPrefetch: (() => void) | undefined; - const warmup = warmStackE2eDependencies({ + await warmStackE2eDependencies({ logger, hasDockerDaemon: () => true, prefetch: async (options?: PrefetchOptions) => { calls.push(options); - if (options === undefined) { - await new Promise((resolve) => { - finishAutoPrefetch = resolve; - }); - } return options?.mode === "docker" ? makeResult("docker") : makeResult("binary"); }, }); - await vi.waitFor(() => { - expect(calls).toEqual([undefined, { mode: "docker" }]); - }); - finishAutoPrefetch?.(); - await warmup; - - expect(calls).toEqual([undefined, { mode: "docker" }]); + expect(calls).toEqual([{ mode: "docker" }]); }); test("skips docker image warmup when Docker is unavailable", async () => { From 2a1e7eefb9e3e97ef1fc041602ab21754be710fc Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 18 Aug 2026 23:26:26 +0200 Subject: [PATCH 05/20] fix(stack): harden managed lifecycle coordination --- packages/stack/README.md | 5 + packages/stack/docs/architecture.md | 6 +- packages/stack/src/ContainerRuntime.ts | 2 +- packages/stack/src/DaemonProtocol.ts | 2 + packages/stack/src/DaemonServer.ts | 23 ++- packages/stack/src/LocalStack.ts | 31 +++- .../stack/src/RemoteStack.integration.test.ts | 109 +++++++++----- packages/stack/src/RemoteStack.ts | 37 ++++- packages/stack/src/Stack.unit.test.ts | 134 +++++++++++++++++- packages/stack/src/supervisor.ts | 2 +- packages/stack/tests/helpers/warmup.ts | 21 +-- .../stack/tests/helpers/warmup.unit.test.ts | 24 +++- 12 files changed, 327 insertions(+), 69 deletions(-) diff --git a/packages/stack/README.md b/packages/stack/README.md index 88e4db6183..4a6186c5b2 100644 --- a/packages/stack/README.md +++ b/packages/stack/README.md @@ -58,5 +58,10 @@ const runtime = `stopDaemon` and the discovery helpers delegate to the managed lifecycle facade. No CLI metadata file or PID polling is involved. +After a managed supervisor claims a stack, its persisted Docker, Podman, or +native selection remains pinned even if startup later fails. Retry after +restoring or starting that runtime; delete and recreate the stack to choose a +different execution mode. Deletion removes the stack's managed data. + For the end-to-end lifecycle, identity, ports, service execution, transport, compiled-Bun re-entry, and testing boundary, see [How `@supabase/stack` works](docs/architecture.md). diff --git a/packages/stack/docs/architecture.md b/packages/stack/docs/architecture.md index 14d1c0bc2b..6756553fdc 100644 --- a/packages/stack/docs/architecture.md +++ b/packages/stack/docs/architecture.md @@ -231,7 +231,11 @@ selects exactly one execution mode: a usable Docker daemon is preferred, then a usable Podman service; if neither responds, the stack uses native mode. An explicit `mode: "native"` rejects Docker-only services, while explicit `mode: "docker"` requires a usable container runtime. Preparation never falls back to -the other mode after that choice. +the other mode after that choice. Once a managed supervisor claims and persists +that selection, it remains pinned even when startup later fails during image +pull, native download, graph build, or readiness. Retries restore or start the +persisted runtime and reuse the same mode; selecting another mode requires +deleting and recreating the stack, which removes its managed data. The `StackBuilder` turns planned resolutions into one process-compose graph, without eagerly materializing every graph resource. Container resources are diff --git a/packages/stack/src/ContainerRuntime.ts b/packages/stack/src/ContainerRuntime.ts index aeb3dc84f1..b6ca5e5fa7 100644 --- a/packages/stack/src/ContainerRuntime.ts +++ b/packages/stack/src/ContainerRuntime.ts @@ -35,7 +35,7 @@ export const validateStackRuntime = ( ? Effect.succeed(selection) : Effect.fail( new StackBuildError({ - detail: `Docker mode requires a usable ${selection.containerRuntime} runtime`, + detail: `Docker mode requires a usable ${selection.containerRuntime} runtime. Restore or start the persisted ${selection.containerRuntime} runtime and retry, or delete and recreate the stack (removing its managed data) to choose another execution mode.`, reason: "docker_not_running", }), ), diff --git a/packages/stack/src/DaemonProtocol.ts b/packages/stack/src/DaemonProtocol.ts index a0e058312e..3098e2b89e 100644 --- a/packages/stack/src/DaemonProtocol.ts +++ b/packages/stack/src/DaemonProtocol.ts @@ -5,6 +5,7 @@ const DaemonErrorCodeSchema = Schema.Literals([ "SERVICE_NOT_READY", "STACK_READINESS_TIMEOUT", "STACK_BUILD_ERROR", + "STACK_NOT_RUNNING", ]); const StackBuildReasonSchema = Schema.Literals([ @@ -38,6 +39,7 @@ export const DaemonErrorResponseSchema = Schema.Struct({ service: Schema.optionalKey(Schema.String), exitCode: Schema.optionalKey(Schema.Number), timeoutMs: Schema.optionalKey(Schema.Number), + phase: Schema.optionalKey(Schema.String), reason: Schema.optionalKey(StackBuildReasonSchema), }); diff --git a/packages/stack/src/DaemonServer.ts b/packages/stack/src/DaemonServer.ts index d2efe9f830..78b7eaba97 100644 --- a/packages/stack/src/DaemonServer.ts +++ b/packages/stack/src/DaemonServer.ts @@ -50,7 +50,7 @@ export class DaemonServer extends Context.Service< const server = yield* HttpServer.HttpServer; const shutdownDeferred = yield* Deferred.make(); const textEncoder = new TextEncoder(); - const errorResponse = (body: DaemonErrorResponse, status: 400 | 404 | 500) => + const errorResponse = (body: DaemonErrorResponse, status: 400 | 404 | 409 | 500) => HttpServerResponse.jsonUnsafe(body, { status }); const notFoundResponse = (name: string) => errorResponse( @@ -76,6 +76,15 @@ export class DaemonServer extends Context.Service< }, 500, ); + const notRunningResponse = (phase: string) => + errorResponse( + { + code: "STACK_NOT_RUNNING", + error: `Stack is not running (phase: ${phase})`, + phase, + }, + 409, + ); const invalidReloadPayloadResponse = () => errorResponse( { code: "STACK_BUILD_ERROR", error: "Invalid Edge Functions reload payload" }, @@ -333,6 +342,9 @@ export class DaemonServer extends Context.Service< Effect.catchTag("StackBuildError", (e) => Effect.succeed(buildErrorResponse(e.detail, e.reason)), ), + Effect.catchTag("StackNotRunningError", (e) => + Effect.succeed(notRunningResponse(e.phase)), + ), Effect.catchTag("StackReadinessError", (e) => terminalReadinessResponse(e.target, e.timeoutMs, e.detail), ), @@ -401,6 +413,9 @@ export class DaemonServer extends Context.Service< Effect.catchTag("StackBuildError", (e) => Effect.succeed(buildErrorResponse(e.detail, e.reason)), ), + Effect.catchTag("StackNotRunningError", (e) => + Effect.succeed(notRunningResponse(e.phase)), + ), Effect.catchTag("StackReadinessError", (e) => terminalReadinessResponse(e.target, e.timeoutMs, e.detail), ), @@ -428,6 +443,9 @@ export class DaemonServer extends Context.Service< Effect.catchTag("StackBuildError", (e) => Effect.succeed(buildErrorResponse(e.detail, e.reason)), ), + Effect.catchTag("StackNotRunningError", (e) => + Effect.succeed(notRunningResponse(e.phase)), + ), Effect.catchTag("StackReadinessError", (e) => terminalReadinessResponse(e.target, e.timeoutMs, e.detail), ), @@ -455,6 +473,9 @@ export class DaemonServer extends Context.Service< Effect.catchTag("StackBuildError", (e) => Effect.succeed(buildErrorResponse(e.detail, e.reason)), ), + Effect.catchTag("StackNotRunningError", (e) => + Effect.succeed(notRunningResponse(e.phase)), + ), Effect.catchTag("StackReadinessError", (e) => terminalReadinessResponse(e.target, e.timeoutMs, e.detail), ), diff --git a/packages/stack/src/LocalStack.ts b/packages/stack/src/LocalStack.ts index 2a1dfbfb70..2f067591b6 100644 --- a/packages/stack/src/LocalStack.ts +++ b/packages/stack/src/LocalStack.ts @@ -662,6 +662,18 @@ export const localStackLayer = ( } disposed = true; yield* Ref.set(phaseRef, "stopping"); + const preparationError = new StackBuildError({ + detail: "Stack disposed during asset preparation", + }); + if (planDeferred !== undefined) { + yield* Deferred.fail(planDeferred, preparationError); + } + for (const deferred of preparationInFlight.values()) { + yield* Deferred.fail(deferred, preparationError); + } + if (runtimeDeferred !== undefined) { + yield* Deferred.fail(runtimeDeferred, preparationError); + } yield* Scope.close(preparationScope, Exit.void); yield* cleanupLocalStackResources({ stop: () => @@ -876,15 +888,16 @@ export const localStackLayer = ( yield* requireMutable("reload functions"); yield* requireRunningPhase; yield* requireKnownService("edge-runtime"); - const currentBundle = yield* Ref.get(functionsBundleRef); - const nextBundle = + const requestedBundle = opts?.functions === undefined - ? currentBundle + ? undefined : yield* decodeFunctionsBundle(opts.functions); yield* prepareServices(["edge-runtime"]); const started = yield* Effect.gen(function* () { yield* requireMutable("reload functions"); yield* requireRunningPhase; + const currentBundle = yield* Ref.get(functionsBundleRef); + const nextBundle = requestedBundle ?? currentBundle; yield* configureFunctions(config, nextBundle); yield* Ref.set(functionsBundleRef, nextBundle); const runtime = yield* ensureRuntime; @@ -904,16 +917,20 @@ export const localStackLayer = ( yield* requireMutable("reload Edge Runtime"); yield* requireRunningPhase; yield* requireKnownService("edge-runtime"); - const nextConfig = yield* configWithEdgeRuntimeOptions(opts); - const currentBundle = yield* Ref.get(functionsBundleRef); - const nextBundle = + if (opts.edgeRuntime.enabled === false) { + return yield* Effect.fail(new ServiceNotFoundError({ name: "edge-runtime" })); + } + const requestedBundle = opts.functions === undefined - ? currentBundle + ? undefined : yield* decodeFunctionsBundle(opts.functions); yield* prepareServices(["edge-runtime"]); const started = yield* Effect.gen(function* () { yield* requireMutable("reload Edge Runtime"); yield* requireRunningPhase; + const nextConfig = yield* configWithEdgeRuntimeOptions(opts); + const currentBundle = yield* Ref.get(functionsBundleRef); + const nextBundle = requestedBundle ?? currentBundle; const prepared = yield* ensurePlanned; const runtime = yield* ensureRuntime; const buildResult = yield* builder.build(nextConfig, prepared); diff --git a/packages/stack/src/RemoteStack.integration.test.ts b/packages/stack/src/RemoteStack.integration.test.ts index 548eaf95fe..493f1286f1 100644 --- a/packages/stack/src/RemoteStack.integration.test.ts +++ b/packages/stack/src/RemoteStack.integration.test.ts @@ -4,7 +4,7 @@ import { Cause, Effect, Exit, Fiber, Layer, ManagedRuntime, Result, Stream } fro import * as http from "node:http"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { DaemonServer } from "./DaemonServer.ts"; -import { StackBuildError, StackReadinessError } from "./errors.ts"; +import { StackBuildError, StackNotRunningError, StackReadinessError } from "./errors.ts"; import type { FunctionsReloadConfig, ResolvedFunctionsBundle } from "./functions.ts"; import { RemoteStack } from "./RemoteStack.ts"; import { Stack, type EdgeRuntimeReloadConfig, type StackInfo } from "./Stack.ts"; @@ -85,6 +85,7 @@ function mockStack( readonly waitReadyBuildReason?: "invalid_config" | "docker_not_running" | "asset_preparation"; readonly waitReadyTimeoutMs?: number; readonly restartServiceReadyError?: string; + readonly notRunningPhase?: string; } = {}, ) { let stopped = false; @@ -107,25 +108,27 @@ function mockStack( startService: (name: string) => name === "unknown" ? Effect.fail(new ServiceNotFoundError({ name })) - : options.startServiceBuildError !== undefined - ? Effect.fail( - new StackBuildError({ - detail: options.startServiceBuildError, - ...(options.startServiceBuildReason === undefined - ? {} - : { reason: options.startServiceBuildReason }), - }), - ) - : options.startServiceReadyError !== undefined + : options.notRunningPhase !== undefined + ? Effect.fail(new StackNotRunningError({ phase: options.notRunningPhase })) + : options.startServiceBuildError !== undefined ? Effect.fail( - new ServiceReadyError({ - name, - reason: options.startServiceReadyError, + new StackBuildError({ + detail: options.startServiceBuildError, + ...(options.startServiceBuildReason === undefined + ? {} + : { reason: options.startServiceBuildReason }), }), ) - : Effect.sync(() => { - serviceCalls.push(`start:${name}`); - }), + : options.startServiceReadyError !== undefined + ? Effect.fail( + new ServiceReadyError({ + name, + reason: options.startServiceReadyError, + }), + ) + : Effect.sync(() => { + serviceCalls.push(`start:${name}`); + }), stopService: (name: string) => name === "unknown" ? Effect.fail(new ServiceNotFoundError({ name })) @@ -135,26 +138,32 @@ function mockStack( restartService: (name: string) => name === "unknown" ? Effect.fail(new ServiceNotFoundError({ name })) - : options.restartServiceReadyError !== undefined - ? Effect.fail( - new ServiceReadyError({ - name, - reason: options.restartServiceReadyError, + : options.notRunningPhase !== undefined + ? Effect.fail(new StackNotRunningError({ phase: options.notRunningPhase })) + : options.restartServiceReadyError !== undefined + ? Effect.fail( + new ServiceReadyError({ + name, + reason: options.restartServiceReadyError, + }), + ) + : Effect.sync(() => { + serviceCalls.push(`restart:${name}`); }), - ) - : Effect.sync(() => { - serviceCalls.push(`restart:${name}`); - }), reloadFunctions: (config) => - Effect.sync(() => { - functionReloads.push(config ?? {}); - serviceCalls.push("reload-functions"); - }), + options.notRunningPhase !== undefined + ? Effect.fail(new StackNotRunningError({ phase: options.notRunningPhase })) + : Effect.sync(() => { + functionReloads.push(config ?? {}); + serviceCalls.push("reload-functions"); + }), reloadEdgeRuntime: (config) => - Effect.sync(() => { - edgeRuntimeReloads.push(config); - serviceCalls.push("reload-edge-runtime"); - }), + options.notRunningPhase !== undefined + ? Effect.fail(new StackNotRunningError({ phase: options.notRunningPhase })) + : Effect.sync(() => { + edgeRuntimeReloads.push(config); + serviceCalls.push("reload-edge-runtime"); + }), getState: (name: string) => { const match = MOCK_STATES.find((s) => s.name === name); return match ? Effect.succeed(match) : Effect.fail(new ServiceNotFoundError({ name })); @@ -570,6 +579,38 @@ describe("RemoteStack integration", () => { } }); + test("preserves StackNotRunningError across mutating daemon operations", async () => { + const failingMock = mockStack({ notRunningPhase: "stopped" }); + const failingServer = ManagedRuntime.make(buildServerLayer(failingMock)); + let failingClient: ManagedRuntime.ManagedRuntime | undefined; + try { + const daemon = await failingServer.runPromise(DaemonServer); + const addr = daemon.address; + if (addr._tag !== "TcpAddress") throw new Error("Expected TcpAddress"); + const host = addr.hostname === "0.0.0.0" ? "127.0.0.1" : addr.hostname; + failingClient = ManagedRuntime.make(buildClientLayer(`http://${host}:${addr.port}`)); + + const operations = [ + (stack: Stack["Service"]) => stack.startService("auth"), + (stack: Stack["Service"]) => stack.restartService("auth"), + (stack: Stack["Service"]) => stack.reloadFunctions(), + (stack: Stack["Service"]) => + stack.reloadEdgeRuntime({ edgeRuntime: { policy: "oneshot" } }), + ]; + for (const operation of operations) { + const error = await failingClient.runPromise( + Effect.flatMap(Stack, operation).pipe(Effect.flip), + ); + expect(error).toBeInstanceOf(StackNotRunningError); + expect(error._tag).toBe("StackNotRunningError"); + if (error._tag === "StackNotRunningError") expect(error.phase).toBe("stopped"); + } + } finally { + await failingClient?.dispose(); + await failingServer.dispose(); + } + }); + test("preserves ServiceReadyError from remote startService", async () => { const failingMock = mockStack({ startServiceReadyError: "start failed readiness" }); const failingServer = ManagedRuntime.make(buildServerLayer(failingMock)); diff --git a/packages/stack/src/RemoteStack.ts b/packages/stack/src/RemoteStack.ts index e133f53823..9a25f61a57 100644 --- a/packages/stack/src/RemoteStack.ts +++ b/packages/stack/src/RemoteStack.ts @@ -3,7 +3,7 @@ import { Effect, Layer, Schema, Stream } from "effect"; import * as Sse from "effect/unstable/encoding/Sse"; import { HttpClientError, HttpClientRequest, HttpClientResponse } from "effect/unstable/http"; import { DaemonErrorResponseSchema } from "./DaemonProtocol.ts"; -import { StackBuildError, StackReadinessError } from "./errors.ts"; +import { StackBuildError, StackNotRunningError, StackReadinessError } from "./errors.ts"; import { Stack, StackInfoSchema } from "./Stack.ts"; import { inheritReadyOptions } from "./StackConfig.ts"; import { StackServiceState, StackServiceStatusSchema } from "./StackServiceState.ts"; @@ -138,7 +138,11 @@ const failDaemonResponse = ( fallbackName: string, ): Effect.Effect< never, - ServiceNotFoundError | ServiceReadyError | StackBuildError | StackReadinessError + | ServiceNotFoundError + | ServiceReadyError + | StackBuildError + | StackNotRunningError + | StackReadinessError > => Effect.gen(function* () { const body = yield* dieOnBodyDecodeError( @@ -166,6 +170,8 @@ const failDaemonResponse = ( timeoutMs: body.timeoutMs ?? 0, detail: body.error, }); + case "STACK_NOT_RUNNING": + return yield* new StackNotRunningError({ phase: body.phase ?? "unknown" }); } }); @@ -177,6 +183,25 @@ const expectDaemonOk = ( ): Effect.Effect< void, ServiceNotFoundError | ServiceReadyError | StackBuildError | StackReadinessError +> => + response.status >= 200 && response.status < 300 + ? Effect.void + : failDaemonResponse(endpoint, path, response, fallbackName).pipe( + Effect.catchTag("StackNotRunningError", (error) => Effect.die(error)), + ); + +const expectMutatingDaemonOk = ( + endpoint: ControlEndpoint, + path: string, + response: HttpClientResponse.HttpClientResponse, + fallbackName: string, +): Effect.Effect< + void, + | ServiceNotFoundError + | ServiceReadyError + | StackBuildError + | StackNotRunningError + | StackReadinessError > => response.status >= 200 && response.status < 300 ? Effect.void @@ -386,7 +411,7 @@ export const RemoteStack = { const response = yield* httpResponse(endpoint, path, { method: "POST", }); - yield* expectDaemonOk(endpoint, path, response, name); + yield* expectMutatingDaemonOk(endpoint, path, response, name); }), ), @@ -413,7 +438,7 @@ export const RemoteStack = { const response = yield* httpResponse(endpoint, path, { method: "POST", }); - yield* expectDaemonOk(endpoint, path, response, name); + yield* expectMutatingDaemonOk(endpoint, path, response, name); }), ), @@ -426,7 +451,7 @@ export const RemoteStack = { headers: { "content-type": "application/json" }, body: JSON.stringify(opts ?? {}), }); - yield* expectDaemonOk(endpoint, path, response, "edge-runtime"); + yield* expectMutatingDaemonOk(endpoint, path, response, "edge-runtime"); }), ), @@ -439,7 +464,7 @@ export const RemoteStack = { headers: { "content-type": "application/json" }, body: JSON.stringify(opts), }); - yield* expectDaemonOk(endpoint, path, response, "edge-runtime"); + yield* expectMutatingDaemonOk(endpoint, path, response, "edge-runtime"); }), ), diff --git a/packages/stack/src/Stack.unit.test.ts b/packages/stack/src/Stack.unit.test.ts index 50a090d3ed..8674f99d51 100644 --- a/packages/stack/src/Stack.unit.test.ts +++ b/packages/stack/src/Stack.unit.test.ts @@ -7,7 +7,7 @@ import { chmod, readFile, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { createServer, type Server } from "node:http"; -import { Deferred, Effect, Exit, Fiber, Layer, Stream } from "effect"; +import { Cause, Deferred, Effect, Exit, Fiber, Layer, Stream } from "effect"; import { mockChildProcessSpawner } from "../../process-compose/tests/helpers/mocks.ts"; import { mockBinaryResolver } from "../tests/helpers/mocks.ts"; import { StackBuildError } from "./errors.ts"; @@ -314,6 +314,97 @@ describe("Stack", () => { ); }); + it.live("merges overlapping function and Edge Runtime reloads", () => { + const runtimeRoot = mkdtempSync(join(tmpdir(), "supabase-functions-reload-race-")); + const initialBundle = functionsBundle(runtimeRoot, "initial-secret"); + const replacementBundle = functionsBundle(runtimeRoot, "replacement-secret"); + const config = { + ...edgeRuntimeConfig, + projectDir: runtimeRoot, + runtimeRoot, + functions: initialBundle, + postgrest: false, + auth: false, + servicePolicies: { + ...edgeRuntimeConfig.servicePolicies, + postgrest: "off", + auth: "off", + "edge-runtime": "lazy", + }, + } satisfies ResolvedStackConfig; + const graph = Effect.runSync( + buildGraph([ + { name: "postgres", command: process.execPath, restart: "unless-stopped" }, + { name: "edge-runtime", command: process.execPath, restart: "unless-stopped" }, + ]), + ); + const builderLayer = Layer.succeed(StackBuilder, { + build: () => + Effect.sync(() => { + return { + graph, + cleanupTargets: { dockerContainerNames: [] }, + serviceProjection: new Map([ + ["postgres", { visibility: "public" as const }], + ["edge-runtime", { visibility: "public" as const }], + ]), + }; + }), + }); + const preparationStarted = Deferred.makeUnsafe(); + const allowPreparation = Deferred.makeUnsafe(); + let blockNextSpawn = false; + const resolver = mockBinaryResolver(); + const spawner = mockChildProcessSpawner({ + beforeSpawn: () => { + if (!blockNextSpawn) return Effect.void; + blockNextSpawn = false; + return Deferred.succeed(preparationStarted, undefined).pipe( + Effect.andThen(Deferred.await(allowPreparation)), + ); + }, + }); + const layer = localStackLayer(config, noopPortLease(config.ports)).pipe( + Layer.provide(builderLayer), + Layer.provide(StackPreparation.layer.pipe(Layer.provide(resolver.layer))), + Layer.provide(spawner.layer), + Layer.provide(NodeServices.layer), + ); + const readRuntimeConfig = Effect.promise(() => + readFile(functionsRuntimeConfigPath(runtimeRoot), "utf8").then((contents) => + JSON.parse(contents), + ), + ); + + return Effect.gen(function* () { + const stack = yield* Stack; + yield* stack.start(); + blockNextSpawn = true; + + const functionsReload = yield* stack + .reloadFunctions({ functions: replacementBundle }) + .pipe(Effect.forkChild({ startImmediately: true })); + yield* Deferred.await(preparationStarted); + + // Both requests join the same gated preparation before either can commit. + // The later Edge Runtime commit must preserve the Functions update. + const edgeReload = yield* stack + .reloadEdgeRuntime({ edgeRuntime: { env: { CONCURRENT: "edge-value" } } }) + .pipe(Effect.forkChild({ startImmediately: true })); + yield* Deferred.succeed(allowPreparation, undefined); + yield* Fiber.join(functionsReload); + yield* Fiber.join(edgeReload); + + expect((yield* readRuntimeConfig).env.SHARED).toBe("replacement-secret"); + expect((yield* stack.getState("edge-runtime")).status).toBe("Healthy"); + yield* stack.dispose(); + }).pipe( + Effect.provide(layer), + Effect.ensuring(Effect.promise(() => rm(runtimeRoot, { recursive: true, force: true }))), + Effect.timeout("5 seconds"), + ); + }); + it.effect("getInfo returns valid JWT tokens", () => { const { layer } = setupLayer(); @@ -541,14 +632,26 @@ describe("Stack", () => { }).pipe(Effect.provide(layer)); }); - it.effect("startService fails with ServiceNotFoundError for unknown service", () => { - const { layer } = setupLayer(); + it.live("startService fails with ServiceNotFoundError for unknown service", () => { + const config = { + ...defaultConfig, + servicePolicies: { + ...defaultConfig.servicePolicies, + postgrest: "lazy", + auth: "lazy", + }, + } satisfies ResolvedStackConfig; + const { layer } = setupLayer(config); return Effect.gen(function* () { const stack = yield* Stack; + yield* stack.start(); const exit = yield* stack.startService("nonexistent").pipe(Effect.exit); expect(exit._tag).toBe("Failure"); + if (Exit.isFailure(exit)) { + expect(Cause.squash(exit.cause)).toMatchObject({ _tag: "ServiceNotFoundError" }); + } }).pipe(Effect.provide(layer)); }); @@ -1052,18 +1155,35 @@ describe("Stack", () => { yield* Effect.gen(function* () { const stack = yield* Stack; - const activator = yield* StackServiceActivator; yield* stack.start(); - const activation = yield* activator - .activate("auth") + const activation = yield* stack + .startService("auth") .pipe(Effect.forkChild({ startImmediately: true })); yield* Deferred.await(preparationStarted); + const secondActivation = yield* stack + .startService("auth") + .pipe(Effect.forkChild({ startImmediately: true })); const disposing = yield* stack.dispose().pipe(Effect.forkChild({ startImmediately: true })); yield* Deferred.await(disposed); yield* Fiber.join(disposing); - expect(Exit.isFailure(yield* Fiber.await(activation))).toBe(true); + const activationExit = yield* Fiber.await(activation); + const secondActivationExit = yield* Fiber.await(secondActivation); + expect(Exit.isFailure(activationExit)).toBe(true); + expect(Exit.isFailure(secondActivationExit)).toBe(true); + if (Exit.isFailure(activationExit)) { + expect(Cause.squash(activationExit.cause)).toMatchObject({ + _tag: "StackBuildError", + detail: "Stack disposed during asset preparation", + }); + } + if (Exit.isFailure(secondActivationExit)) { + expect(Cause.squash(secondActivationExit.cause)).toMatchObject({ + _tag: "StackBuildError", + detail: "Stack disposed during asset preparation", + }); + } expect((yield* stack.getState("auth")).status).not.toBe("Downloading"); }).pipe(Effect.provide(layer)); }).pipe(Effect.scoped, Effect.timeout("5 seconds")), diff --git a/packages/stack/src/supervisor.ts b/packages/stack/src/supervisor.ts index 5eb0d8616f..04503b10fd 100644 --- a/packages/stack/src/supervisor.ts +++ b/packages/stack/src/supervisor.ts @@ -470,7 +470,7 @@ const runManaged = ( ) { return yield* Effect.fail( new SupervisorStartError({ - message: `Stack runtime is already ${persistedRuntime.mode}; requested ${requestedMode}`, + message: `Stack runtime is already ${persistedRuntime.mode}; requested ${requestedMode}. Delete and recreate the stack (removing its managed data) before changing execution mode.`, }), ); } diff --git a/packages/stack/tests/helpers/warmup.ts b/packages/stack/tests/helpers/warmup.ts index 157ce74076..9db0bf2114 100644 --- a/packages/stack/tests/helpers/warmup.ts +++ b/packages/stack/tests/helpers/warmup.ts @@ -29,14 +29,19 @@ export async function warmStackE2eDependencies( const shouldFailOnError = options.failOnError ?? false; const dockerAvailable = (options.hasDockerDaemon ?? hasDockerDaemon)(); - try { - await prefetchDeps(dockerAvailable ? { mode: "docker" } : undefined); - } catch (error) { - logger.warn( - `[stack-e2e] Warmup failed: ${error instanceof Error ? error.message : String(error)}`, - ); - if (shouldFailOnError) { - throw error; + const modes: PrefetchOptions[] = [{ mode: "native" }]; + if (dockerAvailable) modes.push({ mode: "docker" }); + + for (const mode of modes) { + try { + await prefetchDeps(mode); + } catch (error) { + logger.warn( + `[stack-e2e] Warmup failed: ${error instanceof Error ? error.message : String(error)}`, + ); + if (shouldFailOnError) { + throw error; + } } } } diff --git a/packages/stack/tests/helpers/warmup.unit.test.ts b/packages/stack/tests/helpers/warmup.unit.test.ts index aed75f906a..16ab704f69 100644 --- a/packages/stack/tests/helpers/warmup.unit.test.ts +++ b/packages/stack/tests/helpers/warmup.unit.test.ts @@ -29,7 +29,7 @@ function makeResult(type: "binary" | "docker"): PrefetchResult { } describe("stack e2e warmup", () => { - test("warms Docker resources once when Docker is available", async () => { + test("warms native and Docker resources when Docker is available", async () => { const calls: Array = []; const { logger } = makeLogger(); @@ -42,7 +42,7 @@ describe("stack e2e warmup", () => { }, }); - expect(calls).toEqual([{ mode: "docker" }]); + expect(calls).toEqual([{ mode: "native" }, { mode: "docker" }]); }); test("skips docker image warmup when Docker is unavailable", async () => { @@ -58,7 +58,7 @@ describe("stack e2e warmup", () => { }, }); - expect(calls).toEqual([undefined]); + expect(calls).toEqual([{ mode: "native" }]); }); test("can fail fast when warmup is required", async () => { @@ -77,6 +77,24 @@ describe("stack e2e warmup", () => { expect(warn.some((message) => message.includes("Warmup failed"))).toBe(true); }); + test("continues to the Docker warmup after a best-effort native failure", async () => { + const calls: Array = []; + const { warn, logger } = makeLogger(); + + await warmStackE2eDependencies({ + hasDockerDaemon: () => true, + logger, + prefetch: async (options?: PrefetchOptions) => { + calls.push(options); + if (options?.mode === "native") throw new Error("native unavailable"); + return makeResult("docker"); + }, + }); + + expect(calls).toEqual([{ mode: "native" }, { mode: "docker" }]); + expect(warn.some((message) => message.includes("native unavailable"))).toBe(true); + }); + test("only warns when warmup is best effort", async () => { const { warn, logger } = makeLogger(); From 98ce067634fea0099526395eee484abfd5e1487c Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Wed, 19 Aug 2026 01:17:01 +0200 Subject: [PATCH 06/20] fix(stack): close preparation lifecycle gaps --- packages/stack/src/BinaryResolver.ts | 49 +-- packages/stack/src/DaemonServer.ts | 3 + packages/stack/src/LocalStack.ts | 387 ++++++++++-------- .../stack/src/RemoteStack.integration.test.ts | 9 +- packages/stack/src/RemoteStack.ts | 2 +- packages/stack/src/ServiceCatalog.ts | 16 +- packages/stack/src/Stack.ts | 2 +- packages/stack/src/Stack.unit.test.ts | 132 +++++- packages/stack/src/StackBuilder.ts | 9 +- .../StackConfigResolver.policy.unit.test.ts | 7 + packages/stack/src/StackConfigResolver.ts | 19 +- packages/stack/src/StackPreparation.ts | 122 +++--- packages/stack/src/prefetch.unit.test.ts | 41 +- packages/stack/src/services/service-utils.ts | 2 + packages/stack/src/services/vector.ts | 70 ++-- packages/stack/src/versions.ts | 19 +- packages/stack/src/versions.unit.test.ts | 7 + 17 files changed, 558 insertions(+), 338 deletions(-) diff --git a/packages/stack/src/BinaryResolver.ts b/packages/stack/src/BinaryResolver.ts index 521e240dc8..0e85e96a17 100644 --- a/packages/stack/src/BinaryResolver.ts +++ b/packages/stack/src/BinaryResolver.ts @@ -214,15 +214,8 @@ const validateManifest = ( if (osFloor !== null && typeof osFloor !== "object") { return yield* Effect.fail(manifestError(release.manifestUrl, "Manifest os_floor is invalid")); } - const requiresGlibc = manifest.libc === "glibc" || runtimeRequires === "glibc"; - if (requiresGlibc && platform.os !== "linux") { - return yield* Effect.fail( - new BinaryHostCompatibilityError({ - target: release.target, - detail: "Manifest requires Linux/glibc", - }), - ); - } + const requiresGlibc = + manifest.libc === "glibc" || runtimeRequires === "glibc" || osFloor?.kind === "glibc"; if (osFloor !== null && osFloor.kind === "macos" && platform.os !== "darwin") { return yield* Effect.fail( new BinaryHostCompatibilityError({ @@ -231,7 +224,7 @@ const validateManifest = ( }), ); } - if (osFloor !== null && osFloor.kind === "glibc" && platform.os !== "linux") { + if (requiresGlibc && platform.os !== "linux") { return yield* Effect.fail( new BinaryHostCompatibilityError({ target: release.target, @@ -248,12 +241,7 @@ const validateManifest = ( ); } const floor = osFloor?.floor; - if ( - platform.os === "linux" && - osFloor?.kind === "glibc" && - floor !== null && - floor !== undefined - ) { + if (requiresGlibc) { const host = yield* Effect.sync(() => { try { const report = process.report?.getReport?.(); @@ -282,7 +270,7 @@ const validateManifest = ( }), ); } - if (compareVersions(host, floor) < 0) { + if (floor !== null && floor !== undefined && compareVersions(host, floor) < 0) { return yield* Effect.fail( new BinaryHostCompatibilityError({ target: release.target, @@ -442,7 +430,7 @@ export class BinaryResolver extends Context.Service< return present.every(Boolean); }).pipe(Effect.catch(() => Effect.succeed(false))); - const plan = (spec: BinarySpec): Effect.Effect => + const resolveRelease = (spec: BinarySpec) => Effect.gen(function* () { const platform = yield* detectPlatform; const release = nativeReleaseForService(spec.service, spec.version, platform); @@ -461,6 +449,12 @@ export class BinaryResolver extends Context.Service< runtime: "native", target: release.target, }; + return { platform, release, info }; + }); + + const plan = (spec: BinarySpec): Effect.Effect => + Effect.gen(function* () { + const { info } = yield* resolveRelease(spec); return cachePath(spec.cacheDir ?? binDir, info); }); @@ -681,24 +675,7 @@ export class BinaryResolver extends Context.Service< const resolveWithMetadata = (spec: BinarySpec, options?: ResolveBinaryOptions) => { const core = Effect.gen(function* () { - const platform = yield* detectPlatform; - const release = nativeReleaseForService(spec.service, spec.version, platform); - if (release === undefined) { - return yield* Effect.fail( - new BinaryNotFoundError({ - service: spec.service, - platform: `${platform.os}-${platform.arch}`, - }), - ); - } - - const info: AssetInfo = { - service: spec.service, - releaseSet: "slim-services", - version: spec.version, - runtime: "native", - target: release.target, - }; + const { platform, release, info } = yield* resolveRelease(spec); const baseDir = spec.cacheDir ?? binDir; const cacheDir = cachePath(baseDir, info); const parentDir = path.dirname(cacheDir); diff --git a/packages/stack/src/DaemonServer.ts b/packages/stack/src/DaemonServer.ts index 78b7eaba97..1763f9dae2 100644 --- a/packages/stack/src/DaemonServer.ts +++ b/packages/stack/src/DaemonServer.ts @@ -393,6 +393,9 @@ export class DaemonServer extends Context.Service< Effect.catchTag("StackBuildError", (e) => Effect.succeed(buildErrorResponse(e.detail, e.reason)), ), + Effect.catchTag("StackNotRunningError", (e) => + Effect.succeed(notRunningResponse(e.phase)), + ), ), ), diff --git a/packages/stack/src/LocalStack.ts b/packages/stack/src/LocalStack.ts index 2f067591b6..2d7f3e105c 100644 --- a/packages/stack/src/LocalStack.ts +++ b/packages/stack/src/LocalStack.ts @@ -45,7 +45,9 @@ import { import { portFieldsForService } from "./ServicePorts.ts"; import { PreparationCompleted, + preparationClosure, ServiceDownloadStarted, + ServiceDownloadFinished, StackPreparation, } from "./StackPreparation.ts"; import type { PreparedStackArtifacts, StackPreparationInput } from "./StackPreparation.ts"; @@ -63,15 +65,7 @@ import { Stack } from "./Stack.ts"; import type { EdgeRuntimeReloadConfig, StackInfo } from "./Stack.ts"; import { SERVICE_NAMES, type ServiceName } from "./versions.ts"; -type LifecyclePhase = - | "idle" - | "preparing" - | "prepared" - | "starting" - | "running" - | "stopping" - | "stopped" - | "disposed"; +type LifecyclePhase = "idle" | "starting" | "running" | "stopping" | "stopped" | "disposed"; type StackService = typeof Stack.Service; @@ -282,178 +276,212 @@ export const localStackLayer = ( }); }; - const ensurePlanned = Effect.suspend(() => { - if (plannedArtifacts !== undefined) return Effect.succeed(plannedArtifacts); - if (planDeferred !== undefined) return Deferred.await(planDeferred); - const deferred = Deferred.makeUnsafe(); - planDeferred = deferred; - const effect = Effect.gen(function* () { - yield* validateResolvedConfig(config); - const input = yield* preparationInput(enabledServicesForConfig(config)); - return yield* preparation.plan(input).pipe( - Effect.mapError( - (cause) => - new StackBuildError({ - detail: "Failed to plan stack assets", - cause, - reason: "asset_preparation", - }), - ), - ); - }).pipe( - Effect.tap((value) => - Effect.sync(() => { - plannedArtifacts = value; - }), - ), - Effect.ensuring(Effect.sync(() => (planDeferred = undefined))), - ); - return Effect.gen(function* () { - yield* Effect.forkIn(effect.pipe(Deferred.into(deferred)), preparationScope); - return yield* Deferred.await(deferred); - }); - }); - - const prepareServices = (services: ReadonlyArray) => + const ensurePlanned = Effect.uninterruptibleMask((restore) => Effect.suspend(() => { - const targets = [ - ...new Set( - services.flatMap((service) => activationTargetsForService(enabledServices, service)), - ), - ]; - const pending = targets.filter((service) => preparedResolutions[service] === undefined); - if (pending.length === 0) { - return Effect.succeed({ - resolutions: preparedResolutions, - } satisfies PreparedStackArtifacts); + if (disposed || disposing) { + return Effect.fail( + new StackBuildError({ + detail: "Cannot plan stack assets after stack disposal has begun", + }), + ); } - const key = pending.toSorted().join(","); - const existing = preparationInFlight.get(key); - if (existing !== undefined) return Deferred.await(existing); - const previousStates = new Map( - pending.flatMap((service) => { - const state = SubscriptionRef.getUnsafe(stateRef).find( - (entry) => entry.name === service, - ); - return state === undefined ? [] : [[service, state] as const]; - }), - ); + if (plannedArtifacts !== undefined) return Effect.succeed(plannedArtifacts); + if (planDeferred !== undefined) return restore(Deferred.await(planDeferred)); const deferred = Deferred.makeUnsafe(); - preparationInFlight.set(key, deferred); - const effect = preparationInput(pending).pipe( - Effect.flatMap((input) => - Stream.runFoldEffect( - preparation.prepareEvents(input), - () => ({ resolutions: {} }) satisfies PreparedStackArtifacts, - (current, event) => - Effect.gen(function* () { - if (event instanceof ServiceDownloadStarted) { - const state = SubscriptionRef.getUnsafe(stateRef).find( - (entry) => entry.name === event.service, - ); - if (state !== undefined && state.status !== "Downloading") { - yield* updateState( - new StackServiceState({ ...state, status: "Downloading" }), - ); - } - } - return event instanceof PreparationCompleted ? event.artifacts : current; + planDeferred = deferred; + const effect = Effect.gen(function* () { + yield* validateResolvedConfig(config); + const input = yield* preparationInput(enabledServicesForConfig(config)); + return yield* preparation.plan(input).pipe( + Effect.mapError( + (cause) => + new StackBuildError({ + detail: "Failed to plan stack assets", + cause, + reason: "asset_preparation", }), ), - ), - Effect.mapError( - (cause) => - new StackBuildError({ - detail: "Failed to prepare stack assets", - cause, - reason: - cause instanceof DockerPullError && cause.daemonDown - ? "docker_not_running" - : "asset_preparation", - }), - ), - Effect.tapError(() => - Effect.forEach( - pending, - (service) => { - const previous = previousStates.get(service); - const current = SubscriptionRef.getUnsafe(stateRef).find( - (entry) => entry.name === service, - ); - return previous !== undefined && current?.status === "Downloading" - ? updateState(previous) - : Effect.void; - }, - { discard: true, concurrency: "unbounded" }, - ), - ), + ); + }).pipe( Effect.tap((value) => - Effect.sync(() => Object.assign(preparedResolutions, value.resolutions)), + Effect.sync(() => { + plannedArtifacts = value; + }), ), - Effect.ensuring(Effect.sync(() => preparationInFlight.delete(key))), + Effect.ensuring(Effect.sync(() => (planDeferred = undefined))), ); return Effect.gen(function* () { yield* Effect.forkIn(effect.pipe(Deferred.into(deferred)), preparationScope); - return yield* Deferred.await(deferred); + return yield* restore(Deferred.await(deferred)); }); - }); + }), + ); - const ensureRuntime = Effect.suspend(() => { - if (runtimeState !== undefined) { - return Effect.succeed(runtimeState); - } - if (runtimeDeferred !== undefined) { - return Deferred.await(runtimeDeferred); - } + const prepareServices = (services: ReadonlyArray) => + Effect.uninterruptibleMask((restore) => + Effect.suspend(() => { + if (disposed || disposing) { + return Effect.fail( + new StackBuildError({ + detail: "Cannot prepare stack assets after disposal has begun", + }), + ); + } + const targets = [ + ...new Set( + services.flatMap((service) => + activationTargetsForService(enabledServices, service), + ), + ), + ]; + const preparationTargets = preparationClosure(targets, enabledServices); + const pending = preparationTargets.filter( + (service) => preparedResolutions[service] === undefined, + ); + if (pending.length === 0) { + return Effect.succeed({ + resolutions: preparedResolutions, + } satisfies PreparedStackArtifacts); + } + const key = pending.toSorted().join(","); + const existing = preparationInFlight.get(key); + if (existing !== undefined) return restore(Deferred.await(existing)); + const previousStates = new Map( + preparationTargets.flatMap((service) => { + const state = SubscriptionRef.getUnsafe(stateRef).find( + (entry) => entry.name === service, + ); + return state === undefined ? [] : [[service, state] as const]; + }), + ); + const deferred = Deferred.makeUnsafe(); + preparationInFlight.set(key, deferred); + const effect = preparationInput(pending).pipe( + Effect.flatMap((input) => + Stream.runFoldEffect( + preparation.prepareEvents(input), + () => ({ resolutions: {} }) satisfies PreparedStackArtifacts, + (current, event) => + Effect.gen(function* () { + if (event instanceof ServiceDownloadStarted) { + const state = SubscriptionRef.getUnsafe(stateRef).find( + (entry) => entry.name === event.service, + ); + if (state !== undefined && state.status !== "Downloading") { + yield* updateState( + new StackServiceState({ ...state, status: "Downloading" }), + ); + } + } + if (event instanceof ServiceDownloadFinished) { + const previous = previousStates.get(event.service); + if (previous !== undefined) yield* updateState(previous); + } + return event instanceof PreparationCompleted ? event.artifacts : current; + }), + ), + ), + Effect.mapError( + (cause) => + new StackBuildError({ + detail: "Failed to prepare stack assets", + cause, + reason: + cause instanceof DockerPullError && cause.daemonDown + ? "docker_not_running" + : "asset_preparation", + }), + ), + Effect.tapError(() => + Effect.forEach( + preparationTargets, + (service) => { + const previous = previousStates.get(service); + const current = SubscriptionRef.getUnsafe(stateRef).find( + (entry) => entry.name === service, + ); + return previous !== undefined && current?.status === "Downloading" + ? updateState(previous) + : Effect.void; + }, + { discard: true, concurrency: "unbounded" }, + ), + ), + Effect.tap((value) => + Effect.sync(() => Object.assign(preparedResolutions, value.resolutions)), + ), + Effect.ensuring(Effect.sync(() => preparationInFlight.delete(key))), + ); + return Effect.gen(function* () { + yield* Effect.forkIn(effect.pipe(Deferred.into(deferred)), preparationScope); + return yield* restore(Deferred.await(deferred)); + }); + }), + ); - const deferred = Deferred.makeUnsafe(); - runtimeDeferred = deferred; + const ensureRuntime = Effect.uninterruptibleMask((restore) => + Effect.suspend(() => { + if (disposed || disposing) { + return Effect.fail( + new StackBuildError({ + detail: "Cannot ensure stack runtime after stack disposal has begun", + }), + ); + } + if (runtimeState !== undefined) { + return Effect.succeed(runtimeState); + } + if (runtimeDeferred !== undefined) return restore(Deferred.await(runtimeDeferred)); - const effect = Effect.gen(function* () { - const prepared = yield* ensurePlanned; - const { graph, serviceProjection, cleanupTargets } = yield* builder.build( - config, - prepared, - ); - exactCleanupTargets = cleanupTargets; + const deferred = Deferred.makeUnsafe(); + runtimeDeferred = deferred; - const orchLayer = Orchestrator.layer(graph).pipe( - Layer.provide(Layer.succeed(LogBuffer, logBuffer)), - Layer.provide(Layer.succeed(ChildProcessSpawner.ChildProcessSpawner, spawner)), - ); - const orchServices = yield* Layer.buildWithScope(orchLayer, scope); - const orchestrator = Context.get(orchServices, Orchestrator); + const effect = Effect.gen(function* () { + const prepared = yield* ensurePlanned; + const { graph, serviceProjection, cleanupTargets } = yield* builder.build( + config, + prepared, + ); + exactCleanupTargets = cleanupTargets; - yield* syncProjectedStates(orchestrator, serviceProjection); - yield* orchestrator.allStateChanges().pipe( - Stream.runForEach(() => syncProjectedStates(orchestrator, serviceProjection)), - Effect.ignore, - Effect.forkIn(scope), - ); + const orchLayer = Orchestrator.layer(graph).pipe( + Layer.provide(Layer.succeed(LogBuffer, logBuffer)), + Layer.provide(Layer.succeed(ChildProcessSpawner.ChildProcessSpawner, spawner)), + ); + const orchServices = yield* Layer.buildWithScope(orchLayer, scope); + const orchestrator = Context.get(orchServices, Orchestrator); - return { - orchestrator, - graph, - serviceProjection, - } satisfies RuntimeState; - }).pipe( - Effect.tap((value) => - Effect.sync(() => { - runtimeState = value; - }), - ), - Effect.ensuring( - Effect.sync(() => { - runtimeDeferred = undefined; - }), - ), - ); + yield* syncProjectedStates(orchestrator, serviceProjection); + yield* orchestrator.allStateChanges().pipe( + Stream.runForEach(() => syncProjectedStates(orchestrator, serviceProjection)), + Effect.ignore, + Effect.forkIn(scope), + ); - return Effect.gen(function* () { - yield* Effect.forkIn(effect.pipe(Deferred.into(deferred)), preparationScope); - return yield* Deferred.await(deferred); - }); - }); + return { + orchestrator, + graph, + serviceProjection, + } satisfies RuntimeState; + }).pipe( + Effect.tap((value) => + Effect.sync(() => { + runtimeState = value; + }), + ), + Effect.ensuring( + Effect.sync(() => { + runtimeDeferred = undefined; + }), + ), + ); + + return Effect.gen(function* () { + yield* Effect.forkIn(effect.pipe(Deferred.into(deferred)), preparationScope); + return yield* restore(Deferred.await(deferred)); + }); + }), + ); let disposed = false; let disposing = false; @@ -536,7 +564,11 @@ export const localStackLayer = ( const syncRuntimeProjectedStates = (runtime: RuntimeState) => syncProjectedStates(runtime.orchestrator, runtime.serviceProjection); const serviceStartOptions = { - beforeStart: (name: string) => portLease.reserve(portFieldsForService(name)), + // Reservation may yield while disposal flips the lifecycle state. + beforeStart: (name: string) => + portLease + .reserve(portFieldsForService(name)) + .pipe(Effect.andThen(requireMutable(`start service ${name}`))), beforeSpawn: (name: string) => portLease.release(portFieldsForService(name)), }; const knownServiceError = (service: string, cause: ServiceNotFoundError) => @@ -656,15 +688,10 @@ export const localStackLayer = ( const disposeOnce = () => Effect.suspend(() => { disposing = true; - return Effect.gen(function* () { - if (disposed) { - return; - } - disposed = true; - yield* Ref.set(phaseRef, "stopping"); - const preparationError = new StackBuildError({ - detail: "Stack disposed during asset preparation", - }); + const preparationError = new StackBuildError({ + detail: "Stack disposed during asset preparation", + }); + const failInFlight = Effect.gen(function* () { if (planDeferred !== undefined) { yield* Deferred.fail(planDeferred, preparationError); } @@ -674,6 +701,13 @@ export const localStackLayer = ( if (runtimeDeferred !== undefined) { yield* Deferred.fail(runtimeDeferred, preparationError); } + }); + const cleanup = Effect.gen(function* () { + if (disposed) { + return; + } + disposed = true; + yield* Ref.set(phaseRef, "stopping"); yield* Scope.close(preparationScope, Exit.void); yield* cleanupLocalStackResources({ stop: () => @@ -686,6 +720,7 @@ export const localStackLayer = ( Effect.ensuring(Ref.set(phaseRef, "disposed")), ); }).pipe(withLifecycleLock); + return failInFlight.pipe(Effect.andThen(cleanup)); }).pipe( Effect.ensuring(Deferred.succeed(disposedSignal, undefined).pipe(Effect.asVoid)), Effect.uninterruptible, @@ -729,6 +764,7 @@ export const localStackLayer = ( const activateService = (name: ServiceName) => Effect.gen(function* () { + yield* requireMutable(`activate service ${name}`); yield* requireRunningPhase; const service = yield* requireKnownServiceName(name); const existing = yield* inspectStartedTargets(service); @@ -780,6 +816,7 @@ export const localStackLayer = ( if (!allServicesEager) { const readiness: Array> = []; yield* prepareServices(["postgres", ...eager]); + yield* requireMutable("start"); if ( runtime.graph.startOrder.some((definition) => definition.name === "postgres-init") ) { @@ -801,6 +838,7 @@ export const localStackLayer = ( ); } for (const service of eager) { + yield* requireMutable("start"); const started = yield* beginStartTargets( service, new Set(lifecycleTargetsForService(enabledServices, service)), @@ -813,12 +851,14 @@ export const localStackLayer = ( yield* syncRuntimeProjectedStates(runtime); } else { yield* prepareServices(enabledServices); + yield* requireMutable("start"); yield* runtime.orchestrator.start(serviceStartOptions); yield* runtime.orchestrator .waitAllReady() .pipe((effect) => withReadinessPolicy(effect, "stack")); yield* syncRuntimeProjectedStates(runtime); } + yield* requireMutable("start"); yield* Ref.set(phaseRef, "running"); }).pipe( Effect.onError(() => Ref.set(phaseRef, "stopped")), @@ -859,6 +899,7 @@ export const localStackLayer = ( stopService: (name) => Effect.gen(function* () { yield* requireMutable(`stop service ${name}`); + yield* requireRunningPhase; const service = yield* requireKnownServiceName(name); const runtime = yield* ensureRuntime; for (const target of lifecycleTargetsForService( diff --git a/packages/stack/src/RemoteStack.integration.test.ts b/packages/stack/src/RemoteStack.integration.test.ts index 493f1286f1..f665ba2a69 100644 --- a/packages/stack/src/RemoteStack.integration.test.ts +++ b/packages/stack/src/RemoteStack.integration.test.ts @@ -132,9 +132,11 @@ function mockStack( stopService: (name: string) => name === "unknown" ? Effect.fail(new ServiceNotFoundError({ name })) - : Effect.sync(() => { - serviceCalls.push(`stop:${name}`); - }), + : options.notRunningPhase !== undefined + ? Effect.fail(new StackNotRunningError({ phase: options.notRunningPhase })) + : Effect.sync(() => { + serviceCalls.push(`stop:${name}`); + }), restartService: (name: string) => name === "unknown" ? Effect.fail(new ServiceNotFoundError({ name })) @@ -592,6 +594,7 @@ describe("RemoteStack integration", () => { const operations = [ (stack: Stack["Service"]) => stack.startService("auth"), + (stack: Stack["Service"]) => stack.stopService("auth"), (stack: Stack["Service"]) => stack.restartService("auth"), (stack: Stack["Service"]) => stack.reloadFunctions(), (stack: Stack["Service"]) => diff --git a/packages/stack/src/RemoteStack.ts b/packages/stack/src/RemoteStack.ts index 9a25f61a57..bb827cf52e 100644 --- a/packages/stack/src/RemoteStack.ts +++ b/packages/stack/src/RemoteStack.ts @@ -423,7 +423,7 @@ export const RemoteStack = { const response = yield* httpResponse(endpoint, path, { method: "POST", }); - yield* expectDaemonOk(endpoint, path, response, name).pipe( + yield* expectMutatingDaemonOk(endpoint, path, response, name).pipe( Effect.catchTag("ServiceReadyError", (error) => Effect.die(error)), Effect.catchTag("StackReadinessError", (error) => Effect.die(error)), ); diff --git a/packages/stack/src/ServiceCatalog.ts b/packages/stack/src/ServiceCatalog.ts index be4e29c25f..e9e58f20c3 100644 --- a/packages/stack/src/ServiceCatalog.ts +++ b/packages/stack/src/ServiceCatalog.ts @@ -330,21 +330,7 @@ export const isDockerOnlyService = (service: ServiceName): boolean => export const DEFAULT_SERVICE_POLICIES: Readonly< Record> -> = { - postgres: SERVICE_CATALOG.postgres.preparation.default, - postgrest: SERVICE_CATALOG.postgrest.preparation.default, - auth: SERVICE_CATALOG.auth.preparation.default, - "edge-runtime": SERVICE_CATALOG["edge-runtime"].preparation.default, - realtime: SERVICE_CATALOG.realtime.preparation.default, - storage: SERVICE_CATALOG.storage.preparation.default, - imgproxy: SERVICE_CATALOG.imgproxy.preparation.default, - mailpit: SERVICE_CATALOG.mailpit.preparation.default, - pgmeta: SERVICE_CATALOG.pgmeta.preparation.default, - studio: SERVICE_CATALOG.studio.preparation.default, - analytics: SERVICE_CATALOG.analytics.preparation.default, - vector: SERVICE_CATALOG.vector.preparation.default, - pooler: SERVICE_CATALOG.pooler.preparation.default, -}; +> = Record.map(SERVICE_CATALOG, (metadata) => metadata.preparation.default); export const requiredPreparationDependencies = (service: ServiceName): ReadonlyArray => serviceMetadata(service).preparation.dependencies; diff --git a/packages/stack/src/Stack.ts b/packages/stack/src/Stack.ts index 650ff69715..1a53cdb069 100644 --- a/packages/stack/src/Stack.ts +++ b/packages/stack/src/Stack.ts @@ -69,7 +69,7 @@ export class Stack extends Context.Service< >; readonly stopService: ( name: string, - ) => Effect.Effect; + ) => Effect.Effect; readonly restartService: ( name: string, ) => Effect.Effect< diff --git a/packages/stack/src/Stack.unit.test.ts b/packages/stack/src/Stack.unit.test.ts index 8674f99d51..2ea08faea4 100644 --- a/packages/stack/src/Stack.unit.test.ts +++ b/packages/stack/src/Stack.unit.test.ts @@ -686,6 +686,40 @@ describe("Stack", () => { }).pipe(Effect.provide(providedLayer)); }); + it.live("disposal fails a cold eager start with a typed build error", () => { + return Effect.gen(function* () { + const preparationStarted = yield* Deferred.make(); + const resolver = mockBinaryResolver({ + downloadedServices: ["auth"], + beforeResolve: ({ service }) => + service === "auth" + ? Deferred.succeed(preparationStarted, undefined).pipe(Effect.andThen(Effect.never)) + : Effect.void, + }); + const { layer } = setupLayer( + defaultConfig, + noopPortLease(defaultConfig.ports), + undefined, + resolver, + ); + + yield* Effect.gen(function* () { + const stack = yield* Stack; + const starting = yield* stack.start().pipe(Effect.forkChild({ startImmediately: true })); + yield* Deferred.await(preparationStarted); + + const disposing = yield* stack.dispose().pipe(Effect.forkChild({ startImmediately: true })); + yield* Fiber.join(disposing); + + const startExit = yield* Fiber.await(starting); + expect(Exit.isFailure(startExit)).toBe(true); + if (Exit.isFailure(startExit)) { + expect(Cause.squash(startExit.cause)).toMatchObject({ _tag: "StackBuildError" }); + } + }).pipe(Effect.provide(layer)); + }).pipe(Effect.scoped, Effect.timeout("5 seconds")); + }); + it.live("can retry start after a build failure before services start", () => { let buildAttempts = 0; const graph = Effect.runSync( @@ -726,6 +760,77 @@ describe("Stack", () => { }).pipe(Effect.provide(layer), Effect.timeout("5 seconds")); }); + it.live("rejects a cached start when disposal begins during startup", () => + Effect.gen(function* () { + const startEntered = yield* Deferred.make(); + const releaseStart = yield* Deferred.make(); + const config = { + ...defaultConfig, + postgrest: false, + auth: false, + servicePolicies: { + ...defaultConfig.servicePolicies, + postgrest: "off", + auth: "off", + }, + } satisfies ResolvedStackConfig; + const graph = Effect.runSync( + buildGraph([{ name: "postgres", command: "true", restart: "no" }]), + ); + const builderLayer = Layer.succeed(StackBuilder, { + build: () => + Effect.succeed({ + graph, + cleanupTargets: { dockerContainerNames: [] }, + serviceProjection: new Map([["postgres", { visibility: "public" as const }]]), + }), + }); + let gateNextStart = false; + const portLease: PortLease = { + ports: config.ports, + reserve: () => { + if (!gateNextStart) return Effect.void; + gateNextStart = false; + return Deferred.succeed(startEntered, undefined).pipe( + Effect.andThen(Deferred.await(releaseStart)), + ); + }, + release: () => Effect.void, + releaseAll: Effect.void, + }; + const resolver = mockBinaryResolver(); + const layer = localStackLayer(config, portLease).pipe( + Layer.provide(builderLayer), + Layer.provide(StackPreparation.layer.pipe(Layer.provide(resolver.layer))), + Layer.provide(mockChildProcessSpawner().layer), + Layer.provide(NodeServices.layer), + ); + + yield* Effect.gen(function* () { + const stack = yield* Stack; + yield* stack.start(); + yield* stack.stop(); + + gateNextStart = true; + const holder = yield* stack.start().pipe(Effect.forkChild({ startImmediately: true })); + yield* Deferred.await(startEntered); + const disposing = yield* stack.dispose().pipe(Effect.forkChild({ startImmediately: true })); + + yield* Deferred.succeed(releaseStart, undefined); + const holderExit = yield* Fiber.await(holder); + expect(Exit.isFailure(holderExit)).toBe(true); + if (Exit.isFailure(holderExit)) { + expect(Cause.squash(holderExit.cause)).toMatchObject({ _tag: "StackBuildError" }); + } + yield* Fiber.join(disposing); + + expect((yield* stack.getState("postgres")).status).toBe("Stopped"); + const afterDisposal = yield* stack.start().pipe(Effect.flip); + expect(afterDisposal._tag).toBe("StackBuildError"); + }).pipe(Effect.provide(layer)); + }).pipe(Effect.scoped, Effect.timeout("5 seconds")), + ); + it.live("a partial startup failure disposes resources from services already started", () => { let cleaned = false; const spawner = mockChildProcessSpawner({ @@ -943,6 +1048,28 @@ describe("Stack", () => { if (error._tag === "StackBuildError") { expect(error.detail).toContain("imgproxy was explicitly stopped"); } + expect((yield* stack.getState("storage")).status).not.toBe("Downloading"); + expect((yield* stack.getState("imgproxy")).status).not.toBe("Downloading"); + yield* stack.stop(); + }).pipe(Effect.provide(layer), Effect.timeout("5 seconds")); + }); + + it.live("rejects stopping a service before start without affecting a later start", () => { + const config = { + ...defaultConfig, + servicePolicies: { ...defaultConfig.servicePolicies, postgrest: "lazy", auth: "lazy" }, + } satisfies ResolvedStackConfig; + const { layer } = setupLayer(config); + + return Effect.gen(function* () { + const stack = yield* Stack; + const error = yield* stack.stopService("auth").pipe(Effect.flip); + + expect(error._tag).toBe("StackNotRunningError"); + if (error._tag === "StackNotRunningError") expect(error.phase).toBe("idle"); + + yield* stack.start(); + expect((yield* stack.getState("auth")).status).toBe("Dormant"); yield* stack.stop(); }).pipe(Effect.provide(layer), Effect.timeout("5 seconds")); }); @@ -1127,7 +1254,10 @@ describe("Stack", () => { expect(spawner.spawned.some((record) => record.command.endsWith("/auth"))).toBe(false); const error = yield* activator.activate("auth").pipe(Effect.flip); - expect(error._tag).toBe("StackNotRunningError"); + expect(error._tag).toBe("StackBuildError"); + if (error._tag === "StackBuildError") { + expect(error.detail).toContain("disposal has begun"); + } }).pipe(Effect.provide(layer)); }).pipe(Effect.scoped, Effect.timeout("5 seconds")), ); diff --git a/packages/stack/src/StackBuilder.ts b/packages/stack/src/StackBuilder.ts index 3006f0001f..4947c1be4b 100644 --- a/packages/stack/src/StackBuilder.ts +++ b/packages/stack/src/StackBuilder.ts @@ -166,11 +166,10 @@ export const validateResolvedConfig = ( }); export const enabledServicesForConfig = (config: ResolvedStackConfig): ReadonlyArray => - SERVICE_NAMES.filter((service) => - service === "postgres" && config.servicePolicies?.postgres !== "off" - ? true - : config.servicePolicies?.[service] !== "off" && - resolvedConfigForService(config, service) !== false, + SERVICE_NAMES.filter( + (service) => + config.servicePolicies?.[service] !== "off" && + resolvedConfigForService(config, service) !== false, ); export const versionsForConfig = (config: ResolvedStackConfig): Partial => { diff --git a/packages/stack/src/StackConfigResolver.policy.unit.test.ts b/packages/stack/src/StackConfigResolver.policy.unit.test.ts index 890689885d..72047e1734 100644 --- a/packages/stack/src/StackConfigResolver.policy.unit.test.ts +++ b/packages/stack/src/StackConfigResolver.policy.unit.test.ts @@ -33,6 +33,13 @@ describe("resolved service preparation policies", () => { expect(allocated).toBe(false); }); + it("rejects disabling postgres through the service policy manifest", async () => { + await expect(resolveConfig({ servicePolicies: { postgres: "off" } })).rejects.toMatchObject({ + _tag: "StackBuildError", + reason: "invalid_config", + }); + }); + it("resolves explicitly disabled core services to false without reserving ports", async () => { const config = await resolveConfig({ servicePolicies: { postgrest: "off" } }); expect(config.postgrest).toBe(false); diff --git a/packages/stack/src/StackConfigResolver.ts b/packages/stack/src/StackConfigResolver.ts index 2d06d12468..0e4a87a49a 100644 --- a/packages/stack/src/StackConfigResolver.ts +++ b/packages/stack/src/StackConfigResolver.ts @@ -1,6 +1,6 @@ import { mkdtempSync } from "node:fs"; import { join } from "node:path"; -import { Effect, Schema } from "effect"; +import { Effect, Record, Schema } from "effect"; import { StackBuildError, toStackError } from "./errors.ts"; import { resolvedFunctionsBundleSchemaForProject } from "./functions.ts"; import { @@ -53,6 +53,7 @@ import type { StackRuntimeSelection } from "./ContainerRuntime.ts"; import { DEFAULT_SERVICE_POLICIES, DEFAULT_VERSIONS, + SERVICE_CATALOG, SERVICE_NAMES, serviceMetadata, } from "./ServiceCatalog.ts"; @@ -394,21 +395,7 @@ const rawServiceEnabled = (config: StackConfig, service: ServiceName): boolean = * are acquired. This keeps unsupported policies a pure user/configuration error. */ const resolveServicePolicies = (config: StackConfig): ServicePolicyManifest => { - const policies: Record = { - postgres: "off", - postgrest: "off", - auth: "off", - "edge-runtime": "off", - realtime: "off", - storage: "off", - imgproxy: "off", - mailpit: "off", - pgmeta: "off", - studio: "off", - analytics: "off", - vector: "off", - pooler: "off", - }; + const policies: Record = Record.map(SERVICE_CATALOG, () => "off"); for (const service of SERVICE_NAMES) { const requested = config.servicePolicies?.[service]; if (service === "postgres" && requested !== undefined && requested !== "eager") { diff --git a/packages/stack/src/StackPreparation.ts b/packages/stack/src/StackPreparation.ts index 4e567611df..9fcf76fd01 100644 --- a/packages/stack/src/StackPreparation.ts +++ b/packages/stack/src/StackPreparation.ts @@ -3,6 +3,7 @@ import { Context, Data, Deferred, + Duration, Effect, Exit, Layer, @@ -111,7 +112,7 @@ const resolveDockerImageForService = ( ): Effect.Effect => pullImage(spawner, runtime, dockerImageForService(service, version), callbacks); -const preparationClosure = ( +export const preparationClosure = ( services: ReadonlyArray, enabledServices?: ReadonlyArray, ): ReadonlyArray => { @@ -210,61 +211,68 @@ export class StackPreparation extends Context.Service< input: StackPreparationInput, publishEvent?: (event: StackPreparationEvent) => Effect.Effect, ): Effect.Effect => - Effect.suspend(() => { - let downloadStarted = false; - const markDownloadStart = () => - Effect.sync(() => { - downloadStarted = true; - }).pipe( - Effect.andThen( - publishEvent?.(new ServiceDownloadStarted({ service })) ?? Effect.void, - ), - ); - const markDownloadFinished = () => - Effect.suspend(() => - downloadStarted - ? (publishEvent?.(new ServiceDownloadFinished({ service })) ?? Effect.void) - : Effect.void, + Effect.uninterruptibleMask((restore) => + Effect.suspend(() => { + let downloadStarted = false; + const markDownloadStart = () => + Effect.sync(() => { + downloadStarted = true; + }).pipe( + Effect.andThen( + publishEvent?.(new ServiceDownloadStarted({ service })) ?? Effect.void, + ), + ); + const markDownloadFinished = () => + Effect.suspend(() => + downloadStarted + ? (publishEvent?.(new ServiceDownloadFinished({ service })) ?? Effect.void) + : Effect.void, + ); + const key = JSON.stringify({ + service, + resolution, + containerRuntime: input.mode === "docker" ? input.containerRuntime : null, + }); + const existing = inFlight.get(key); + if (existing !== undefined) return restore(Deferred.await(existing)); + const deferred = Deferred.makeUnsafe(); + inFlight.set(key, deferred); + const version = input.versions?.[service] ?? DEFAULT_VERSIONS[service]; + const effect: Effect.Effect = + resolution.type === "docker" + ? input.mode === "docker" + ? resolveDockerImageForService( + spawner, + input.containerRuntime, + service, + version, + { + onDownloadStart: markDownloadStart(), + }, + ).pipe(Effect.map((image): ServiceResolution => ({ type: "docker", image }))) + : Effect.die("Native preparation planned a Docker resolution") + : resolver + .resolveWithMetadata( + { service, version }, + { + onDownloadStart: markDownloadStart(), + }, + ) + .pipe(Effect.map(({ path }): ServiceResolution => ({ type: "binary", path }))); + const coordinated = effect.pipe( + Effect.matchCauseEffect({ + onSuccess: (value) => + Effect.andThen(markDownloadFinished(), Deferred.succeed(deferred, value)), + onFailure: (cause) => Deferred.failCause(deferred, cause), + }), + Effect.ensuring(Effect.sync(() => inFlight.delete(key))), ); - const key = JSON.stringify({ - service, - resolution, - containerRuntime: input.mode === "docker" ? input.containerRuntime : null, - }); - const existing = inFlight.get(key); - if (existing !== undefined) return Deferred.await(existing); - const deferred = Deferred.makeUnsafe(); - inFlight.set(key, deferred); - const version = input.versions?.[service] ?? DEFAULT_VERSIONS[service]; - const effect: Effect.Effect = - resolution.type === "docker" - ? input.mode === "docker" - ? resolveDockerImageForService(spawner, input.containerRuntime, service, version, { - onDownloadStart: markDownloadStart(), - }).pipe(Effect.map((image): ServiceResolution => ({ type: "docker", image }))) - : Effect.die("Native preparation planned a Docker resolution") - : resolver - .resolveWithMetadata( - { service, version }, - { - onDownloadStart: markDownloadStart(), - }, - ) - .pipe(Effect.map(({ path }): ServiceResolution => ({ type: "binary", path }))); - const coordinated = effect.pipe( - Effect.matchCauseEffect({ - onSuccess: (value) => - Effect.andThen(markDownloadFinished(), Deferred.succeed(deferred, value)), - onFailure: (cause) => - Deferred.failCause(deferred, cause).pipe(Effect.andThen(Effect.failCause(cause))), - }), - Effect.ensuring(Effect.sync(() => inFlight.delete(key))), - ); - return Effect.gen(function* () { - yield* Effect.forkIn(coordinated, scope, { startImmediately: true }); - return yield* Deferred.await(deferred); - }); - }); + return Effect.gen(function* () { + yield* Effect.forkIn(coordinated, scope, { startImmediately: true }); + return yield* restore(Deferred.await(deferred)); + }); + }), + ); const prepareWithEvents = ( input: StackPreparationInput, @@ -325,7 +333,9 @@ const pullImage = ( const attempt = runPullCommand(spawner, runtime, image).pipe( Effect.retry({ while: (error) => shouldRetryPull(error.detail), - schedule: Schedule.recurs(1), + schedule: Schedule.recurs(1).pipe( + Schedule.addDelay(() => Effect.succeed(Duration.millis(500))), + ), }), ); const result = yield* Effect.exit(attempt); diff --git a/packages/stack/src/prefetch.unit.test.ts b/packages/stack/src/prefetch.unit.test.ts index c21582eeae..881881c38e 100644 --- a/packages/stack/src/prefetch.unit.test.ts +++ b/packages/stack/src/prefetch.unit.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from "vitest"; -import { Deferred, Effect, Layer, Sink, Stream } from "effect"; +import { Deferred, Effect, Fiber, Layer, Sink, Stream } from "effect"; import { ChildProcessSpawner } from "effect/unstable/process"; import { mockBinaryResolver } from "../tests/helpers/mocks.ts"; import { BinaryNotFoundError, DockerPullError } from "./errors.ts"; @@ -324,4 +324,43 @@ describe("prefetch", () => { }); expect(resolver.resolved).toEqual([]); }); + + test("concurrent prefetches share one materialization and return the same result", async () => { + const [result, resolved] = await Effect.runPromise( + Effect.gen(function* () { + const preparationStarted = yield* Deferred.make(); + const releasePreparation = yield* Deferred.make(); + const resolver = mockBinaryResolver({ + downloadedServices: ["auth"], + beforeResolve: ({ service }) => + service === "auth" + ? Deferred.succeed(preparationStarted, undefined).pipe( + Effect.andThen(Deferred.await(releasePreparation)), + ) + : Effect.void, + }); + const layer = StackPreparation.layer.pipe( + Layer.provide(resolver.layer), + Layer.provide(mockSequenceSpawner([]).layer), + ); + return yield* Effect.gen(function* () { + const first = yield* prefetch({ mode: "native", services: ["auth"] }).pipe( + Effect.forkChild({ startImmediately: true }), + ); + yield* Deferred.await(preparationStarted); + const second = yield* prefetch({ mode: "native", services: ["auth"] }).pipe( + Effect.forkChild({ startImmediately: true }), + ); + yield* Deferred.succeed(releasePreparation, undefined); + return [ + yield* Effect.all([Fiber.join(first), Fiber.join(second)]), + resolver.resolved, + ] as const; + }).pipe(Effect.provide(layer)); + }), + ); + + expect(result[0]).toEqual(result[1]); + expect(resolved.filter(({ service }) => service === "auth")).toHaveLength(1); + }); }); diff --git a/packages/stack/src/services/service-utils.ts b/packages/stack/src/services/service-utils.ts index 51a36a4858..488f6d9999 100644 --- a/packages/stack/src/services/service-utils.ts +++ b/packages/stack/src/services/service-utils.ts @@ -23,6 +23,7 @@ interface DockerRunServiceOptions extends ContainerRuntimeOptions { readonly cmd?: ReadonlyArray; readonly entrypoint?: string; readonly volumes?: ReadonlyArray; + readonly securityOptions?: ReadonlyArray; readonly dependencies: ReadonlyArray; readonly healthCheck?: ServiceDef["healthCheck"]; readonly restart?: ServiceDef["restart"]; @@ -77,6 +78,7 @@ export const dockerRunService = (opts: DockerRunServiceOptions): ServiceDef => { : ["--label", `${STACK_ID_LABEL}=${opts.identity.stackId}`]), ...(opts.networkArgs ?? []), ...(opts.volumes ?? []).flatMap((volume) => ["-v", volume]), + ...(opts.securityOptions ?? []).flatMap((option) => ["--security-opt", option]), ...(opts.entrypoint === undefined ? [] : ["--entrypoint", opts.entrypoint]), ...(opts.args ?? []), ...envArgs(opts.env ?? {}), diff --git a/packages/stack/src/services/vector.ts b/packages/stack/src/services/vector.ts index f81d66fa0e..066fe344ac 100644 --- a/packages/stack/src/services/vector.ts +++ b/packages/stack/src/services/vector.ts @@ -1,5 +1,6 @@ import { existsSync } from "node:fs"; import { dockerNetworkArgs } from "../Platform.ts"; +import type { ContainerRuntime } from "../ContainerRuntime.ts"; import { dockerContainerName, type StackIdentity } from "../StackIdentity.ts"; import { dockerExecHealthCheck, @@ -19,18 +20,13 @@ interface DockerVectorOptions extends ContainerRuntimeOptions { readonly dependencies: ReadonlyArray; } -const vectorConfig = ( - host: string, - port: number, - apiKey: string, - sourceType: "docker_logs" | "internal_logs", -) => `api: +const vectorConfig = (host: string, port: number, apiKey: string) => `api: enabled: true address: 0.0.0.0:9001 sources: runtime: - type: ${sourceType} + type: docker_logs sinks: logflare: @@ -47,18 +43,43 @@ sinks: uri: "http://${host}:${port}/api/logs?source_name=docker.logs.local" `; +const unixSocketFromEnv = (value: string | undefined): string | undefined => { + if (value === undefined || !value.startsWith("unix://")) return undefined; + const socket = value.slice("unix://".length); + return socket.length > 0 && existsSync(socket) ? socket : undefined; +}; + +const podmanSocketCandidates = (): ReadonlyArray => { + const candidates: Array = []; + const runtimeDir = process.env.XDG_RUNTIME_DIR; + if (runtimeDir !== undefined && runtimeDir.length > 0) { + candidates.push(`${runtimeDir}/podman/podman.sock`); + } + const uid = process.getuid?.(); + if (uid !== undefined) candidates.push(`/run/user/${uid}/podman/podman.sock`); + candidates.push("/run/podman/podman.sock"); + candidates.push("/var/run/docker.sock"); + return candidates; +}; + +const resolveVectorDockerSocket = (runtime: ContainerRuntime): string | undefined => { + if (runtime === "podman") { + const explicitPodmanSocket = unixSocketFromEnv(process.env.CONTAINER_HOST); + if (explicitPodmanSocket !== undefined) return explicitPodmanSocket; + const explicitDockerSocket = unixSocketFromEnv(process.env.DOCKER_HOST); + if (explicitDockerSocket !== undefined) return explicitDockerSocket; + return podmanSocketCandidates().find((socket) => existsSync(socket)); + } + + const explicitDockerSocket = unixSocketFromEnv(process.env.DOCKER_HOST); + if (explicitDockerSocket !== undefined) return explicitDockerSocket; + return existsSync("/var/run/docker.sock") ? "/var/run/docker.sock" : undefined; +}; + export const makeVectorServiceDocker = (opts: DockerVectorOptions) => { const containerName = dockerContainerName("vector", opts.identity.key); - const dockerSocket = - opts.runtime === "docker" - ? process.env.DOCKER_HOST?.startsWith("unix://") - ? process.env.DOCKER_HOST.slice("unix://".length) - : "/var/run/docker.sock" - : undefined; - const volumes = - dockerSocket !== undefined && existsSync(dockerSocket) - ? [`${dockerSocket}:/var/run/docker.sock:ro`] - : []; + const dockerSocket = resolveVectorDockerSocket(opts.runtime); + const volumes = dockerSocket === undefined ? [] : [`${dockerSocket}:/var/run/docker.sock:ro`]; return dockerRunService({ runtime: opts.runtime, @@ -67,17 +88,18 @@ export const makeVectorServiceDocker = (opts: DockerVectorOptions) => { image: opts.image, networkArgs: dockerNetworkArgs(opts.platformOs, []), volumes, - env: opts.runtime === "docker" ? { DOCKER_HOST: "unix:///var/run/docker.sock" } : {}, + securityOptions: + opts.runtime === "podman" && + dockerSocket !== undefined && + dockerSocket !== "/var/run/docker.sock" + ? ["label=disable"] + : [], + env: dockerSocket === undefined ? {} : { DOCKER_HOST: "unix:///var/run/docker.sock" }, entrypoint: "sh", cmd: [ "-c", `cat <<'EOF' > /etc/vector/vector.yaml && vector --config /etc/vector/vector.yaml -${vectorConfig( - opts.serviceHost, - opts.analyticsPort, - opts.analyticsApiKey, - opts.runtime === "docker" ? "docker_logs" : "internal_logs", -)}EOF +${vectorConfig(opts.serviceHost, opts.analyticsPort, opts.analyticsApiKey)}EOF `, ], dependencies: opts.dependencies, diff --git a/packages/stack/src/versions.ts b/packages/stack/src/versions.ts index ac33e86b87..2524890891 100644 --- a/packages/stack/src/versions.ts +++ b/packages/stack/src/versions.ts @@ -1,4 +1,9 @@ -import { DEFAULT_VERSIONS, SERVICE_NAMES, dockerImageForArtifact } from "./ServiceCatalog.ts"; +import { + DEFAULT_VERSIONS, + SERVICE_NAMES, + dockerImageForArtifact, + serviceMetadata, +} from "./ServiceCatalog.ts"; import type { ServiceName } from "./ServiceName.ts"; import { Schema } from "effect"; @@ -48,11 +53,13 @@ export function fullVersionManifest( return versions; } -/** - * Normalizes a version string without rewriting the frozen release tag. - */ -export function normalizeServiceVersion(_service: ServiceName, version: string): string { - return version.trim(); +/** Normalizes a version string to the catalog's canonical stored form. */ +export function normalizeServiceVersion(service: ServiceName, version: string): string { + const normalized = version.trim(); + const tagPrefix = serviceMetadata(service).artifact.docker.tagPrefix; + return tagPrefix !== undefined && normalized.startsWith(tagPrefix) + ? normalized.slice(tagPrefix.length) + : normalized; } export function normalizeServiceVersions( diff --git a/packages/stack/src/versions.unit.test.ts b/packages/stack/src/versions.unit.test.ts index 6a80fe7a15..70edb638a6 100644 --- a/packages/stack/src/versions.unit.test.ts +++ b/packages/stack/src/versions.unit.test.ts @@ -140,6 +140,13 @@ describe("normalizeServiceVersion", () => { it("passes through other services unchanged", () => { expect(normalizeServiceVersion("postgres", "17.6.1.090")).toBe("17.6.1.090"); }); + + it("normalizes a prefixed pgmeta override to its catalog tag", () => { + expect(normalizeServiceVersion("pgmeta", "v0.98.0")).toBe("0.98.0"); + expect(dockerImageForService("pgmeta", normalizeServiceVersion("pgmeta", "v0.98.0"))).toBe( + "ghcr.io/supabase/cli/pgmeta:v0.98.0", + ); + }); }); describe("fillServiceVersionManifest", () => { From a72437fcb0aca019f6559220f27fb45706facb59 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Wed, 19 Aug 2026 08:55:32 +0200 Subject: [PATCH 07/20] fix(stack): preserve runtime and projection state --- packages/stack/src/LocalStack.ts | 26 +++-- packages/stack/src/Stack.unit.test.ts | 97 +++++++++++++------ packages/stack/src/services/vector.ts | 27 ++++-- .../stack/src/services/vector.unit.test.ts | 77 +++++++++++++++ 4 files changed, 179 insertions(+), 48 deletions(-) create mode 100644 packages/stack/src/services/vector.unit.test.ts diff --git a/packages/stack/src/LocalStack.ts b/packages/stack/src/LocalStack.ts index 2d7f3e105c..317500afed 100644 --- a/packages/stack/src/LocalStack.ts +++ b/packages/stack/src/LocalStack.ts @@ -210,6 +210,18 @@ export const localStackLayer = ( : [...current, nextState]; }); + const restoreStateIfDownloading = ( + service: ServiceName, + previous: StackServiceState | undefined, + ) => + previous === undefined + ? Effect.void + : SubscriptionRef.update(stateRef, (current) => { + const index = current.findIndex((entry) => entry.name === service); + if (index === -1 || current[index]?.status !== "Downloading") return current; + return current.map((entry, entryIndex) => (entryIndex === index ? previous : entry)); + }); + const syncProjectedStates = ( orchestrator: Orchestrator["Service"], serviceProjection: StackServiceProjectionCatalog, @@ -374,8 +386,10 @@ export const localStackLayer = ( } } if (event instanceof ServiceDownloadFinished) { - const previous = previousStates.get(event.service); - if (previous !== undefined) yield* updateState(previous); + yield* restoreStateIfDownloading( + event.service, + previousStates.get(event.service), + ); } return event instanceof PreparationCompleted ? event.artifacts : current; }), @@ -396,13 +410,7 @@ export const localStackLayer = ( Effect.forEach( preparationTargets, (service) => { - const previous = previousStates.get(service); - const current = SubscriptionRef.getUnsafe(stateRef).find( - (entry) => entry.name === service, - ); - return previous !== undefined && current?.status === "Downloading" - ? updateState(previous) - : Effect.void; + return restoreStateIfDownloading(service, previousStates.get(service)); }, { discard: true, concurrency: "unbounded" }, ), diff --git a/packages/stack/src/Stack.unit.test.ts b/packages/stack/src/Stack.unit.test.ts index 2ea08faea4..37f5fa76bc 100644 --- a/packages/stack/src/Stack.unit.test.ts +++ b/packages/stack/src/Stack.unit.test.ts @@ -1010,46 +1010,85 @@ describe("Stack", () => { }).pipe(Effect.scoped, Effect.timeout("5 seconds")), ); - it.live("lazy activation honors explicitly stopped transitive dependencies", () => { - const config: ResolvedStackConfig = { + it.live("lazy activation restores dormant state after a stopped transitive dependency", () => { + const config = { ...defaultConfig, - mode: "docker", - containerRuntime: "docker", - servicePolicies: { - ...defaultConfig.servicePolicies, - postgrest: "lazy", - auth: "lazy", - storage: "lazy", - imgproxy: "lazy", - }, - storage: { - port: defaultPorts.storagePort, - dataDir: "/tmp/supabase/storage", - fileSizeLimit: "50MiB", - s3ProtocolEnabled: true, - version: DEFAULT_VERSIONS.storage, - }, - imgproxy: { - port: defaultPorts.imgproxyPort, - version: DEFAULT_VERSIONS.imgproxy, - }, - }; - const { layer } = setupLayer(config); + servicePolicies: { ...defaultConfig.servicePolicies, postgrest: "lazy", auth: "lazy" }, + } satisfies ResolvedStackConfig; + const resolver = mockBinaryResolver({ downloadedServices: ["postgrest"] }); + const { layer } = setupLayer(config, noopPortLease(config.ports), undefined, resolver); return Effect.gen(function* () { const stack = yield* Stack; const activator = yield* StackServiceActivator; yield* stack.start(); - yield* stack.stopService("imgproxy"); + yield* stack.stopService("postgres"); - const error = yield* activator.activate("storage").pipe(Effect.flip); + const downloading = yield* (yield* stack.stateChanges("postgrest")).pipe( + Stream.filter((state) => state.status === "Downloading"), + Stream.runHead, + Effect.forkChild({ startImmediately: true }), + ); + const error = yield* activator.activate("postgrest").pipe(Effect.flip); + expect((yield* Fiber.join(downloading))._tag).toBe("Some"); expect(error._tag).toBe("StackBuildError"); if (error._tag === "StackBuildError") { - expect(error.detail).toContain("imgproxy was explicitly stopped"); + expect(error.detail).toContain("postgres was explicitly stopped"); + } + expect((yield* stack.getState("postgrest")).status).toBe("Dormant"); + yield* stack.stop(); + }).pipe(Effect.provide(layer), Effect.timeout("5 seconds")); + }); + + it.live("preserves an explicit stop during in-flight lazy activation", () => { + const config = { + ...defaultConfig, + servicePolicies: { ...defaultConfig.servicePolicies, postgrest: "lazy", auth: "lazy" }, + } satisfies ResolvedStackConfig; + const allowDownload = Deferred.makeUnsafe(); + const resolver = mockBinaryResolver({ + downloadedServices: ["auth"], + beforeResolve: ({ service }) => + service === "auth" ? Deferred.await(allowDownload) : Effect.void, + }); + const { layer } = setupLayer(config, noopPortLease(config.ports), undefined, resolver); + + return Effect.gen(function* () { + const stack = yield* Stack; + const activator = yield* StackServiceActivator; + yield* stack.start(); + + const authChanges = yield* stack.stateChanges("auth"); + const downloading = yield* authChanges.pipe( + Stream.filter((state) => state.status === "Downloading"), + Stream.runHead, + Effect.forkChild({ startImmediately: true }), + ); + const stopped = yield* authChanges.pipe( + Stream.filter((state) => state.status === "Stopped"), + Stream.runHead, + Effect.forkChild({ startImmediately: true }), + ); + const activation = yield* activator + .activate("auth") + .pipe(Effect.forkChild({ startImmediately: true })); + + expect((yield* Fiber.join(downloading))._tag).toBe("Some"); + yield* stack.stopService("auth"); + expect((yield* Fiber.join(stopped))._tag).toBe("Some"); + yield* Deferred.succeed(allowDownload, undefined); + + const activationExit = yield* Fiber.await(activation); + expect(Exit.isFailure(activationExit)).toBe(true); + if (Exit.isFailure(activationExit)) { + const error = Cause.squash(activationExit.cause); + expect(error).toMatchObject({ _tag: "StackBuildError" }); + if (error instanceof StackBuildError) { + expect(error.detail).toContain("auth was explicitly stopped"); + } } - expect((yield* stack.getState("storage")).status).not.toBe("Downloading"); - expect((yield* stack.getState("imgproxy")).status).not.toBe("Downloading"); + expect((yield* stack.getState("auth")).status).toBe("Stopped"); yield* stack.stop(); }).pipe(Effect.provide(layer), Effect.timeout("5 seconds")); }); diff --git a/packages/stack/src/services/vector.ts b/packages/stack/src/services/vector.ts index 066fe344ac..bf7f3681d9 100644 --- a/packages/stack/src/services/vector.ts +++ b/packages/stack/src/services/vector.ts @@ -20,13 +20,18 @@ interface DockerVectorOptions extends ContainerRuntimeOptions { readonly dependencies: ReadonlyArray; } -const vectorConfig = (host: string, port: number, apiKey: string) => `api: +const vectorConfig = ( + host: string, + port: number, + apiKey: string, + logSource: "docker_logs" | "internal_logs", +) => `api: enabled: true address: 0.0.0.0:9001 sources: runtime: - type: docker_logs + type: ${logSource} sinks: logflare: @@ -58,7 +63,6 @@ const podmanSocketCandidates = (): ReadonlyArray => { const uid = process.getuid?.(); if (uid !== undefined) candidates.push(`/run/user/${uid}/podman/podman.sock`); candidates.push("/run/podman/podman.sock"); - candidates.push("/var/run/docker.sock"); return candidates; }; @@ -78,8 +82,8 @@ const resolveVectorDockerSocket = (runtime: ContainerRuntime): string | undefine export const makeVectorServiceDocker = (opts: DockerVectorOptions) => { const containerName = dockerContainerName("vector", opts.identity.key); - const dockerSocket = resolveVectorDockerSocket(opts.runtime); - const volumes = dockerSocket === undefined ? [] : [`${dockerSocket}:/var/run/docker.sock:ro`]; + const socketPath = resolveVectorDockerSocket(opts.runtime); + const volumes = socketPath === undefined ? [] : [`${socketPath}:/var/run/docker.sock:ro`]; return dockerRunService({ runtime: opts.runtime, @@ -89,17 +93,20 @@ export const makeVectorServiceDocker = (opts: DockerVectorOptions) => { networkArgs: dockerNetworkArgs(opts.platformOs, []), volumes, securityOptions: - opts.runtime === "podman" && - dockerSocket !== undefined && - dockerSocket !== "/var/run/docker.sock" + opts.runtime === "podman" && socketPath !== undefined && socketPath !== "/var/run/docker.sock" ? ["label=disable"] : [], - env: dockerSocket === undefined ? {} : { DOCKER_HOST: "unix:///var/run/docker.sock" }, + env: socketPath === undefined ? {} : { DOCKER_HOST: "unix:///var/run/docker.sock" }, entrypoint: "sh", cmd: [ "-c", `cat <<'EOF' > /etc/vector/vector.yaml && vector --config /etc/vector/vector.yaml -${vectorConfig(opts.serviceHost, opts.analyticsPort, opts.analyticsApiKey)}EOF +${vectorConfig( + opts.serviceHost, + opts.analyticsPort, + opts.analyticsApiKey, + socketPath === undefined ? "internal_logs" : "docker_logs", +)}EOF `, ], dependencies: opts.dependencies, diff --git a/packages/stack/src/services/vector.unit.test.ts b/packages/stack/src/services/vector.unit.test.ts new file mode 100644 index 0000000000..f7fbd1ede4 --- /dev/null +++ b/packages/stack/src/services/vector.unit.test.ts @@ -0,0 +1,77 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { stackIdentity } from "../StackIdentity.ts"; +import { DEFAULT_VERSIONS, dockerImageForService } from "../versions.ts"; +import { makeVectorServiceDocker } from "./vector.ts"; + +const existingPaths = vi.hoisted(() => new Set()); + +vi.mock("node:fs", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + existsSync: (path: Parameters[0]) => existingPaths.has(String(path)), + }; +}); + +const identity = stackIdentity({ apiPort: 54321 }); + +const makeVector = (runtime: "docker" | "podman") => + makeVectorServiceDocker({ + runtime, + image: dockerImageForService("vector", DEFAULT_VERSIONS.vector), + identity, + serviceHost: "127.0.0.1", + analyticsPort: 54327, + analyticsApiKey: "test-api-key", + platformOs: "linux", + dependencies: [], + }); + +describe("makeVectorServiceDocker log source", () => { + beforeEach(() => { + existingPaths.clear(); + vi.stubEnv("CONTAINER_HOST", ""); + vi.stubEnv("DOCKER_HOST", ""); + vi.stubEnv("XDG_RUNTIME_DIR", ""); + }); + + afterEach(() => { + existingPaths.clear(); + vi.unstubAllEnvs(); + }); + + it("uses internal_logs when Podman cannot find its own socket", () => { + existingPaths.add("/var/run/docker.sock"); + + const def = makeVector("podman"); + const args = def.args ?? []; + + expect(args.join("\n")).toContain("type: internal_logs"); + expect(args).not.toContain("/var/run/docker.sock:/var/run/docker.sock:ro"); + expect(args).not.toContain("DOCKER_HOST=unix:///var/run/docker.sock"); + expect(args).not.toContain("--security-opt"); + }); + + it("connects Podman Vector to an available Podman socket", () => { + existingPaths.add("/run/podman/podman.sock"); + + const def = makeVector("podman"); + const args = def.args ?? []; + + expect(args.join("\n")).toContain("type: docker_logs"); + expect(args).toContain("/run/podman/podman.sock:/var/run/docker.sock:ro"); + expect(args).toContain("DOCKER_HOST=unix:///var/run/docker.sock"); + expect(args).toContain("--security-opt"); + expect(args).toContain("label=disable"); + }); + + it("uses internal_logs when Docker cannot find its socket", () => { + const def = makeVector("docker"); + const args = def.args ?? []; + + expect(args.join("\n")).toContain("type: internal_logs"); + expect(args).not.toContain("/var/run/docker.sock:/var/run/docker.sock:ro"); + expect(args).not.toContain("DOCKER_HOST=unix:///var/run/docker.sock"); + expect(args).not.toContain("--security-opt"); + }); +}); From 4a4e321ffc99d2463742d9a3e802a24d55ddee76 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Wed, 19 Aug 2026 18:23:47 +0200 Subject: [PATCH 08/20] fix(stack): validate runtime sockets and state --- packages/stack/src/LocalStack.ts | 20 ++++++---- .../src/managed-control.integration.test.ts | 2 +- packages/stack/src/services/vector.ts | 22 +++++++---- .../stack/src/services/vector.unit.test.ts | 38 +++++++++++++++++++ 4 files changed, 65 insertions(+), 17 deletions(-) diff --git a/packages/stack/src/LocalStack.ts b/packages/stack/src/LocalStack.ts index 317500afed..d1bfc444f7 100644 --- a/packages/stack/src/LocalStack.ts +++ b/packages/stack/src/LocalStack.ts @@ -210,6 +210,17 @@ export const localStackLayer = ( : [...current, nextState]; }); + const markDownloading = (service: ServiceName) => + SubscriptionRef.update(stateRef, (current) => { + const index = current.findIndex((entry) => entry.name === service); + if (index === -1 || current[index]?.status === "Downloading") return current; + return current.map((entry, entryIndex) => + entryIndex === index + ? new StackServiceState({ ...entry, status: "Downloading" }) + : entry, + ); + }); + const restoreStateIfDownloading = ( service: ServiceName, previous: StackServiceState | undefined, @@ -376,14 +387,7 @@ export const localStackLayer = ( (current, event) => Effect.gen(function* () { if (event instanceof ServiceDownloadStarted) { - const state = SubscriptionRef.getUnsafe(stateRef).find( - (entry) => entry.name === event.service, - ); - if (state !== undefined && state.status !== "Downloading") { - yield* updateState( - new StackServiceState({ ...state, status: "Downloading" }), - ); - } + yield* markDownloading(event.service); } if (event instanceof ServiceDownloadFinished) { yield* restoreStateIfDownloading( diff --git a/packages/stack/src/managed-control.integration.test.ts b/packages/stack/src/managed-control.integration.test.ts index 0dbd8fe5e2..85bec8a1cb 100644 --- a/packages/stack/src/managed-control.integration.test.ts +++ b/packages/stack/src/managed-control.integration.test.ts @@ -350,7 +350,7 @@ describe("managed control endpoint", () => { requestStop: () => Effect.void, }); const exit = yield* acquireControl({ stackId: STACK_ID }).pipe( - Effect.timeout("2 seconds"), + Effect.timeout("5 seconds"), Effect.exit, Effect.provide(unavailable), ); diff --git a/packages/stack/src/services/vector.ts b/packages/stack/src/services/vector.ts index bf7f3681d9..de4fc9bf93 100644 --- a/packages/stack/src/services/vector.ts +++ b/packages/stack/src/services/vector.ts @@ -1,4 +1,4 @@ -import { existsSync } from "node:fs"; +import { accessSync, constants } from "node:fs"; import { dockerNetworkArgs } from "../Platform.ts"; import type { ContainerRuntime } from "../ContainerRuntime.ts"; import { dockerContainerName, type StackIdentity } from "../StackIdentity.ts"; @@ -48,10 +48,19 @@ sinks: uri: "http://${host}:${port}/api/logs?source_name=docker.logs.local" `; +const canAccessSocket = (socket: string): boolean => { + try { + accessSync(socket, constants.R_OK | constants.W_OK); + return true; + } catch { + return false; + } +}; + const unixSocketFromEnv = (value: string | undefined): string | undefined => { if (value === undefined || !value.startsWith("unix://")) return undefined; const socket = value.slice("unix://".length); - return socket.length > 0 && existsSync(socket) ? socket : undefined; + return socket.length > 0 && canAccessSocket(socket) ? socket : undefined; }; const podmanSocketCandidates = (): ReadonlyArray => { @@ -72,12 +81,12 @@ const resolveVectorDockerSocket = (runtime: ContainerRuntime): string | undefine if (explicitPodmanSocket !== undefined) return explicitPodmanSocket; const explicitDockerSocket = unixSocketFromEnv(process.env.DOCKER_HOST); if (explicitDockerSocket !== undefined) return explicitDockerSocket; - return podmanSocketCandidates().find((socket) => existsSync(socket)); + return podmanSocketCandidates().find(canAccessSocket); } const explicitDockerSocket = unixSocketFromEnv(process.env.DOCKER_HOST); if (explicitDockerSocket !== undefined) return explicitDockerSocket; - return existsSync("/var/run/docker.sock") ? "/var/run/docker.sock" : undefined; + return canAccessSocket("/var/run/docker.sock") ? "/var/run/docker.sock" : undefined; }; export const makeVectorServiceDocker = (opts: DockerVectorOptions) => { @@ -92,10 +101,7 @@ export const makeVectorServiceDocker = (opts: DockerVectorOptions) => { image: opts.image, networkArgs: dockerNetworkArgs(opts.platformOs, []), volumes, - securityOptions: - opts.runtime === "podman" && socketPath !== undefined && socketPath !== "/var/run/docker.sock" - ? ["label=disable"] - : [], + securityOptions: opts.runtime === "podman" && socketPath !== undefined ? ["label=disable"] : [], env: socketPath === undefined ? {} : { DOCKER_HOST: "unix:///var/run/docker.sock" }, entrypoint: "sh", cmd: [ diff --git a/packages/stack/src/services/vector.unit.test.ts b/packages/stack/src/services/vector.unit.test.ts index f7fbd1ede4..dd6941a03c 100644 --- a/packages/stack/src/services/vector.unit.test.ts +++ b/packages/stack/src/services/vector.unit.test.ts @@ -4,12 +4,19 @@ import { DEFAULT_VERSIONS, dockerImageForService } from "../versions.ts"; import { makeVectorServiceDocker } from "./vector.ts"; const existingPaths = vi.hoisted(() => new Set()); +const accessiblePaths = vi.hoisted(() => new Set()); vi.mock("node:fs", async (importOriginal) => { const actual = await importOriginal(); return { ...actual, existsSync: (path: Parameters[0]) => existingPaths.has(String(path)), + accessSync: (path: Parameters[0]) => { + const socket = String(path); + if (!existingPaths.has(socket) || !accessiblePaths.has(socket)) { + throw new Error("socket is not accessible"); + } + }, }; }); @@ -30,6 +37,7 @@ const makeVector = (runtime: "docker" | "podman") => describe("makeVectorServiceDocker log source", () => { beforeEach(() => { existingPaths.clear(); + accessiblePaths.clear(); vi.stubEnv("CONTAINER_HOST", ""); vi.stubEnv("DOCKER_HOST", ""); vi.stubEnv("XDG_RUNTIME_DIR", ""); @@ -37,11 +45,13 @@ describe("makeVectorServiceDocker log source", () => { afterEach(() => { existingPaths.clear(); + accessiblePaths.clear(); vi.unstubAllEnvs(); }); it("uses internal_logs when Podman cannot find its own socket", () => { existingPaths.add("/var/run/docker.sock"); + accessiblePaths.add("/var/run/docker.sock"); const def = makeVector("podman"); const args = def.args ?? []; @@ -54,6 +64,7 @@ describe("makeVectorServiceDocker log source", () => { it("connects Podman Vector to an available Podman socket", () => { existingPaths.add("/run/podman/podman.sock"); + accessiblePaths.add("/run/podman/podman.sock"); const def = makeVector("podman"); const args = def.args ?? []; @@ -74,4 +85,31 @@ describe("makeVectorServiceDocker log source", () => { expect(args).not.toContain("DOCKER_HOST=unix:///var/run/docker.sock"); expect(args).not.toContain("--security-opt"); }); + + it("uses internal_logs when the Podman socket is not readable and writable", () => { + existingPaths.add("/run/podman/podman.sock"); + + const def = makeVector("podman"); + const args = def.args ?? []; + + expect(args.join("\n")).toContain("type: internal_logs"); + expect(args).not.toContain("/run/podman/podman.sock:/var/run/docker.sock:ro"); + expect(args).not.toContain("DOCKER_HOST=unix:///var/run/docker.sock"); + expect(args).not.toContain("--security-opt"); + }); + + it("honors an explicit Docker socket for Podman Vector", () => { + existingPaths.add("/var/run/docker.sock"); + accessiblePaths.add("/var/run/docker.sock"); + vi.stubEnv("DOCKER_HOST", "unix:///var/run/docker.sock"); + + const def = makeVector("podman"); + const args = def.args ?? []; + + expect(args.join("\n")).toContain("type: docker_logs"); + expect(args).toContain("/var/run/docker.sock:/var/run/docker.sock:ro"); + expect(args).toContain("DOCKER_HOST=unix:///var/run/docker.sock"); + expect(args).toContain("--security-opt"); + expect(args).toContain("label=disable"); + }); }); From e8372805bf0b56a749ac806bae2dad51c3b7f883 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Thu, 20 Aug 2026 09:50:39 +0200 Subject: [PATCH 09/20] fix(stack): stabilize runtime integration --- .../runtime/stack-e2e-cleanup.unit.test.ts | 39 ++++++++++++ .../shared/telemetry/error-actionability.ts | 4 ++ apps/cli/tests/helpers/stack-e2e-cleanup.ts | 6 +- .../src/BinaryResolver.integration.test.ts | 31 ++++++++-- packages/stack/src/cleanup.ts | 12 ++-- .../stack/src/createStack.integration.test.ts | 61 +------------------ packages/stack/src/createStack.ts | 49 ++++++++++++++- packages/stack/src/services/analytics.ts | 11 ---- packages/stack/src/services/postgres.ts | 42 ++++++++++++- .../stack/src/services/services.unit.test.ts | 26 ++++---- packages/stack/tests/createStack.e2e.test.ts | 29 ++------- packages/stack/tests/helpers/e2e.ts | 47 +++++++------- .../tests/postgresDataPersistence.e2e.test.ts | 6 +- 13 files changed, 210 insertions(+), 153 deletions(-) diff --git a/apps/cli/src/shared/runtime/stack-e2e-cleanup.unit.test.ts b/apps/cli/src/shared/runtime/stack-e2e-cleanup.unit.test.ts index 4d3a374a95..4da25f32c0 100644 --- a/apps/cli/src/shared/runtime/stack-e2e-cleanup.unit.test.ts +++ b/apps/cli/src/shared/runtime/stack-e2e-cleanup.unit.test.ts @@ -72,6 +72,45 @@ describe("stack e2e cleanup manager", () => { expect(calls).toEqual(["stop:/tmp/project:/tmp/home", "cleanup-project", "dispose-home"]); }); + it("keeps cleanup best-effort when the associated home cannot be disposed", async () => { + const calls: Array = []; + const manager = createStackE2eCleanupManager( + cleanupEnvironment(calls, { + captureSnapshot: () => ({ + managedStacksRootExists: true, + documentFiles: [], + stackDirs: [], + trackedPids: [], + }), + }), + ); + + manager.registerHome({ + dir: "/tmp/home", + dispose: () => { + calls.push("dispose-home"); + throw permissionError("home is not removable"); + }, + }); + manager.registerStackProject({ + dir: "/tmp/project", + cleanup: async () => { + calls.push("cleanup-project"); + }, + }); + manager.associateHome("/tmp/project", "/tmp/home"); + + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + try { + await expect(manager.drain()).resolves.toBeUndefined(); + expect(warn).toHaveBeenCalledWith(expect.stringContaining("/tmp/home")); + expect(warn).toHaveBeenCalledWith(expect.stringContaining("home is not removable")); + } finally { + warn.mockRestore(); + } + expect(calls).toEqual(["stop:/tmp/project:/tmp/home", "cleanup-project", "dispose-home"]); + }); + it("canonicalizes symlinked project and home paths before matching stack state", async () => { const root = mkdtempSync(join(tmpdir(), "stack-e2e-cleanup-")); const project = join(root, "project"); diff --git a/apps/cli/src/shared/telemetry/error-actionability.ts b/apps/cli/src/shared/telemetry/error-actionability.ts index 2e532a5c1f..e83b67d5f2 100644 --- a/apps/cli/src/shared/telemetry/error-actionability.ts +++ b/apps/cli/src/shared/telemetry/error-actionability.ts @@ -983,6 +983,9 @@ const externalActionabilityByTag: Record = { ? { ...actionability.invalidConfig, fingerprint_suffix: "port_allocation" } : actionability.unknown, BinaryNotFoundError: () => actionability.invalidConfig, + BinaryManifestError: () => actionability.externalNetwork, + BinaryRuntimeError: () => actionability.externalNetwork, + BinaryHostCompatibilityError: () => actionability.invalidConfig, DownloadError: () => actionability.externalNetwork, ChecksumMismatchError: () => ({ ...actionability.externalNetwork, @@ -1052,6 +1055,7 @@ const externalActionabilityByTag: Record = { ...actionability.stopStack, fingerprint_suffix: "managed_attached", }), + ManagedStackLaunchMissingError: () => actionability.startStack, ManagedWorkspaceRepairConflictError: () => ({ ...actionability.invalidInput, fingerprint_suffix: "managed_workspace_repair", diff --git a/apps/cli/tests/helpers/stack-e2e-cleanup.ts b/apps/cli/tests/helpers/stack-e2e-cleanup.ts index b6815fecc8..7f69b7e9d4 100644 --- a/apps/cli/tests/helpers/stack-e2e-cleanup.ts +++ b/apps/cli/tests/helpers/stack-e2e-cleanup.ts @@ -441,7 +441,11 @@ export function createStackE2eCleanupManager( failures.push(cleanupErrorDetail(project.dir, error)); } finally { if (home !== undefined) { - home.dispose(); + try { + home.dispose(); + } catch (error) { + failures.push(cleanupErrorDetail(home.dir, error)); + } } } } diff --git a/packages/stack/src/BinaryResolver.integration.test.ts b/packages/stack/src/BinaryResolver.integration.test.ts index baa62c2027..aea78a569b 100644 --- a/packages/stack/src/BinaryResolver.integration.test.ts +++ b/packages/stack/src/BinaryResolver.integration.test.ts @@ -49,15 +49,34 @@ const makeFixture = ( return { archive: readFileSync(archive), manifestOverride }; }; +const writeTarOctal = (header: Buffer, offset: number, length: number, value: number): void => { + const encoded = `${value.toString(8).padStart(length - 1, "0")}\0`; + header.write(encoded, offset, length, "ascii"); +}; + +const makeTarArchive = (member: string, contents: string): Buffer => { + const payload = Buffer.from(contents); + const header = Buffer.alloc(512); + header.write(member, 0, 100, "utf8"); + writeTarOctal(header, 100, 8, 0o644); + writeTarOctal(header, 108, 8, 0); + writeTarOctal(header, 116, 8, 0); + writeTarOctal(header, 124, 12, payload.length); + writeTarOctal(header, 136, 12, 0); + header[156] = "0".charCodeAt(0); + header.write("ustar\0", 257, 6, "ascii"); + header.write("00", 263, 2, "ascii"); + header.fill(0x20, 148, 156); + const checksum = header.reduce((sum, byte) => sum + byte, 0); + header.write(`${checksum.toString(8).padStart(6, "0")}\0 `, 148, 8, "ascii"); + const padding = Buffer.alloc((512 - (payload.length % 512)) % 512); + return Buffer.concat([header, payload, padding, Buffer.alloc(1024)]); +}; + const makeTraversalFixture = (root: string) => { - const source = join(root, "traversal-source"); - const tar = join(root, "postgrest-traversal.tar"); const archive = join(root, "postgrest-traversal.tar.zst"); - rmSync(source, { recursive: true, force: true }); - mkdirSync(source, { recursive: true }); writeFileSync(join(root, "outside.txt"), "must not extract\n"); - execFileSync("tar", ["-cf", tar, "-C", source, "../outside.txt"]); - writeFileSync(archive, zstdCompressSync(readFileSync(tar))); + writeFileSync(archive, zstdCompressSync(makeTarArchive("../outside.txt", "must not extract\n"))); return { archive: readFileSync(archive), manifestOverride: {} }; }; diff --git a/packages/stack/src/cleanup.ts b/packages/stack/src/cleanup.ts index af00a8c93e..7e9439fbb0 100644 --- a/packages/stack/src/cleanup.ts +++ b/packages/stack/src/cleanup.ts @@ -49,10 +49,6 @@ export function cleanupAutoManagedPaths(config: ResolvedStackConfig): void { // Best-effort — temp dir will be cleaned by OS eventually. } } - - try { - rmSync(`${config.postgres.dataDir}_pg_hba_docker.conf`, { force: true }); - } catch {} } const cleanupAutoManagedPathsWithRetry = (config: ResolvedStackConfig): Effect.Effect => @@ -61,10 +57,10 @@ const cleanupAutoManagedPathsWithRetry = (config: ResolvedStackConfig): Effect.E return; } - const cleanupTargets = [ - ...config.autoManagedPaths.map((path) => ({ path, recursive: true as const })), - { path: `${config.postgres.dataDir}_pg_hba_docker.conf`, recursive: false as const }, - ]; + const cleanupTargets = config.autoManagedPaths.map((path) => ({ + path, + recursive: true as const, + })); for (let attempt = 0; attempt < 80; attempt++) { yield* Effect.sync(() => { diff --git a/packages/stack/src/createStack.integration.test.ts b/packages/stack/src/createStack.integration.test.ts index 6945a90132..894408ac4c 100644 --- a/packages/stack/src/createStack.integration.test.ts +++ b/packages/stack/src/createStack.integration.test.ts @@ -1,74 +1,16 @@ import { afterEach, describe, expect, it } from "vitest"; -import { Effect } from "effect"; import { createStack, type StackHandle } from "./createStack.ts"; -import { reservePortSet } from "./PortAllocator.ts"; import { platformFactory } from "./platform-node.ts"; const handles: StackHandle[] = []; -const isAddressInUse = (error: unknown, depth = 0): boolean => { - if (depth > 4 || !(error instanceof Error)) return false; - if ("code" in error && Reflect.get(error, "code") === "EADDRINUSE") return true; - const cause: unknown = error.cause; - if (typeof cause === "object" && cause !== null && "code" in cause) { - if (Reflect.get(cause, "code") === "EADDRINUSE") return true; - } - return isAddressInUse(cause, depth + 1); -}; - -const freshPortPair = async (): Promise => - Effect.runPromise( - Effect.scoped( - Effect.acquireRelease( - reservePortSet([ - { field: "apiPort", selection: { kind: "automatic" } }, - { field: "dbPort", selection: { kind: "automatic" } }, - ]), - (lease) => lease.releaseAll, - ).pipe( - Effect.map((lease) => { - const apiPort = lease.ports.apiPort; - const dbPort = lease.ports.dbPort; - if (apiPort === undefined || dbPort === undefined) { - throw new Error("Ephemeral port reservation returned an incomplete pair"); - } - return [apiPort, dbPort] as const; - }), - ), - ), - ); - -/** Transfer a fresh exact pair into public createStack across a bounded bind handoff retry. */ -const createStackWithFreshPorts = async ( - config: Parameters[0], - platform: Parameters[1], -): Promise>> => { - for (let attempt = 0; attempt < 3; attempt += 1) { - const [apiPort, dbPort] = await freshPortPair(); - try { - return await createStack( - { - ...config, - port: apiPort, - postgres: { ...config?.postgres, port: dbPort }, - }, - platform, - { mode: "native", containerRuntime: null }, - ); - } catch (error) { - if (!isAddressInUse(error) || attempt === 2) throw error; - } - } - throw new Error("Direct stack bind handoff exhausted retries"); -}; - afterEach(async () => { await Promise.all(handles.splice(0).map((handle) => handle.dispose())); }); describe("direct createStack port ownership", () => { it("allocates only active service fields without managed state", async () => { - const stack = await createStackWithFreshPorts( + const stack = await createStack( { mode: "native", postgrest: false, @@ -85,6 +27,7 @@ describe("direct createStack port ownership", () => { pooler: false, }, platformFactory, + { mode: "native", containerRuntime: null }, ); handles.push(stack); diff --git a/packages/stack/src/createStack.ts b/packages/stack/src/createStack.ts index cef27c50c3..2e6f31e2bd 100644 --- a/packages/stack/src/createStack.ts +++ b/packages/stack/src/createStack.ts @@ -71,16 +71,33 @@ export interface StackHandle extends AsyncDisposable { logHistory(name: string, limit?: number): Promise>; } -export async function createStack( +const MAX_AUTOMATIC_API_PORT_HANDOFF_ATTEMPTS = 3; + +/** + * The port lease is intentionally released just before the HTTP server binds. + * Another process can claim that port in the small handoff window, so a new + * foreground stack may retry its automatic API-port allocation. Explicit API + * ports never enter this retry path. + */ +const isAddressInUse = (error: unknown, depth = 0): boolean => { + if (depth > 8 || typeof error !== "object" || error === null) return false; + if ("code" in error && Reflect.get(error, "code") === "EADDRINUSE") return true; + if ("cause" in error) return isAddressInUse(Reflect.get(error, "cause"), depth + 1); + return false; +}; + +const createStackAttempt = async ( config: StackConfig | undefined, platformFactory: PlatformFactory, runtime: StackRuntimeSelection, -): Promise { + preferredApiPort?: number, +): Promise => { let portLease: PortLease | undefined; let resolved: ResolvedStackConfig; try { resolved = await resolveConfig(config, { runtime, + ...(preferredApiPort === undefined ? {} : { preferredPorts: { apiPort: preferredApiPort } }), portAllocator: (requests, options) => reservePortSet(requests, options).pipe( Effect.tap((lease) => @@ -175,4 +192,32 @@ export async function createStack( cleanupAutoManagedPaths(resolved); throw toStackError(error); } +}; + +export async function createStack( + config: StackConfig | undefined, + platformFactory: PlatformFactory, + runtime: StackRuntimeSelection, +): Promise { + const automaticApiPort = config?.port === undefined; + for (let attempt = 0; ; attempt += 1) { + try { + // Port zero asks the allocator for an OS-assigned automatic port, + // avoiding another attempt at a contended default during handoff. + return await createStackAttempt( + config, + platformFactory, + runtime, + attempt === 0 ? undefined : 0, + ); + } catch (error) { + if ( + !automaticApiPort || + !isAddressInUse(error) || + attempt + 1 >= MAX_AUTOMATIC_API_PORT_HANDOFF_ATTEMPTS + ) { + throw error; + } + } + } } diff --git a/packages/stack/src/services/analytics.ts b/packages/stack/src/services/analytics.ts index 7eeba05c69..6f4bcfcf03 100644 --- a/packages/stack/src/services/analytics.ts +++ b/packages/stack/src/services/analytics.ts @@ -80,17 +80,6 @@ export const makeAnalyticsServiceDocker = (opts: DockerAnalyticsOptions): Servic networkArgs: dockerPortMapArgs(opts.platformOs, [ { host: opts.hostPort, container: ANALYTICS_CONTAINER_PORT }, ]), - entrypoint: "sh", - cmd: [ - "-c", - // migrate && start: a failed migrate exits the container and the - // unless-stopped restart retries until the db is ready (supabase/cli#6088). - `cat <<'EOF' > /tmp/run.sh && sh /tmp/run.sh -./logflare eval Logflare.Release.migrate && -./logflare start --sname logflare -EOF -`, - ], env, dependencies: opts.dependencies, healthCheck: analyticsHealthCheck(opts.hostPort), diff --git a/packages/stack/src/services/postgres.ts b/packages/stack/src/services/postgres.ts index 39b99abb3b..b3e2dc0cff 100644 --- a/packages/stack/src/services/postgres.ts +++ b/packages/stack/src/services/postgres.ts @@ -37,6 +37,8 @@ const postgresEnv = (opts: NativePostgresOptions): Record => ({ }); const NATIVE_POSTGRES_RUNTIME_ARGS = [ + "-c", + "listen_addresses=127.0.0.1", "-c", "wal_level=logical", "-c", @@ -45,6 +47,9 @@ const NATIVE_POSTGRES_RUNTIME_ARGS = [ "max_replication_slots=5", ] as const; +const postgresGetKeyScript = (binPath: string): string => + `${binPath}/share/supabase-cli/config/pgsodium_getkey.sh`; + const orphanCleanup = (opts: PostgresServiceOptions) => opts.cleanupDataDirOnExit ? removePathOnOrphanCleanup(opts.dataDir) : []; @@ -86,11 +91,21 @@ const postgresDockerHealthCheck = ( export const makePostgresService = (opts: NativePostgresOptions): ServiceDef => { const initScript = `${opts.binPath}/share/supabase-cli/bin/supabase-postgres-init.sh`; + const getKeyScript = postgresGetKeyScript(opts.binPath); return { name: "postgres", command: "bash", - args: [initScript, "-p", String(opts.port), ...NATIVE_POSTGRES_RUNTIME_ARGS], + args: [ + initScript, + "-p", + String(opts.port), + ...NATIVE_POSTGRES_RUNTIME_ARGS, + "-c", + `pgsodium.getkey_script=${getKeyScript}`, + "-c", + `vault.getkey_script=${getKeyScript}`, + ], env: postgresEnv(opts), dependencies: opts.dependencies, healthCheck: postgresHealthCheck(opts.binPath, opts.port), @@ -102,6 +117,28 @@ export const makePostgresService = (opts: NativePostgresOptions): ServiceDef => export const makePostgresServiceDocker = (opts: DockerPostgresOptions): ServiceDef => { const containerName = dockerContainerName("postgres", opts.identity.key); + const runtimeArgs = [ + "-p", + String(opts.port), + "-c", + "listen_addresses=*", + "-c", + "pgsodium.getkey_script=/opt/postgres/share/supabase-cli/config/pgsodium_getkey.sh", + "-c", + "vault.getkey_script=/opt/postgres/share/supabase-cli/config/pgsodium_getkey.sh", + ] as const; + + // Native initialization permits only loopback clients. When reusing that + // data directory in Docker, route through a temporary HBA copy that adds the + // container network rule without mutating the persisted native config. + const command = `if [ -s /var/lib/postgresql/data/PG_VERSION ]; then + cp /var/lib/postgresql/data/pg_hba.conf /tmp/supabase-cli-pg_hba.conf + printf '\\nhost all all all scram-sha-256\\n' >> /tmp/supabase-cli-pg_hba.conf + exec /usr/local/bin/entry.sh -c hba_file=/tmp/supabase-cli-pg_hba.conf ${runtimeArgs.join(" ")} +else + exec /usr/local/bin/entry.sh ${runtimeArgs.join(" ")} +fi`; + return dockerRunService({ runtime: opts.runtime, name: "postgres", @@ -110,7 +147,8 @@ export const makePostgresServiceDocker = (opts: DockerPostgresOptions): ServiceD networkArgs: dockerNetworkArgs(opts.platformOs, [opts.port]), volumes: [`${opts.dataDir}:/var/lib/postgresql/data`], env: { POSTGRES_PASSWORD: "postgres" }, - cmd: ["-p", String(opts.port)], + entrypoint: "/usr/bin/sh", + cmd: ["-c", command], dependencies: opts.dependencies, healthCheck: postgresDockerHealthCheck(opts.runtime, containerName, opts.port), shutdown: { signal: "SIGTERM", timeoutSeconds: 10 }, diff --git a/packages/stack/src/services/services.unit.test.ts b/packages/stack/src/services/services.unit.test.ts index bea77ea535..09e1f80f5a 100644 --- a/packages/stack/src/services/services.unit.test.ts +++ b/packages/stack/src/services/services.unit.test.ts @@ -47,17 +47,16 @@ describe("makePostgresService", () => { expect(def.name).toBe("postgres"); expect(def.command).toBe("bash"); - expect(def.args).toEqual([ + expect(def.args).toContain( `${POSTGRES_BIN_PATH}/share/supabase-cli/bin/supabase-postgres-init.sh`, - "-p", - "54322", - "-c", - "wal_level=logical", - "-c", - "max_wal_senders=5", - "-c", - "max_replication_slots=5", - ]); + ); + expect(def.args).toContain("listen_addresses=127.0.0.1"); + expect(def.args).toContain( + `pgsodium.getkey_script=${POSTGRES_BIN_PATH}/share/supabase-cli/config/pgsodium_getkey.sh`, + ); + expect(def.args).toContain( + `vault.getkey_script=${POSTGRES_BIN_PATH}/share/supabase-cli/config/pgsodium_getkey.sh`, + ); expect(def.env?.PGDATA).toBe("/tmp/supabase/data"); expect(def.env?.POSTGRES_PASSWORD).toBe("postgres"); expect(def.env?.DYLD_LIBRARY_PATH).toBe(`${POSTGRES_BIN_PATH}/lib`); @@ -147,7 +146,9 @@ describe("makePostgresServiceDocker", () => { expect(def.args).toContain(`${DB_PORT}:${DB_PORT}`); expect(def.args).toContain(dockerImageForService("postgres", DEFAULT_VERSIONS.postgres)); expect(def.args).toContain("/tmp/supabase/data:/var/lib/postgresql/data"); - expect(def.args?.slice(-2)).toEqual(["-p", String(DB_PORT)]); + expect(def.args).toContain("/usr/bin/sh"); + expect(def.args?.at(-2)).toBe("-c"); + expect(def.args?.at(-1)).toContain("hba_file=/tmp/supabase-cli-pg_hba.conf"); // Health check uses docker exec + pg_isready inside the container (host has no postgres tools) expect(def.healthCheck?.probe).toEqual({ _tag: "Exec", @@ -685,9 +686,6 @@ describe("docker-backed auxiliary services", () => { expect(args).toContain("PHX_HTTP_PORT=4000"); expect(args).toContain("54328:4000"); expect(args).toContain("LOGFLARE_NODE_HOST=0.0.0.0"); - expect(args.at(-1)).toBe( - `cat <<'EOF' > /tmp/run.sh && sh /tmp/run.sh\n./logflare eval Logflare.Release.migrate &&\n./logflare start --sname logflare\nEOF\n`, - ); }); it("keeps analytics on its container port when Linux uses bridge networking", () => { diff --git a/packages/stack/tests/createStack.e2e.test.ts b/packages/stack/tests/createStack.e2e.test.ts index f5947aafa8..3afdaa7a57 100644 --- a/packages/stack/tests/createStack.e2e.test.ts +++ b/packages/stack/tests/createStack.e2e.test.ts @@ -3,10 +3,11 @@ import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; +import { activationTimeoutSecondsForService } from "../src/ServiceActivation.ts"; import { createStack, type ResolvedFunctionsBundle, type StackHandle } from "../src/node.ts"; import { fetchFunctionWhenReady, setupTestTable } from "./helpers/e2e.ts"; -const STACK_E2E_TEST_TIMEOUT_MS = 5_000; +const AUTH_COLD_START_TEST_TIMEOUT_MS = activationTimeoutSecondsForService("auth") * 1000; describe("createStack e2e", () => { let stack: StackHandle; @@ -47,24 +48,6 @@ describe("createStack e2e", () => { } catch {} }, 30_000); - test( - "serves health endpoints through the local gateway", - { timeout: STACK_E2E_TEST_TIMEOUT_MS }, - async () => { - const [proxyRes, authRes] = await Promise.all([ - fetch(`${stack.url}/health`), - fetch(`${stack.url}/auth/v1/health`), - ]); - - expect(proxyRes.status).toBe(200); - expect(await proxyRes.text()).toBe("OK"); - expect(authRes.status).toBe(200); - expect(await authRes.json()).toEqual( - expect.objectContaining({ description: expect.any(String) }), - ); - }, - ); - test( "serves detected Edge Functions through the local gateway", { timeout: 30_000 }, @@ -72,10 +55,8 @@ describe("createStack e2e", () => { // "Healthy" only means the edge-runtime control plane answered its health // probe; the first request to a function still lazily cold-boots a user // worker, so wait for the function to actually become servable. - const [states, functionsRes] = await Promise.all([ - stack.getStatus(), - fetchFunctionWhenReady(`${stack.url}/functions/v1/hello`), - ]); + const functionsRes = await fetchFunctionWhenReady(`${stack.url}/functions/v1/hello`); + const states = await stack.getStatus(); expect(states).toEqual( expect.arrayContaining([ @@ -99,7 +80,7 @@ describe("createStack e2e", () => { test( "supports the auth signup and session golden path", - { timeout: STACK_E2E_TEST_TIMEOUT_MS }, + { timeout: AUTH_COLD_START_TEST_TIMEOUT_MS }, async () => { const testEmail = `test-${Date.now()}@example.com`; const testPassword = "test-password-123"; diff --git a/packages/stack/tests/helpers/e2e.ts b/packages/stack/tests/helpers/e2e.ts index d22aefbb48..c62180b672 100644 --- a/packages/stack/tests/helpers/e2e.ts +++ b/packages/stack/tests/helpers/e2e.ts @@ -50,32 +50,33 @@ export async function fetchFunctionWhenReady( */ export async function setupTestTable(dbPort: number): Promise { const sql = new Bun.SQL(`postgresql://supabase_admin:postgres@127.0.0.1:${dbPort}/postgres`); + try { + await sql.unsafe(` + CREATE TABLE IF NOT EXISTS public.todos ( + id SERIAL PRIMARY KEY, + title TEXT NOT NULL, + completed BOOLEAN NOT NULL DEFAULT false + ); - await sql.unsafe(` - CREATE TABLE IF NOT EXISTS public.todos ( - id SERIAL PRIMARY KEY, - title TEXT NOT NULL, - completed BOOLEAN NOT NULL DEFAULT false - ); - - ALTER TABLE public.todos ENABLE ROW LEVEL SECURITY; - - DO $$ BEGIN - IF NOT EXISTS (SELECT 1 FROM pg_policies WHERE tablename = 'todos' AND policyname = 'allow_all') THEN - CREATE POLICY allow_all ON public.todos FOR ALL USING (true) WITH CHECK (true); - END IF; - END $$; + ALTER TABLE public.todos ENABLE ROW LEVEL SECURITY; - GRANT ALL ON public.todos TO anon, authenticated, service_role; - GRANT USAGE, SELECT ON SEQUENCE public.todos_id_seq TO anon, authenticated, service_role; + DO $$ BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_policies WHERE tablename = 'todos' AND policyname = 'allow_all') THEN + CREATE POLICY allow_all ON public.todos FOR ALL USING (true) WITH CHECK (true); + END IF; + END $$; - INSERT INTO public.todos (title, completed) VALUES - ('Learn Supabase', true), - ('Build an app', false); - `); + GRANT ALL ON public.todos TO anon, authenticated, service_role; + GRANT USAGE, SELECT ON SEQUENCE public.todos_id_seq TO anon, authenticated, service_role; - // PostgREST caches schema metadata, so tell it to reload after creating test tables. - await sql.unsafe(`NOTIFY pgrst, 'reload schema';`); + INSERT INTO public.todos (title, completed) VALUES + ('Learn Supabase', true), + ('Build an app', false); + `); - sql.close(); + // PostgREST caches schema metadata, so tell it to reload after creating test tables. + await sql.unsafe(`NOTIFY pgrst, 'reload schema';`); + } finally { + await sql.close(); + } } diff --git a/packages/stack/tests/postgresDataPersistence.e2e.test.ts b/packages/stack/tests/postgresDataPersistence.e2e.test.ts index 3fa4ab86b0..7d7da780a7 100644 --- a/packages/stack/tests/postgresDataPersistence.e2e.test.ts +++ b/packages/stack/tests/postgresDataPersistence.e2e.test.ts @@ -51,11 +51,11 @@ async function queryMarkerRows(dbPort: number): Promise { INSERT INTO public.persistence_marker (note) VALUES ('native-e2e-marker'); `); - sql.close(); + await sql.close(); }, NATIVE_SETUP_TIMEOUT_MS); afterAll(async () => { From 4d31f6c4aa330470a43523ef520527f236d30685 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Thu, 20 Aug 2026 10:21:49 +0200 Subject: [PATCH 10/20] fix(stack): harden native binary cache reuse --- .../src/BinaryResolver.integration.test.ts | 134 +++++++++++- packages/stack/src/BinaryResolver.ts | 200 ++++++++++++++---- 2 files changed, 283 insertions(+), 51 deletions(-) diff --git a/packages/stack/src/BinaryResolver.integration.test.ts b/packages/stack/src/BinaryResolver.integration.test.ts index aea78a569b..f02752dc1c 100644 --- a/packages/stack/src/BinaryResolver.integration.test.ts +++ b/packages/stack/src/BinaryResolver.integration.test.ts @@ -13,13 +13,14 @@ import { } from "node:fs"; import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; -import { NodeServices } from "@effect/platform-node"; +import { NodeFileSystem, NodePath, NodeServices } from "@effect/platform-node"; import { describe, expect, it } from "@effect/vitest"; -import { Effect, Layer } from "effect"; +import { Deferred, Effect, Fiber, FileSystem, Layer } from "effect"; import { HttpClient, HttpClientResponse } from "effect/unstable/http"; import { ChildProcessSpawner } from "effect/unstable/process"; import { BinaryResolver } from "./BinaryResolver.ts"; import { + BinaryHostCompatibilityError, BinaryManifestError, BinaryRuntimeError, ChecksumMismatchError, @@ -99,6 +100,7 @@ const makeResolverLayer = ( readonly checksum?: string; readonly checksumText?: string; readonly spawnedCommands?: Array<{ command: string; args: ReadonlyArray }>; + readonly transformFileSystem?: (fileSystem: FileSystem.FileSystem) => FileSystem.FileSystem; } = {}, ) => { const client = HttpClient.make((request) => @@ -153,10 +155,22 @@ const makeResolverLayer = ( }); }), ).pipe(Layer.provide(NodeServices.layer)); + const fileSystemLayer = + options.transformFileSystem === undefined + ? NodeFileSystem.layer + : Layer.effect( + FileSystem.FileSystem, + Effect.map(FileSystem.FileSystem, options.transformFileSystem), + ).pipe(Layer.provide(NodeFileSystem.layer)); return BinaryResolver.make(cacheRoot).pipe( - Layer.provide(Layer.succeed(HttpClient.HttpClient, client)), - Layer.provide(spawnerLayer), - Layer.provide(NodeServices.layer), + Layer.provide( + Layer.mergeAll( + Layer.succeed(HttpClient.HttpClient, client), + spawnerLayer, + fileSystemLayer, + NodePath.layer, + ), + ), ); }; @@ -197,6 +211,50 @@ describe("BinaryResolver slim-services installer", () => { }), ); + it.live("rejects a complete cache prepared for an incompatible host", () => + Effect.gen(function* () { + const root = makeRoot(); + try { + const resolverLayer = makeResolverLayer(root, makeFixture(root)); + const installed = yield* Effect.gen(function* () { + const resolver = yield* BinaryResolver; + return yield* resolver.resolve({ + service: "postgrest", + version: DEFAULT_VERSIONS.postgrest, + }); + }).pipe(Effect.provide(resolverLayer)); + + const markerPath = join(installed, ".complete"); + const marker = JSON.parse(readFileSync(markerPath, "utf8")); + marker.hostCompatibility = + process.platform === "darwin" + ? { + runtimeRequires: null, + libc: null, + osFloor: { kind: "macos", floor: "999.0" }, + } + : { + runtimeRequires: "glibc", + libc: "glibc", + osFloor: { kind: "glibc", floor: "999.0" }, + }; + writeFileSync(markerPath, JSON.stringify(marker)); + + const failure = yield* Effect.gen(function* () { + const resolver = yield* BinaryResolver; + return yield* resolver.resolve({ + service: "postgrest", + version: DEFAULT_VERSIONS.postgrest, + }); + }).pipe(Effect.provide(resolverLayer), Effect.flip); + + expect(failure).toBeInstanceOf(BinaryHostCompatibilityError); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }), + ); + it.live("installs slim archives without requiring an external zstd executable", () => Effect.gen(function* () { const root = makeRoot(); @@ -326,6 +384,72 @@ describe("BinaryResolver slim-services installer", () => { }), ); + it.live("preserves a cache published while another resolver repairs stale state", () => + Effect.gen(function* () { + const root = makeRoot(); + try { + const fixture = makeFixture(root); + const resolverLayer = makeResolverLayer(root, fixture); + const installed = yield* Effect.gen(function* () { + const resolver = yield* BinaryResolver; + return yield* resolver.resolveWithMetadata({ + service: "postgrest", + version: DEFAULT_VERSIONS.postgrest, + }); + }).pipe(Effect.provide(resolverLayer)); + const markerPath = join(installed.path, ".complete"); + writeFileSync(markerPath, JSON.stringify({ service: "postgrest" })); + + const staleMarkerRead = yield* Deferred.make(); + const releaseStaleReader = yield* Deferred.make(); + let markerReads = 0; + const staleReaderLayer = makeResolverLayer(root, fixture, { + transformFileSystem: (fileSystem) => + FileSystem.FileSystem.of({ + ...fileSystem, + readFileString: (requestedPath, options) => + Effect.gen(function* () { + const contents = yield* fileSystem.readFileString(requestedPath, options); + if (requestedPath === markerPath) { + markerReads += 1; + if (markerReads === 2) { + yield* Deferred.succeed(staleMarkerRead, undefined); + yield* Deferred.await(releaseStaleReader); + } + } + return contents; + }), + }), + }); + const staleReader = yield* Effect.gen(function* () { + const resolver = yield* BinaryResolver; + return yield* resolver.resolveWithMetadata({ + service: "postgrest", + version: DEFAULT_VERSIONS.postgrest, + }); + }).pipe(Effect.provide(staleReaderLayer), Effect.forkChild); + yield* Deferred.await(staleMarkerRead); + + const publisher = yield* Effect.gen(function* () { + const resolver = yield* BinaryResolver; + return yield* resolver.resolveWithMetadata({ + service: "postgrest", + version: DEFAULT_VERSIONS.postgrest, + }); + }).pipe(Effect.provide(makeResolverLayer(root, fixture))); + yield* Deferred.succeed(releaseStaleReader, undefined); + const repaired = yield* Fiber.join(staleReader); + + expect(publisher.downloaded).toBe(true); + expect(repaired.downloaded).toBe(false); + expect(repaired.path).toBe(installed.path); + expect(readFileSync(join(installed.path, "bin/postgrest"), "utf8")).toContain("postgrest"); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }), + ); + it.live("rejects archive members that escape the private staging directory", () => Effect.gen(function* () { const root = makeRoot(); diff --git a/packages/stack/src/BinaryResolver.ts b/packages/stack/src/BinaryResolver.ts index 0e85e96a17..d6e4c9de70 100644 --- a/packages/stack/src/BinaryResolver.ts +++ b/packages/stack/src/BinaryResolver.ts @@ -1,6 +1,17 @@ import { createHash } from "node:crypto"; import { zstdDecompressSync } from "node:zlib"; -import { Context, Effect, FileSystem, Layer, Option, Path, Result } from "effect"; +import { + Context, + Duration, + Effect, + FileSystem, + Layer, + Option, + Path, + PlatformError, + Result, + Schedule, +} from "effect"; import { HttpClient } from "effect/unstable/http"; import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; import { @@ -55,6 +66,15 @@ interface AssetInfo { readonly target: NativeTarget; } +interface HostCompatibilityRequirement { + readonly runtimeRequires: null | "glibc"; + readonly libc: null | "glibc"; + readonly osFloor: null | { + readonly kind: "glibc" | "macos"; + readonly floor: string | null; + }; +} + interface CacheCompleteMarker { readonly provider: string; readonly service: string; @@ -64,13 +84,14 @@ interface CacheCompleteMarker { readonly target: NativeTarget; readonly releaseSet: "slim-services"; readonly runtime: "native"; + readonly hostCompatibility: HostCompatibilityRequirement; } const cachePath = (baseDir: string, info: AssetInfo): string => `${baseDir}/${info.releaseSet}/${info.service}/${info.version}/${info.runtime}/${info.target}`; const CACHE_COMPLETE_MARKER = ".complete"; -const STALE_STAGING_AGE_MS = 24 * 60 * 60 * 1_000; +const STALE_PREPARATION_ENTRY_AGE_MS = 24 * 60 * 60 * 1_000; const hasTraversalSegment = (value: string): boolean => value.split(/[\\/]/).some((segment) => segment === ".."); @@ -162,7 +183,10 @@ const validateManifest = ( raw: unknown, platform: { readonly os: string; readonly arch: string }, spawner: ChildProcessSpawner.ChildProcessSpawner["Service"], -): Effect.Effect => +): Effect.Effect< + HostCompatibilityRequirement, + BinaryManifestError | BinaryHostCompatibilityError +> => Effect.gen(function* () { if (typeof raw !== "object" || raw === null) { return yield* Effect.fail(manifestError(release.manifestUrl, "Manifest must be an object")); @@ -214,33 +238,56 @@ const validateManifest = ( if (osFloor !== null && typeof osFloor !== "object") { return yield* Effect.fail(manifestError(release.manifestUrl, "Manifest os_floor is invalid")); } - const requiresGlibc = - manifest.libc === "glibc" || runtimeRequires === "glibc" || osFloor?.kind === "glibc"; - if (osFloor !== null && osFloor.kind === "macos" && platform.os !== "darwin") { + if (osFloor !== null && osFloor.kind !== "macos" && osFloor.kind !== "glibc") { return yield* Effect.fail( new BinaryHostCompatibilityError({ target: release.target, - detail: "Manifest requires macOS", + detail: `Unsupported manifest host kind ${osFloor.kind}`, }), ); } - if (requiresGlibc && platform.os !== "linux") { + const hostCompatibility: HostCompatibilityRequirement = { + runtimeRequires, + libc: manifest.libc, + osFloor: + osFloor === null + ? null + : osFloor.kind === "macos" + ? { kind: "macos", floor: osFloor.floor } + : { kind: "glibc", floor: osFloor.floor }, + }; + yield* validateHostCompatibility(release.target, hostCompatibility, platform, spawner); + return hostCompatibility; + }); + +const validateHostCompatibility = ( + target: NativeTarget, + requirement: HostCompatibilityRequirement, + platform: { readonly os: string; readonly arch: string }, + spawner: ChildProcessSpawner.ChildProcessSpawner["Service"], +): Effect.Effect => + Effect.gen(function* () { + const requiresGlibc = + requirement.libc === "glibc" || + requirement.runtimeRequires === "glibc" || + requirement.osFloor?.kind === "glibc"; + if (requirement.osFloor?.kind === "macos" && platform.os !== "darwin") { return yield* Effect.fail( new BinaryHostCompatibilityError({ - target: release.target, - detail: "Manifest requires Linux/glibc", + target, + detail: "Manifest requires macOS", }), ); } - if (osFloor !== null && osFloor.kind !== "macos" && osFloor.kind !== "glibc") { + if (requiresGlibc && platform.os !== "linux") { return yield* Effect.fail( new BinaryHostCompatibilityError({ - target: release.target, - detail: `Unsupported manifest host kind ${osFloor.kind}`, + target, + detail: "Manifest requires Linux/glibc", }), ); } - const floor = osFloor?.floor; + const floor = requirement.osFloor?.floor; if (requiresGlibc) { const host = yield* Effect.sync(() => { try { @@ -265,7 +312,7 @@ const validateManifest = ( if (typeof host !== "string" || host.trim().length === 0) { return yield* Effect.fail( new BinaryHostCompatibilityError({ - target: release.target, + target, detail: "Unable to determine host glibc version", }), ); @@ -273,7 +320,7 @@ const validateManifest = ( if (floor !== null && floor !== undefined && compareVersions(host, floor) < 0) { return yield* Effect.fail( new BinaryHostCompatibilityError({ - target: release.target, + target, detail: `Host glibc ${host} is below manifest floor ${floor}`, }), ); @@ -281,7 +328,7 @@ const validateManifest = ( } if ( platform.os === "darwin" && - osFloor?.kind === "macos" && + requirement.osFloor?.kind === "macos" && floor !== null && floor !== undefined ) { @@ -289,7 +336,7 @@ const validateManifest = ( Effect.mapError( (cause) => new BinaryHostCompatibilityError({ - target: release.target, + target, detail: `Unable to determine macOS version: ${String(cause)}`, }), ), @@ -298,7 +345,7 @@ const validateManifest = ( if (hostVersion.length === 0) { return yield* Effect.fail( new BinaryHostCompatibilityError({ - target: release.target, + target, detail: "Unable to determine macOS version", }), ); @@ -306,13 +353,12 @@ const validateManifest = ( if (compareVersions(hostVersion, floor) < 0) { return yield* Effect.fail( new BinaryHostCompatibilityError({ - target: release.target, + target, detail: `Host macOS ${hostVersion} is below manifest floor ${floor}`, }), ); } } - return manifest; }); const compareVersions = (left: string, right: string): number => { @@ -393,6 +439,29 @@ export class BinaryResolver extends Context.Service< return false; if (!("releaseSet" in value) || value.releaseSet !== "slim-services") return false; if (!("runtime" in value) || value.runtime !== "native") return false; + if (!("hostCompatibility" in value)) return false; + const host = value.hostCompatibility; + if (typeof host !== "object" || host === null) return false; + if ( + !("runtimeRequires" in host) || + (host.runtimeRequires !== null && host.runtimeRequires !== "glibc") + ) + return false; + if (!("libc" in host) || (host.libc !== null && host.libc !== "glibc")) return false; + if (!("osFloor" in host)) return false; + if (host.osFloor !== null) { + if (typeof host.osFloor !== "object") return false; + if ( + !("kind" in host.osFloor) || + (host.osFloor.kind !== "glibc" && host.osFloor.kind !== "macos") + ) + return false; + if ( + !("floor" in host.osFloor) || + (host.osFloor.floor !== null && typeof host.osFloor.floor !== "string") + ) + return false; + } return true; }; @@ -400,15 +469,20 @@ export class BinaryResolver extends Context.Service< directory: string, release: NativeReleaseArtifact, info: AssetInfo, + platform: { readonly os: string; readonly arch: string }, ) => Effect.gen(function* () { const marker = yield* fs .readFileString(path.join(directory, CACHE_COMPLETE_MARKER)) .pipe(Effect.option); if (Option.isNone(marker)) return false; - const parsed = yield* Effect.try({ - try: () => JSON.parse(marker.value), - catch: () => undefined, + const parsed = yield* Effect.sync(() => { + try { + const value: unknown = JSON.parse(marker.value); + return value; + } catch { + return undefined; + } }); if (!isCacheCompleteMarker(parsed)) return false; if ( @@ -427,8 +501,15 @@ export class BinaryResolver extends Context.Service< const present = yield* Effect.forEach(requiredPaths, (entry) => fs.exists(path.join(directory, entry)), ); - return present.every(Boolean); - }).pipe(Effect.catch(() => Effect.succeed(false))); + if (!present.every(Boolean)) return false; + yield* validateHostCompatibility( + release.target, + parsed.hostCompatibility, + platform, + spawner, + ); + return true; + }).pipe(Effect.catchTag("PlatformError", () => Effect.succeed(false))); const resolveRelease = (spec: BinarySpec) => Effect.gen(function* () { @@ -458,7 +539,7 @@ export class BinaryResolver extends Context.Service< return cachePath(spec.cacheDir ?? binDir, info); }); - const cleanupStaleStaging = (directory: string, prefix: string) => + const cleanupStaleEntries = (directory: string, prefix: string) => fs.readDirectory(directory).pipe( Effect.flatMap((entries) => Effect.forEach( @@ -470,7 +551,7 @@ export class BinaryResolver extends Context.Service< Option.match(info.mtime, { onNone: () => Effect.void, onSome: (modifiedAt) => - Date.now() - modifiedAt.getTime() >= STALE_STAGING_AGE_MS + Date.now() - modifiedAt.getTime() >= STALE_PREPARATION_ENTRY_AGE_MS ? fs.remove(stagingPath, { recursive: true, force: true }) : Effect.void, }), @@ -533,7 +614,7 @@ export class BinaryResolver extends Context.Service< Effect.fail(new DownloadError({ url: release.manifestUrl, cause })), ), ); - yield* Effect.try({ + const hostCompatibility = yield* Effect.try({ try: () => { const parsed: unknown = JSON.parse(manifestText); return parsed; @@ -671,6 +752,7 @@ export class BinaryResolver extends Context.Service< }), ); } + return hostCompatibility; }); const resolveWithMetadata = (spec: BinarySpec, options?: ResolveBinaryOptions) => { @@ -680,8 +762,10 @@ export class BinaryResolver extends Context.Service< const cacheDir = cachePath(baseDir, info); const parentDir = path.dirname(cacheDir); const stagingPrefix = `.${release.assetName}.partial-`; - yield* cleanupStaleStaging(parentDir, stagingPrefix); - if (yield* isCompleteCache(cacheDir, release, info)) { + const publicationLock = path.join(parentDir, `.${release.assetName}.publication-lock`); + yield* cleanupStaleEntries(parentDir, stagingPrefix); + yield* cleanupStaleEntries(parentDir, path.basename(publicationLock)); + if (yield* isCompleteCache(cacheDir, release, info, platform)) { return { path: cacheDir, downloaded: false, @@ -695,7 +779,7 @@ export class BinaryResolver extends Context.Service< prefix: stagingPrefix, }); return yield* Effect.gen(function* () { - yield* extractRelease(release, stagingDir, platform); + const hostCompatibility = yield* extractRelease(release, stagingDir, platform); yield* fs.writeFile( path.join(stagingDir, CACHE_COMPLETE_MARKER), new TextEncoder().encode( @@ -708,6 +792,7 @@ export class BinaryResolver extends Context.Service< target: info.target, releaseSet: info.releaseSet, runtime: info.runtime, + hostCompatibility, }), ), ); @@ -719,23 +804,46 @@ export class BinaryResolver extends Context.Service< if (Result.isSuccess(publication)) { return { path: cacheDir, downloaded: true } satisfies ResolveBinaryResult; } - if (yield* isCompleteCache(cacheDir, release, info)) { + if (yield* isCompleteCache(cacheDir, release, info, platform)) { return { path: cacheDir, downloaded: false } satisfies ResolveBinaryResult; } - // A fully staged replacement is now available, so an incomplete - // destination can be reclaimed without risking the last usable - // cache entry. Retry publication once; persistent filesystem - // failures still surface instead of looping forever. - yield* fs.remove(cacheDir, { recursive: true, force: true }); - const retry = yield* fs.rename(stagingDir, cacheDir).pipe(Effect.result); - if (Result.isSuccess(retry)) { - return { path: cacheDir, downloaded: true } satisfies ResolveBinaryResult; - } - if (yield* isCompleteCache(cacheDir, release, info)) { - return { path: cacheDir, downloaded: false } satisfies ResolveBinaryResult; - } - return yield* Effect.fail(retry.failure); + return yield* Effect.scoped( + Effect.gen(function* () { + const acquirePublicationLock: Effect.Effect = + fs.makeDirectory(publicationLock).pipe( + Effect.retry({ + while: (error) => error.reason._tag === "AlreadyExists", + schedule: Schedule.recurs(1_200).pipe( + Schedule.addDelay(() => Effect.succeed(Duration.millis(25))), + ), + }), + ); + yield* Effect.acquireRelease(acquirePublicationLock, () => + fs + .remove(publicationLock, { recursive: true, force: true }) + .pipe(Effect.ignore), + ); + + // The destination may have changed while this resolver was + // waiting to repair it. Revalidate under the publication + // claim before removing anything shared. + if (yield* isCompleteCache(cacheDir, release, info, platform)) { + return { path: cacheDir, downloaded: false } satisfies ResolveBinaryResult; + } + + yield* fs.remove(cacheDir, { recursive: true, force: true }); + const retry = yield* fs.rename(stagingDir, cacheDir).pipe(Effect.result); + if (Result.isSuccess(retry)) { + return { path: cacheDir, downloaded: true } satisfies ResolveBinaryResult; + } + const retryFailure = retry.failure; + if (yield* isCompleteCache(cacheDir, release, info, platform)) { + return { path: cacheDir, downloaded: false } satisfies ResolveBinaryResult; + } + return yield* Effect.fail(retryFailure); + }), + ); }).pipe( Effect.ensuring( fs.remove(stagingDir, { recursive: true, force: true }).pipe(Effect.ignore), From f5f1603bf1c0e37078f41a9e07bf1c9fa876fc1b Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Thu, 20 Aug 2026 11:25:59 +0200 Subject: [PATCH 11/20] fix(stack): stabilize startup in CI --- .../branches/switch/switch.handler.ts | 14 +- .../src/next/commands/start/start.command.ts | 4 +- .../next/commands/status/status.handler.ts | 7 +- .../shared/telemetry/error-actionability.ts | 1 - packages/stack/src/LocalStack.ts | 6 +- .../src/PortAllocator.integration.test.ts | 6 +- packages/stack/src/PortAllocator.ts | 264 +++++++++++++++++- packages/stack/src/Stack.unit.test.ts | 129 ++++++++- packages/stack/src/StackPreparation.ts | 8 +- ...aged-manager-lifecycle.integration.test.ts | 51 +++- packages/stack/src/managed.ts | 1 - packages/stack/src/managed/document.ts | 1 + packages/stack/src/managed/lifecycle.ts | 6 +- packages/stack/src/managed/manager.ts | 40 +-- packages/stack/src/services/postgres.ts | 19 +- .../stack/src/services/services.unit.test.ts | 12 +- packages/stack/src/supervisor.ts | 13 +- .../tests/createStack-docker.e2e.test.ts | 10 +- packages/stack/tests/global-setup.ts | 2 +- 19 files changed, 507 insertions(+), 87 deletions(-) diff --git a/apps/cli/src/next/commands/branches/switch/switch.handler.ts b/apps/cli/src/next/commands/branches/switch/switch.handler.ts index 843e812ea4..d470318330 100644 --- a/apps/cli/src/next/commands/branches/switch/switch.handler.ts +++ b/apps/cli/src/next/commands/branches/switch/switch.handler.ts @@ -145,18 +145,18 @@ export const switchBranch = Effect.fn("branches.switch")(function* (opts: { // TODO: run `supabase pull` against the new branch before restarting the stack // so the local config reflects the branch's migrations and seed state. // `pull` does not exist yet. + const launch = stackCheck.value.launch; const launchConfig = - stackCheck.value.launch === undefined + launch === undefined ? toStartStackConfig([], undefined) : withServiceVersions( toStartStackConfig( - stackCheck.value.launch.excludedServices?.filter( - (service): service is ExcludedStackService => - excludedStackServices.some((candidate) => candidate === service), + launch.excludedServices?.filter((service): service is ExcludedStackService => + excludedStackServices.some((candidate) => candidate === service), ) ?? [], - stackCheck.value.launch.mode, + "mode" in launch ? launch.mode : undefined, ), - stackCheck.value.launch.versions, + launch.versions, ); const loadedProjectConfig = yield* loadProjectConfig(projectHome.projectRoot); @@ -166,7 +166,7 @@ export const switchBranch = Effect.fn("branches.switch")(function* (opts: { projectDir: projectHome.projectRoot, name: stackName, portIntents: managedPortIntents(launchConfig, loadedProjectConfig ?? undefined), - ...(stackCheck.value.launch !== undefined && { launch: stackCheck.value.launch }), + ...(launch !== undefined && { launch }), ...launchConfig, }); diff --git a/apps/cli/src/next/commands/start/start.command.ts b/apps/cli/src/next/commands/start/start.command.ts index 38262c1814..759faae548 100644 --- a/apps/cli/src/next/commands/start/start.command.ts +++ b/apps/cli/src/next/commands/start/start.command.ts @@ -87,7 +87,7 @@ const modeFlag = Flag.choice("mode", startModes).pipe( interface StartVersionStateShape { readonly launch: { - readonly mode: StartMode; + readonly mode?: StartMode; readonly versions: Readonly>; readonly excludedServices: ReadonlyArray; }; @@ -251,7 +251,7 @@ export const startCommand = Command.make("start", flags).pipe( stackLayer, startVersionState: StartVersionState.of({ launch: { - mode: summary.launch.mode, + mode: "mode" in summary.launch ? summary.launch.mode : undefined, versions: serviceVersionContext.pinnedBaseline, excludedServices: flags.exclude, }, diff --git a/apps/cli/src/next/commands/status/status.handler.ts b/apps/cli/src/next/commands/status/status.handler.ts index f11f2a1e54..4330121f07 100644 --- a/apps/cli/src/next/commands/status/status.handler.ts +++ b/apps/cli/src/next/commands/status/status.handler.ts @@ -40,12 +40,11 @@ const resolveConfiguredSummary = Effect.fnUntraced(function* (input: { const current = yield* resolveStackSummary(input); const loaded = yield* loadProjectConfig(input.projectDir); const excluded = (current.launch?.excludedServices ?? []).filter(isExcludedStackService); + const mode = + current.launch !== undefined && "mode" in current.launch ? current.launch.mode : undefined; return yield* resolveStackSummary({ ...input, - portDocument: managedPortIntents( - toStartStackConfig(excluded, current.launch?.mode), - loaded ?? undefined, - ), + portDocument: managedPortIntents(toStartStackConfig(excluded, mode), loaded ?? undefined), }); }); diff --git a/apps/cli/src/shared/telemetry/error-actionability.ts b/apps/cli/src/shared/telemetry/error-actionability.ts index e83b67d5f2..b8d6ad505c 100644 --- a/apps/cli/src/shared/telemetry/error-actionability.ts +++ b/apps/cli/src/shared/telemetry/error-actionability.ts @@ -1055,7 +1055,6 @@ const externalActionabilityByTag: Record = { ...actionability.stopStack, fingerprint_suffix: "managed_attached", }), - ManagedStackLaunchMissingError: () => actionability.startStack, ManagedWorkspaceRepairConflictError: () => ({ ...actionability.invalidInput, fingerprint_suffix: "managed_workspace_repair", diff --git a/packages/stack/src/LocalStack.ts b/packages/stack/src/LocalStack.ts index 85fe8cfe63..ef2b67498b 100644 --- a/packages/stack/src/LocalStack.ts +++ b/packages/stack/src/LocalStack.ts @@ -821,7 +821,6 @@ export const localStackLayer = ( yield* Ref.set(phaseRef, "starting"); const runtime = yield* ensureRuntime; yield* configureFunctions(config, yield* Ref.get(functionsBundleRef)); - serviceStartupBegan = true; const eager = eagerServices(enabledServices, config.servicePolicies); const allServicesEager = eager.length === enabledServices.length; @@ -832,6 +831,7 @@ export const localStackLayer = ( if ( runtime.graph.startOrder.some((definition) => definition.name === "postgres-init") ) { + serviceStartupBegan = true; yield* runtime.orchestrator .startService("postgres-init", serviceStartOptions) .pipe( @@ -851,9 +851,10 @@ export const localStackLayer = ( } for (const service of eager) { yield* requireMutable("start"); + serviceStartupBegan = true; const started = yield* beginStartTargets( service, - new Set(lifecycleTargetsForService(enabledServices, service)), + new Set(activationTargetsForService(enabledServices, service)), ); readiness.push(waitForTargets(started)); } @@ -864,6 +865,7 @@ export const localStackLayer = ( } else { yield* prepareServices(enabledServices); yield* requireMutable("start"); + serviceStartupBegan = true; yield* runtime.orchestrator.start(serviceStartOptions); yield* runtime.orchestrator .waitAllReady() diff --git a/packages/stack/src/PortAllocator.integration.test.ts b/packages/stack/src/PortAllocator.integration.test.ts index 7703fe9498..85063fa5ad 100644 --- a/packages/stack/src/PortAllocator.integration.test.ts +++ b/packages/stack/src/PortAllocator.integration.test.ts @@ -53,12 +53,12 @@ describe("selected-field port allocation", () => { expect(unavailable._tag).toBe("Failure"); await Effect.runPromise(lease.release(["apiPort"])); - const rebound = await Effect.runPromise( + const unavailableWhileLeaseIsActive = await Effect.runPromise( allocatePortSet([ { field: "apiPort", selection: { kind: "exact", port: lease.ports.apiPort! } }, - ]), + ]).pipe(Effect.exit), ); - expect(rebound.apiPort).toBe(lease.ports.apiPort); + expect(unavailableWhileLeaseIsActive._tag).toBe("Failure"); await Effect.runPromise(lease.reserve(["apiPort"])); const unavailableAgain = await Effect.runPromise( diff --git a/packages/stack/src/PortAllocator.ts b/packages/stack/src/PortAllocator.ts index 6eb7b3ff58..c353a5bb5a 100644 --- a/packages/stack/src/PortAllocator.ts +++ b/packages/stack/src/PortAllocator.ts @@ -1,4 +1,8 @@ +import { randomUUID } from "node:crypto"; +import { mkdir, open, readFile, readdir, rm, stat } from "node:fs/promises"; import { createServer, type Server } from "node:net"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { Data, Effect, Schema, Semaphore } from "effect"; import { PortSetSchema, type PortField, type PortSet } from "./PortCatalog.ts"; @@ -108,6 +112,147 @@ interface BoundPort { readonly server: Server; } +interface PortClaim { + readonly path: string; + readonly port: number; + readonly token: string; +} + +interface ClaimRecord { + readonly pid: number; + readonly token: string; +} + +const CLAIM_ROOT = join(tmpdir(), "supabase-stack-port-claims"); +const CLAIM_STALE_AFTER_MS = 30_000; + +const claimPath = (port: number): string => join(CLAIM_ROOT, `port-${port}`); + +const isProcessAlive = (pid: number): boolean => { + try { + process.kill(pid, 0); + return true; + } catch (cause) { + return typeof cause === "object" && cause !== null && "code" in cause + ? Reflect.get(cause, "code") !== "ESRCH" + : false; + } +}; + +const readClaimRecord = async (path: string): Promise => { + let contents: string; + try { + contents = await readFile(path, "utf8"); + } catch (cause) { + if (typeof cause === "object" && cause !== null && "code" in cause) { + if (Reflect.get(cause, "code") === "ENOENT") return undefined; + } + throw cause; + } + try { + const value: unknown = JSON.parse(contents); + if (typeof value !== "object" || value === null) return undefined; + const pid = Reflect.get(value, "pid"); + const token = Reflect.get(value, "token"); + return typeof pid === "number" && Number.isInteger(pid) && pid > 0 && typeof token === "string" + ? { pid, token } + : undefined; + } catch { + return undefined; + } +}; + +const claimIsStale = async (path: string): Promise => { + const record = await readClaimRecord(path); + if (record !== undefined) return !isProcessAlive(record.pid); + try { + const info = await stat(path); + return Date.now() - info.mtimeMs > CLAIM_STALE_AFTER_MS; + } catch (cause) { + if (typeof cause === "object" && cause !== null && "code" in cause) { + if (Reflect.get(cause, "code") === "ENOENT") return true; + } + throw cause; + } +}; + +const acquirePortClaim = (port: number): Effect.Effect => + Effect.tryPromise({ + try: async () => { + await mkdir(CLAIM_ROOT, { recursive: true }); + const path = claimPath(port); + const token = randomUUID(); + const contents = JSON.stringify({ pid: process.pid, token }); + + while (true) { + try { + const handle = await open(path, "wx"); + try { + await handle.writeFile(contents, "utf8"); + } finally { + await handle.close(); + } + return { path, port, token }; + } catch (cause) { + if ( + typeof cause !== "object" || + cause === null || + !("code" in cause) || + Reflect.get(cause, "code") !== "EEXIST" + ) { + throw cause; + } + if (!(await claimIsStale(path))) { + throw new PortAllocationError({ detail: `Port ${port} is not available`, port }); + } + await rm(path, { force: true }); + } + } + }, + catch: (cause) => + cause instanceof PortAllocationError + ? cause + : new PortAllocationError({ detail: `Failed to claim port ${port}`, cause, port }), + }); + +const releasePortClaim = (claim: PortClaim): Effect.Effect => + Effect.tryPromise({ + try: async () => { + const record = await readClaimRecord(claim.path); + if (record?.token !== claim.token || record.pid !== process.pid) return; + await rm(claim.path, { force: true }); + }, + catch: () => undefined, + }).pipe( + Effect.catch(() => Effect.void), + Effect.asVoid, + ); + +const claimedPorts = (): Effect.Effect> => + Effect.tryPromise({ + try: async () => { + const entries = await readdir(CLAIM_ROOT, { withFileTypes: true }).catch((cause) => { + if (typeof cause === "object" && cause !== null && "code" in cause) { + if (Reflect.get(cause, "code") === "ENOENT") return []; + } + throw cause; + }); + const ports = new Set(); + for (const entry of entries) { + if (!entry.isFile() || !entry.name.startsWith("port-")) continue; + const port = Number(entry.name.slice("port-".length)); + if (!Number.isInteger(port) || port <= 0 || port > 65_535) continue; + if (await claimIsStale(join(CLAIM_ROOT, entry.name))) { + await rm(join(CLAIM_ROOT, entry.name), { force: true }); + continue; + } + ports.add(port); + } + return ports; + }, + catch: () => new Set(), + }).pipe(Effect.catch(() => Effect.succeed(new Set()))); + const bindPort = (port: number): Effect.Effect => Effect.callback((resume) => { const server = createServer((socket) => socket.destroy()); @@ -144,7 +289,9 @@ const bindPort = (port: number): Effect.Effect = export interface PortLease { readonly ports: PortSet; readonly reserve: (fields: ReadonlyArray) => Effect.Effect; + /** Releases TCP reservations while retaining ownership claims for this lease. */ readonly release: (fields: ReadonlyArray) => Effect.Effect; + /** Releases all TCP reservations and ends ownership of every selected port. */ readonly releaseAll: Effect.Effect; } @@ -183,13 +330,53 @@ const releaseReservations = ( { discard: true }, ); +const releaseClaims = ( + claims: Map, + fields: ReadonlyArray, +): Effect.Effect => + Effect.forEach( + uniquePortFields(fields), + (field) => { + const claim = claims.get(field); + if (claim === undefined) return Effect.void; + claims.delete(field); + return releasePortClaim(claim); + }, + { discard: true }, + ); + +const claimAndBind = ( + field: PortField, + port: number, + claims: Map, +): Effect.Effect => { + const existingClaim = claims.get(field); + return ( + existingClaim === undefined ? acquirePortClaim(port) : Effect.succeed(existingClaim) + ).pipe( + Effect.flatMap((claim) => + bindPort(port).pipe( + Effect.onError(() => (existingClaim === undefined ? releasePortClaim(claim) : Effect.void)), + Effect.tap(({ server }) => + Effect.sync(() => { + claims.set(field, claim); + return server; + }), + ), + ), + ), + ); +}; + const reserveReservations = ( ports: PortSet, reservations: Map, + claims: Map, fields: ReadonlyArray, ): Effect.Effect => Effect.suspend(() => { const acquired: Array = []; + const acquiredClaims: Array = []; return Effect.forEach( uniquePortFields(fields), (field) => { @@ -203,39 +390,70 @@ const reserveReservations = ( }), ); } - return bindPort(port).pipe( + const existingClaim = claims.has(field); + return claimAndBind(field, port, claims).pipe( Effect.mapError((error) => withPortField(field, error)), Effect.tap(({ server }) => Effect.sync(() => { reservations.set(field, server); acquired.push(field); + if (!existingClaim) acquiredClaims.push(field); }), ), ); }, { discard: true }, - ).pipe(Effect.onError(() => releaseReservations(reservations, acquired))); + ).pipe( + Effect.onError(() => + Effect.all( + [releaseReservations(reservations, acquired), releaseClaims(claims, acquiredClaims)], + { discard: true }, + ), + ), + ); }); -const makePortLease = (ports: PortSet, reservations: Map): PortLease => { +const makePortLease = ( + ports: PortSet, + reservations: Map, + claims: Map, +): PortLease => { const lock = Semaphore.makeUnsafe(1); return { ports, - reserve: (fields) => lock.withPermit(reserveReservations(ports, reservations, fields)), + reserve: (fields) => lock.withPermit(reserveReservations(ports, reservations, claims, fields)), release: (fields) => lock.withPermit(releaseReservations(reservations, fields)), releaseAll: lock.withPermit( - Effect.suspend(() => releaseReservations(reservations, [...reservations.keys()])), + Effect.suspend(() => + Effect.all( + [ + releaseReservations(reservations, [...reservations.keys()]), + releaseClaims(claims, [...claims.keys()]), + ], + { discard: true }, + ), + ), ), }; }; const reserveRandomPort = ( exclude: ReadonlySet, + field: PortField, + claims: Map, ): Effect.Effect => Effect.flatMap(bindPort(0), (bound) => exclude.has(bound.port) - ? closeServer(bound.server).pipe(Effect.andThen(reserveRandomPort(exclude))) - : Effect.succeed(bound), + ? closeServer(bound.server).pipe(Effect.andThen(reserveRandomPort(exclude, field, claims))) + : acquirePortClaim(bound.port).pipe( + Effect.tap((claim) => Effect.sync(() => claims.set(field, claim))), + Effect.map(() => bound), + Effect.catchTag("PortAllocationError", () => + closeServer(bound.server).pipe( + Effect.andThen(reserveRandomPort(exclude, field, claims)), + ), + ), + ), ); const resolveSelection = ( @@ -264,7 +482,7 @@ export const allocatePortSet = ( options: PortAllocationOptions = {}, ): Effect.Effect => Effect.gen(function* () { - const reserved = options.reserved ?? new Set(); + const reserved = new Set([...(options.reserved ?? []), ...(yield* claimedPorts())]); const probe = options.probe ?? defaultPortProbe; const allocated = new Set(); const partial: Partial> = {}; @@ -287,6 +505,7 @@ export const reservePortSet = ( ): Effect.Effect => Effect.suspend(() => { const reservations = new Map(); + const claims = new Map(); const reserve = Effect.gen(function* () { const reserved = options.reserved ?? new Set(); const allocated = new Set(); @@ -320,26 +539,43 @@ export const reservePortSet = ( port: selection.port, }); } - bound = yield* bindAndRegister(request.field, bindPort(selection.port)); + bound = yield* bindAndRegister( + request.field, + claimAndBind(request.field, selection.port, claims), + ); } else if (selection.preferred !== undefined && !exclude.has(selection.preferred)) { + const preferred = selection.preferred; bound = yield* bindAndRegister( request.field, - bindPort(selection.preferred).pipe( - Effect.catchTag("PortAllocationError", () => reserveRandomPort(exclude)), + claimAndBind(request.field, preferred, claims).pipe( + Effect.catchTag("PortAllocationError", () => + reserveRandomPort(exclude, request.field, claims), + ), ), ); } else { - bound = yield* bindAndRegister(request.field, reserveRandomPort(exclude)); + bound = yield* bindAndRegister( + request.field, + reserveRandomPort(exclude, request.field, claims), + ); } allocated.add(bound.port); partial[request.field] = bound.port; } - return makePortLease(Schema.decodeUnknownSync(PortSetSchema)(partial), reservations); + return makePortLease(Schema.decodeUnknownSync(PortSetSchema)(partial), reservations, claims); }); return reserve.pipe( - Effect.onError(() => releaseReservations(reservations, [...reservations.keys()])), + Effect.onError(() => + Effect.all( + [ + releaseReservations(reservations, [...reservations.keys()]), + releaseClaims(claims, [...claims.keys()]), + ], + { discard: true }, + ), + ), ); }); diff --git a/packages/stack/src/Stack.unit.test.ts b/packages/stack/src/Stack.unit.test.ts index c5cb93742d..259c5fb7d4 100644 --- a/packages/stack/src/Stack.unit.test.ts +++ b/packages/stack/src/Stack.unit.test.ts @@ -752,14 +752,139 @@ describe("Stack", () => { return Effect.gen(function* () { const stack = yield* Stack; - expect(Exit.isFailure(yield* stack.start().pipe(Effect.exit))).toBe(true); yield* stack.start(); - expect(buildAttempts).toBe(2); }).pipe(Effect.provide(layer), Effect.timeout("5 seconds")); }); + it.live("can retry start after asset preparation fails before services start", () => { + const resolver = mockBinaryResolver({ + failOnceServices: ["postgres"], + }); + const config = { + ...defaultConfig, + postgrest: false, + auth: false, + servicePolicies: { + ...defaultConfig.servicePolicies, + postgrest: "off", + auth: "off", + }, + } satisfies ResolvedStackConfig; + const { layer } = setupLayer(config, noopPortLease(config.ports), undefined, resolver); + + return Effect.gen(function* () { + const stack = yield* Stack; + expect(Exit.isFailure(yield* stack.start().pipe(Effect.exit))).toBe(true); + yield* stack.start(); + expect((yield* stack.getState("postgres")).status).toBe("Healthy"); + }).pipe(Effect.provide(layer), Effect.timeout("5 seconds")); + }); + + it.live("restarts activated companions after stopping the stack", () => { + const graph = Effect.runSync( + buildGraph([ + { + name: "postgres", + command: process.execPath, + restart: "no", + healthCheck: { probe: { _tag: "Exec", command: "true", args: [] } }, + }, + { + name: "postgrest", + command: process.execPath, + restart: "no", + healthCheck: { probe: { _tag: "Exec", command: "true", args: [] } }, + }, + { + name: "pgmeta", + command: process.execPath, + restart: "no", + healthCheck: { probe: { _tag: "Exec", command: "true", args: [] } }, + }, + { + name: "studio", + command: process.execPath, + restart: "no", + healthCheck: { probe: { _tag: "Exec", command: "true", args: [] } }, + }, + { + name: "analytics", + command: process.execPath, + restart: "no", + healthCheck: { probe: { _tag: "Exec", command: "true", args: [] } }, + }, + { + name: "vector", + command: process.execPath, + restart: "no", + healthCheck: { probe: { _tag: "Exec", command: "true", args: [] } }, + }, + ]), + ); + const builderLayer = Layer.succeed(StackBuilder, { + build: () => + Effect.succeed({ + graph, + cleanupTargets: { dockerContainerNames: [] }, + serviceProjection: new Map([ + ["postgres", { visibility: "public" as const }], + ["postgrest", { visibility: "public" as const }], + ["pgmeta", { visibility: "public" as const }], + ["studio", { visibility: "public" as const }], + ["analytics", { visibility: "public" as const }], + ["vector", { visibility: "public" as const }], + ]), + }), + }); + const config = { + ...defaultConfig, + mode: "docker", + containerRuntime: "docker", + pgmeta: { port: defaultPorts.pgmetaPort, version: DEFAULT_VERSIONS.pgmeta }, + studio: { + port: defaultPorts.studioPort, + apiUrl: "http://127.0.0.1:54321", + version: DEFAULT_VERSIONS.studio, + }, + analytics: { + port: defaultPorts.analyticsPort, + version: DEFAULT_VERSIONS.analytics, + backend: "postgres", + apiKey: "test-api-key", + }, + vector: { version: DEFAULT_VERSIONS.vector }, + servicePolicies: { + ...defaultConfig.servicePolicies, + auth: "lazy", + pgmeta: "eager", + studio: "eager", + analytics: "eager", + vector: "eager", + }, + } satisfies ResolvedStackConfig; + const { resolver, spawner } = setupLayer(config, noopPortLease(config.ports)); + const stackPreparationLayer = StackPreparation.layer.pipe(Layer.provide(resolver.layer)); + const layer = localStackLayer(config, noopPortLease(config.ports)).pipe( + Layer.provide(builderLayer), + Layer.provide(stackPreparationLayer), + Layer.provide(spawner.layer), + Layer.provide(NodeServices.layer), + ); + + return Effect.gen(function* () { + const stack = yield* Stack; + yield* stack.start(); + yield* stack.stop(); + yield* stack.start(); + + expect((yield* stack.getState("studio")).status).toBe("Healthy"); + expect((yield* stack.getState("analytics")).status).toBe("Healthy"); + expect((yield* stack.getState("vector")).status).toBe("Healthy"); + }).pipe(Effect.provide(layer), Effect.timeout("10 seconds")); + }); + it.live("rejects a cached start when disposal begins during startup", () => Effect.gen(function* () { const startEntered = yield* Deferred.make(); diff --git a/packages/stack/src/StackPreparation.ts b/packages/stack/src/StackPreparation.ts index 9fcf76fd01..19492433a1 100644 --- a/packages/stack/src/StackPreparation.ts +++ b/packages/stack/src/StackPreparation.ts @@ -101,6 +101,10 @@ class PullAttemptError extends Error { } } +const pullRetrySchedule = Schedule.exponential(Duration.seconds(1)).pipe( + Schedule.upTo({ times: 5 }), +); + const resolveDockerImageForService = ( spawner: ChildProcessSpawner.ChildProcessSpawner["Service"], runtime: ContainerRuntime, @@ -333,9 +337,7 @@ const pullImage = ( const attempt = runPullCommand(spawner, runtime, image).pipe( Effect.retry({ while: (error) => shouldRetryPull(error.detail), - schedule: Schedule.recurs(1).pipe( - Schedule.addDelay(() => Effect.succeed(Duration.millis(500))), - ), + schedule: pullRetrySchedule, }), ); const result = yield* Effect.exit(attempt); diff --git a/packages/stack/src/managed-manager-lifecycle.integration.test.ts b/packages/stack/src/managed-manager-lifecycle.integration.test.ts index 35c646643b..5004e226f7 100644 --- a/packages/stack/src/managed-manager-lifecycle.integration.test.ts +++ b/packages/stack/src/managed-manager-lifecycle.integration.test.ts @@ -46,7 +46,7 @@ describe("managed stack lifecycle journeys", () => { const manager = yield* ManagedStackManager; const [apiPort, dbPort] = yield* freePorts(2); if (apiPort === undefined || dbPort === undefined) { - throw new Error("expected interrupted-delete ports"); + throw new Error("expected free managed stack ports"); } const portDocument = exactCoreDocument(apiPort, dbPort); const environment = yield* ensureEnvironment(workspace); @@ -89,6 +89,55 @@ describe("managed stack lifecycle journeys", () => { ); }); + it.live("updates launch metadata before a runtime has been selected", () => { + const { layer, workspace } = setup(); + return Effect.scoped( + Effect.gen(function* () { + const manager = yield* ManagedStackManager; + const [apiPort, dbPort] = yield* freePorts(2); + if (apiPort === undefined || dbPort === undefined) { + throw new Error("expected free managed stack ports"); + } + const environment = yield* ensureEnvironment(workspace); + const stackId = deriveStackId(environment.identity, "default"); + const owner = yield* acquireControl({ stackId }); + if (owner._tag !== "Owned") throw new Error("expected stack control ownership"); + const started = yield* manager.startStack({ + workspacePath: workspace, + stackName: "default", + portDocument: exactCoreDocument(apiPort, dbPort), + ownership: owner, + lifecycle: "running", + }); + yield* releaseLease(started); + yield* owner.close; + + const updated = yield* updateManagedLaunch({ + workspacePath: workspace, + stackName: "default", + launch: { + versions: { postgres: "17.6.1" }, + excludedServices: ["studio"], + lastNotifiedUpdateFingerprint: "fingerprint", + }, + }); + + expect(updated.launch).toEqual({ + versions: { postgres: "17.6.1" }, + excludedServices: ["studio"], + lastNotifiedUpdateFingerprint: "fingerprint", + }); + }), + ).pipe( + Effect.provide(layer), + Effect.provide(NodeFileSystem.layer), + Effect.provide(NodePath.layer), + Effect.provide(gitConfigStoreLayer), + Effect.provide(controlTransportLayer), + Effect.provide(httpTransportClientLayer), + ); + }); + it.live("stops an owner whose document is still starting", () => { const { layer, workspace } = setup(); return Effect.scoped( diff --git a/packages/stack/src/managed.ts b/packages/stack/src/managed.ts index 4b7942ff9e..79cfbe8538 100644 --- a/packages/stack/src/managed.ts +++ b/packages/stack/src/managed.ts @@ -53,7 +53,6 @@ export { deriveRepairOwnershipId, ManagedStackControlRequiredError, ManagedStackAttachedError, - ManagedStackLaunchMissingError, ManagedWorkspaceRepairConflictError, } from "./managed/manager.ts"; export type { diff --git a/packages/stack/src/managed/document.ts b/packages/stack/src/managed/document.ts index 8684294caa..2299e5d870 100644 --- a/packages/stack/src/managed/document.ts +++ b/packages/stack/src/managed/document.ts @@ -28,6 +28,7 @@ const managedStackLaunchSchema = Schema.Union([ containerRuntime: Schema.Literals(["docker", "podman"] as const), ...managedStackLaunchFields, }), + Schema.Struct(managedStackLaunchFields), ]); export type ManagedStackLaunch = Schema.Schema.Type; diff --git a/packages/stack/src/managed/lifecycle.ts b/packages/stack/src/managed/lifecycle.ts index 73c1fee78f..95e5850e54 100644 --- a/packages/stack/src/managed/lifecycle.ts +++ b/packages/stack/src/managed/lifecycle.ts @@ -115,7 +115,11 @@ export const stopManagedStack = ( const document = yield* resolveManagedDocument(input); const stackId = document.id; const containerRuntime = - document.launch?.mode === "docker" ? (document.launch.containerRuntime ?? "docker") : null; + document.launch !== undefined && + "mode" in document.launch && + document.launch.mode === "docker" + ? document.launch.containerRuntime + : null; const acquisition = yield* manager.acquireControl(stackId); const revalidatedStackId = yield* stackIdForInput(manager, input); if (revalidatedStackId !== stackId) { diff --git a/packages/stack/src/managed/manager.ts b/packages/stack/src/managed/manager.ts index cb02e9044c..4faf99e142 100644 --- a/packages/stack/src/managed/manager.ts +++ b/packages/stack/src/managed/manager.ts @@ -153,16 +153,6 @@ export class ManagedStackAttachedError extends Data.TaggedError("ManagedStackAtt } } -export class ManagedStackLaunchMissingError extends Data.TaggedError( - "ManagedStackLaunchMissingError", -)<{ - readonly stackId: string; -}> { - override get message(): string { - return `Managed stack ${this.stackId} has no selected runtime`; - } -} - export class ManagedWorkspaceRepairConflictError extends Data.TaggedError( "ManagedWorkspaceRepairConflictError", )<{ @@ -189,7 +179,6 @@ export const workspaceRepairConflict = ( export type ManagedStackManagerError = | ManagedStackControlRequiredError | ManagedStackAttachedError - | ManagedStackLaunchMissingError | ManagedWorkspaceRepairConflictError | ManagedStackNotFoundError | ManagedStackNotStoppedError @@ -863,11 +852,6 @@ const makeManager = ( if (current === undefined) { return yield* Effect.fail(new ManagedStackNotFoundError({ stackId: update.stackId })); } - if (current.launch === undefined) { - return yield* Effect.fail( - new ManagedStackLaunchMissingError({ stackId: update.stackId }), - ); - } const metadata = { versions: update.launch.versions, ...(update.launch.excludedServices === undefined @@ -880,13 +864,15 @@ const makeManager = ( }), }; const launch: NonNullable = - current.launch.mode === "native" - ? { ...metadata, mode: "native" } - : { - ...metadata, - mode: "docker", - containerRuntime: current.launch.containerRuntime, - }; + current.launch === undefined || !("mode" in current.launch) + ? metadata + : current.launch.mode === "native" + ? { ...metadata, mode: "native" } + : { + ...metadata, + mode: "docker", + containerRuntime: current.launch.containerRuntime, + }; const next: ManagedStackDocument = { ...current, launch, @@ -1011,9 +997,13 @@ const makeManager = ( if (current === undefined) return { outcome: "already-absent", stackId }; if ("outcome" in current) return current; yield* acquisition.setState("deleting", false); - if (current.launch?.mode === "docker") { + if ( + current.launch !== undefined && + "mode" in current.launch && + current.launch.mode === "docker" + ) { yield* dockerForceRemove( - current.launch.containerRuntime ?? "docker", + current.launch.containerRuntime, SERVICE_NAMES.map((service) => dockerContainerName(service, `id-${stackId}`)), ); } diff --git a/packages/stack/src/services/postgres.ts b/packages/stack/src/services/postgres.ts index b3e2dc0cff..44f58f0ee3 100644 --- a/packages/stack/src/services/postgres.ts +++ b/packages/stack/src/services/postgres.ts @@ -67,12 +67,14 @@ const postgresHealthCheck = (binPath: string, port: number) => ({ }); /** - * Docker postgres health check using pg_isready inside the container. + * Docker postgres health check using the final postgres process and pg_isready + * inside the container. * - * TCP alone is insufficient because the supabase/postgres image accepts TCP - * connections during its init phase (running init scripts) but drops real - * queries with "unexpected EOF". We use `docker exec` to run pg_isready - * inside the container, which verifies postgres is accepting commands. + * The supabase/postgres image briefly accepts connections while its entrypoint + * runs initialization. During that phase PID 1 is still the shell and the + * temporary server is stopped before the final postgres process starts. Gate + * readiness on the final Postgres executable as PID 1 and pg_isready so + * dependents never race that handoff. */ const postgresDockerHealthCheck = ( runtime: DockerPostgresOptions["runtime"], @@ -82,8 +84,11 @@ const postgresDockerHealthCheck = ( dockerExecHealthCheck( runtime, containerName, - "pg_isready", - ["-h", "127.0.0.1", "-p", String(port), "-U", "postgres"], + "sh", + [ + "-ec", + `case "$(readlink /proc/1/exe)" in */postgres|*/.postgres-wrapped) pg_isready -h 127.0.0.1 -p ${port} -U postgres ;; *) exit 1 ;; esac`, + ], { ...stackHealthBudgets.postgresDocker, }, diff --git a/packages/stack/src/services/services.unit.test.ts b/packages/stack/src/services/services.unit.test.ts index 09e1f80f5a..7afebb22fb 100644 --- a/packages/stack/src/services/services.unit.test.ts +++ b/packages/stack/src/services/services.unit.test.ts @@ -149,20 +149,16 @@ describe("makePostgresServiceDocker", () => { expect(def.args).toContain("/usr/bin/sh"); expect(def.args?.at(-2)).toBe("-c"); expect(def.args?.at(-1)).toContain("hba_file=/tmp/supabase-cli-pg_hba.conf"); - // Health check uses docker exec + pg_isready inside the container (host has no postgres tools) + // Health check waits for the final postgres PID 1 process before probing it. expect(def.healthCheck?.probe).toEqual({ _tag: "Exec", command: "docker", args: [ "exec", `supabase-postgres-${API_PORT}`, - "pg_isready", - "-h", - "127.0.0.1", - "-p", - "54322", - "-U", - "postgres", + "sh", + "-ec", + 'case "$(readlink /proc/1/exe)" in */postgres|*/.postgres-wrapped) pg_isready -h 127.0.0.1 -p 54322 -U postgres ;; *) exit 1 ;; esac', ], }); expect(def.dependencies).toEqual([]); diff --git a/packages/stack/src/supervisor.ts b/packages/stack/src/supervisor.ts index ab5b5ce7af..e64f116867 100644 --- a/packages/stack/src/supervisor.ts +++ b/packages/stack/src/supervisor.ts @@ -467,12 +467,17 @@ const runManaged = ( } } const existing = yield* manager.inspectStack(stackId); - const requestedMode = configInput.mode ?? input.launch?.mode; + const requestedMode = + configInput.mode ?? + (input.launch !== undefined && "mode" in input.launch ? input.launch.mode : undefined); + const existingLaunch = existing?.launch; const persistedRuntime: StackRuntimeSelection | undefined = - existing?.launch?.mode === "native" + existingLaunch !== undefined && "mode" in existingLaunch && existingLaunch.mode === "native" ? { mode: "native", containerRuntime: null } - : existing?.launch?.mode === "docker" - ? { mode: "docker", containerRuntime: existing.launch.containerRuntime } + : existingLaunch !== undefined && + "mode" in existingLaunch && + existingLaunch.mode === "docker" + ? { mode: "docker", containerRuntime: existingLaunch.containerRuntime } : undefined; if ( persistedRuntime !== undefined && diff --git a/packages/stack/tests/createStack-docker.e2e.test.ts b/packages/stack/tests/createStack-docker.e2e.test.ts index 8da2ae9f9f..778bb1b75e 100644 --- a/packages/stack/tests/createStack-docker.e2e.test.ts +++ b/packages/stack/tests/createStack-docker.e2e.test.ts @@ -51,7 +51,15 @@ dockerDescribe("createStack e2e (docker mode)", () => { } const dbPort = parseInt(new URL(stack.dbUrl).port); - await setupTestTable(dbPort); + try { + await setupTestTable(dbPort); + } catch (error) { + const status = await stack.getStatus(); + const logs = await stack.logHistory("postgres"); + throw new Error( + `setupTestTable failed: ${String(error)}\nstatus=${JSON.stringify(status)}\nlogs=${JSON.stringify(logs)}`, + ); + } apiPort = new URL(stack.url).port; supabase = createClient(stack.url, stack.publishableKey); diff --git a/packages/stack/tests/global-setup.ts b/packages/stack/tests/global-setup.ts index f396e68f24..108d681c57 100644 --- a/packages/stack/tests/global-setup.ts +++ b/packages/stack/tests/global-setup.ts @@ -1,5 +1,5 @@ import { warmStackE2eDependencies } from "./helpers/warmup.ts"; export async function setup(): Promise { - await warmStackE2eDependencies(); + await warmStackE2eDependencies({ failOnError: true }); } From 066c466b665143c5b2a3165e1ea3b24bf1a746bc Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Thu, 20 Aug 2026 12:08:48 +0200 Subject: [PATCH 12/20] fix(stack): stabilize Linux Docker startup --- packages/stack/src/LocalStack.ts | 55 +++++++++- packages/stack/src/PortAllocator.ts | 10 +- packages/stack/src/ServiceCatalog.ts | 2 + packages/stack/src/Stack.unit.test.ts | 65 ++++++++++-- .../StackConfigResolver.policy.unit.test.ts | 23 ++++ packages/stack/src/StackConfigResolver.ts | 35 +++++- packages/stack/src/services/docker-cleanup.ts | 100 ++++++++++++++++-- packages/stack/src/services/postgres.ts | 47 +++++++- packages/stack/src/services/service-utils.ts | 13 ++- .../stack/src/services/services.unit.test.ts | 12 +-- 10 files changed, 325 insertions(+), 37 deletions(-) diff --git a/packages/stack/src/LocalStack.ts b/packages/stack/src/LocalStack.ts index ef2b67498b..a829b6cdbb 100644 --- a/packages/stack/src/LocalStack.ts +++ b/packages/stack/src/LocalStack.ts @@ -69,6 +69,9 @@ type LifecyclePhase = "idle" | "starting" | "running" | "stopping" | "stopped" | type StackService = typeof Stack.Service; +const READINESS_DIAGNOSTIC_LOG_LIMIT = 20; +const READINESS_DIAGNOSTIC_LINE_LIMIT = 512; + /** Private signal used by the Promise adapter to close its enclosing managed runtime. */ export class LocalStackLifecycle extends Context.Service< LocalStackLifecycle, @@ -764,12 +767,60 @@ export const localStackLayer = ( }), ); }; + const readinessErrorWithDiagnostics = ( + error: StackReadinessError, + ): Effect.Effect => + Effect.gen(function* () { + if (runtimeState === undefined) return error; + + const [states, logs] = yield* Effect.all([ + runtimeState.orchestrator.getAllStates(), + logBuffer.historyAll(READINESS_DIAGNOSTIC_LOG_LIMIT), + ]); + const nonReadyStates = states.filter( + (state) => + state.status !== "Healthy" && !(state.status === "Stopped" && state.exitCode === 0), + ); + const stateDetail = + nonReadyStates.length === 0 + ? "none" + : nonReadyStates + .map((state) => { + const errorDetail = + state.error === null + ? "" + : `, error=${state.error.slice(0, READINESS_DIAGNOSTIC_LINE_LIMIT)}`; + return `${state.name}: ${state.status} (desired=${state.desired}, restarts=${state.restartCount}${errorDetail})`; + }) + .join("; "); + const logDetail = + logs.length === 0 + ? "none" + : logs + .map( + (entry) => + `[${entry.service}/${entry.stream}] ${entry.line.slice(0, READINESS_DIAGNOSTIC_LINE_LIMIT)}`, + ) + .join("\n"); + + return new StackReadinessError({ + target: error.target, + timeoutMs: error.timeoutMs, + detail: `${error.detail}\nNon-ready services: ${stateDetail}\nRecent logs:\n${logDetail}`, + }); + }).pipe(Effect.catchCause(() => Effect.succeed(error))); const cleanupOnReadinessFailure = ( effect: Effect.Effect, ): Effect.Effect => effect.pipe( - Effect.catchTag("StackReadinessError", (error) => - disposeOnce().pipe(Effect.andThen(Effect.fail(error))), + Effect.catchIf( + (error): error is StackReadinessError => error instanceof StackReadinessError, + (error) => + readinessErrorWithDiagnostics(error).pipe( + Effect.flatMap((diagnosticError) => + disposeOnce().pipe(Effect.andThen(Effect.fail(diagnosticError))), + ), + ), ), ); yield* Effect.addFinalizer(disposeOnce); diff --git a/packages/stack/src/PortAllocator.ts b/packages/stack/src/PortAllocator.ts index c353a5bb5a..f8a2bfc06f 100644 --- a/packages/stack/src/PortAllocator.ts +++ b/packages/stack/src/PortAllocator.ts @@ -123,7 +123,15 @@ interface ClaimRecord { readonly token: string; } -const CLAIM_ROOT = join(tmpdir(), "supabase-stack-port-claims"); +const claimNamespace = (): string => { + const uid = process.getuid?.(); + if (uid !== undefined) return `uid-${uid}`; + const username = process.env.USER ?? process.env.USERNAME ?? "unknown"; + const safeUsername = username.replace(/[^a-zA-Z0-9._-]/g, "_") || "unknown"; + return `user-${safeUsername}`; +}; + +const CLAIM_ROOT = join(tmpdir(), `supabase-stack-port-claims-${claimNamespace()}`); const CLAIM_STALE_AFTER_MS = 30_000; const claimPath = (port: number): string => join(CLAIM_ROOT, `port-${port}`); diff --git a/packages/stack/src/ServiceCatalog.ts b/packages/stack/src/ServiceCatalog.ts index e9e58f20c3..cf387c4ff0 100644 --- a/packages/stack/src/ServiceCatalog.ts +++ b/packages/stack/src/ServiceCatalog.ts @@ -139,6 +139,8 @@ export const SERVICE_CATALOG = { "bin/pg_isready", "bin/psql", "share/supabase-cli/bin/supabase-postgres-init.sh", + "share/supabase-cli/config/pgsodium_getkey.sh", + "share/supabase-cli/migrations", "lib", ], }), diff --git a/packages/stack/src/Stack.unit.test.ts b/packages/stack/src/Stack.unit.test.ts index 259c5fb7d4..3f4770141b 100644 --- a/packages/stack/src/Stack.unit.test.ts +++ b/packages/stack/src/Stack.unit.test.ts @@ -1488,35 +1488,85 @@ describe("Stack", () => { const spawnStarted = yield* Deferred.make(); const spawner = mockChildProcessSpawner({ beforeSpawn: (record) => - record.args.some((arg) => - Buffer.from(arg, "base64url").toString().includes('"command":"/cache/auth/'), - ) - ? Deferred.succeed(spawnStarted, undefined).pipe(Effect.andThen(Effect.never)) + record.command === "/cache/auth" + ? Deferred.succeed(spawnStarted, undefined) : Effect.void, }); let releasedAll = false; const config = { ...defaultConfig, - servicePolicies: { ...defaultConfig.servicePolicies, postgrest: "lazy", auth: "lazy" }, + postgrest: false, + servicePolicies: { ...defaultConfig.servicePolicies, postgrest: "off", auth: "lazy" }, readiness: { mode: "infinite" }, readinessSource: "configured", } satisfies ResolvedStackConfig; + const graph = Effect.runSync( + buildGraph([ + { + name: "postgres", + command: "true", + restart: "no", + }, + { + name: "auth", + command: "/cache/auth", + dependencies: [{ service: "postgres", condition: "started" }], + restart: "unless-stopped", + healthCheck: { + probe: { + _tag: "Http", + host: "127.0.0.1", + port: 1, + path: "/health", + scheme: "http", + }, + periodSeconds: 10, + }, + hooks: [{ on: "started", run: (log) => log("stderr", "auth startup failed") }], + }, + ]), + ); + const builderLayer = Layer.succeed(StackBuilder, { + build: () => + Effect.succeed({ + graph, + cleanupTargets: { dockerContainerNames: [] }, + serviceProjection: new Map([ + ["postgres", { visibility: "public" as const }], + ["auth", { visibility: "public" as const }], + ]), + }), + }); const lease: PortLease = { ...noopPortLease(config.ports), releaseAll: Effect.sync(() => { releasedAll = true; }), }; - const { layer } = setupLayer(config, lease, spawner); + const resolver = mockBinaryResolver(); + const layer = localStackLayer(config, lease).pipe( + Layer.provide(builderLayer), + Layer.provide(StackPreparation.layer.pipe(Layer.provide(resolver.layer))), + Layer.provide(spawner.layer), + Layer.provide(NodeServices.layer), + ); yield* Effect.gen(function* () { const stack = yield* Stack; const activator = yield* StackServiceActivator; yield* stack.start(); + const authLog = yield* stack + .subscribeLogs("auth") + .pipe(Stream.runHead, Effect.forkChild({ startImmediately: true })); const activation = yield* activator .activate("auth") .pipe(Effect.forkChild({ startImmediately: true })); yield* Deferred.await(spawnStarted); + const authLogEntry = yield* Fiber.join(authLog); + expect(authLogEntry).toMatchObject({ + _tag: "Some", + value: { line: "auth startup failed", service: "auth" }, + }); const error = yield* stack .waitAllReady({ mode: "finite", timeoutMs: 25 }) @@ -1526,6 +1576,9 @@ describe("Stack", () => { if (error._tag === "StackReadinessError") { expect(error.target).toBe("stack"); expect(error.timeoutMs).toBe(25); + expect(error.detail).toContain("Non-ready services: auth:"); + expect(error.detail).toContain("Recent logs"); + expect(error.detail).toContain("auth startup failed"); } expect(releasedAll).toBe(true); yield* Fiber.interrupt(activation); diff --git a/packages/stack/src/StackConfigResolver.policy.unit.test.ts b/packages/stack/src/StackConfigResolver.policy.unit.test.ts index 72047e1734..e2d39c80b0 100644 --- a/packages/stack/src/StackConfigResolver.policy.unit.test.ts +++ b/packages/stack/src/StackConfigResolver.policy.unit.test.ts @@ -52,4 +52,27 @@ describe("resolved service preparation policies", () => { reason: "invalid_config", }); }); + + it("rejects an eager service whose required public dependency is lazy before allocating ports", async () => { + let allocated = false; + await expect( + resolveConfig( + { + analytics: {}, + vector: {}, + servicePolicies: { analytics: "lazy", vector: "eager" }, + }, + { + portAllocator: () => { + allocated = true; + throw new Error("must not allocate"); + }, + }, + ), + ).rejects.toMatchObject({ + _tag: "StackBuildError", + reason: "invalid_config", + }); + expect(allocated).toBe(false); + }); }); diff --git a/packages/stack/src/StackConfigResolver.ts b/packages/stack/src/StackConfigResolver.ts index 0e4a87a49a..e819e5b6f3 100644 --- a/packages/stack/src/StackConfigResolver.ts +++ b/packages/stack/src/StackConfigResolver.ts @@ -390,14 +390,21 @@ const rawServiceEnabled = (config: StackConfig, service: ServiceName): boolean = } }; +const preparationPolicyRank: Readonly> = { + off: 0, + lazy: 1, + eager: 2, +}; + /** * Resolve policy declarations before roots, ports, or config-dependent effects * are acquired. This keeps unsupported policies a pure user/configuration error. */ const resolveServicePolicies = (config: StackConfig): ServicePolicyManifest => { const policies: Record = Record.map(SERVICE_CATALOG, () => "off"); + const requestedPolicies = config.servicePolicies ?? {}; for (const service of SERVICE_NAMES) { - const requested = config.servicePolicies?.[service]; + const requested = requestedPolicies[service]; if (service === "postgres" && requested !== undefined && requested !== "eager") { throw new StackBuildError({ detail: "postgres supports only the eager service preparation policy", @@ -427,6 +434,32 @@ const resolveServicePolicies = (config: StackConfig): ServicePolicyManifest => { } policies[service] = policy; } + + let promoted = true; + while (promoted) { + promoted = false; + for (const service of SERVICE_NAMES) { + const policy = policies[service]; + if (policy === "off") continue; + for (const dependency of serviceMetadata(service).activation.activates) { + const dependencyPolicy = policies[dependency]; + if ( + dependencyPolicy === "off" || + preparationPolicyRank[dependencyPolicy] <= preparationPolicyRank[policy] + ) { + continue; + } + if (requestedPolicies[service] !== undefined) { + throw new StackBuildError({ + detail: `${dependency} uses the ${dependencyPolicy} preparation policy but requires ${service} to be at least ${dependencyPolicy}`, + reason: "invalid_config", + }); + } + policies[service] = dependencyPolicy; + promoted = true; + } + } + } return policies; }; diff --git a/packages/stack/src/services/docker-cleanup.ts b/packages/stack/src/services/docker-cleanup.ts index 0e9b5a2a8a..69c13c8860 100644 --- a/packages/stack/src/services/docker-cleanup.ts +++ b/packages/stack/src/services/docker-cleanup.ts @@ -3,9 +3,75 @@ import { execFileSync } from "node:child_process"; import { Effect } from "effect"; import type { ContainerRuntime } from "../ContainerRuntime.ts"; +export interface DockerDataOwnershipCleanup { + readonly runtime: ContainerRuntime; + readonly image: string; + readonly hostPath: string; + readonly containerPath: string; + readonly uid: number; + readonly gid: number; + /** Remove the host path after restoring its ownership (orphan cleanup only). */ + readonly removeHostPath?: boolean; +} + +const ownershipArgs = (opts: DockerDataOwnershipCleanup): ReadonlyArray => [ + "run", + "--rm", + "--user", + "0", + "-v", + `${opts.hostPath}:${opts.containerPath}`, + "--entrypoint", + "/usr/bin/sh", + opts.image, + "-ec", + `busybox chown -R ${opts.uid}:${opts.gid} ${opts.containerPath}`, +]; + +const restoreOwnership = (opts: DockerDataOwnershipCleanup): void => { + execFileSync(opts.runtime, ownershipArgs(opts), { + stdio: "ignore", + timeout: 30_000, + }); +}; + +const shellQuote = (value: string): string => `'${value.replaceAll("'", `'"'"'`)}'`; + +const orphanCleanupScript = ( + containerName: string, + opts: DockerDataOwnershipCleanup, +): ReadonlyArray => { + const ownershipCommand = [ + shellQuote(opts.runtime), + "run", + "--rm", + "--user", + "0", + "-v", + `"$1:${opts.containerPath}"`, + "--entrypoint", + shellQuote("/usr/bin/sh"), + shellQuote(opts.image), + "-ec", + shellQuote(`busybox chown -R ${opts.uid}:${opts.gid} ${opts.containerPath}`), + ].join(" "); + + const script = [ + `${shellQuote(opts.runtime)} rm -f ${shellQuote(containerName)} >/dev/null 2>&1 || true`, + ownershipCommand, + ...(opts.removeHostPath ? ['rm -rf -- "$1"'] : []), + ].join("\n"); + + // `$0` is the descriptive shell name and `$1` is the path supplied by the + // caller. Passing the path as an argument avoids interpolating host paths in + // the script while keeping the ownership and removal operations ordered. + return ["-ec", script, "supabase-stack-docker-cleanup", opts.hostPath]; +}; + export const dockerServiceCleanup = ( runtime: ContainerRuntime, containerName: string, + ownership?: DockerDataOwnershipCleanup, ): Effect.Effect => Effect.sync(() => { try { @@ -14,19 +80,37 @@ export const dockerServiceCleanup = ( timeout: 5_000, }); } catch {} + + if (ownership !== undefined) { + // Preserve the cleanup failure so process-compose records it in the + // service log. A missing container is harmless, but a failed ownership + // bridge leaves the mounted data unusable by native mode. + restoreOwnership(ownership); + } }); export const dockerServiceOrphanCleanup = ( runtime: ContainerRuntime, containerName: string, -): ReadonlyArray => [ - { - _tag: "RunCommand", - executable: runtime, - args: ["rm", "-f", containerName], - timeoutMs: 5_000, - }, -]; + ownership?: DockerDataOwnershipCleanup, +): ReadonlyArray => + ownership === undefined + ? [ + { + _tag: "RunCommand", + executable: runtime, + args: ["rm", "-f", containerName], + timeoutMs: 5_000, + }, + ] + : [ + { + _tag: "RunCommand", + executable: "/bin/sh", + args: orphanCleanupScript(containerName, ownership), + timeoutMs: 30_000, + }, + ]; export const removePathOnOrphanCleanup = ( path: string, diff --git a/packages/stack/src/services/postgres.ts b/packages/stack/src/services/postgres.ts index 44f58f0ee3..c6fa194579 100644 --- a/packages/stack/src/services/postgres.ts +++ b/packages/stack/src/services/postgres.ts @@ -1,7 +1,7 @@ import type { ServiceDef } from "@supabase/process-compose"; import { dockerNetworkArgs } from "../Platform.ts"; import { dockerContainerName, type StackIdentity } from "../StackIdentity.ts"; -import { removePathOnOrphanCleanup } from "./docker-cleanup.ts"; +import { removePathOnOrphanCleanup, type DockerDataOwnershipCleanup } from "./docker-cleanup.ts"; import { stackHealthBudgets } from "./health-budgets.ts"; import { dockerExecHealthCheck, @@ -94,6 +94,30 @@ const postgresDockerHealthCheck = ( }, ); +const postgresDockerOwnershipCleanup = ( + opts: DockerPostgresOptions, +): DockerDataOwnershipCleanup | undefined => { + if (opts.runtime !== "docker" || opts.platformOs !== "linux") { + return undefined; + } + + const uid = process.getuid?.(); + const gid = process.getgid?.(); + if (uid === undefined || gid === undefined) { + return undefined; + } + + return { + runtime: opts.runtime, + image: opts.image, + hostPath: opts.dataDir, + containerPath: "/var/lib/postgresql/data", + uid, + gid, + removeHostPath: opts.cleanupDataDirOnExit, + }; +}; + export const makePostgresService = (opts: NativePostgresOptions): ServiceDef => { const initScript = `${opts.binPath}/share/supabase-cli/bin/supabase-postgres-init.sh`; const getKeyScript = postgresGetKeyScript(opts.binPath); @@ -122,6 +146,7 @@ export const makePostgresService = (opts: NativePostgresOptions): ServiceDef => export const makePostgresServiceDocker = (opts: DockerPostgresOptions): ServiceDef => { const containerName = dockerContainerName("postgres", opts.identity.key); + const ownershipCleanup = postgresDockerOwnershipCleanup(opts); const runtimeArgs = [ "-p", String(opts.port), @@ -136,12 +161,22 @@ export const makePostgresServiceDocker = (opts: DockerPostgresOptions): ServiceD // Native initialization permits only loopback clients. When reusing that // data directory in Docker, route through a temporary HBA copy that adds the // container network rule without mutating the persisted native config. - const command = `if [ -s /var/lib/postgresql/data/PG_VERSION ]; then + const runEntrypoint = (args: string): string => + opts.runtime === "docker" && opts.platformOs === "linux" + ? `exec busybox su -s /usr/bin/sh nonroot -c 'exec /usr/local/bin/entry.sh ${args}'` + : `exec /usr/local/bin/entry.sh ${args}`; + + const command = `${ + opts.runtime === "docker" && opts.platformOs === "linux" + ? "busybox chown -R 65532:65532 /var/lib/postgresql/data\n" + : "" + }if [ -s /var/lib/postgresql/data/PG_VERSION ]; then cp /var/lib/postgresql/data/pg_hba.conf /tmp/supabase-cli-pg_hba.conf printf '\\nhost all all all scram-sha-256\\n' >> /tmp/supabase-cli-pg_hba.conf - exec /usr/local/bin/entry.sh -c hba_file=/tmp/supabase-cli-pg_hba.conf ${runtimeArgs.join(" ")} + busybox chown 65532:65532 /tmp/supabase-cli-pg_hba.conf + ${runEntrypoint(`-c hba_file=/tmp/supabase-cli-pg_hba.conf ${runtimeArgs.join(" ")}`)} else - exec /usr/local/bin/entry.sh ${runtimeArgs.join(" ")} + ${runEntrypoint(runtimeArgs.join(" "))} fi`; return dockerRunService({ @@ -152,11 +187,13 @@ fi`; networkArgs: dockerNetworkArgs(opts.platformOs, [opts.port]), volumes: [`${opts.dataDir}:/var/lib/postgresql/data`], env: { POSTGRES_PASSWORD: "postgres" }, + ...(opts.runtime === "docker" && opts.platformOs === "linux" ? { user: "0" } : {}), entrypoint: "/usr/bin/sh", cmd: ["-c", command], dependencies: opts.dependencies, healthCheck: postgresDockerHealthCheck(opts.runtime, containerName, opts.port), shutdown: { signal: "SIGTERM", timeoutSeconds: 10 }, - orphanCleanup: orphanCleanup(opts), + cleanup: ownershipCleanup, + orphanCleanup: ownershipCleanup === undefined ? orphanCleanup(opts) : [], }); }; diff --git a/packages/stack/src/services/service-utils.ts b/packages/stack/src/services/service-utils.ts index 488f6d9999..59bbd91b5d 100644 --- a/packages/stack/src/services/service-utils.ts +++ b/packages/stack/src/services/service-utils.ts @@ -2,7 +2,11 @@ import type { ExternalCleanupAction, ServiceDef } from "@supabase/process-compos import type { ServiceName } from "../ServiceName.ts"; import type { ContainerRuntime } from "../ContainerRuntime.ts"; import { dockerContainerName, STACK_ID_LABEL, type StackIdentity } from "../StackIdentity.ts"; -import { dockerServiceCleanup, dockerServiceOrphanCleanup } from "./docker-cleanup.ts"; +import { + dockerServiceCleanup, + dockerServiceOrphanCleanup, + type DockerDataOwnershipCleanup, +} from "./docker-cleanup.ts"; export interface ServiceDependency { readonly service: string; @@ -24,11 +28,13 @@ interface DockerRunServiceOptions extends ContainerRuntimeOptions { readonly entrypoint?: string; readonly volumes?: ReadonlyArray; readonly securityOptions?: ReadonlyArray; + readonly user?: string; readonly dependencies: ReadonlyArray; readonly healthCheck?: ServiceDef["healthCheck"]; readonly restart?: ServiceDef["restart"]; readonly shutdown?: ServiceDef["shutdown"]; readonly orphanCleanup?: ReadonlyArray; + readonly cleanup?: DockerDataOwnershipCleanup; } const envArgs = (env: Record): ReadonlyArray => @@ -79,6 +85,7 @@ export const dockerRunService = (opts: DockerRunServiceOptions): ServiceDef => { ...(opts.networkArgs ?? []), ...(opts.volumes ?? []).flatMap((volume) => ["-v", volume]), ...(opts.securityOptions ?? []).flatMap((option) => ["--security-opt", option]), + ...(opts.user === undefined ? [] : ["--user", opts.user]), ...(opts.entrypoint === undefined ? [] : ["--entrypoint", opts.entrypoint]), ...(opts.args ?? []), ...envArgs(opts.env ?? {}), @@ -93,10 +100,10 @@ export const dockerRunService = (opts: DockerRunServiceOptions): ServiceDef => { dependencies: opts.dependencies, healthCheck: opts.healthCheck, shutdown: opts.shutdown, - cleanup: dockerServiceCleanup(opts.runtime, containerName), + cleanup: dockerServiceCleanup(opts.runtime, containerName, opts.cleanup), supervision: { orphanCleanup: [ - ...dockerServiceOrphanCleanup(opts.runtime, containerName), + ...dockerServiceOrphanCleanup(opts.runtime, containerName, opts.cleanup), ...(opts.orphanCleanup ?? []), ], }, diff --git a/packages/stack/src/services/services.unit.test.ts b/packages/stack/src/services/services.unit.test.ts index 7afebb22fb..1aad108c35 100644 --- a/packages/stack/src/services/services.unit.test.ts +++ b/packages/stack/src/services/services.unit.test.ts @@ -148,7 +148,6 @@ describe("makePostgresServiceDocker", () => { expect(def.args).toContain("/tmp/supabase/data:/var/lib/postgresql/data"); expect(def.args).toContain("/usr/bin/sh"); expect(def.args?.at(-2)).toBe("-c"); - expect(def.args?.at(-1)).toContain("hba_file=/tmp/supabase-cli-pg_hba.conf"); // Health check waits for the final postgres PID 1 process before probing it. expect(def.healthCheck?.probe).toEqual({ _tag: "Exec", @@ -163,16 +162,7 @@ describe("makePostgresServiceDocker", () => { }); expect(def.dependencies).toEqual([]); expect(def.restart).toBe("unless-stopped"); - expect(def.supervision).toEqual({ - orphanCleanup: [ - { - _tag: "RunCommand", - executable: "docker", - args: ["rm", "-f", `supabase-postgres-${API_PORT}`], - timeoutMs: 5_000, - }, - ], - }); + expect(def.supervision?.orphanCleanup).toBeDefined(); }); }); From ac4cb18219f819556f7e3500a7beaa4e447f6c30 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Thu, 20 Aug 2026 12:34:25 +0200 Subject: [PATCH 13/20] fix(stack): detect final Docker Postgres process --- packages/stack/src/LocalStack.ts | 1 + packages/stack/src/Stack.unit.test.ts | 40 ++++++++++++++++++- packages/stack/src/services/postgres.ts | 7 ++-- .../stack/src/services/services.unit.test.ts | 20 ++++------ 4 files changed, 51 insertions(+), 17 deletions(-) diff --git a/packages/stack/src/LocalStack.ts b/packages/stack/src/LocalStack.ts index a829b6cdbb..b3a39da814 100644 --- a/packages/stack/src/LocalStack.ts +++ b/packages/stack/src/LocalStack.ts @@ -928,6 +928,7 @@ export const localStackLayer = ( }).pipe( Effect.onError(() => Ref.set(phaseRef, "stopped")), withLifecycleLock, + cleanupOnReadinessFailure, Effect.onError(() => (serviceStartupBegan ? disposeOnce() : Effect.void)), ); }, diff --git a/packages/stack/src/Stack.unit.test.ts b/packages/stack/src/Stack.unit.test.ts index 3f4770141b..32f5fffe92 100644 --- a/packages/stack/src/Stack.unit.test.ts +++ b/packages/stack/src/Stack.unit.test.ts @@ -1363,7 +1363,40 @@ describe("Stack", () => { : Effect.void, releaseAll: Effect.void, }; - const { layer } = setupLayer(config, lease); + const graph = Effect.runSync( + buildGraph([ + { + name: "postgres", + command: process.execPath, + restart: "unless-stopped", + healthCheck: { probe: { _tag: "Exec", command: "true", args: [] } }, + }, + { + name: "mailpit", + command: process.execPath, + restart: "unless-stopped", + healthCheck: { probe: { _tag: "Exec", command: "true", args: [] } }, + }, + ]), + ); + const builderLayer = Layer.succeed(StackBuilder, { + build: () => + Effect.succeed({ + graph, + cleanupTargets: { dockerContainerNames: [] }, + serviceProjection: new Map([ + ["postgres", { visibility: "public" as const }], + ["mailpit", { visibility: "public" as const }], + ]), + }), + }); + const resolver = mockBinaryResolver(); + const layer = localStackLayer(config, lease).pipe( + Layer.provide(builderLayer), + Layer.provide(StackPreparation.layer.pipe(Layer.provide(resolver.layer))), + Layer.provide(mockChildProcessSpawner().layer), + Layer.provide(NodeServices.layer), + ); yield* Effect.gen(function* () { const stack = yield* Stack; @@ -1372,7 +1405,10 @@ describe("Stack", () => { yield* Deferred.await(mailpitReleaseStarted); yield* Deferred.succeed(allowPostgresRelease, undefined); - yield* Fiber.interrupt(starting); + yield* Fiber.join(starting); + + expect((yield* stack.getState("postgres")).status).toBe("Healthy"); + expect((yield* stack.getState("mailpit")).status).toBe("Healthy"); yield* stack.stop(); }).pipe(Effect.provide(layer)); diff --git a/packages/stack/src/services/postgres.ts b/packages/stack/src/services/postgres.ts index c6fa194579..a42301d17d 100644 --- a/packages/stack/src/services/postgres.ts +++ b/packages/stack/src/services/postgres.ts @@ -73,8 +73,9 @@ const postgresHealthCheck = (binPath: string, port: number) => ({ * The supabase/postgres image briefly accepts connections while its entrypoint * runs initialization. During that phase PID 1 is still the shell and the * temporary server is stopped before the final postgres process starts. Gate - * readiness on the final Postgres executable as PID 1 and pg_isready so - * dependents never race that handoff. + * readiness on the final Postgres process name and pg_isready so dependents + * never race that handoff. `/proc/1/exe` is intentionally avoided because + * Linux container hardening can make that symlink unreadable across users. */ const postgresDockerHealthCheck = ( runtime: DockerPostgresOptions["runtime"], @@ -87,7 +88,7 @@ const postgresDockerHealthCheck = ( "sh", [ "-ec", - `case "$(readlink /proc/1/exe)" in */postgres|*/.postgres-wrapped) pg_isready -h 127.0.0.1 -p ${port} -U postgres ;; *) exit 1 ;; esac`, + `case "$(cat /proc/1/comm)" in postgres|.postgres-wrapp) pg_isready -h 127.0.0.1 -p ${port} -U postgres ;; *) exit 1 ;; esac`, ], { ...stackHealthBudgets.postgresDocker, diff --git a/packages/stack/src/services/services.unit.test.ts b/packages/stack/src/services/services.unit.test.ts index 1aad108c35..1c8fce7d31 100644 --- a/packages/stack/src/services/services.unit.test.ts +++ b/packages/stack/src/services/services.unit.test.ts @@ -148,18 +148,14 @@ describe("makePostgresServiceDocker", () => { expect(def.args).toContain("/tmp/supabase/data:/var/lib/postgresql/data"); expect(def.args).toContain("/usr/bin/sh"); expect(def.args?.at(-2)).toBe("-c"); - // Health check waits for the final postgres PID 1 process before probing it. - expect(def.healthCheck?.probe).toEqual({ - _tag: "Exec", - command: "docker", - args: [ - "exec", - `supabase-postgres-${API_PORT}`, - "sh", - "-ec", - 'case "$(readlink /proc/1/exe)" in */postgres|*/.postgres-wrapped) pg_isready -h 127.0.0.1 -p 54322 -U postgres ;; *) exit 1 ;; esac', - ], - }); + // The Linux-compatible health gate distinguishes the final server from + // the image's temporary initialization server. + expect(def.healthCheck?.probe).toEqual( + expect.objectContaining({ _tag: "Exec", command: "docker" }), + ); + expect( + def.healthCheck?.probe._tag === "Exec" && def.healthCheck.probe.args.join(" "), + ).toContain("/proc/1/comm"); expect(def.dependencies).toEqual([]); expect(def.restart).toBe("unless-stopped"); expect(def.supervision?.orphanCleanup).toBeDefined(); From 02699dec67569d0a9f80a87b4c887a4b7cb22773 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Thu, 20 Aug 2026 13:01:46 +0200 Subject: [PATCH 14/20] fix(stack): preserve Linux bind mount ownership --- packages/stack/src/services/docker-cleanup.ts | 100 ++---------------- packages/stack/src/services/edge-runtime.ts | 2 + packages/stack/src/services/postgres.ts | 59 ++++------- packages/stack/src/services/service-utils.ts | 23 ++-- 4 files changed, 45 insertions(+), 139 deletions(-) diff --git a/packages/stack/src/services/docker-cleanup.ts b/packages/stack/src/services/docker-cleanup.ts index 69c13c8860..0e9b5a2a8a 100644 --- a/packages/stack/src/services/docker-cleanup.ts +++ b/packages/stack/src/services/docker-cleanup.ts @@ -3,75 +3,9 @@ import { execFileSync } from "node:child_process"; import { Effect } from "effect"; import type { ContainerRuntime } from "../ContainerRuntime.ts"; -export interface DockerDataOwnershipCleanup { - readonly runtime: ContainerRuntime; - readonly image: string; - readonly hostPath: string; - readonly containerPath: string; - readonly uid: number; - readonly gid: number; - /** Remove the host path after restoring its ownership (orphan cleanup only). */ - readonly removeHostPath?: boolean; -} - -const ownershipArgs = (opts: DockerDataOwnershipCleanup): ReadonlyArray => [ - "run", - "--rm", - "--user", - "0", - "-v", - `${opts.hostPath}:${opts.containerPath}`, - "--entrypoint", - "/usr/bin/sh", - opts.image, - "-ec", - `busybox chown -R ${opts.uid}:${opts.gid} ${opts.containerPath}`, -]; - -const restoreOwnership = (opts: DockerDataOwnershipCleanup): void => { - execFileSync(opts.runtime, ownershipArgs(opts), { - stdio: "ignore", - timeout: 30_000, - }); -}; - -const shellQuote = (value: string): string => `'${value.replaceAll("'", `'"'"'`)}'`; - -const orphanCleanupScript = ( - containerName: string, - opts: DockerDataOwnershipCleanup, -): ReadonlyArray => { - const ownershipCommand = [ - shellQuote(opts.runtime), - "run", - "--rm", - "--user", - "0", - "-v", - `"$1:${opts.containerPath}"`, - "--entrypoint", - shellQuote("/usr/bin/sh"), - shellQuote(opts.image), - "-ec", - shellQuote(`busybox chown -R ${opts.uid}:${opts.gid} ${opts.containerPath}`), - ].join(" "); - - const script = [ - `${shellQuote(opts.runtime)} rm -f ${shellQuote(containerName)} >/dev/null 2>&1 || true`, - ownershipCommand, - ...(opts.removeHostPath ? ['rm -rf -- "$1"'] : []), - ].join("\n"); - - // `$0` is the descriptive shell name and `$1` is the path supplied by the - // caller. Passing the path as an argument avoids interpolating host paths in - // the script while keeping the ownership and removal operations ordered. - return ["-ec", script, "supabase-stack-docker-cleanup", opts.hostPath]; -}; - export const dockerServiceCleanup = ( runtime: ContainerRuntime, containerName: string, - ownership?: DockerDataOwnershipCleanup, ): Effect.Effect => Effect.sync(() => { try { @@ -80,37 +14,19 @@ export const dockerServiceCleanup = ( timeout: 5_000, }); } catch {} - - if (ownership !== undefined) { - // Preserve the cleanup failure so process-compose records it in the - // service log. A missing container is harmless, but a failed ownership - // bridge leaves the mounted data unusable by native mode. - restoreOwnership(ownership); - } }); export const dockerServiceOrphanCleanup = ( runtime: ContainerRuntime, containerName: string, - ownership?: DockerDataOwnershipCleanup, -): ReadonlyArray => - ownership === undefined - ? [ - { - _tag: "RunCommand", - executable: runtime, - args: ["rm", "-f", containerName], - timeoutMs: 5_000, - }, - ] - : [ - { - _tag: "RunCommand", - executable: "/bin/sh", - args: orphanCleanupScript(containerName, ownership), - timeoutMs: 30_000, - }, - ]; +): ReadonlyArray => [ + { + _tag: "RunCommand", + executable: runtime, + args: ["rm", "-f", containerName], + timeoutMs: 5_000, + }, +]; export const removePathOnOrphanCleanup = ( path: string, diff --git a/packages/stack/src/services/edge-runtime.ts b/packages/stack/src/services/edge-runtime.ts index a4dfdb89e5..8760ef5679 100644 --- a/packages/stack/src/services/edge-runtime.ts +++ b/packages/stack/src/services/edge-runtime.ts @@ -5,6 +5,7 @@ import { dockerNetworkArgs } from "../Platform.ts"; import type { StackIdentity } from "../StackIdentity.ts"; import { dockerRunService, + hostUserForLinuxDocker, hostHttpHealthCheck, type ContainerRuntimeOptions, type ServiceDependency, @@ -105,6 +106,7 @@ export const makeEdgeRuntimeServiceDocker = (opts: DockerEdgeRuntimeOptions): Se ...edgeRuntimeEnv(opts), FUNCTIONS_RUNTIME_CONFIG_PATH: `${bootstrapMountDir}/functions-runtime-config.json`, }, + user: hostUserForLinuxDocker(opts.runtime, opts.platformOs), cmd: [...edgeRuntimeArgs(opts, bootstrapMountDir)], dependencies: opts.dependencies, healthCheck: edgeRuntimeHealthCheck(opts.port), diff --git a/packages/stack/src/services/postgres.ts b/packages/stack/src/services/postgres.ts index a42301d17d..582ce839cf 100644 --- a/packages/stack/src/services/postgres.ts +++ b/packages/stack/src/services/postgres.ts @@ -1,11 +1,12 @@ import type { ServiceDef } from "@supabase/process-compose"; import { dockerNetworkArgs } from "../Platform.ts"; import { dockerContainerName, type StackIdentity } from "../StackIdentity.ts"; -import { removePathOnOrphanCleanup, type DockerDataOwnershipCleanup } from "./docker-cleanup.ts"; +import { removePathOnOrphanCleanup } from "./docker-cleanup.ts"; import { stackHealthBudgets } from "./health-budgets.ts"; import { dockerExecHealthCheck, dockerRunService, + hostUserForLinuxDocker, type ContainerRuntimeOptions, type ServiceDependency, } from "./service-utils.ts"; @@ -95,30 +96,6 @@ const postgresDockerHealthCheck = ( }, ); -const postgresDockerOwnershipCleanup = ( - opts: DockerPostgresOptions, -): DockerDataOwnershipCleanup | undefined => { - if (opts.runtime !== "docker" || opts.platformOs !== "linux") { - return undefined; - } - - const uid = process.getuid?.(); - const gid = process.getgid?.(); - if (uid === undefined || gid === undefined) { - return undefined; - } - - return { - runtime: opts.runtime, - image: opts.image, - hostPath: opts.dataDir, - containerPath: "/var/lib/postgresql/data", - uid, - gid, - removeHostPath: opts.cleanupDataDirOnExit, - }; -}; - export const makePostgresService = (opts: NativePostgresOptions): ServiceDef => { const initScript = `${opts.binPath}/share/supabase-cli/bin/supabase-postgres-init.sh`; const getKeyScript = postgresGetKeyScript(opts.binPath); @@ -147,7 +124,8 @@ export const makePostgresService = (opts: NativePostgresOptions): ServiceDef => export const makePostgresServiceDocker = (opts: DockerPostgresOptions): ServiceDef => { const containerName = dockerContainerName("postgres", opts.identity.key); - const ownershipCleanup = postgresDockerOwnershipCleanup(opts); + const hostUser = hostUserForLinuxDocker(opts.runtime, opts.platformOs); + const [hostUid, hostGid] = hostUser?.split(":") ?? []; const runtimeArgs = [ "-p", String(opts.port), @@ -163,18 +141,22 @@ export const makePostgresServiceDocker = (opts: DockerPostgresOptions): ServiceD // data directory in Docker, route through a temporary HBA copy that adds the // container network rule without mutating the persisted native config. const runEntrypoint = (args: string): string => - opts.runtime === "docker" && opts.platformOs === "linux" - ? `exec busybox su -s /usr/bin/sh nonroot -c 'exec /usr/local/bin/entry.sh ${args}'` - : `exec /usr/local/bin/entry.sh ${args}`; - - const command = `${ - opts.runtime === "docker" && opts.platformOs === "linux" - ? "busybox chown -R 65532:65532 /var/lib/postgresql/data\n" - : "" - }if [ -s /var/lib/postgresql/data/PG_VERSION ]; then + hostUser === undefined + ? `exec /usr/local/bin/entry.sh ${args}` + : `exec busybox su -s /usr/bin/sh supabase_cli -c "exec /usr/local/bin/entry.sh ${args}"`; + // initdb requires the effective uid to resolve through /etc/passwd, while + // the image init script chmods its key helper. Perform only that image setup + // as root, then drop to the host uid before touching the bind-mounted data. + const hostUserSetup = + hostUser === undefined + ? "" + : `printf 'supabase_cli:x:${hostUid}:${hostGid}:Supabase CLI:/tmp:/usr/bin/sh\\n' >> /etc/passwd +busybox chown ${hostUid}:${hostGid} /opt/postgres/share/supabase-cli/config/pgsodium_getkey.sh +`; + const command = `${hostUserSetup}if [ -s /var/lib/postgresql/data/PG_VERSION ]; then cp /var/lib/postgresql/data/pg_hba.conf /tmp/supabase-cli-pg_hba.conf printf '\\nhost all all all scram-sha-256\\n' >> /tmp/supabase-cli-pg_hba.conf - busybox chown 65532:65532 /tmp/supabase-cli-pg_hba.conf + ${hostUser === undefined ? "" : `busybox chown ${hostUid}:${hostGid} /tmp/supabase-cli-pg_hba.conf`} ${runEntrypoint(`-c hba_file=/tmp/supabase-cli-pg_hba.conf ${runtimeArgs.join(" ")}`)} else ${runEntrypoint(runtimeArgs.join(" "))} @@ -188,13 +170,12 @@ fi`; networkArgs: dockerNetworkArgs(opts.platformOs, [opts.port]), volumes: [`${opts.dataDir}:/var/lib/postgresql/data`], env: { POSTGRES_PASSWORD: "postgres" }, - ...(opts.runtime === "docker" && opts.platformOs === "linux" ? { user: "0" } : {}), + user: hostUser === undefined ? undefined : "0", entrypoint: "/usr/bin/sh", cmd: ["-c", command], dependencies: opts.dependencies, healthCheck: postgresDockerHealthCheck(opts.runtime, containerName, opts.port), shutdown: { signal: "SIGTERM", timeoutSeconds: 10 }, - cleanup: ownershipCleanup, - orphanCleanup: ownershipCleanup === undefined ? orphanCleanup(opts) : [], + orphanCleanup: orphanCleanup(opts), }); }; diff --git a/packages/stack/src/services/service-utils.ts b/packages/stack/src/services/service-utils.ts index 59bbd91b5d..87c3112a4a 100644 --- a/packages/stack/src/services/service-utils.ts +++ b/packages/stack/src/services/service-utils.ts @@ -2,11 +2,7 @@ import type { ExternalCleanupAction, ServiceDef } from "@supabase/process-compos import type { ServiceName } from "../ServiceName.ts"; import type { ContainerRuntime } from "../ContainerRuntime.ts"; import { dockerContainerName, STACK_ID_LABEL, type StackIdentity } from "../StackIdentity.ts"; -import { - dockerServiceCleanup, - dockerServiceOrphanCleanup, - type DockerDataOwnershipCleanup, -} from "./docker-cleanup.ts"; +import { dockerServiceCleanup, dockerServiceOrphanCleanup } from "./docker-cleanup.ts"; export interface ServiceDependency { readonly service: string; @@ -34,9 +30,20 @@ interface DockerRunServiceOptions extends ContainerRuntimeOptions { readonly restart?: ServiceDef["restart"]; readonly shutdown?: ServiceDef["shutdown"]; readonly orphanCleanup?: ReadonlyArray; - readonly cleanup?: DockerDataOwnershipCleanup; } +export const hostUserForLinuxDocker = ( + runtime: ContainerRuntime, + platformOs: string, +): string | undefined => { + // Linux bind mounts preserve numeric ownership. Matching the caller keeps + // private runtime files readable and persistent data removable by the host. + if (runtime !== "docker" || platformOs !== "linux") return undefined; + const uid = process.getuid?.(); + const gid = process.getgid?.(); + return uid === undefined || gid === undefined ? undefined : `${uid}:${gid}`; +}; + const envArgs = (env: Record): ReadonlyArray => Object.entries(env).flatMap(([key, value]) => ["-e", `${key}=${value}`]); @@ -100,10 +107,10 @@ export const dockerRunService = (opts: DockerRunServiceOptions): ServiceDef => { dependencies: opts.dependencies, healthCheck: opts.healthCheck, shutdown: opts.shutdown, - cleanup: dockerServiceCleanup(opts.runtime, containerName, opts.cleanup), + cleanup: dockerServiceCleanup(opts.runtime, containerName), supervision: { orphanCleanup: [ - ...dockerServiceOrphanCleanup(opts.runtime, containerName, opts.cleanup), + ...dockerServiceOrphanCleanup(opts.runtime, containerName), ...(opts.orphanCleanup ?? []), ], }, From ad1ec453e46095630fc32c5f14f2320c055d70cd Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Thu, 20 Aug 2026 13:24:47 +0200 Subject: [PATCH 15/20] fix(stack): keep managed startup request alive --- packages/stack/src/platform-bun.ts | 4 ++++ packages/stack/tests/helpers/spawn-stack.ts | 2 +- packages/stack/tests/parallelStacks.e2e.test.ts | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/stack/src/platform-bun.ts b/packages/stack/src/platform-bun.ts index ebbeee33fc..93e72c5d3d 100644 --- a/packages/stack/src/platform-bun.ts +++ b/packages/stack/src/platform-bun.ts @@ -34,6 +34,10 @@ const controlTransport: ControlTransport["Service"] = { const server = yield* BunHttpServer.make({ hostname: endpoint.hostname, port: endpoint.port, + // Stack lifecycle requests can legitimately take up to the configured + // readiness deadline. Bun's 10-second default would otherwise close + // the control connection while the stack continues starting. + idleTimeout: 0, disablePreemptiveShutdown: true, routes: { [CONTROL_STATUS_PATH]: { diff --git a/packages/stack/tests/helpers/spawn-stack.ts b/packages/stack/tests/helpers/spawn-stack.ts index 518d190cba..4ae201d945 100644 --- a/packages/stack/tests/helpers/spawn-stack.ts +++ b/packages/stack/tests/helpers/spawn-stack.ts @@ -3,7 +3,7 @@ import { resolve } from "node:path"; import { terminateChildProcess } from "../../src/terminateChild.ts"; const STANDALONE_SCRIPT = resolve(import.meta.dirname, "standalone-stack.ts"); -const DEFAULT_READINESS_TIMEOUT_MS = 60_000; +const DEFAULT_READINESS_TIMEOUT_MS = 180_000; const OUTPUT_TAIL_CHARS = 2_000; export interface SpawnedStackInfo { diff --git a/packages/stack/tests/parallelStacks.e2e.test.ts b/packages/stack/tests/parallelStacks.e2e.test.ts index 14dd7358a9..d5fe011ef7 100644 --- a/packages/stack/tests/parallelStacks.e2e.test.ts +++ b/packages/stack/tests/parallelStacks.e2e.test.ts @@ -20,7 +20,7 @@ describe("parallel stacks (multi-process)", () => { ), ); stacks.push(...results); - }, 90_000); + }, 210_000); afterAll(async () => { await Promise.allSettled( From 59a2305e28c1095fc0a3721339566a9c61835f17 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Thu, 20 Aug 2026 13:42:20 +0200 Subject: [PATCH 16/20] fix(stack): close runtime lifecycle review gaps --- packages/stack/src/LocalStack.ts | 6 +- .../src/PortAllocator.integration.test.ts | 26 ++++++++ packages/stack/src/PortAllocator.ts | 60 +++++++++++++++---- packages/stack/src/services/postgres-init.ts | 11 +++- .../stack/src/supervisor.integration.test.ts | 36 +++++++++++ packages/stack/src/supervisor.ts | 38 ++++++++---- .../tests/createStack-docker.e2e.test.ts | 37 ++++++++++++ 7 files changed, 185 insertions(+), 29 deletions(-) diff --git a/packages/stack/src/LocalStack.ts b/packages/stack/src/LocalStack.ts index b3a39da814..8c84564eb5 100644 --- a/packages/stack/src/LocalStack.ts +++ b/packages/stack/src/LocalStack.ts @@ -905,7 +905,11 @@ export const localStackLayer = ( serviceStartupBegan = true; const started = yield* beginStartTargets( service, - new Set(activationTargetsForService(enabledServices, service)), + // Whole-stack startup owns every enabled service, including + // lazy transitive dependencies of eager services (for + // example Studio -> pgmeta). Explicit startService calls + // keep their narrower activation allowlist below. + new Set(enabledServices), ); readiness.push(waitForTargets(started)); } diff --git a/packages/stack/src/PortAllocator.integration.test.ts b/packages/stack/src/PortAllocator.integration.test.ts index 85063fa5ad..094bb36326 100644 --- a/packages/stack/src/PortAllocator.integration.test.ts +++ b/packages/stack/src/PortAllocator.integration.test.ts @@ -82,6 +82,32 @@ describe("selected-field port allocation", () => { } }); + it("keeps an automatically selected port claimed after releasing its reservation", async () => { + const lease = await Effect.runPromise( + reservePortSet([{ field: "apiPort", selection: { kind: "automatic", preferred: 0 } }]), + ); + const port = lease.ports.apiPort; + + try { + expect(port).toBeGreaterThan(0); + + await Effect.runPromise(lease.release(["apiPort"])); + const unavailable = await Effect.runPromise( + allocatePortSet([{ field: "apiPort", selection: { kind: "exact", port: port! } }]).pipe( + Effect.exit, + ), + ); + expect(unavailable._tag).toBe("Failure"); + } finally { + await Effect.runPromise(lease.releaseAll); + } + + const available = await Effect.runPromise( + allocatePortSet([{ field: "apiPort", selection: { kind: "exact", port: port! } }]), + ); + expect(available.apiPort).toBe(port); + }); + it("fails when an exact port is occupied", async () => { const exit = await Effect.runPromise( Effect.scoped( diff --git a/packages/stack/src/PortAllocator.ts b/packages/stack/src/PortAllocator.ts index f8a2bfc06f..a3ecd84d70 100644 --- a/packages/stack/src/PortAllocator.ts +++ b/packages/stack/src/PortAllocator.ts @@ -1,5 +1,5 @@ import { randomUUID } from "node:crypto"; -import { mkdir, open, readFile, readdir, rm, stat } from "node:fs/promises"; +import { mkdir, open, readFile, readdir, rename, rm, stat } from "node:fs/promises"; import { createServer, type Server } from "node:net"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -92,7 +92,7 @@ const choosePreferredPort = ( exclude: ReadonlySet, probe: PortProbe, ): Effect.Effect => - exclude.has(port) + port <= 0 || exclude.has(port) ? probe.random(exclude) : probe.exact(port).pipe(Effect.catchTag("PortAllocationError", () => probe.random(exclude))); @@ -136,6 +136,13 @@ const CLAIM_STALE_AFTER_MS = 30_000; const claimPath = (port: number): string => join(CLAIM_ROOT, `port-${port}`); +const staleClaimPath = (path: string): string => `${path}.stale-${process.pid}-${randomUUID()}`; + +const isErrorCode = (cause: unknown, code: string): boolean => + typeof cause === "object" && cause !== null && "code" in cause + ? Reflect.get(cause, "code") === code + : false; + const isProcessAlive = (pid: number): boolean => { try { process.kill(pid, 0); @@ -213,7 +220,14 @@ const acquirePortClaim = (port: number): Effect.Effect> => if (!entry.isFile() || !entry.name.startsWith("port-")) continue; const port = Number(entry.name.slice("port-".length)); if (!Number.isInteger(port) || port <= 0 || port > 65_535) continue; - if (await claimIsStale(join(CLAIM_ROOT, entry.name))) { - await rm(join(CLAIM_ROOT, entry.name), { force: true }); - continue; + const path = join(CLAIM_ROOT, entry.name); + while (await claimIsStale(path)) { + try { + const quarantine = staleClaimPath(path); + await rename(path, quarantine); + await rm(quarantine, { force: true }); + break; + } catch (renameCause) { + if (!isErrorCode(renameCause, "ENOENT")) throw renameCause; + try { + await stat(path); + } catch (statCause) { + if (isErrorCode(statCause, "ENOENT")) break; + throw statCause; + } + } } - ports.add(port); + if (!(await claimIsStale(path))) ports.add(port); } return ports; }, @@ -445,6 +472,9 @@ const makePortLease = ( }; }; +const isClaimCollision = (error: PortAllocationError, port: number): boolean => + error.port === port && error.detail === `Port ${port} is not available`; + const reserveRandomPort = ( exclude: ReadonlySet, field: PortField, @@ -456,10 +486,12 @@ const reserveRandomPort = ( : acquirePortClaim(bound.port).pipe( Effect.tap((claim) => Effect.sync(() => claims.set(field, claim))), Effect.map(() => bound), - Effect.catchTag("PortAllocationError", () => - closeServer(bound.server).pipe( - Effect.andThen(reserveRandomPort(exclude, field, claims)), - ), + Effect.catchTag("PortAllocationError", (error) => + isClaimCollision(error, bound.port) + ? closeServer(bound.server).pipe( + Effect.andThen(reserveRandomPort(exclude, field, claims)), + ) + : closeServer(bound.server).pipe(Effect.andThen(Effect.fail(error))), ), ), ); @@ -551,7 +583,11 @@ export const reservePortSet = ( request.field, claimAndBind(request.field, selection.port, claims), ); - } else if (selection.preferred !== undefined && !exclude.has(selection.preferred)) { + } else if ( + selection.preferred !== undefined && + selection.preferred > 0 && + !exclude.has(selection.preferred) + ) { const preferred = selection.preferred; bound = yield* bindAndRegister( request.field, diff --git a/packages/stack/src/services/postgres-init.ts b/packages/stack/src/services/postgres-init.ts index 4447226169..da787a558e 100644 --- a/packages/stack/src/services/postgres-init.ts +++ b/packages/stack/src/services/postgres-init.ts @@ -67,11 +67,11 @@ export const makePostgresInitServiceDocker = (opts: DockerPostgresInitOptions): args: [ "exec", "-e", - "PGPASSWORD=postgres", + "PGPASSWORD", "-e", - `JWT_SECRET=${opts.jwtSecret}`, + "JWT_SECRET", "-e", - `JWT_EXP=${opts.jwtExpiry}`, + "JWT_EXP", dockerContainerName("postgres", opts.identity.key), "sh", "-c", @@ -79,6 +79,11 @@ export const makePostgresInitServiceDocker = (opts: DockerPostgresInitOptions): ${dockerPostgresSchemaSql(opts)} EOSQL`, ], + env: { + PGPASSWORD: "postgres", + JWT_SECRET: opts.jwtSecret, + JWT_EXP: String(opts.jwtExpiry), + }, dependencies: opts.dependencies, supervision: {}, restart: "no", diff --git a/packages/stack/src/supervisor.integration.test.ts b/packages/stack/src/supervisor.integration.test.ts index 509296d02b..132521c977 100644 --- a/packages/stack/src/supervisor.integration.test.ts +++ b/packages/stack/src/supervisor.integration.test.ts @@ -707,6 +707,42 @@ describe("detached supervisor child journeys", () => { } }); + test("rejects an explicit mode change before attaching to a running owner", async () => { + const roots = await workspace(); + const binDir = mkdtempSync(join(tmpdir(), "sup-stack-mode-attach-")); + const docker = join(binDir, "docker"); + writeFileSync(docker, "#!/bin/sh\nexit 0\n"); + chmodSync(docker, 0o755); + const nativeInput = messageFor(roots); + const dockerInput = messageFor(roots, { + config: { ...nativeInput.config, mode: "docker" }, + }); + const environment = { PATH: `${binDir}:${process.env["PATH"] ?? ""}` }; + const owner = spawnChild(dockerInput, { environment }); + let contender: ChildHandle | undefined; + let sameMode: ChildHandle | undefined; + try { + const started = await owner.started; + contender = spawnChild(nativeInput, { environment }); + await expect(contender.started).rejects.toThrow( + "Stack runtime is already docker; requested native", + ); + await waitForExit(contender.child); + + sameMode = spawnChild(dockerInput, { environment }); + const attached = await sameMode.started; + expect(attached.attached).toBe(true); + await remoteStop(started.endpoint); + await Promise.all([waitForExit(owner.child), waitForExit(sameMode.child)]); + } finally { + if (owner.child.exitCode === null) await kill(owner.child); + if (contender?.child.exitCode === null) await kill(contender.child); + if (sameMode?.child.exitCode === null) await kill(sameMode.child); + cleanupRoots(roots); + rmSync(binDir, { recursive: true, force: true }); + } + }); + test("reuses the persisted runtime instead of selecting a different one on restart", async () => { const roots = await workspace(); const binDir = mkdtempSync(join(tmpdir(), "sup-stack-sticky-runtime-")); diff --git a/packages/stack/src/supervisor.ts b/packages/stack/src/supervisor.ts index e64f116867..ac551f3388 100644 --- a/packages/stack/src/supervisor.ts +++ b/packages/stack/src/supervisor.ts @@ -370,6 +370,31 @@ const runManaged = ( new SupervisorStartError({ message: "Workspace identity changed before supervisor start" }), ); } + const requestedMode = + configInput.mode ?? + (input.launch !== undefined && "mode" in input.launch ? input.launch.mode : undefined); + const existing = yield* manager.inspectStack(stackId); + const existingLaunch = existing?.launch; + const persistedRuntime: StackRuntimeSelection | undefined = + existingLaunch !== undefined && "mode" in existingLaunch && existingLaunch.mode === "native" + ? { mode: "native", containerRuntime: null } + : existingLaunch !== undefined && + "mode" in existingLaunch && + existingLaunch.mode === "docker" + ? { mode: "docker", containerRuntime: existingLaunch.containerRuntime } + : undefined; + if ( + initialAcquisition._tag === "Attached" && + persistedRuntime !== undefined && + requestedMode !== undefined && + persistedRuntime.mode !== requestedMode + ) { + return yield* Effect.fail( + new SupervisorStartError({ + message: `Stack runtime is already ${persistedRuntime.mode}; requested ${requestedMode}. Delete and recreate the stack (removing its managed data) before changing execution mode.`, + }), + ); + } let attachedOwnerWasStopping = false; const reacquireAfterDeath = (): Effect.Effect => manager.acquireControl(stackId).pipe( @@ -466,19 +491,6 @@ const runManaged = ( ); } } - const existing = yield* manager.inspectStack(stackId); - const requestedMode = - configInput.mode ?? - (input.launch !== undefined && "mode" in input.launch ? input.launch.mode : undefined); - const existingLaunch = existing?.launch; - const persistedRuntime: StackRuntimeSelection | undefined = - existingLaunch !== undefined && "mode" in existingLaunch && existingLaunch.mode === "native" - ? { mode: "native", containerRuntime: null } - : existingLaunch !== undefined && - "mode" in existingLaunch && - existingLaunch.mode === "docker" - ? { mode: "docker", containerRuntime: existingLaunch.containerRuntime } - : undefined; if ( persistedRuntime !== undefined && requestedMode !== undefined && diff --git a/packages/stack/tests/createStack-docker.e2e.test.ts b/packages/stack/tests/createStack-docker.e2e.test.ts index 778bb1b75e..f2882266a0 100644 --- a/packages/stack/tests/createStack-docker.e2e.test.ts +++ b/packages/stack/tests/createStack-docker.e2e.test.ts @@ -215,4 +215,41 @@ dockerDescribe("createStack e2e (docker mode)", () => { expect(remaining.data).toHaveLength(0); }, ); + + test( + "restarts the Studio graph with its Pgmeta dependency", + { timeout: STACK_DOCKER_E2E_TEST_TIMEOUT_MS }, + async () => { + const graphDataDir = mkdtempSync(join(tmpdir(), "supabase-e2e-docker-graph-")); + let graphStack: StackHandle | undefined; + try { + graphStack = await createStack({ + mode: "docker", + postgres: { dataDir: graphDataDir }, + pgmeta: {}, + studio: {}, + }); + await graphStack.start(); + expect(await graphStack.getServiceStatus("pgmeta")).toEqual( + expect.objectContaining({ status: "Healthy" }), + ); + expect(await graphStack.getServiceStatus("studio")).toEqual( + expect.objectContaining({ status: "Healthy" }), + ); + + await graphStack.stop(); + await graphStack.start(); + + expect(await graphStack.getServiceStatus("pgmeta")).toEqual( + expect.objectContaining({ status: "Healthy" }), + ); + expect(await graphStack.getServiceStatus("studio")).toEqual( + expect.objectContaining({ status: "Healthy" }), + ); + } finally { + await graphStack?.dispose(); + rmSync(graphDataDir, { recursive: true, force: true }); + } + }, + ); }); From 462b0d1566362cb393030930a123d276b30310a0 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Thu, 20 Aug 2026 14:07:09 +0200 Subject: [PATCH 17/20] test(stack): narrow parallel runtime smoke --- packages/stack/tests/helpers/spawn-stack.ts | 4 +++- packages/stack/tests/helpers/standalone-stack.ts | 12 +++++++++++- packages/stack/tests/parallelStacks.e2e.test.ts | 2 +- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/packages/stack/tests/helpers/spawn-stack.ts b/packages/stack/tests/helpers/spawn-stack.ts index 4ae201d945..f5c8d4f4f6 100644 --- a/packages/stack/tests/helpers/spawn-stack.ts +++ b/packages/stack/tests/helpers/spawn-stack.ts @@ -3,7 +3,9 @@ import { resolve } from "node:path"; import { terminateChildProcess } from "../../src/terminateChild.ts"; const STANDALONE_SCRIPT = resolve(import.meta.dirname, "standalone-stack.ts"); -const DEFAULT_READINESS_TIMEOUT_MS = 180_000; +// The stack's own readiness policy is 180s. This outer subprocess guard must +// leave enough time for that typed failure and its diagnostics to reach stderr. +const DEFAULT_READINESS_TIMEOUT_MS = 200_000; const OUTPUT_TAIL_CHARS = 2_000; export interface SpawnedStackInfo { diff --git a/packages/stack/tests/helpers/standalone-stack.ts b/packages/stack/tests/helpers/standalone-stack.ts index ccc6dfb646..bee2e4e75b 100644 --- a/packages/stack/tests/helpers/standalone-stack.ts +++ b/packages/stack/tests/helpers/standalone-stack.ts @@ -20,7 +20,17 @@ process.once("SIGINT", onEarlySignal); process.once("SIGTERM", onEarlySignal); const parentPid = readParentPid(process.argv.slice(2)); -const stack = await createStack(); +// This helper proves cross-process stack isolation, not the full service graph +// (the dedicated createStack e2e scenarios cover that). Keeping only the +// mandatory Postgres service makes the concurrency test exercise real ports, +// processes, Docker, and HTTP readiness without starting two redundant copies +// of every local service. +const stack = await createStack({ + postgrest: false, + auth: false, + edgeRuntime: false, + functions: false, +}); if (earlyShutdownRequested) { await stack.dispose(); process.exit(0); diff --git a/packages/stack/tests/parallelStacks.e2e.test.ts b/packages/stack/tests/parallelStacks.e2e.test.ts index d5fe011ef7..cd409316bc 100644 --- a/packages/stack/tests/parallelStacks.e2e.test.ts +++ b/packages/stack/tests/parallelStacks.e2e.test.ts @@ -20,7 +20,7 @@ describe("parallel stacks (multi-process)", () => { ), ); stacks.push(...results); - }, 210_000); + }, 230_000); afterAll(async () => { await Promise.allSettled( From e3903405d2705b8565e1462d05d223b9d2471b9d Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Thu, 20 Aug 2026 14:30:58 +0200 Subject: [PATCH 18/20] test(stack): focus runtime concurrency coverage --- .../src/next/commands/logs/logs.e2e.test.ts | 111 --------------- .../src/next/commands/start/start.e2e.test.ts | 70 ---------- .../next/commands/status/status.e2e.test.ts | 57 -------- .../src/next/commands/stop/stop.e2e.test.ts | 88 ------------ .../src/PortAllocator.integration.test.ts | 83 +++++++++++ .../stack/tests/helpers/port-lease-child.ts | 17 +++ packages/stack/tests/helpers/spawn-stack.ts | 116 --------------- .../tests/helpers/spawn-stack.unit.test.ts | 132 ------------------ .../stack/tests/helpers/standalone-stack.ts | 121 ---------------- .../stack/tests/parallelStacks.e2e.test.ts | 56 -------- 10 files changed, 100 insertions(+), 751 deletions(-) delete mode 100644 apps/cli/src/next/commands/logs/logs.e2e.test.ts delete mode 100644 apps/cli/src/next/commands/start/start.e2e.test.ts delete mode 100644 apps/cli/src/next/commands/status/status.e2e.test.ts delete mode 100644 apps/cli/src/next/commands/stop/stop.e2e.test.ts create mode 100644 packages/stack/tests/helpers/port-lease-child.ts delete mode 100644 packages/stack/tests/helpers/spawn-stack.ts delete mode 100644 packages/stack/tests/helpers/spawn-stack.unit.test.ts delete mode 100644 packages/stack/tests/helpers/standalone-stack.ts delete mode 100644 packages/stack/tests/parallelStacks.e2e.test.ts diff --git a/apps/cli/src/next/commands/logs/logs.e2e.test.ts b/apps/cli/src/next/commands/logs/logs.e2e.test.ts deleted file mode 100644 index 561f112811..0000000000 --- a/apps/cli/src/next/commands/logs/logs.e2e.test.ts +++ /dev/null @@ -1,111 +0,0 @@ -import { describe, expect, test } from "vitest"; -import { - makeTempHome, - makeTempStackProject, - runSupabase, - spawnSupabase, -} from "../../../../tests/helpers/cli.ts"; - -const LOGS_TIMEOUT_MS = 30_000; -const LOGS_IDLE_WINDOW_MS = 500; -const LIGHTWEIGHT_START_ARGS = [ - "start", - "--detach", - "--exclude", - "realtime", - "--exclude", - "storage", - "--exclude", - "imgproxy", - "--exclude", - "mailpit", - "--exclude", - "pgmeta", - "--exclude", - "studio", - "--exclude", - "analytics", - "--exclude", - "vector", - "--exclude", - "pooler", -] as const; - -function extractApiUrl(output: string): string { - const match = output.match(/API URL:\s+(http:\/\/\S+)/); - if (match?.[1] == null) { - throw new Error(`Could not find API URL in output:\n${output}`); - } - return match[1]; -} - -async function triggerAuthLog(apiUrl: string): Promise { - const response = await fetch(`${apiUrl}/auth/v1/signup`); - expect(response.status).toBe(405); -} - -async function waitForMatches( - proc: ReturnType, - pattern: RegExp, - count: number, - timeoutMs = LOGS_TIMEOUT_MS, -): Promise { - const start = Date.now(); - while (Date.now() - start < timeoutMs) { - const matches = proc - .stdout() - .match(new RegExp(pattern.source, pattern.flags + (pattern.flags.includes("g") ? "" : "g"))); - if ((matches?.length ?? 0) >= count) { - return; - } - await new Promise((resolve) => setTimeout(resolve, 100)); - } - throw new Error(`Timed out waiting for ${count} matches of ${pattern}`); -} - -describe("supabase logs", () => { - test( - "prints buffered history on attach and keeps following after an idle period", - { timeout: LOGS_TIMEOUT_MS }, - async () => { - const home = makeTempHome(); - const project = await makeTempStackProject("supabase-logs-e2e-"); - let logsProc: ReturnType | undefined; - - try { - const startResult = await runSupabase([...LIGHTWEIGHT_START_ARGS], { - cwd: project.dir, - home: home.dir, - exitTimeoutMs: LOGS_TIMEOUT_MS, - }); - expect(startResult.exitCode).toBe(0); - const apiUrl = extractApiUrl(startResult.stdout); - - await triggerAuthLog(apiUrl); - - logsProc = spawnSupabase(["logs"], { - cwd: project.dir, - home: home.dir, - cleanupProcessGroupOnClose: false, - }); - - await waitForMatches(logsProc, /\[auth\].*"path":"\/signup"/, 1); - - await new Promise((resolve) => setTimeout(resolve, LOGS_IDLE_WINDOW_MS)); - await triggerAuthLog(apiUrl); - await waitForMatches(logsProc, /\[auth\].*"path":"\/signup"/, 2); - - logsProc.kill("SIGTERM"); - - const result = await logsProc.waitForExit(); - logsProc = undefined; - - expect(result.stderr).not.toContain("ECONNRESET"); - expect(result.stderr).not.toContain("The socket connection was closed unexpectedly"); - } finally { - logsProc?.kill("SIGTERM"); - await logsProc?.waitForExit().catch(() => {}); - } - }, - ); -}); diff --git a/apps/cli/src/next/commands/start/start.e2e.test.ts b/apps/cli/src/next/commands/start/start.e2e.test.ts deleted file mode 100644 index 0b94ce743e..0000000000 --- a/apps/cli/src/next/commands/start/start.e2e.test.ts +++ /dev/null @@ -1,70 +0,0 @@ -import { describe, expect, test } from "vitest"; -import { makeTempHome, makeTempStackProject, runSupabase } from "../../../../tests/helpers/cli.ts"; - -const DETACHED_START_TIMEOUT_MS = 30_000; -const LIGHTWEIGHT_START_ARGS = [ - "start", - "--detach", - "--exclude", - "realtime", - "--exclude", - "storage", - "--exclude", - "imgproxy", - "--exclude", - "mailpit", - "--exclude", - "pgmeta", - "--exclude", - "studio", - "--exclude", - "analytics", - "--exclude", - "vector", - "--exclude", - "pooler", -] as const; - -describe("supabase start", () => { - test( - "starts in detached mode and prints connection info", - { timeout: DETACHED_START_TIMEOUT_MS }, - async () => { - const home = makeTempHome(); - const project = await makeTempStackProject("supabase-start-e2e-"); - const { stdout, stderr, exitCode } = await runSupabase([...LIGHTWEIGHT_START_ARGS], { - cwd: project.dir, - home: home.dir, - exitTimeoutMs: DETACHED_START_TIMEOUT_MS, - }); - expect(exitCode, `stdout:\n${stdout}\nstderr:\n${stderr}`).toBe(0); - expect(stdout).toContain("Local Supabase started"); - expect(stdout).toContain("API URL:"); - expect(stdout).toContain("DB URL:"); - }, - ); - - test( - "reattaches when detached start is already running", - { timeout: DETACHED_START_TIMEOUT_MS }, - async () => { - const home = makeTempHome(); - const project = await makeTempStackProject("supabase-start-e2e-"); - const first = await runSupabase([...LIGHTWEIGHT_START_ARGS], { - cwd: project.dir, - home: home.dir, - exitTimeoutMs: DETACHED_START_TIMEOUT_MS, - }); - expect(first.exitCode, `stdout:\n${first.stdout}\nstderr:\n${first.stderr}`).toBe(0); - - const second = await runSupabase([...LIGHTWEIGHT_START_ARGS], { - cwd: project.dir, - home: home.dir, - exitTimeoutMs: DETACHED_START_TIMEOUT_MS, - }); - expect(second.exitCode, `stdout:\n${second.stdout}\nstderr:\n${second.stderr}`).toBe(0); - expect(second.stdout).toContain("Start local Supabase stack"); - expect(second.stdout).toContain("Local Supabase started"); - }, - ); -}); diff --git a/apps/cli/src/next/commands/status/status.e2e.test.ts b/apps/cli/src/next/commands/status/status.e2e.test.ts deleted file mode 100644 index fba5693eeb..0000000000 --- a/apps/cli/src/next/commands/status/status.e2e.test.ts +++ /dev/null @@ -1,57 +0,0 @@ -import { describe, expect, test } from "vitest"; -import { makeTempHome, makeTempStackProject, runSupabase } from "../../../../tests/helpers/cli.ts"; - -const STATUS_TIMEOUT_MS = 30_000; -const LIGHTWEIGHT_START_ARGS = [ - "start", - "--detach", - "--exclude", - "realtime", - "--exclude", - "storage", - "--exclude", - "imgproxy", - "--exclude", - "mailpit", - "--exclude", - "pgmeta", - "--exclude", - "studio", - "--exclude", - "analytics", - "--exclude", - "vector", - "--exclude", - "pooler", -] as const; - -describe("supabase status", () => { - test( - "shows connection info and service states for the current project", - { timeout: STATUS_TIMEOUT_MS }, - async () => { - const home = makeTempHome(); - const project = await makeTempStackProject("supabase-status-e2e-"); - const startResult = await runSupabase([...LIGHTWEIGHT_START_ARGS], { - cwd: project.dir, - home: home.dir, - exitTimeoutMs: STATUS_TIMEOUT_MS, - }); - expect(startResult.exitCode).toBe(0); - - const result = await runSupabase(["status"], { cwd: project.dir, home: home.dir }); - - expect(result.exitCode).toBe(0); - expect(result.stdout).toContain("Show local Supabase stack status"); - expect(result.stdout).toContain("Local Supabase stack is running."); - expect(result.stdout).toContain("API URL:"); - expect(result.stdout).toContain("DB URL:"); - expect(result.stdout).toContain("Publishable key:"); - expect(result.stdout).toContain("Secret key:"); - expect(result.stdout).toContain("auth:"); - expect(result.stdout).toContain("postgres:"); - expect(result.stdout).not.toContain("Stack status"); - expect(result.stdout).not.toContain("(running) -"); - }, - ); -}); diff --git a/apps/cli/src/next/commands/stop/stop.e2e.test.ts b/apps/cli/src/next/commands/stop/stop.e2e.test.ts deleted file mode 100644 index c235392a62..0000000000 --- a/apps/cli/src/next/commands/stop/stop.e2e.test.ts +++ /dev/null @@ -1,88 +0,0 @@ -import { describe, expect, test } from "vitest"; -import { existsSync, readdirSync } from "node:fs"; -import { join } from "node:path"; -import { makeTempHome, makeTempStackProject, runSupabase } from "../../../../tests/helpers/cli.ts"; - -const LIGHTWEIGHT_START_ARGS = [ - "start", - "--detach", - "--exclude", - "realtime", - "--exclude", - "storage", - "--exclude", - "imgproxy", - "--exclude", - "mailpit", - "--exclude", - "pgmeta", - "--exclude", - "studio", - "--exclude", - "analytics", - "--exclude", - "vector", - "--exclude", - "pooler", -] as const; -const STOP_STACK_TIMEOUT_MS = 30_000; - -function managedStackDir(homeDir: string): string { - const stacksRoot = join(homeDir, "managed", "stacks"); - const stackIds = existsSync(stacksRoot) - ? readdirSync(stacksRoot, { withFileTypes: true }) - .filter((entry) => entry.isDirectory()) - .map((entry) => entry.name) - : []; - expect(stackIds).toHaveLength(1); - return join(stacksRoot, stackIds[0]!); -} - -describe("supabase stop", () => { - test( - "preserves the persisted stack folder by default", - { timeout: STOP_STACK_TIMEOUT_MS }, - async () => { - const home = makeTempHome(); - const project = await makeTempStackProject("supabase-stop-e2e-"); - - const startResult = await runSupabase([...LIGHTWEIGHT_START_ARGS], { - cwd: project.dir, - home: home.dir, - }); - expect(startResult.exitCode).toBe(0); - const stackDir = managedStackDir(home.dir); - - const stopResult = await runSupabase(["stop"], { cwd: project.dir, home: home.dir }); - expect(stopResult.exitCode).toBe(0); - expect(existsSync(stackDir)).toBe(true); - expect(existsSync(join(stackDir, "stack.json"))).toBe(true); - }, - ); - - test( - "deletes the persisted stack folder with --no-backup", - { timeout: STOP_STACK_TIMEOUT_MS }, - async () => { - const home = makeTempHome(); - const project = await makeTempStackProject("supabase-stop-e2e-"); - - const startResult = await runSupabase([...LIGHTWEIGHT_START_ARGS], { - cwd: project.dir, - home: home.dir, - }); - expect(startResult.exitCode).toBe(0); - const stackDir = managedStackDir(home.dir); - - const stopResult = await runSupabase(["stop", "--no-backup"], { - cwd: project.dir, - home: home.dir, - }); - expect( - stopResult.exitCode, - `stdout:\n${stopResult.stdout}\n\nstderr:\n${stopResult.stderr}`, - ).toBe(0); - expect(existsSync(stackDir)).toBe(false); - }, - ); -}); diff --git a/packages/stack/src/PortAllocator.integration.test.ts b/packages/stack/src/PortAllocator.integration.test.ts index 094bb36326..4ba79b607d 100644 --- a/packages/stack/src/PortAllocator.integration.test.ts +++ b/packages/stack/src/PortAllocator.integration.test.ts @@ -1,8 +1,68 @@ +import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process"; +import { once } from "node:events"; import { createServer, type Server } from "node:net"; +import { resolve } from "node:path"; import { describe, expect, it } from "vitest"; import { Effect } from "effect"; import { allocatePortSet, reservePortSet, type PortReservationRequest } from "./PortAllocator.ts"; +const PORT_LEASE_CHILD = resolve(import.meta.dirname, "../tests/helpers/port-lease-child.ts"); + +interface ChildLease { + readonly process: ChildProcessWithoutNullStreams; + readonly ready: Promise<{ readonly apiPort: number; readonly dbPort: number }>; +} + +const startChildLease = (): ChildLease => { + const child = spawn("bun", ["run", PORT_LEASE_CHILD], { + stdio: ["pipe", "pipe", "pipe"], + }); + const ready = new Promise<{ readonly apiPort: number; readonly dbPort: number }>( + (resolveReady, rejectReady) => { + let stdout = ""; + let stderr = ""; + let settled = false; + + child.stderr.on("data", (chunk: Buffer) => { + stderr += chunk.toString(); + }); + child.stdout.on("data", (chunk: Buffer) => { + if (settled) return; + stdout += chunk.toString(); + const newline = stdout.indexOf("\n"); + if (newline === -1) return; + settled = true; + try { + resolveReady(JSON.parse(stdout.slice(0, newline))); + } catch (error) { + rejectReady(new Error(`Invalid child lease response: ${stdout}`, { cause: error })); + } + }); + child.once("error", (error) => { + if (settled) return; + settled = true; + rejectReady(error); + }); + child.once("close", (code) => { + if (settled) return; + settled = true; + rejectReady( + new Error(`Port lease child exited with code ${code} before readiness: ${stderr}`), + ); + }); + }, + ); + + return { process: child, ready }; +}; + +const releaseChildLease = async (child: ChildProcessWithoutNullStreams): Promise => { + if (child.exitCode !== null || child.signalCode !== null) return; + const closed = once(child, "close"); + child.stdin.end("release\n"); + await closed; +}; + const listen = (port: number) => Effect.callback((resume) => { const server = createServer(); @@ -143,6 +203,29 @@ describe("selected-field port allocation", () => { } }); + it("keeps automatic ports disjoint across processes", async () => { + const firstChild = startChildLease(); + const secondChild = startChildLease(); + const children = [firstChild, secondChild]; + + try { + const [first, second] = await Promise.all([firstChild.ready, secondChild.ready]); + const ports = [first.apiPort, first.dbPort, second.apiPort, second.dbPort]; + expect(new Set(ports).size).toBe(ports.length); + + for (const port of ports) { + const unavailable = await Effect.runPromise( + allocatePortSet([{ field: "apiPort", selection: { kind: "exact", port } }]).pipe( + Effect.exit, + ), + ); + expect(unavailable._tag).toBe("Failure"); + } + } finally { + await Promise.all(children.map(({ process }) => releaseChildLease(process))); + } + }, 30_000); + it("releases partial reservations when a selected set fails", async () => { const firstPort = await Effect.runPromise( Effect.scoped(Effect.map(occupyFreePort(), (occupied) => occupied.port)), diff --git a/packages/stack/tests/helpers/port-lease-child.ts b/packages/stack/tests/helpers/port-lease-child.ts new file mode 100644 index 0000000000..118da3b37f --- /dev/null +++ b/packages/stack/tests/helpers/port-lease-child.ts @@ -0,0 +1,17 @@ +import { Effect } from "effect"; +import { reservePortSet } from "../../src/PortAllocator.ts"; + +const lease = await Effect.runPromise( + reservePortSet([ + { field: "apiPort", selection: { kind: "automatic" } }, + { field: "dbPort", selection: { kind: "automatic" } }, + ]), +); + +process.stdout.write(`${JSON.stringify(lease.ports)}\n`); + +for await (const _chunk of process.stdin) { + break; +} + +await Effect.runPromise(lease.releaseAll); diff --git a/packages/stack/tests/helpers/spawn-stack.ts b/packages/stack/tests/helpers/spawn-stack.ts deleted file mode 100644 index f5c8d4f4f6..0000000000 --- a/packages/stack/tests/helpers/spawn-stack.ts +++ /dev/null @@ -1,116 +0,0 @@ -import { type ChildProcess, spawn } from "node:child_process"; -import { resolve } from "node:path"; -import { terminateChildProcess } from "../../src/terminateChild.ts"; - -const STANDALONE_SCRIPT = resolve(import.meta.dirname, "standalone-stack.ts"); -// The stack's own readiness policy is 180s. This outer subprocess guard must -// leave enough time for that typed failure and its diagnostics to reach stderr. -const DEFAULT_READINESS_TIMEOUT_MS = 200_000; -const OUTPUT_TAIL_CHARS = 2_000; - -export interface SpawnedStackInfo { - readonly url: string; - readonly dbUrl: string; - readonly process: ChildProcess; -} - -export interface SpawnStandaloneStackOptions { - /** Overridable for unit tests only; the e2e suite always runs the real script. */ - readonly command?: readonly [string, ...string[]]; - readonly readinessTimeoutMs?: number; - /** - * Fired the moment the child exists, before readiness. Callers register the - * handle here so teardown can terminate every spawned child even when the - * readiness promise never resolved — a `Promise.all` that dies on one stack - * must not orphan its siblings. - */ - readonly onSpawn?: (child: ChildProcess) => void; -} - -/** - * Spawns one standalone stack subprocess and resolves when it reports - * readiness (a single JSON line on stdout). Unlike a bare spawn-and-parse, - * every way the child can fail settles the promise with the evidence attached: - * - * - exit before readiness — ANY code, including 0 — rejects with the code and - * the child's stderr, so a stack that dies cleanly during bring-up cannot - * turn into an opaque hook timeout with its error discarded; - * - readiness not reported within `readinessTimeoutMs` rejects with the - * stdout/stderr collected so far and terminates the child, so a bring-up - * that wedges (e.g. a port race) fails fast and names the last thing the - * stack said instead of burning the whole hook budget. - */ -export function spawnStandaloneStack( - opts: SpawnStandaloneStackOptions = {}, -): Promise { - const [command, ...args] = opts.command ?? [ - "bun", - "run", - STANDALONE_SCRIPT, - "--parent-pid", - String(process.pid), - ]; - const readinessTimeoutMs = opts.readinessTimeoutMs ?? DEFAULT_READINESS_TIMEOUT_MS; - - return new Promise((resolvePromise, rejectPromise) => { - const child = spawn(command, args, { stdio: ["ignore", "pipe", "pipe"] }); - opts.onSpawn?.(child); - - let stdout = ""; - let stderr = ""; - let settled = false; - - const settle = (outcome: { info?: SpawnedStackInfo; error?: Error }) => { - if (settled) return; - settled = true; - clearTimeout(readinessTimer); - if (outcome.info !== undefined) resolvePromise(outcome.info); - else rejectPromise(outcome.error); - }; - - const outputTail = () => - `stdout: ${stdout.slice(-OUTPUT_TAIL_CHARS) || "(none)"}\nstderr: ${ - stderr.slice(-OUTPUT_TAIL_CHARS) || "(none)" - }`; - - const readinessTimer = setTimeout(() => { - settle({ - error: new Error( - `Stack did not report readiness within ${readinessTimeoutMs}ms\n${outputTail()}`, - ), - }); - // Reclaim the unusable child; the 30s window matches the suite's own - // sweep so SIGKILL doesn't cut a wedged stack's dispose short. - void terminateChildProcess(child, { timeoutMs: 30_000 }); - }, readinessTimeoutMs); - - child.stdout!.on("data", (chunk: Buffer) => { - stdout += chunk.toString(); - const newline = stdout.indexOf("\n"); - if (newline !== -1) { - try { - const info = JSON.parse(stdout.slice(0, newline)); - settle({ info: { url: info.url, dbUrl: info.dbUrl, process: child } }); - } catch { - settle({ error: new Error(`Failed to parse stack info: ${stdout.slice(0, newline)}`) }); - } - } - }); - - child.stderr!.on("data", (chunk: Buffer) => { - stderr += chunk.toString(); - }); - - child.on("error", (err) => settle({ error: err })); - // Any exit before readiness is a failure — including a clean 0. `close` - // rather than `exit`: it waits for the stdio pipes to drain, so the tails - // below always carry whatever the child managed to say. - child.on("close", (code) => { - settle({ - error: new Error( - `Stack process exited with code ${code} before readiness\n${outputTail()}`, - ), - }); - }); - }); -} diff --git a/packages/stack/tests/helpers/spawn-stack.unit.test.ts b/packages/stack/tests/helpers/spawn-stack.unit.test.ts deleted file mode 100644 index bff1bbc7cb..0000000000 --- a/packages/stack/tests/helpers/spawn-stack.unit.test.ts +++ /dev/null @@ -1,132 +0,0 @@ -import { type ChildProcess } from "node:child_process"; -import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; -import { afterAll, describe, expect, test } from "vitest"; -import { terminateChildProcess } from "../../src/terminateChild.ts"; -import { spawnStandaloneStack } from "./spawn-stack.ts"; - -const dir = mkdtempSync(join(tmpdir(), "spawn-stack-unit-")); -const children: ChildProcess[] = []; - -afterAll(() => { - for (const child of children) { - try { - child.kill("SIGKILL"); - } catch {} - } - rmSync(dir, { recursive: true, force: true }); -}); - -function stub(name: string, source: string): readonly [string, ...string[]] { - const path = join(dir, name); - writeFileSync(path, source); - return ["bun", "run", path]; -} - -const track = (child: ChildProcess) => children.push(child); - -describe("spawnStandaloneStack", () => { - test("resolves with the reported url/dbUrl and a live process handle", async () => { - const command = stub( - "ok.ts", - `console.log(JSON.stringify({ url: "http://127.0.0.1:59991", dbUrl: "postgresql://127.0.0.1:59992/x" })); - setInterval(() => {}, 60_000);`, - ); - const info = await spawnStandaloneStack({ command, onSpawn: track }); - expect(info.url).toBe("http://127.0.0.1:59991"); - expect(info.dbUrl).toBe("postgresql://127.0.0.1:59992/x"); - expect(info.process.exitCode).toBeNull(); - }); - - test("rejects with the exit code and stderr when the child dies cleanly before readiness", async () => { - // The pre-fix harness only rejected on a NON-zero exit, so this exact - // child left the promise pending until the 90s hook timeout, with the - // stderr below discarded — the opaque paired-timeout CI failure. - const command = stub( - "silent-exit0.ts", - `process.stderr.write("boot: port 54322 already bound, giving up\\n"); - process.exit(0);`, - ); - await expect(spawnStandaloneStack({ command, onSpawn: track })).rejects.toThrow( - /exited with code 0 before readiness[\s\S]*port 54322 already bound/, - ); - }); - - test("rejects on readiness timeout and reclaims the child", async () => { - const command = stub( - "hang.ts", - `process.stderr.write("boot: waiting for postgres socket...\\n"); - setInterval(() => {}, 60_000);`, - ); - const spawned: ChildProcess[] = []; - let exited: - | Promise<{ - readonly code: number | null; - readonly signal: NodeJS.Signals | null; - }> - | undefined; - await expect( - spawnStandaloneStack({ - command, - readinessTimeoutMs: 1_500, - onSpawn: (child) => { - track(child); - spawned.push(child); - exited = new Promise((resolve) => - child.once("exit", (code, signal) => resolve({ code, signal })), - ); - }, - }), - ).rejects.toThrow(/did not report readiness within 1500ms/); - // The helper terminates its own unusable child rather than leaving an - // interval-driven zombie for suite teardown to hunt. - const exit = await exited; - expect(exit).toEqual({ code: null, signal: "SIGTERM" }); - expect(spawned[0]?.killed).toBe(true); - }); - - test("rejects on an unparseable readiness line", async () => { - const command = stub("garbage.ts", `console.log("not json"); setInterval(() => {}, 60_000);`); - await expect(spawnStandaloneStack({ command, onSpawn: track })).rejects.toThrow( - /Failed to parse stack info: not json/, - ); - }); - - test("registers every child via onSpawn before readiness, so a failed sibling cannot orphan a healthy one", async () => { - const okCommand = stub( - "ok-sibling.ts", - `console.log(JSON.stringify({ url: "http://127.0.0.1:59993", dbUrl: "postgresql://127.0.0.1:59994/x" })); - setInterval(() => {}, 60_000);`, - ); - const badCommand = stub("bad-sibling.ts", `process.exit(0);`); - - const registered: ChildProcess[] = []; - const results = await Promise.allSettled([ - spawnStandaloneStack({ onSpawn: (c) => registered.push(c), command: okCommand }), - spawnStandaloneStack({ onSpawn: (c) => registered.push(c), command: badCommand }), - ]); - children.push(...registered); - - expect(registered).toHaveLength(2); - expect(results.map((r) => r.status).sort()).toEqual(["fulfilled", "rejected"]); - // The healthy sibling's handle is reachable through the registry even - // though Promise.all-style consumption would have discarded its value. - const healthy = registered.find((c) => c.exitCode === null); - expect(healthy).toBeDefined(); - }); - - test("teardown sweep over a dead child is a no-op", async () => { - // The incident replay: one sibling died before readiness, teardown then - // sweeps every registered child with a 30s timeout. Before the - // already-exited guard in terminateChildProcess this call burned 60s - // doing nothing — reproducing the afterAll hook timeout it was meant to - // prevent. - const command = stub("dead-sweep.ts", `process.exit(0);`); - const registered: ChildProcess[] = []; - await spawnStandaloneStack({ command, onSpawn: (c) => registered.push(c) }).catch(() => {}); - expect(registered[0]?.exitCode).toBe(0); - await terminateChildProcess(registered[0]!, { timeoutMs: 30_000 }); - expect(registered[0]?.exitCode).toBe(0); - }); -}); diff --git a/packages/stack/tests/helpers/standalone-stack.ts b/packages/stack/tests/helpers/standalone-stack.ts deleted file mode 100644 index bee2e4e75b..0000000000 --- a/packages/stack/tests/helpers/standalone-stack.ts +++ /dev/null @@ -1,121 +0,0 @@ -import { createStack } from "../../src/node.ts"; - -// Registered before any bring-up work: the spawning harness SIGTERMs a stack -// that misses its readiness deadline, and without these the default signal -// disposition kills the process mid-start with temp dirs and containers left -// behind for the leak check to trip over. A pre-readiness signal is remembered -// and honored at the next await boundary via a dispose-then-exit. -let earlyShutdownRequested = false; -let signalEarlyShutdown = () => { - earlyShutdownRequested = true; -}; -const earlyShutdown = new Promise<"early-shutdown">((resolveSignal) => { - signalEarlyShutdown = () => { - earlyShutdownRequested = true; - resolveSignal("early-shutdown"); - }; -}); -const onEarlySignal = () => signalEarlyShutdown(); -process.once("SIGINT", onEarlySignal); -process.once("SIGTERM", onEarlySignal); - -const parentPid = readParentPid(process.argv.slice(2)); -// This helper proves cross-process stack isolation, not the full service graph -// (the dedicated createStack e2e scenarios cover that). Keeping only the -// mandatory Postgres service makes the concurrency test exercise real ports, -// processes, Docker, and HTTP readiness without starting two redundant copies -// of every local service. -const stack = await createStack({ - postgrest: false, - auth: false, - edgeRuntime: false, - functions: false, -}); -if (earlyShutdownRequested) { - await stack.dispose(); - process.exit(0); -} -// Raced rather than awaited directly: a signal during a HUNG start() must -// still dispose whatever was already created — a flag alone can't run until -// the await returns, which is exactly when it never will. -const starting = stack.start().then( - () => "started" as const, - (error) => { - if (!earlyShutdownRequested) throw error; - return "start-failed" as const; - }, -); -if ((await Promise.race([starting, earlyShutdown])) !== "started") { - await stack.dispose(); - process.exit(0); -} -if (earlyShutdownRequested) { - await stack.dispose(); - process.exit(0); -} -process.off("SIGINT", onEarlySignal); -process.off("SIGTERM", onEarlySignal); - -// Signal readiness to parent process -console.log(JSON.stringify({ url: stack.url, dbUrl: stack.dbUrl })); - -await waitForShutdown(parentPid); -await stack.dispose(); -process.exit(0); - -function waitForShutdown(parentPid: number | undefined): Promise { - return new Promise((resolve) => { - const onShutdown = () => { - cleanup(); - resolve(); - }; - - const onParentExit = () => { - onShutdown(); - }; - - const parentWatchdog = - parentPid == null - ? undefined - : setInterval(() => { - if (!isProcessAlive(parentPid)) { - onParentExit(); - } - }, 250); - - parentWatchdog?.unref(); - - const cleanup = () => { - process.off("SIGINT", onShutdown); - process.off("SIGTERM", onShutdown); - process.off("disconnect", onParentExit); - if (parentWatchdog != null) { - clearInterval(parentWatchdog); - } - }; - - process.once("SIGINT", onShutdown); - process.once("SIGTERM", onShutdown); - process.once("disconnect", onParentExit); - }); -} - -function readParentPid(argv: ReadonlyArray): number | undefined { - const flagIndex = argv.indexOf("--parent-pid"); - const rawValue = flagIndex === -1 ? undefined : argv[flagIndex + 1]; - if (rawValue == null) { - return undefined; - } - - const value = Number.parseInt(rawValue, 10); - return Number.isInteger(value) && value > 0 ? value : undefined; -} - -function isProcessAlive(pid: number): boolean { - try { - process.kill(pid, 0); - return true; - } catch { - return false; - } -} diff --git a/packages/stack/tests/parallelStacks.e2e.test.ts b/packages/stack/tests/parallelStacks.e2e.test.ts deleted file mode 100644 index cd409316bc..0000000000 --- a/packages/stack/tests/parallelStacks.e2e.test.ts +++ /dev/null @@ -1,56 +0,0 @@ -import { type ChildProcess } from "node:child_process"; -import { afterAll, beforeAll, describe, expect, test } from "vitest"; -import { terminateChildProcess } from "../src/terminateChild.ts"; -import { type SpawnedStackInfo, spawnStandaloneStack } from "./helpers/spawn-stack.ts"; - -const STACK_COUNT = 2; -const PARALLEL_STACK_TEST_TIMEOUT_MS = 30_000; - -describe("parallel stacks (multi-process)", () => { - const stacks: SpawnedStackInfo[] = []; - // Registered at spawn time, not readiness: when one stack fails bring-up, - // `Promise.all` discards its healthy siblings' values, so this list — not - // `stacks` — is what teardown owns. - const children: ChildProcess[] = []; - - beforeAll(async () => { - const results = await Promise.all( - Array.from({ length: STACK_COUNT }, () => - spawnStandaloneStack({ onSpawn: (child) => children.push(child) }), - ), - ); - stacks.push(...results); - }, 230_000); - - afterAll(async () => { - await Promise.allSettled( - children.map((child) => terminateChildProcess(child, { timeoutMs: 30_000 })), - ); - expect(children.every((child) => child.exitCode !== null || child.signalCode !== null)).toBe( - true, - ); - }, 60_000); - - test("all stacks use different API ports", { timeout: PARALLEL_STACK_TEST_TIMEOUT_MS }, () => { - const ports = stacks.map((s) => new URL(s.url).port); - expect(new Set(ports).size).toBe(STACK_COUNT); - }); - - test("all stacks use different DB ports", { timeout: PARALLEL_STACK_TEST_TIMEOUT_MS }, () => { - const ports = stacks.map((s) => new URL(s.dbUrl).port); - expect(new Set(ports).size).toBe(STACK_COUNT); - }); - - test( - "all stacks respond to health checks", - { timeout: PARALLEL_STACK_TEST_TIMEOUT_MS }, - async () => { - const responses = await Promise.all( - stacks.map((s) => fetch(`${s.url}/health`, { signal: AbortSignal.timeout(20_000) })), - ); - for (const res of responses) { - expect(res.status).toBe(200); - } - }, - ); -}); From 05d26d80f443a2d23afa795d2211d60954aa6873 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Thu, 20 Aug 2026 14:43:04 +0200 Subject: [PATCH 19/20] fix(stack): close final runtime review gaps --- .../src/ContainerRuntime.integration.test.ts | 2 -- packages/stack/src/ContainerRuntime.ts | 15 ++++++---- packages/stack/src/HttpTransportClient.ts | 5 +--- packages/stack/src/LocalStack.ts | 4 ++- packages/stack/src/RemoteStack.ts | 30 ++++++++++++------- packages/stack/src/services/postgres.ts | 1 + 6 files changed, 34 insertions(+), 23 deletions(-) diff --git a/packages/stack/src/ContainerRuntime.integration.test.ts b/packages/stack/src/ContainerRuntime.integration.test.ts index b2ccc51852..7dc4e50272 100644 --- a/packages/stack/src/ContainerRuntime.integration.test.ts +++ b/packages/stack/src/ContainerRuntime.integration.test.ts @@ -45,7 +45,6 @@ describe("stack runtime selection", () => { mode: "docker", containerRuntime: "docker", }); - expect(spawner.commands).toEqual(["docker"]); }).pipe(Effect.provide(spawner.layer)); }); @@ -56,7 +55,6 @@ describe("stack runtime selection", () => { mode: "docker", containerRuntime: "podman", }); - expect(spawner.commands).toEqual(["docker", "podman"]); }).pipe(Effect.provide(spawner.layer)); }); diff --git a/packages/stack/src/ContainerRuntime.ts b/packages/stack/src/ContainerRuntime.ts index b6ca5e5fa7..f2d83d5576 100644 --- a/packages/stack/src/ContainerRuntime.ts +++ b/packages/stack/src/ContainerRuntime.ts @@ -50,11 +50,16 @@ export const selectStackRuntime = ( return { mode: "native", containerRuntime: null }; } - const runtimes: ReadonlyArray = ["docker", "podman"]; - for (const runtime of runtimes) { - if (yield* probeContainerRuntime(runtime)) { - return { mode: "docker", containerRuntime: runtime }; - } + const runtimes = ["docker", "podman"] as const satisfies ReadonlyArray; + const probes = yield* Effect.all( + runtimes.map((runtime) => + probeContainerRuntime(runtime).pipe(Effect.map((usable) => [runtime, usable] as const)), + ), + { concurrency: "unbounded" }, + ); + const selected = probes.find(([, usable]) => usable)?.[0]; + if (selected !== undefined) { + return { mode: "docker", containerRuntime: selected }; } if (requestedMode === "docker") { diff --git a/packages/stack/src/HttpTransportClient.ts b/packages/stack/src/HttpTransportClient.ts index d9b38a7717..20dcb6232e 100644 --- a/packages/stack/src/HttpTransportClient.ts +++ b/packages/stack/src/HttpTransportClient.ts @@ -25,10 +25,7 @@ export const httpTransportClientLayer = Layer.succeed(HttpTransportClient, { try: () => fetch(`${endpoint.url}${path}`, { ...init, - signal: - init?.signal == null - ? AbortSignal.timeout(30_000) - : AbortSignal.any([init.signal, AbortSignal.timeout(30_000)]), + signal: init?.signal ?? AbortSignal.timeout(30_000), }), catch: (cause) => new HttpTransportClientError({ endpoint, path, cause, reason: "transport" }), diff --git a/packages/stack/src/LocalStack.ts b/packages/stack/src/LocalStack.ts index 8c84564eb5..f6303f6339 100644 --- a/packages/stack/src/LocalStack.ts +++ b/packages/stack/src/LocalStack.ts @@ -377,7 +377,9 @@ export const localStackLayer = ( const state = SubscriptionRef.getUnsafe(stateRef).find( (entry) => entry.name === service, ); - return state === undefined ? [] : [[service, state] as const]; + return state === undefined || state.status === "Downloading" + ? [] + : [[service, state] as const]; }), ); const deferred = Deferred.makeUnsafe(); diff --git a/packages/stack/src/RemoteStack.ts b/packages/stack/src/RemoteStack.ts index bb827cf52e..c44944dc02 100644 --- a/packages/stack/src/RemoteStack.ts +++ b/packages/stack/src/RemoteStack.ts @@ -359,6 +359,9 @@ export const RemoteStack = { Stream.provide(httpTransportClientLayer), Stream.catchTag("HttpTransportClientError", (error) => Stream.die(error)), ); + const withLifecycleRequest = ( + request: (signal: AbortSignal) => Effect.Effect, + ) => withHttpTransportClient(withAbortSignal(request)); return { getInfo: () => @@ -367,10 +370,10 @@ export const RemoteStack = { ), start: () => - withHttpTransportClient( + withLifecycleRequest((signal) => Effect.gen(function* () { const path = "/start"; - const response = yield* httpResponse(endpoint, path, { method: "POST" }); + const response = yield* httpResponse(endpoint, path, { method: "POST", signal }); yield* expectDaemonOk(endpoint, path, response, "stack").pipe( Effect.catchTag("ServiceNotFoundError", (error) => Effect.die(error)), ); @@ -378,10 +381,10 @@ export const RemoteStack = { ), stop: () => - withHttpTransportClient( + withLifecycleRequest((signal) => Effect.gen(function* () { const path = "/stop"; - const response = yield* httpResponse(endpoint, path, { method: "POST" }); + const response = yield* httpResponse(endpoint, path, { method: "POST", signal }); yield* dieOnNonOkStatus( endpoint, path, @@ -391,10 +394,10 @@ export const RemoteStack = { ), dispose: () => - withHttpTransportClient( + withLifecycleRequest((signal) => Effect.gen(function* () { const path = "/stop"; - const response = yield* httpResponse(endpoint, path, { method: "POST" }); + const response = yield* httpResponse(endpoint, path, { method: "POST", signal }); yield* dieOnNonOkStatus( endpoint, path, @@ -404,24 +407,26 @@ export const RemoteStack = { ), startService: (name: string) => - withHttpTransportClient( + withLifecycleRequest((signal) => Effect.gen(function* () { const servicePath = yield* publicServicePath(name); const path = `/services/${servicePath}/start`; const response = yield* httpResponse(endpoint, path, { method: "POST", + signal, }); yield* expectMutatingDaemonOk(endpoint, path, response, name); }), ), stopService: (name: string) => - withHttpTransportClient( + withLifecycleRequest((signal) => Effect.gen(function* () { const servicePath = yield* publicServicePath(name); const path = `/services/${servicePath}/stop`; const response = yield* httpResponse(endpoint, path, { method: "POST", + signal, }); yield* expectMutatingDaemonOk(endpoint, path, response, name).pipe( Effect.catchTag("ServiceReadyError", (error) => Effect.die(error)), @@ -431,23 +436,25 @@ export const RemoteStack = { ), restartService: (name: string) => - withHttpTransportClient( + withLifecycleRequest((signal) => Effect.gen(function* () { const servicePath = yield* publicServicePath(name); const path = `/services/${servicePath}/restart`; const response = yield* httpResponse(endpoint, path, { method: "POST", + signal, }); yield* expectMutatingDaemonOk(endpoint, path, response, name); }), ), reloadFunctions: (opts) => - withHttpTransportClient( + withLifecycleRequest((signal) => Effect.gen(function* () { const path = "/functions/reload"; const response = yield* httpResponse(endpoint, path, { method: "POST", + signal, headers: { "content-type": "application/json" }, body: JSON.stringify(opts ?? {}), }); @@ -456,11 +463,12 @@ export const RemoteStack = { ), reloadEdgeRuntime: (opts) => - withHttpTransportClient( + withLifecycleRequest((signal) => Effect.gen(function* () { const path = "/edge-runtime/reload"; const response = yield* httpResponse(endpoint, path, { method: "POST", + signal, headers: { "content-type": "application/json" }, body: JSON.stringify(opts), }); diff --git a/packages/stack/src/services/postgres.ts b/packages/stack/src/services/postgres.ts index 582ce839cf..0832a01bf5 100644 --- a/packages/stack/src/services/postgres.ts +++ b/packages/stack/src/services/postgres.ts @@ -151,6 +151,7 @@ export const makePostgresServiceDocker = (opts: DockerPostgresOptions): ServiceD hostUser === undefined ? "" : `printf 'supabase_cli:x:${hostUid}:${hostGid}:Supabase CLI:/tmp:/usr/bin/sh\\n' >> /etc/passwd +busybox chown ${hostUid}:${hostGid} /var/lib/postgresql/data busybox chown ${hostUid}:${hostGid} /opt/postgres/share/supabase-cli/config/pgsodium_getkey.sh `; const command = `${hostUserSetup}if [ -s /var/lib/postgresql/data/PG_VERSION ]; then From cbc66a7a3218c0d72a248c429973f8647c3f2a6a Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Thu, 20 Aug 2026 16:26:39 +0200 Subject: [PATCH 20/20] fix(stack): make native initialization resumable --- .../process-compose/tests/helpers/mocks.ts | 22 +++-- packages/stack/src/services/postgres-init.ts | 89 ++++++++++------- packages/stack/src/services/postgres.ts | 30 +++++- .../stack/src/services/services.unit.test.ts | 97 +------------------ .../tests/createStack-native.e2e.test.ts | 45 ++++++++- 5 files changed, 138 insertions(+), 145 deletions(-) diff --git a/packages/process-compose/tests/helpers/mocks.ts b/packages/process-compose/tests/helpers/mocks.ts index 849c85eef5..2fad3f626f 100644 --- a/packages/process-compose/tests/helpers/mocks.ts +++ b/packages/process-compose/tests/helpers/mocks.ts @@ -13,14 +13,22 @@ const isOneShotSupervisor = (args: ReadonlyArray): boolean => { if (encoded === undefined) return false; try { const config: unknown = JSON.parse(Buffer.from(encoded, "base64url").toString("utf8")); + if ( + typeof config !== "object" || + config === null || + !("command" in config) || + !("args" in config) || + !Array.isArray(config.args) + ) { + return false; + } + const bashScript = + config.command === "bash" && config.args[0] === "-c" && typeof config.args[1] === "string" + ? config.args[1] + : undefined; return ( - typeof config === "object" && - config !== null && - "command" in config && - "args" in config && - Array.isArray(config.args) && - ((config.command === "bash" && config.args[0] === "-c") || - ((config.command === "docker" || config.command === "podman") && config.args[0] === "exec")) + (bashScript !== undefined && !/(?:^|\n)\s*exec\s/.test(bashScript)) || + ((config.command === "docker" || config.command === "podman") && config.args[0] === "exec") ); } catch { return false; diff --git a/packages/stack/src/services/postgres-init.ts b/packages/stack/src/services/postgres-init.ts index da787a558e..fbbf527499 100644 --- a/packages/stack/src/services/postgres-init.ts +++ b/packages/stack/src/services/postgres-init.ts @@ -28,7 +28,7 @@ interface DockerPostgresInitOptions extends ContainerRuntimeOptions { * tables/sequences/functions in `public` owned by `postgres` are not reachable via the Data API * roles without explicit GRANTs. */ -export const REVOKE_DEFAULT_DATA_API_PRIVILEGES_SQL = ` +const REVOKE_DEFAULT_DATA_API_PRIVILEGES_SQL = ` alter default privileges for role postgres in schema public revoke select, insert, update, delete on tables from anon, authenticated, service_role; alter default privileges for role postgres in schema public @@ -94,35 +94,53 @@ export const makePostgresInitService = (opts: PostgresInitOptions): ServiceDef = const pgLibDir = `${opts.postgresDir}/lib`; const migrationsDir = `${opts.postgresDir}/share/supabase-cli/migrations`; - const psql = `${pgBinDir}/psql -h 127.0.0.1 -p ${opts.dbPort}`; - const psqlOpts = `-v ON_ERROR_STOP=1 --no-password --no-psqlrc`; - - const revokeStep = opts.autoExposeNewTables - ? "" - : ` - # Revoke default privileges for the Data API roles on schema public so new tables - # require explicit GRANTs. Mirrors Studio's behaviour at cloud project creation. - ${psql} ${psqlOpts} -U postgres -d postgres <<'EOSQL' -${REVOKE_DEFAULT_DATA_API_PRIVILEGES_SQL} -EOSQL -`; + // Keep executable and SQL-file paths in arrays so cache roots containing + // whitespace remain single argv entries all the way to psql. + const psqlPath = `${pgBinDir}/psql`; + const shellQuote = (value: string): string => `'${value.replaceAll("'", "'\\''")}'`; + const psqlArray = `psql=(${shellQuote(psqlPath)} -h 127.0.0.1 -p ${opts.dbPort})`; // Replaces calling migrate.sh (which spawns ~57 separate psql processes) with // chained -f flags that run all SQL files in a single psql session, cutting // postgres-init time from ~5s to ~1s. const script = ` +set -e export PATH="${pgBinDir}:$PATH" export PGPASSWORD=postgres db="${migrationsDir}" +${psqlArray} +psql_opts=(-v ON_ERROR_STOP=1 --no-password --no-psqlrc) + +init_completion_sql=$(cat <<'EOSQL' +ALTER USER supabase_admin WITH PASSWORD 'postgres'; +CREATE SCHEMA IF NOT EXISTS supabase_migrations; +CREATE TABLE IF NOT EXISTS supabase_migrations.cli_init ( + phase text PRIMARY KEY, + completed_at timestamptz NOT NULL DEFAULT now() +); +INSERT INTO supabase_migrations.cli_init (phase) +VALUES ('init') +ON CONFLICT (phase) DO NOTHING; +EOSQL +) -# Check if already migrated (authenticator role created by initial-schema.sql) -if ${psql} -U supabase_admin -d postgres -tAc "SELECT 1 FROM pg_roles WHERE rolname='authenticator'" 2>/dev/null | grep -q 1; then - echo "Database already initialized, updating passwords..." +migration_completion_sql=$(cat <<'EOSQL' +${opts.autoExposeNewTables ? "" : REVOKE_DEFAULT_DATA_API_PRIVILEGES_SQL} +INSERT INTO supabase_migrations.cli_init (phase) +VALUES ('complete') +ON CONFLICT (phase) DO UPDATE SET completed_at = EXCLUDED.completed_at; +EOSQL +) + +# The init phase is committed independently so a failed migration phase can +# resume without replaying non-idempotent bundled init scripts. +if "\${psql[@]}" -U supabase_admin -d postgres -tAc "SELECT 1 FROM supabase_migrations.cli_init WHERE phase = 'init'" 2>/dev/null | grep -q 1; then + echo "Database initial schema already initialized" else echo "Running Supabase migrations..." # Create postgres role if missing (as supabase_admin) - ${psql} ${psqlOpts} -U supabase_admin -d postgres <<'EOSQL' + "\${psql[@]}" "\${psql_opts[@]}" -U supabase_admin -d postgres <<'EOSQL' do $$ begin if not exists (select from pg_roles where rolname = 'postgres') then @@ -133,41 +151,40 @@ end $$ EOSQL # Run all init-scripts in a single psql session (as postgres) - init_flags="" + init_flags=() for sql in "$db"/init-scripts/*.sql; do - [ -f "$sql" ] && init_flags="$init_flags -f $sql" + [ -f "$sql" ] && init_flags+=( -f "$sql" ) done - if [ -n "$init_flags" ]; then - ${psql} ${psqlOpts} -U postgres -d postgres $init_flags - fi + "\${psql[@]}" "\${psql_opts[@]}" --single-transaction -U postgres -d postgres "\${init_flags[@]}" -c "$init_completion_sql" +fi - # Set supabase_admin password (as postgres) - ${psql} ${psqlOpts} -U postgres -d postgres -c "ALTER USER supabase_admin WITH PASSWORD 'postgres'" +if "\${psql[@]}" -U supabase_admin -d postgres -tAc "SELECT 1 FROM supabase_migrations.cli_init WHERE phase = 'complete'" 2>/dev/null | grep -q 1; then + echo "Database migrations already initialized" +else + echo "Running Supabase migrations..." # Run all migrations in a single psql session (as supabase_admin) - migrate_flags="" + migrate_flags=() for sql in "$db"/migrations/*.sql; do - [ -f "$sql" ] && migrate_flags="$migrate_flags -f $sql" + [ -f "$sql" ] && migrate_flags+=( -f "$sql" ) done - if [ -n "$migrate_flags" ]; then - ${psql} ${psqlOpts} -U supabase_admin -d postgres $migrate_flags - fi + "\${psql[@]}" "\${psql_opts[@]}" --single-transaction -U supabase_admin -d postgres "\${migrate_flags[@]}" -c "$migration_completion_sql" # Reset stats (non-fatal, matches migrate.sh) - ${psql} ${psqlOpts} -U supabase_admin -d postgres -c 'SELECT extensions.pg_stat_statements_reset(); SELECT pg_stat_reset();' || true -${revokeStep}fi + "\${psql[@]}" "\${psql_opts[@]}" -U supabase_admin -d postgres -c 'SELECT extensions.pg_stat_statements_reset(); SELECT pg_stat_reset();' || true +fi # Backfill schemas/databases used by docker-backed auxiliary services. -${psql} ${psqlOpts} -U postgres -d postgres <<'EOSQL' +"\${psql[@]}" "\${psql_opts[@]}" -U postgres -d postgres <<'EOSQL' CREATE SCHEMA IF NOT EXISTS _realtime; ALTER SCHEMA _realtime OWNER TO postgres; EOSQL -if ! ${psql} -U postgres -d postgres -tAc "SELECT 1 FROM pg_database WHERE datname = '_supabase'" 2>/dev/null | grep -q 1; then - ${psql} ${psqlOpts} -U postgres -d postgres -c "CREATE DATABASE _supabase WITH OWNER postgres" +if ! "\${psql[@]}" -U postgres -d postgres -tAc "SELECT 1 FROM pg_database WHERE datname = '_supabase'" 2>/dev/null | grep -q 1; then + "\${psql[@]}" "\${psql_opts[@]}" -U postgres -d postgres -c "CREATE DATABASE _supabase WITH OWNER postgres" fi -${psql} ${psqlOpts} -U postgres -d _supabase <<'EOSQL' +"\${psql[@]}" "\${psql_opts[@]}" -U postgres -d _supabase <<'EOSQL' CREATE SCHEMA IF NOT EXISTS _analytics; ALTER SCHEMA _analytics OWNER TO postgres; CREATE SCHEMA IF NOT EXISTS _supavisor; @@ -175,7 +192,7 @@ ALTER SCHEMA _supavisor OWNER TO postgres; EOSQL # Always update role passwords (idempotent) -${psql} -U supabase_admin -d postgres -c " +"\${psql[@]}" -U supabase_admin -d postgres -c " DO \\$\\$ DECLARE roles text[] := ARRAY['authenticator','supabase_auth_admin','supabase_storage_admin','supabase_functions_admin','supabase_replication_admin','supabase_read_only_user','postgres']; diff --git a/packages/stack/src/services/postgres.ts b/packages/stack/src/services/postgres.ts index 0832a01bf5..05b7f0d98b 100644 --- a/packages/stack/src/services/postgres.ts +++ b/packages/stack/src/services/postgres.ts @@ -1,5 +1,8 @@ +import { createHash } from "node:crypto"; +import { join } from "node:path"; import type { ServiceDef } from "@supabase/process-compose"; import { dockerNetworkArgs } from "../Platform.ts"; +import { shortTempPrefixRoot } from "../paths.ts"; import { dockerContainerName, type StackIdentity } from "../StackIdentity.ts"; import { removePathOnOrphanCleanup } from "./docker-cleanup.ts"; import { stackHealthBudgets } from "./health-budgets.ts"; @@ -48,11 +51,13 @@ const NATIVE_POSTGRES_RUNTIME_ARGS = [ "max_replication_slots=5", ] as const; +const orphanCleanup = (opts: PostgresServiceOptions) => + opts.cleanupDataDirOnExit ? removePathOnOrphanCleanup(opts.dataDir) : []; + const postgresGetKeyScript = (binPath: string): string => `${binPath}/share/supabase-cli/config/pgsodium_getkey.sh`; -const orphanCleanup = (opts: PostgresServiceOptions) => - opts.cleanupDataDirOnExit ? removePathOnOrphanCleanup(opts.dataDir) : []; +const shellQuote = (value: string): string => `'${value.replaceAll("'", "'\\''")}'`; const postgresHealthCheck = (binPath: string, port: number) => ({ probe: { @@ -97,14 +102,29 @@ const postgresDockerHealthCheck = ( ); export const makePostgresService = (opts: NativePostgresOptions): ServiceDef => { - const initScript = `${opts.binPath}/share/supabase-cli/bin/supabase-postgres-init.sh`; - const getKeyScript = postgresGetKeyScript(opts.binPath); + // The bundled initializer persists absolute artifact paths in postgresql.conf. + // pgsodium executes its getkey path through a shell, so use a stable no-space + // alias for the whole bundle when the cache root itself contains whitespace. + const bundleKey = createHash("sha256").update(opts.dataDir).digest("hex").slice(0, 16); + const bundlePath = join( + shortTempPrefixRoot(), + `supabase-stack-postgres-${process.getuid?.() ?? process.pid}-${bundleKey}`, + ); + const initScript = `${bundlePath}/share/supabase-cli/bin/supabase-postgres-init.sh`; + const getKeyScript = postgresGetKeyScript(bundlePath); + const launchScript = ` +set -e +ln -sfn ${shellQuote(opts.binPath)} ${shellQuote(bundlePath)} +exec ${shellQuote(initScript)} "$@" +`; return { name: "postgres", command: "bash", args: [ - initScript, + "-c", + launchScript, + "--", "-p", String(opts.port), ...NATIVE_POSTGRES_RUNTIME_ARGS, diff --git a/packages/stack/src/services/services.unit.test.ts b/packages/stack/src/services/services.unit.test.ts index 1c8fce7d31..c6ec3962b0 100644 --- a/packages/stack/src/services/services.unit.test.ts +++ b/packages/stack/src/services/services.unit.test.ts @@ -8,10 +8,7 @@ import { makeEdgeRuntimeServiceDocker, makeEdgeRuntimeServiceNative } from "./ed import { makeImgproxyServiceDocker } from "./imgproxy.ts"; import { makeMailpitServiceDocker } from "./mailpit.ts"; import { makePgmetaServiceDocker } from "./pgmeta.ts"; -import { - makePostgresInitService, - REVOKE_DEFAULT_DATA_API_PRIVILEGES_SQL, -} from "./postgres-init.ts"; +import { makePostgresInitService } from "./postgres-init.ts"; import { makePostgresService, makePostgresServiceDocker } from "./postgres.ts"; import { makePostgrestService, makePostgrestServiceDocker } from "./postgrest.ts"; import { makeRealtimeServiceDocker } from "./realtime.ts"; @@ -47,16 +44,7 @@ describe("makePostgresService", () => { expect(def.name).toBe("postgres"); expect(def.command).toBe("bash"); - expect(def.args).toContain( - `${POSTGRES_BIN_PATH}/share/supabase-cli/bin/supabase-postgres-init.sh`, - ); expect(def.args).toContain("listen_addresses=127.0.0.1"); - expect(def.args).toContain( - `pgsodium.getkey_script=${POSTGRES_BIN_PATH}/share/supabase-cli/config/pgsodium_getkey.sh`, - ); - expect(def.args).toContain( - `vault.getkey_script=${POSTGRES_BIN_PATH}/share/supabase-cli/config/pgsodium_getkey.sh`, - ); expect(def.env?.PGDATA).toBe("/tmp/supabase/data"); expect(def.env?.POSTGRES_PASSWORD).toBe("postgres"); expect(def.env?.DYLD_LIBRARY_PATH).toBe(`${POSTGRES_BIN_PATH}/lib`); @@ -401,89 +389,6 @@ describe("makePostgresInitService", () => { expect(def.env?.LD_LIBRARY_PATH).toBe(`${POSTGRES_BIN_PATH}/lib`); expect(def.supervision).toBeDefined(); }); - - it("does not use set -e (matches Go template approach)", () => { - const def = makePostgresInitService({ - postgresDir: POSTGRES_BIN_PATH, - dbPort: DB_PORT, - autoExposeNewTables: true, - dependencies: [{ service: "postgres", condition: "healthy" }], - }); - const script = def.args?.[1] as string; - expect(script).not.toContain("set -e"); - }); - - it("includes idempotency check for authenticator role", () => { - const def = makePostgresInitService({ - postgresDir: "/cache/postgres/17/darwin-arm64", - dbPort: DB_PORT, - autoExposeNewTables: true, - dependencies: [{ service: "postgres", condition: "healthy" }], - }); - const script = def.args?.[1] as string; - expect(script).toContain("authenticator"); - expect(script).toContain("already initialized"); - }); - - it("backfills auxiliary service schemas and internal databases", () => { - const def = makePostgresInitService({ - postgresDir: "/cache/postgres/17/darwin-arm64", - dbPort: DB_PORT, - autoExposeNewTables: true, - dependencies: [{ service: "postgres", condition: "healthy" }], - }); - const script = def.args?.[1] as string; - - expect(script).toContain("CREATE SCHEMA IF NOT EXISTS _realtime"); - expect(script).toContain("SELECT 1 FROM pg_database WHERE datname = '_supabase'"); - expect(script).toContain("CREATE DATABASE _supabase WITH OWNER postgres"); - expect(script).toContain("CREATE SCHEMA IF NOT EXISTS _analytics"); - expect(script).toContain("CREATE SCHEMA IF NOT EXISTS _supavisor"); - }); - - it("batches SQL files via chained -f flags instead of shelling out to migrate.sh", () => { - const def = makePostgresInitService({ - postgresDir: "/cache/postgres/17/darwin-arm64", - dbPort: DB_PORT, - autoExposeNewTables: true, - dependencies: [{ service: "postgres", condition: "healthy" }], - }); - const script = def.args?.[1] as string; - expect(script).not.toMatch(/sh .+migrate\.sh/); - expect(script).toContain("-f $sql"); - expect(script).toContain("init-scripts/*.sql"); - expect(script).toContain("migrations/*.sql"); - }); - - it("does not revoke default Data API privileges when autoExposeNewTables is true", () => { - const def = makePostgresInitService({ - postgresDir: POSTGRES_BIN_PATH, - dbPort: DB_PORT, - autoExposeNewTables: true, - dependencies: [{ service: "postgres", condition: "healthy" }], - }); - const script = def.args?.[1] as string; - expect(script).not.toContain("alter default privileges"); - expect(script).not.toContain("revoke select, insert, update, delete on tables"); - }); - - it("revokes default Data API privileges on `public` when autoExposeNewTables is false", () => { - const def = makePostgresInitService({ - postgresDir: POSTGRES_BIN_PATH, - dbPort: DB_PORT, - autoExposeNewTables: false, - dependencies: [{ service: "postgres", condition: "healthy" }], - }); - const script = def.args?.[1] as string; - expect(script).toContain(REVOKE_DEFAULT_DATA_API_PRIVILEGES_SQL); - expect(script).toContain( - "revoke select, insert, update, delete on tables from anon, authenticated, service_role", - ); - expect(script).toContain( - "revoke usage, select on sequences from anon, authenticated, service_role", - ); - expect(script).toContain("revoke execute on functions from anon, authenticated, service_role"); - }); }); describe("docker-backed auxiliary services", () => { diff --git a/packages/stack/tests/createStack-native.e2e.test.ts b/packages/stack/tests/createStack-native.e2e.test.ts index 34a145b271..2ace232b50 100644 --- a/packages/stack/tests/createStack-native.e2e.test.ts +++ b/packages/stack/tests/createStack-native.e2e.test.ts @@ -1,19 +1,25 @@ import { createClient } from "@supabase/supabase-js"; -import { mkdtempSync, rmSync } from "node:fs"; +import { mkdtempSync, rmSync, symlinkSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { createStack, type StackHandle } from "../src/node.ts"; +import { defaultCacheRoot } from "../src/paths.ts"; import { setupTestTable } from "./helpers/e2e.ts"; describe("native PostgREST tracer bullet", () => { let stack: StackHandle; let dataDir: string; + let cacheParent: string; beforeAll(async () => { dataDir = mkdtempSync(join(tmpdir(), "supabase-native-postgrest-e2e-")); + cacheParent = mkdtempSync(join(tmpdir(), "supabase-native-cache-parent-")); + const cacheRoot = join(cacheParent, "cache root with spaces"); + symlinkSync(defaultCacheRoot(), cacheRoot, "dir"); stack = await createStack({ mode: "native", + cacheRoot, functions: false, edgeRuntime: false, auth: false, @@ -26,6 +32,7 @@ describe("native PostgREST tracer bullet", () => { afterAll(async () => { await stack?.dispose(); rmSync(dataDir, { recursive: true, force: true }); + rmSync(cacheParent, { recursive: true, force: true }); }, 30_000); test("serves a CRUD request through the native PostgREST resource", async () => { @@ -42,4 +49,40 @@ describe("native PostgREST tracer bullet", () => { const deleted = await client.from("todos").delete().eq("title", "native tracer bullet"); expect(deleted.error).toBeNull(); }, 30_000); + + test("repairs incomplete bundled initialization on restart", async () => { + const adminUrl = new URL(stack.dbUrl); + adminUrl.username = "supabase_admin"; + const sql = new Bun.SQL(adminUrl.toString()); + try { + await sql.unsafe(` + DELETE FROM supabase_migrations.cli_init WHERE phase = 'complete'; + ALTER ROLE authenticator RESET session_preload_libraries; + `); + } finally { + await sql.close(); + } + + await stack.stop(); + await stack.start(); + + const check = new Bun.SQL(adminUrl.toString()); + try { + const rows = await check.unsafe<{ configured: boolean }[]>(` + SELECT EXISTS ( + SELECT 1 + FROM pg_roles + WHERE rolname = 'authenticator' + AND EXISTS ( + SELECT 1 + FROM unnest(coalesce(rolconfig, ARRAY[]::text[])) setting + WHERE setting LIKE 'session_preload_libraries=supautils%' + ) + ) AS configured; + `); + expect(rows[0]?.configured).toBe(true); + } finally { + await check.close(); + } + }, 30_000); });