From 4f816d4bdbcd899d281a7a931e073094ab543fe6 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Fri, 9 Oct 2026 12:22:40 -0600 Subject: [PATCH 1/2] test(runtime): cover MicroSandbox image backend parity --- .github/workflows/pr-ci.yml | 9 + e2e/tests/up/provider_microsandbox.go | 10 +- e2e/tests/up/provider_microsandbox_images.go | 167 ++++++++++++++++++ .../microsandbox-images/.devcontainer.json | 11 ++ .../testdata/microsandbox-images/Dockerfile | 3 + .../features/parity/devcontainer-feature.json | 8 + .../features/parity/install.sh | 4 + .../developing-providers/runtime-protocol.mdx | 8 +- 8 files changed, 216 insertions(+), 4 deletions(-) create mode 100644 e2e/tests/up/provider_microsandbox_images.go create mode 100644 e2e/tests/up/testdata/microsandbox-images/.devcontainer.json create mode 100644 e2e/tests/up/testdata/microsandbox-images/Dockerfile create mode 100644 e2e/tests/up/testdata/microsandbox-images/features/parity/devcontainer-feature.json create mode 100644 e2e/tests/up/testdata/microsandbox-images/features/parity/install.sh diff --git a/.github/workflows/pr-ci.yml b/.github/workflows/pr-ci.yml index 9227a332e..b91c157f9 100644 --- a/.github/workflows/pr-ci.yml +++ b/.github/workflows/pr-ci.yml @@ -642,6 +642,15 @@ jobs: test-timeout: 1200s job-timeout-minutes: 25 + - label: up-provider-microsandbox-images + runner: ubuntu-latest + free-disk-space: false + install-kind: false + requires-secret: false + install-microsandbox: true + test-timeout: 1200s + job-timeout-minutes: 25 + # Snapshot tests - label: snapshot diff --git a/e2e/tests/up/provider_microsandbox.go b/e2e/tests/up/provider_microsandbox.go index 43041f58e..944b7773a 100644 --- a/e2e/tests/up/provider_microsandbox.go +++ b/e2e/tests/up/provider_microsandbox.go @@ -21,7 +21,11 @@ import ( "github.com/onsi/gomega" ) -const osLinux = "linux" +const ( + osLinux = "linux" + microsandboxExternalProvider = "github.com/devsy-org/devsy-provider-microsandbox@v0.1.5" + microsandboxRootUser = "root" +) func skipIfNoMicrosandbox(ctx context.Context) { checkMicrosandboxVersion(ctx) @@ -166,7 +170,7 @@ var _ = ginkgo.Describe( framework.ExpectNoError(err) var devConfig map[string]any framework.ExpectNoError(json.Unmarshal(data, &devConfig)) - devConfig["remoteUser"] = "root" + devConfig["remoteUser"] = microsandboxRootUser data, err = json.Marshal(devConfig) framework.ExpectNoError(err) framework.ExpectNoError(os.WriteFile(configPath, data, 0o600)) @@ -210,7 +214,7 @@ var _ = ginkgo.Describe( ginkgo.SpecTimeout(framework.TimeoutLong())), ginkgo.Entry( "external v0.1.5", - "github.com/devsy-org/devsy-provider-microsandbox@v0.1.5", + microsandboxExternalProvider, "microsandbox-external-parity", ginkgo.SpecTimeout(framework.TimeoutLong()), ), diff --git a/e2e/tests/up/provider_microsandbox_images.go b/e2e/tests/up/provider_microsandbox_images.go new file mode 100644 index 000000000..aafeaef9e --- /dev/null +++ b/e2e/tests/up/provider_microsandbox_images.go @@ -0,0 +1,167 @@ +package up + +import ( + "context" + "encoding/json" + "io" + "log" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "sync/atomic" + + "github.com/devsy-org/devsy/e2e/framework" + "github.com/devsy-org/devsy/pkg/image" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/registry" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" +) + +const microsandboxImageFixture = "tests/up/testdata/microsandbox-images" + +var _ = ginkgo.Describe("microsandbox image backend parity", + ginkgo.Label("up-provider-microsandbox-images"), func() { + for _, provider := range []struct{ name, source string }{ + {"builtin", "microsandbox"}, + {"external", microsandboxExternalProvider}, + } { + ginkgo.Context(provider.name, func() { + var f *framework.Framework + var tempDir string + + ginkgo.BeforeEach(func(ctx context.Context) { + skipIfNoMicrosandbox(ctx) + ginkgo.GinkgoT().Setenv("DEVSY_HOME", ginkgo.GinkgoT().TempDir()) + ginkgo.GinkgoT().Setenv("DEVSY_CONFIG", "") + initialDir, err := os.Getwd() + framework.ExpectNoError(err) + f = framework.NewDefaultFramework(filepath.Join(initialDir, "bin")) + tempDir, err = framework.CopyToTempDir(microsandboxImageFixture) + framework.ExpectNoError(err) + ginkgo.DeferCleanup(framework.CleanupTempDir, initialDir, tempDir) + framework.ExpectNoError( + f.DevsyProviderAdd(ctx, provider.source, "--name", provider.name), + ) + ginkgo.DeferCleanup(f.DevsyProviderDelete, provider.name) + ginkgo.DeferCleanup(f.CleanupWorkspace, tempDir) + }) + + ginkgo.It( + "falls back to the registry when the image is absent from Docker", + func(ctx context.Context) { + ref, manifestReads := microsandboxRegistryImage(ctx) + assertMicrosandboxImageAbsentFromDocker(ctx, ref) + writeMicrosandboxImageConfig(tempDir, ref) + framework.ExpectNoError( + f.DevsyUp(ctx, tempDir, "--devcontainer", ".devcontainer.json"), + ) + out, err := f.DevsySSHOnce(ctx, tempDir, "grep '^ID=' /etc/os-release") + framework.ExpectNoError(err) + gomega.Expect(out).To(gomega.Equal("ID=alpine\n")) + gomega.Expect(manifestReads.Load()).To(gomega.BeNumerically(">", 0)) + assertMicrosandboxImageAbsentFromDocker(ctx, ref) + }, + ginkgo.SpecTimeout(framework.TimeoutLong()), + ) + + ginkgo.It( + "loads a Docker-only image without a registry copy", + func(ctx context.Context) { + ref := "devsy-msb-local:" + filepath.Base(tempDir) + microsandboxDocker( + ctx, + "build", + "--platform", + "linux/"+runtime.GOARCH, + "-t", + ref, + tempDir, + ) + ginkgo.DeferCleanup(func(cleanupCtx context.Context) { + microsandboxDocker(cleanupCtx, "image", "rm", ref) + }) + writeMicrosandboxImageConfig(tempDir, ref) + framework.ExpectNoError( + f.DevsyUp(ctx, tempDir, "--devcontainer", ".devcontainer.json"), + ) + assertMicrosandboxImageMarker(ctx, f, tempDir) + }, + ginkgo.SpecTimeout(framework.TimeoutLong()), + ) + + ginkgo.It( + "builds a Dockerfile and installs a local Dev Container Feature", + func(ctx context.Context) { + framework.ExpectNoError( + f.DevsyUp(ctx, tempDir, "--devcontainer", ".devcontainer.json"), + ) + assertMicrosandboxImageMarker(ctx, f, tempDir) + out, err := f.DevsySSHOnce( + ctx, + tempDir, + "cat /usr/local/share/devsy-feature-parity; printf '%s\\n' \"$DEVSY_FEATURE_PARITY\"", + ) + framework.ExpectNoError(err) + gomega.Expect(out).To(gomega.Equal("feature-parity\nfeature-parity\n")) + }, + ginkgo.SpecTimeout(framework.TimeoutLong()), + ) + }) + } + }) + +func microsandboxRegistryImage(ctx context.Context) (string, *atomic.Int64) { + var manifestReads atomic.Int64 + handler := registry.New(registry.Logger(log.New(io.Discard, "", 0))) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodGet && strings.Contains(r.URL.Path, "/manifests/") { + manifestReads.Add(1) + } + handler.ServeHTTP(w, r) + })) + ginkgo.DeferCleanup(server.Close) + ref, err := name.NewTag(server.Listener.Addr().String() + "/parity:registry") + framework.ExpectNoError(err) + img, err := image.GetImageForArch( + ctx, + "ghcr.io/devsy-org/test-images/base:alpine", + runtime.GOARCH, + ) + framework.ExpectNoError(err) + framework.ExpectNoError(remote.Write(ref, img, remote.WithContext(ctx))) + manifestReads.Store(0) + return ref.Name(), &manifestReads +} + +func writeMicrosandboxImageConfig(dir, ref string) { + data, err := json.Marshal(map[string]string{ + "image": ref, "containerUser": microsandboxRootUser, "remoteUser": microsandboxRootUser, + }) + framework.ExpectNoError(err) + framework.ExpectNoError(os.WriteFile(filepath.Join(dir, ".devcontainer.json"), data, 0o600)) +} + +func assertMicrosandboxImageMarker(ctx context.Context, f *framework.Framework, workspace string) { + out, err := f.DevsySSHOnce(ctx, workspace, + "cat /usr/local/share/devsy-image-parity; printf '%s\\n' \"$DEVSY_IMAGE_PARITY\"") + framework.ExpectNoError(err) + gomega.Expect(out).To(gomega.Equal("dockerfile-parity\ndockerfile-parity\n")) +} + +func assertMicrosandboxImageAbsentFromDocker(ctx context.Context, ref string) { + local := microsandboxDocker(ctx, "image", "ls", "--format", "{{.Repository}}:{{.Tag}}", ref) + gomega.Expect(strings.TrimSpace(local)).To(gomega.BeEmpty()) +} + +func microsandboxDocker(ctx context.Context, args ...string) string { + // #nosec G204 -- fixed Docker executable and controlled image fixture arguments. + out, err := exec.CommandContext(ctx, "docker", args...).CombinedOutput() + gomega.Expect(err).NotTo(gomega.HaveOccurred(), "docker %v: %s", args, out) + return string(out) +} diff --git a/e2e/tests/up/testdata/microsandbox-images/.devcontainer.json b/e2e/tests/up/testdata/microsandbox-images/.devcontainer.json new file mode 100644 index 000000000..b1ae021cb --- /dev/null +++ b/e2e/tests/up/testdata/microsandbox-images/.devcontainer.json @@ -0,0 +1,11 @@ +{ + "name": "MicroSandbox image parity", + "build": { + "dockerfile": "Dockerfile" + }, + "containerUser": "root", + "remoteUser": "root", + "features": { + "./features/parity": {} + } +} diff --git a/e2e/tests/up/testdata/microsandbox-images/Dockerfile b/e2e/tests/up/testdata/microsandbox-images/Dockerfile new file mode 100644 index 000000000..cd02334ed --- /dev/null +++ b/e2e/tests/up/testdata/microsandbox-images/Dockerfile @@ -0,0 +1,3 @@ +FROM ghcr.io/devsy-org/test-images/base:alpine +RUN mkdir -p /usr/local/share && printf '%s\n' dockerfile-parity > /usr/local/share/devsy-image-parity +ENV DEVSY_IMAGE_PARITY=dockerfile-parity diff --git a/e2e/tests/up/testdata/microsandbox-images/features/parity/devcontainer-feature.json b/e2e/tests/up/testdata/microsandbox-images/features/parity/devcontainer-feature.json new file mode 100644 index 000000000..49a54ba90 --- /dev/null +++ b/e2e/tests/up/testdata/microsandbox-images/features/parity/devcontainer-feature.json @@ -0,0 +1,8 @@ +{ + "id": "parity", + "version": "1.0.0", + "name": "MicroSandbox image parity marker", + "containerEnv": { + "DEVSY_FEATURE_PARITY": "feature-parity" + } +} diff --git a/e2e/tests/up/testdata/microsandbox-images/features/parity/install.sh b/e2e/tests/up/testdata/microsandbox-images/features/parity/install.sh new file mode 100644 index 000000000..5c7bbb68a --- /dev/null +++ b/e2e/tests/up/testdata/microsandbox-images/features/parity/install.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +mkdir -p /usr/local/share +printf '%s\n' feature-parity >/usr/local/share/devsy-feature-parity diff --git a/sites/docs-devsy-sh/content/docs/developing-providers/runtime-protocol.mdx b/sites/docs-devsy-sh/content/docs/developing-providers/runtime-protocol.mdx index e359969b9..67b9953be 100644 --- a/sites/docs-devsy-sh/content/docs/developing-providers/runtime-protocol.mdx +++ b/sites/docs-devsy-sh/content/docs/developing-providers/runtime-protocol.mdx @@ -57,4 +57,10 @@ Run it on Linux with KVM or Apple silicon after installing MicroSandbox v0.7.7 o DEVSY_REQUIRE_MICROSANDBOX=true task cli:test:e2e:suite -- up-provider-microsandbox ``` -This baseline does not establish complete parity. Resource limits, hotplug ceilings, storage, ephemeral roots, egress denial, named volumes, tmpfs, alternate mount policies, locally built images, logs, cancellation, and runtime compatibility failures still require coverage before replacing the built-in provider. A green baseline alone does not authorize that cutover. +The separate `up-provider-microsandbox-images` label runs three image cases against each provider: registry fallback with an image absent from Docker, loading a Docker-only image, and a Dockerfile build with a local Dev Container Feature. Guest checks verify the image's filesystem and environment markers and the Feature's installed file and environment. The registry case uses an isolated loopback registry and verifies manifest reads; no external test registry credentials are required. CI gives this matrix a 20-minute test deadline and a 25-minute job deadline. + +```sh +DEVSY_REQUIRE_MICROSANDBOX=true task cli:test:e2e:suite -- up-provider-microsandbox-images +``` + +These scenarios do not establish complete parity. Resource limits, hotplug ceilings, storage, ephemeral roots, egress denial, named volumes, tmpfs, alternate mount policies, prebuilds, dockerless operation, logs, cancellation, and runtime compatibility failures still require coverage before replacing the built-in provider. Green lifecycle and image checks alone do not authorize that cutover. From eab1acacce85a230f8b2176e9cd726b1dac21161 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Fri, 9 Oct 2026 12:27:13 -0600 Subject: [PATCH 2/2] docs(runtime): state image parity Docker prerequisites --- .../content/docs/developing-providers/runtime-protocol.mdx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sites/docs-devsy-sh/content/docs/developing-providers/runtime-protocol.mdx b/sites/docs-devsy-sh/content/docs/developing-providers/runtime-protocol.mdx index 67b9953be..317019b66 100644 --- a/sites/docs-devsy-sh/content/docs/developing-providers/runtime-protocol.mdx +++ b/sites/docs-devsy-sh/content/docs/developing-providers/runtime-protocol.mdx @@ -59,6 +59,8 @@ DEVSY_REQUIRE_MICROSANDBOX=true task cli:test:e2e:suite -- up-provider-microsand The separate `up-provider-microsandbox-images` label runs three image cases against each provider: registry fallback with an image absent from Docker, loading a Docker-only image, and a Dockerfile build with a local Dev Container Feature. Guest checks verify the image's filesystem and environment markers and the Feature's installed file and environment. The registry case uses an isolated loopback registry and verifies manifest reads; no external test registry credentials are required. CI gives this matrix a 20-minute test deadline and a 25-minute job deadline. +The image suite also requires the Docker CLI and a running Docker daemon, in addition to the MicroSandbox and virtualization prerequisites above. + ```sh DEVSY_REQUIRE_MICROSANDBOX=true task cli:test:e2e:suite -- up-provider-microsandbox-images ```