diff --git a/.github/workflows/pr-ci.yml b/.github/workflows/pr-ci.yml index b91c157f99..78a23024f7 100644 --- a/.github/workflows/pr-ci.yml +++ b/.github/workflows/pr-ci.yml @@ -651,6 +651,15 @@ jobs: test-timeout: 1200s job-timeout-minutes: 25 + - label: up-provider-microsandbox-mounts + runner: ubuntu-latest + free-disk-space: false + install-kind: false + requires-secret: false + install-microsandbox: true + test-timeout: 1200s + job-timeout-minutes: 25 + # Snapshot tests - label: snapshot diff --git a/e2e/tests/up/provider_microsandbox.go b/e2e/tests/up/provider_microsandbox.go index 944b7773ad..86a2e7ff1c 100644 --- a/e2e/tests/up/provider_microsandbox.go +++ b/e2e/tests/up/provider_microsandbox.go @@ -23,7 +23,7 @@ import ( const ( osLinux = "linux" - microsandboxExternalProvider = "github.com/devsy-org/devsy-provider-microsandbox@v0.1.5" + microsandboxExternalProvider = "github.com/devsy-org/devsy-provider-microsandbox@v0.1.6" microsandboxRootUser = "root" ) @@ -213,7 +213,7 @@ var _ = ginkgo.Describe( ginkgo.Entry("built-in", "microsandbox", "microsandbox-builtin-parity", ginkgo.SpecTimeout(framework.TimeoutLong())), ginkgo.Entry( - "external v0.1.5", + "external v0.1.6", microsandboxExternalProvider, "microsandbox-external-parity", ginkgo.SpecTimeout(framework.TimeoutLong()), diff --git a/e2e/tests/up/provider_microsandbox_mounts.go b/e2e/tests/up/provider_microsandbox_mounts.go new file mode 100644 index 0000000000..56cfe7543f --- /dev/null +++ b/e2e/tests/up/provider_microsandbox_mounts.go @@ -0,0 +1,242 @@ +package up + +import ( + "context" + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "time" + + "github.com/devsy-org/devsy/e2e/framework" + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" +) + +var _ = ginkgo.Describe("microsandbox mount parity", + ginkgo.Label("up-provider-microsandbox-mounts"), func() { + for _, provider := range []struct{ name, source string }{ + {"builtin", "microsandbox"}, + {"external", microsandboxExternalProvider}, + } { + ginkgo.Context(provider.name, func() { + var f *framework.Framework + var workspace, volume string + + ginkgo.BeforeEach(func(ctx context.Context) { + skipIfNoMicrosandbox(ctx) + ginkgo.GinkgoT().Setenv("DEVSY_HOME", ginkgo.GinkgoT().TempDir()) + ginkgo.GinkgoT().Setenv("DEVSY_CONFIG", "") + initialDir, err := os.Getwd() + framework.ExpectNoError(err) + f = framework.NewDefaultFramework(filepath.Join(initialDir, "bin")) + workspace, err = framework.CreateTempDir() + framework.ExpectNoError(err) + volume = "" + ginkgo.DeferCleanup(framework.CleanupTempDir, initialDir, workspace) + framework.ExpectNoError( + f.DevsyProviderAdd(ctx, provider.source, "--name", provider.name), + ) + ginkgo.DeferCleanup(f.DevsyProviderDelete, provider.name) + // Cleanup is LIFO: tear down the VM before attempting volume removal. + ginkgo.DeferCleanup(func(cleanupCtx context.Context) { + if volume != "" { + microsandboxMountCommand(cleanupCtx, "volume", "rm", volume) + } + }) + ginkgo.DeferCleanup(f.CleanupWorkspace, workspace) + }) + + ginkgo.It( + "shares writable binds and rejects writes to read-only binds", + func(ctx context.Context) { + writable, readonly := ginkgo.GinkgoT().TempDir(), ginkgo.GinkgoT().TempDir() + for _, dir := range []string{writable, readonly} { + framework.ExpectNoError( + os.WriteFile( + filepath.Join(dir, "from-host"), + []byte("host\n"), + 0o600, + ), + ) + } + writeMicrosandboxMountConfig(workspace, []string{ + "type=bind,source=" + writable + ",target=/parity-write", + "type=bind,source=" + readonly + ",target=/parity-read,readonly", + }) + framework.ExpectNoError( + f.DevsyUp(ctx, workspace, "--devcontainer", ".devcontainer.json"), + ) + out, err := f.DevsySSHOnce( + ctx, + workspace, + "cat /parity-write/from-host /parity-read/from-host && printf 'guest\\n' > /parity-write/from-guest", + ) + framework.ExpectNoError(err) + gomega.Expect(out).To(gomega.Equal("host\nhost\n")) + data, err := os.ReadFile( + filepath.Join(writable, "from-guest"), + ) // #nosec G304 -- test-owned bind directory + framework.ExpectNoError(err) + gomega.Expect(string(data)).To(gomega.Equal("guest\n")) + _, err = f.DevsySSHOnce( + ctx, + workspace, + "printf 'forbidden\\n' > /parity-read/from-host", + ) + gomega.Expect(err).To(gomega.HaveOccurred()) + // A subsequent successful read distinguishes mount enforcement from lost SSH connectivity. + out, err = f.DevsySSHOnce(ctx, workspace, "cat /parity-read/from-host") + framework.ExpectNoError(err) + gomega.Expect(out).To(gomega.Equal("host\n")) + data, err = os.ReadFile( + filepath.Join(readonly, "from-host"), + ) // #nosec G304 -- test-owned bind directory + framework.ExpectNoError(err) + gomega.Expect(string(data)).To(gomega.Equal("host\n")) + }, + ginkgo.SpecTimeout(framework.TimeoutLong()), + ) + + ginkgo.It( + "preserves named-volume data and resets tmpfs across restart and recreation", + func(ctx context.Context) { + volume = "devsy-mount-parity-" + filepath.Base(workspace) + microsandboxMountCommand(ctx, "volume", "create", volume) + writeMicrosandboxMountConfig(workspace, []string{ + "type=volume,source=" + volume + ",target=/parity-volume", + "type=tmpfs,target=/parity-scratch", + }) + framework.ExpectNoError( + f.DevsyUp(ctx, workspace, "--devcontainer", ".devcontainer.json"), + ) + out, err := f.DevsySSHOnce(ctx, workspace, + "test \"$(stat -f -c %T /parity-scratch)\" = tmpfs && "+ + "printf 'persistent\\n' > /parity-volume/marker && printf 'scratch\\n' > /parity-scratch/marker") + framework.ExpectNoError(err, out) + framework.ExpectNoError(f.DevsyWorkspaceStop(ctx, workspace)) + framework.ExpectNoError(f.DevsyUp(ctx, workspace)) + assertMicrosandboxPersistentMounts(ctx, f, workspace) + _, err = f.DevsySSHOnce( + ctx, + workspace, + "printf 'scratch-again\\n' > /parity-scratch/marker", + ) + framework.ExpectNoError(err) + framework.ExpectNoError(f.DevsyUpRecreate(ctx, workspace)) + assertMicrosandboxPersistentMounts(ctx, f, workspace) + }, + ginkgo.SpecTimeout(framework.TimeoutLong()), + ) + + ginkgo.DescribeTable( + "honors workspace stat virtualization with private host permissions", + func(ctx context.Context, policy string) { + framework.ExpectNoError(f.DevsyProviderUse(ctx, provider.name, + "--option", "MICROSANDBOX_WORKSPACE_HOST_PERMISSIONS=private", + "--option", "MICROSANDBOX_WORKSPACE_STAT_VIRTUALIZATION="+policy)) + writeMicrosandboxMountConfig(workspace, nil) + framework.ExpectNoError( + f.DevsyUp(ctx, workspace, "--devcontainer", ".devcontainer.json"), + ) + // Create after setup so workspace chown cannot supply the guest fallback identity. + hostFile := filepath.Join(workspace, "policy-file") + framework.ExpectNoError(os.WriteFile(hostFile, []byte("host\n"), 0o600)) + // CI runs as root; keep literal host ownership distinct from the guest fallback. + if os.Geteuid() == 0 { + framework.ExpectNoError(os.Chown(hostFile, 10001, 10002)) + } + //nolint:gosec // explicit host modes are the behavior under test + framework.ExpectNoError(os.Chmod(hostFile, 0o644)) + owner := microsandboxHostOwner(ctx, hostFile) + gomega.Expect(owner).NotTo(gomega.Equal("0:0")) + guestFile := "/workspaces/" + filepath.Base(workspace) + "/policy-file" + expectedOwner := "0:0" + virtualized := policy != "off" + if !virtualized { + expectedOwner = owner + } + out, err := f.DevsySSHOnce(ctx, workspace, "stat -c '%u:%g %a' "+guestFile) + framework.ExpectNoError(err) + gomega.Expect(strings.TrimSpace(out)). + To(gomega.Equal(expectedOwner + " 644")) + if !virtualized { + framework.ExpectNoError(os.Chmod(hostFile, 0o600)) + } else { + _, err = f.DevsySSHOnce(ctx, workspace, "chmod 600 "+guestFile) + framework.ExpectNoError(err) + info, err := os.Stat(hostFile) + framework.ExpectNoError(err) + gomega.Expect(info.Mode().Perm()).To(gomega.Equal(os.FileMode(0o644))) + } + gomega.Expect(microsandboxHostOwner(ctx, hostFile)).To(gomega.Equal(owner)) + if !virtualized { + // MicroSandbox v0.7.7 caches guest attributes for five seconds after stat. + pollCtx, cancel := context.WithTimeout(ctx, 15*time.Second) + defer cancel() + gomega.Eventually(pollCtx, func() string { + out, err := f.DevsySSHOnce( + pollCtx, + workspace, + "stat -c '%u:%g %a' "+guestFile, + ) + if err != nil { + gomega.StopTrying("read guest file attributes").Wrap(err).Now() + } + return strings.TrimSpace(out) + }).WithTimeout(15 * time.Second).WithPolling(time.Second). + Should(gomega.Equal(owner + " 600")) + return + } + out, err = f.DevsySSHOnce(ctx, workspace, "stat -c '%u:%g %a' "+guestFile) + framework.ExpectNoError(err) + gomega.Expect(strings.TrimSpace(out)). + To(gomega.Equal(expectedOwner + " 600")) + }, + ginkgo.Entry("strict", "strict", ginkgo.SpecTimeout(framework.TimeoutLong())), + ginkgo.Entry("relaxed", "relaxed", ginkgo.SpecTimeout(framework.TimeoutLong())), + ginkgo.Entry( + "off exposes host metadata", + "off", + ginkgo.SpecTimeout(framework.TimeoutLong()), + ), + ) + }) + } + }) + +func writeMicrosandboxMountConfig(dir string, mounts []string) { + config := struct { + Image string `json:"image"` + ContainerUser string `json:"containerUser"` + RemoteUser string `json:"remoteUser"` + Mounts []string `json:"mounts,omitempty"` + }{ + Image: "ghcr.io/devsy-org/test-images/base:alpine", + ContainerUser: microsandboxRootUser, RemoteUser: microsandboxRootUser, + Mounts: mounts, + } + data, err := json.Marshal(config) + framework.ExpectNoError(err) + framework.ExpectNoError(os.WriteFile(filepath.Join(dir, ".devcontainer.json"), data, 0o600)) +} + +func assertMicrosandboxPersistentMounts( + ctx context.Context, + f *framework.Framework, + workspace string, +) { + out, err := f.DevsySSHOnce(ctx, workspace, + "test \"$(stat -f -c %T /parity-scratch)\" = tmpfs && "+ + "test ! -e /parity-scratch/marker && cat /parity-volume/marker") + framework.ExpectNoError(err) + gomega.Expect(out).To(gomega.Equal("persistent\n")) +} + +func microsandboxMountCommand(ctx context.Context, args ...string) { + // #nosec G204 -- fixed runtime executable and test-owned volume arguments. + out, err := exec.CommandContext(ctx, "msb", args...).CombinedOutput() + gomega.Expect(err).NotTo(gomega.HaveOccurred(), fmt.Sprintf("msb %v: %s", args, out)) +} diff --git a/providers/microsandbox/provider.yaml b/providers/microsandbox/provider.yaml index 8e438bbf1d..1e61e27e13 100644 --- a/providers/microsandbox/provider.yaml +++ b/providers/microsandbox/provider.yaml @@ -53,7 +53,7 @@ options: displayName: "Require stat virtualization support." - value: relaxed displayName: "Use stat virtualization where supported." - - value: off + - value: "off" displayName: "Expose literal host ownership and modes." INACTIVITY_TIMEOUT: description: "If defined, will automatically stop the microVM after the inactivity period. Examples: 10m, 1h" diff --git a/providers/providers_test.go b/providers/providers_test.go index fe0b8d280a..4e92db7793 100644 --- a/providers/providers_test.go +++ b/providers/providers_test.go @@ -46,3 +46,21 @@ func TestMicrosandboxProviderUsesMicrosandboxDriver(t *testing.T) { ) } } + +func TestMicrosandboxProviderPreservesStatVirtualizationOptions(t *testing.T) { + cfg, err := provider.ParseProvider(strings.NewReader(providers.MicrosandboxProvider)) + if err != nil { + t.Fatalf("parse microsandbox provider: %v", err) + } + option := cfg.Options["MICROSANDBOX_WORKSPACE_STAT_VIRTUALIZATION"] + if option == nil { + t.Fatal("missing workspace stat virtualization option") + } + values := make([]string, 0, len(option.Enum)) + for _, choice := range option.Enum { + values = append(values, choice.Value) + } + if got := strings.Join(values, ","); got != "strict,relaxed,off" { + t.Errorf("workspace stat virtualization values = %q, want strict,relaxed,off", got) + } +} diff --git a/sites/docs-devsy-sh/content/docs/developing-providers/runtime-protocol.mdx b/sites/docs-devsy-sh/content/docs/developing-providers/runtime-protocol.mdx index 317019b66a..f5a0392262 100644 --- a/sites/docs-devsy-sh/content/docs/developing-providers/runtime-protocol.mdx +++ b/sites/docs-devsy-sh/content/docs/developing-providers/runtime-protocol.mdx @@ -47,7 +47,7 @@ For SDK usage and development commands, see the [SDK README](https://github.com/ ## MicroSandbox parity gate -The `up-provider-microsandbox` E2E label runs the same lifecycle and ownership scenario against the built-in provider and the external v0.1.5 release, each with isolated Devsy configuration. CI pins MicroSandbox v0.7.7 by checksum and requires access to KVM; unavailable virtualization fails this job instead of producing a passing skipped test. Each provider scenario has a ten-minute deadline and the CI job has a 25-minute deadline. +The `up-provider-microsandbox` E2E label runs the same lifecycle and ownership scenario against the built-in provider and the external v0.1.6 release, each with isolated Devsy configuration. CI pins MicroSandbox v0.7.7 by checksum and requires access to KVM; unavailable virtualization fails this job instead of producing a passing skipped test. Each provider scenario has a ten-minute deadline and the CI job has a 25-minute deadline. The shared scenario exercises agent delivery, SSH, a 1 MiB binary stdin/stdout round trip with separate stderr and a nonzero guest exit, root workload versus developer identity, bind-mount ownership and mode mirroring, stop/start, recreation, rejection of an identity change without recreation while preserving VM-local data, and deletion of the VM. @@ -65,4 +65,12 @@ The image suite also requires the Docker CLI and a running Docker daemon, in add DEVSY_REQUIRE_MICROSANDBOX=true task cli:test:e2e:suite -- up-provider-microsandbox-images ``` -These scenarios do not establish complete parity. Resource limits, hotplug ceilings, storage, ephemeral roots, egress denial, named volumes, tmpfs, alternate mount policies, prebuilds, dockerless operation, logs, cancellation, and runtime compatibility failures still require coverage before replacing the built-in provider. Green lifecycle and image checks alone do not authorize that cutover. +The `up-provider-microsandbox-mounts` label runs five mount scenarios against each provider. It checks bidirectional bind access, read-only write rejection, named-volume persistence through stop/start and recreation, tmpfs reset, and strict/relaxed/off stat virtualization with private host permissions. Permission checks use host-created files after workspace setup so recursive chown cannot mask the guest ownership fallback. Host mode changes under `off` must become visible after MicroSandbox's five-second guest attribute cache expires. Each scenario has a ten-minute deadline; CI gives the matrix a 20-minute test deadline and a 25-minute job deadline. Test-owned named volumes are removed after workspace cleanup. + +This suite needs the same MicroSandbox and virtualization prerequisites as the lifecycle suite: + +```sh +DEVSY_REQUIRE_MICROSANDBOX=true task cli:test:e2e:suite -- up-provider-microsandbox-mounts +``` + +These scenarios do not establish complete parity. Resource limits, hotplug ceilings, storage capacity, ephemeral roots, egress denial, prebuilds, dockerless operation, logs, cancellation, and runtime compatibility failures still require coverage before replacing the built-in provider. Green lifecycle, image, and mount checks alone do not authorize that cutover.