From 83ba76ece6685b75bf689a3d3b4b40e17c3c1c39 Mon Sep 17 00:00:00 2001 From: Matthew Teufel Date: Mon, 26 Oct 2026 13:35:26 -0700 Subject: [PATCH] fix: filter redundant OCI whiteouts before rootfs copy --- .../pkg/template/build/core/oci/oci.go | 3 + .../pkg/template/build/core/oci/whiteout.go | 61 ++++ .../template/build/core/oci/whiteout_test.go | 282 ++++++++++++++++++ 3 files changed, 346 insertions(+) create mode 100644 packages/orchestrator/pkg/template/build/core/oci/whiteout.go create mode 100644 packages/orchestrator/pkg/template/build/core/oci/whiteout_test.go diff --git a/packages/orchestrator/pkg/template/build/core/oci/oci.go b/packages/orchestrator/pkg/template/build/core/oci/oci.go index e6b25556f8..91c833a52b 100644 --- a/packages/orchestrator/pkg/template/build/core/oci/oci.go +++ b/packages/orchestrator/pkg/template/build/core/oci/oci.go @@ -431,6 +431,9 @@ func createExport(ctx context.Context, logger logger.Logger, srcImage containerr if err != nil { return fmt.Errorf("failed to untar layer %d: %w", i, err) } + if err := markWhiteoutParents(ctx, layerPath); err != nil { + return fmt.Errorf("failed to prepare whiteouts in layer %d: %w", i, err) + } return nil }) diff --git a/packages/orchestrator/pkg/template/build/core/oci/whiteout.go b/packages/orchestrator/pkg/template/build/core/oci/whiteout.go new file mode 100644 index 0000000000..348e3eac9a --- /dev/null +++ b/packages/orchestrator/pkg/template/build/core/oci/whiteout.go @@ -0,0 +1,61 @@ +//go:build linux + +package oci + +import ( + "context" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + + "golang.org/x/sys/unix" +) + +// markWhiteoutParents prepares extracted layers for use as read-only overlay +// lowerdirs. Without an origin marker, overlayfs can enumerate a non-merged +// directory directly, exposing whiteouts that subsequent lookups hide. An empty +// origin makes overlayfs filter those entries without hiding surviving lower +// files, unlike marking the directory opaque. +// Linux treats an empty origin as "copied up but origin unknown" and sets +// OVL_WHITEOUTS even when the directory exists in only one lower layer: +// https://github.com/torvalds/linux/blob/v6.8/fs/overlayfs/inode.c#L1289-L1294 +func markWhiteoutParents(ctx context.Context, layerPath string) error { + marked := make(map[string]struct{}) + + return filepath.WalkDir(layerPath, func(path string, entry fs.DirEntry, err error) error { + if err != nil { + return err + } + if err := ctx.Err(); err != nil { + return err + } + // WalkDir does not follow symlinks. Inspect the final extracted tree so + // entries replaced later in the tar cannot leave stale whiteout paths. + if entry.Type()&os.ModeCharDevice == 0 { + return nil + } + + var stat unix.Stat_t + if err := unix.Lstat(path, &stat); err != nil { + return fmt.Errorf("stat whiteout %s: %w", path, err) + } + if stat.Rdev != 0 || stat.Mode&unix.S_IFMT != unix.S_IFCHR { + return nil + } + + parent := filepath.Dir(path) + if _, ok := marked[parent]; ok { + return nil + } + // Preserve any existing origin, including a real file handle. This + // hint is only for temporary, privately extracted read-only lowerdirs. + if err := unix.Lsetxattr(parent, "trusted.overlay.origin", nil, unix.XATTR_CREATE); err != nil && !errors.Is(err, unix.EEXIST) { + return fmt.Errorf("mark whiteout parent %s: %w", parent, err) + } + marked[parent] = struct{}{} + + return nil + }) +} diff --git a/packages/orchestrator/pkg/template/build/core/oci/whiteout_test.go b/packages/orchestrator/pkg/template/build/core/oci/whiteout_test.go new file mode 100644 index 0000000000..cd8f8ba4eb --- /dev/null +++ b/packages/orchestrator/pkg/template/build/core/oci/whiteout_test.go @@ -0,0 +1,282 @@ +//go:build linux + +package oci + +import ( + "archive/tar" + "bytes" + "context" + "errors" + "io" + "os" + "os/exec" + "path/filepath" + "slices" + "strings" + "testing" + + containerregistry "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/tarball" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/sys/unix" + + "github.com/e2b-dev/infra/packages/orchestrator/pkg/template/build/core/filesystem" + "github.com/e2b-dev/infra/packages/shared/pkg/logger" +) + +func requireWhiteoutPrivileges(t *testing.T) { + t.Helper() + if os.Geteuid() != 0 { + t.Skip("whiteout extraction requires root and CAP_MKNOD/CAP_SYS_ADMIN") + } +} + +func readWhiteoutXattr(t *testing.T, path, name string) []byte { + t.Helper() + value := make([]byte, 256) + n, err := unix.Lgetxattr(path, name, value) + require.NoError(t, err) + + return value[:n] +} + +func TestMarkWhiteoutParents(t *testing.T) { + t.Parallel() + requireWhiteoutPrivileges(t) + + root := t.TempDir() + outside := t.TempDir() + for _, dir := range []string{"whiteouts", "existing", "opaque", "ordinary", "hardlink"} { + require.NoError(t, os.Mkdir(filepath.Join(root, dir), 0o755)) + } + for _, name := range []string{"whiteouts/one", "whiteouts/two", "existing/gone", "opaque/gone", "root-whiteout"} { + require.NoError(t, unix.Mknod(filepath.Join(root, name), unix.S_IFCHR|0o600, 0)) + } + require.NoError(t, os.Link(filepath.Join(root, "whiteouts/one"), filepath.Join(root, "hardlink/gone"))) + require.NoError(t, unix.Mknod(filepath.Join(root, "ordinary/null"), unix.S_IFCHR|0o600, int(unix.Mkdev(1, 3)))) + require.NoError(t, os.WriteFile(filepath.Join(root, "ordinary/empty"), nil, 0o600)) + require.NoError(t, unix.Mknod(filepath.Join(outside, "gone"), unix.S_IFCHR|0o600, 0)) + require.NoError(t, os.Symlink(outside, filepath.Join(root, "outside"))) + require.NoError(t, os.Symlink("missing", filepath.Join(root, "dangling"))) + + origin := []byte("existing-origin") + require.NoError(t, unix.Lsetxattr(filepath.Join(root, "existing"), "trusted.overlay.origin", origin, 0)) + require.NoError(t, unix.Lsetxattr(filepath.Join(root, "opaque"), "trusted.overlay.opaque", []byte("y"), 0)) + require.NoError(t, markWhiteoutParents(t.Context(), root)) + require.NoError(t, markWhiteoutParents(t.Context(), root), "marking is idempotent") + + for _, dir := range []string{"", "whiteouts", "opaque", "hardlink"} { + assert.Empty(t, readWhiteoutXattr(t, filepath.Join(root, dir), "trusted.overlay.origin")) + } + assert.Equal(t, origin, readWhiteoutXattr(t, filepath.Join(root, "existing"), "trusted.overlay.origin")) + assert.Equal(t, []byte("y"), readWhiteoutXattr(t, filepath.Join(root, "opaque"), "trusted.overlay.opaque")) + for _, dir := range []string{outside, filepath.Join(root, "ordinary")} { + _, err := unix.Lgetxattr(dir, "trusted.overlay.origin", nil) + require.ErrorIs(t, err, unix.ENODATA, "must not mark non-whiteout directories or follow links") + } +} + +func TestMarkWhiteoutParentsErrors(t *testing.T) { + t.Parallel() + ctx, cancel := context.WithCancel(t.Context()) + cancel() + require.ErrorIs(t, markWhiteoutParents(ctx, t.TempDir()), context.Canceled) + require.ErrorIs(t, markWhiteoutParents(t.Context(), filepath.Join(t.TempDir(), "missing")), os.ErrNotExist) +} + +func TestMarkWhiteoutParentsReadOnly(t *testing.T) { + t.Parallel() + requireWhiteoutPrivileges(t) + source := t.TempDir() + require.NoError(t, unix.Mknod(filepath.Join(source, "gone"), unix.S_IFCHR|0o600, 0)) + mount := t.TempDir() + require.NoError(t, unix.Mount(source, mount, "", unix.MS_BIND, "")) + t.Cleanup(func() { require.NoError(t, unix.Unmount(mount, 0)) }) + require.NoError(t, unix.Mount("", mount, "", unix.MS_BIND|unix.MS_REMOUNT|unix.MS_RDONLY, "")) + require.ErrorIs(t, markWhiteoutParents(t.Context(), mount), unix.EROFS) +} + +// whiteoutImage builds layers entirely in memory. Regular files contain their +// own path, making unexpected deletion or replacement observable after copying. +func whiteoutImage(t *testing.T, layers ...[]tar.Header) containerregistry.Image { + t.Helper() + img := empty.Image + for _, headers := range layers { + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + for _, header := range headers { + if header.Mode == 0 { + header.Mode = 0o755 + } + var content []byte + if header.Typeflag == tar.TypeReg || header.Typeflag == tar.TypeRegA { + content = []byte(header.Name) + if strings.HasPrefix(filepath.Base(header.Name), ".wh.") { + content = nil + } + header.Size = int64(len(content)) + } + require.NoError(t, tw.WriteHeader(&header)) + _, err := tw.Write(content) + require.NoError(t, err) + } + require.NoError(t, tw.Close()) + layer, err := tarball.LayerFromOpener(func() (io.ReadCloser, error) { + return io.NopCloser(bytes.NewReader(buf.Bytes())), nil + }) + require.NoError(t, err) + img, err = mutate.AppendLayers(img, layer) + require.NoError(t, err) + } + + return img +} + +func mountWhiteoutTestLayers(t *testing.T, layers []string) (string, func()) { + t.Helper() + mount := t.TempDir() + err := filesystem.MountOverlayFS(t.Context(), layers, mount) + legacyAllowed := os.Getenv("E2B_TEST_ALLOW_LEGACY_OVERLAY") == "1" + if legacyAllowed && (errors.Is(err, unix.EINVAL) || errors.Is(err, unix.ENOSYS) || errors.Is(err, unix.EOPNOTSUPP)) { + // The production fsconfig lowerdir+ interface requires Linux >=6.8. + // Explicitly opt in with E2B_TEST_ALLOW_LEGACY_OVERLAY=1 only for + // component coverage on older kernels. Default tests must exercise + // the production mount interface and propagate its errors. + t.Logf("production mount interface unavailable; testing legacy lowerdir mount: %v", err) + err = unix.Mount("overlay", mount, "overlay", unix.MS_RDONLY, "lowerdir="+strings.Join(layers, ":")) + } + require.NoError(t, err) + mounted := true + unmount := func() { + if mounted { + require.NoError(t, unix.Unmount(mount, 0)) + mounted = false + } + } + t.Cleanup(unmount) + + return mount, unmount +} + +func TestCreateExportCopiesWhiteoutDirectories(t *testing.T) { + t.Parallel() + requireWhiteoutPrivileges(t) + _, err := exec.LookPath("rsync") + require.NoError(t, err) + + img := whiteoutImage(t, []tar.Header{ + {Name: "base.txt"}, + {Name: "merged/", Typeflag: tar.TypeDir}, + {Name: "merged/keep"}, + {Name: "merged/deleted"}, + {Name: "opaque/", Typeflag: tar.TypeDir}, + {Name: "opaque/hidden"}, + {Name: "recreated/", Typeflag: tar.TypeDir}, + {Name: "recreated/file"}, + }, []tar.Header{ + {Name: "unique/", Typeflag: tar.TypeDir}, + {Name: "unique/.wh.absent"}, + {Name: "only-whiteouts/", Typeflag: tar.TypeDir}, + {Name: "only-whiteouts/.wh.first"}, + {Name: "only-whiteouts/.wh.second"}, + {Name: "unique/kept"}, + {Name: "unique/hardlink", Typeflag: tar.TypeLink, Linkname: "unique/kept"}, + {Name: "unique/symlink", Typeflag: tar.TypeSymlink, Linkname: "kept"}, + {Name: "same-layer/", Typeflag: tar.TypeDir}, + {Name: "same-layer/.wh.recreated"}, + {Name: "same-layer/recreated"}, + {Name: "merged/", Typeflag: tar.TypeDir}, + {Name: "merged/.wh.deleted"}, + {Name: "opaque/", Typeflag: tar.TypeDir}, + {Name: "opaque/.wh..wh..opq"}, + {Name: "opaque/.wh.absent"}, + {Name: "opaque/kept"}, + {Name: "recreated/", Typeflag: tar.TypeDir}, + {Name: "recreated/.wh.file"}, + }, []tar.Header{ + {Name: "recreated/", Typeflag: tar.TypeDir}, + {Name: "recreated/file"}, + }) + layers, err := createExport(t.Context(), logger.NewNopLogger(), img, t.TempDir()) + require.NoError(t, err) + _, err = unix.Lgetxattr(filepath.Join(layers[1], "same-layer"), "trusted.overlay.origin", nil) + require.ErrorIs(t, err, unix.ENODATA, "a replaced whiteout must not mark its parent") + mount, _ := mountWhiteoutTestLayers(t, layers) + entries, err := os.ReadDir(filepath.Join(mount, "only-whiteouts")) + require.NoError(t, err) + assert.Empty(t, entries, "no ghost whiteouts may be enumerated") + dest := t.TempDir() + require.NoError(t, copyFiles(t.Context(), mount, dest)) + for _, name := range []string{"base.txt", "unique/kept", "merged/keep", "opaque/kept", "recreated/file", "same-layer/recreated"} { + content, err := os.ReadFile(filepath.Join(dest, name)) + require.NoError(t, err) + assert.Equal(t, name, string(content)) + } + for _, name := range []string{"unique/absent", "only-whiteouts/first", "only-whiteouts/second", "merged/deleted", "opaque/hidden", "opaque/absent"} { + _, err := os.Lstat(filepath.Join(dest, name)) + require.ErrorIs(t, err, os.ErrNotExist) + } + kept, err := os.Stat(filepath.Join(dest, "unique/kept")) + require.NoError(t, err) + linked, err := os.Stat(filepath.Join(dest, "unique/hardlink")) + require.NoError(t, err) + assert.True(t, os.SameFile(kept, linked), "rsync must preserve hardlinks") + target, err := os.Readlink(filepath.Join(dest, "unique/symlink")) + require.NoError(t, err) + assert.Equal(t, "kept", target) +} + +func TestWhiteoutOriginPreventsRsyncVanishedFiles(t *testing.T) { + t.Parallel() + requireWhiteoutPrivileges(t) + img := whiteoutImage(t, []tar.Header{{Name: "base.txt"}}, []tar.Header{ + {Name: "unique/", Typeflag: tar.TypeDir}, + {Name: "unique/.wh.absent"}, + {Name: "unique/kept"}, + }) + layers, err := createExport(t.Context(), logger.NewNopLogger(), img, t.TempDir()) + require.NoError(t, err) + parent := filepath.Join(layers[0], "unique") + require.NoError(t, unix.Lremovexattr(parent, "trusted.overlay.origin")) + mount, unmount := mountWhiteoutTestLayers(t, layers) + entries, err := os.ReadDir(filepath.Join(mount, "unique")) + require.NoError(t, err) + var names []string + for _, entry := range entries { + names = append(names, entry.Name()) + } + _, err = os.Lstat(filepath.Join(mount, "unique/absent")) + require.ErrorIs(t, err, os.ErrNotExist) + err = copyFiles(t.Context(), mount, t.TempDir()) + if slices.Contains(names, "absent") { + var exitErr *exec.ExitError + require.ErrorAs(t, err, &exitErr) + assert.Equal(t, 24, exitErr.ExitCode(), "vanished sources must remain fatal") + } else { + // A kernel that filters unmarked whiteouts already must also pass. + t.Log("kernel filters non-merged whiteouts without an origin marker") + require.NoError(t, err) + } + unmount() + + require.NoError(t, markWhiteoutParents(t.Context(), layers[0])) + mount, _ = mountWhiteoutTestLayers(t, layers) + entries, err = os.ReadDir(filepath.Join(mount, "unique")) + require.NoError(t, err) + require.Len(t, entries, 1) + assert.Equal(t, "kept", entries[0].Name()) + require.NoError(t, copyFiles(t.Context(), mount, t.TempDir())) +} + +func TestCopyFilesPropagatesFailure(t *testing.T) { + t.Parallel() + _, err := exec.LookPath("rsync") + require.NoError(t, err) + err = copyFiles(t.Context(), filepath.Join(t.TempDir(), "missing"), t.TempDir()) + var exitErr *exec.ExitError + require.ErrorAs(t, err, &exitErr) + assert.NotZero(t, exitErr.ExitCode()) +}