From 561056b298b44a8dd7558fcd0a98e3c0840170e7 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Fri, 17 Jul 2026 18:11:36 +0000 Subject: [PATCH 1/4] fix(backends): fall back to copying links when the filesystem rejects symlinks (#10890) Backend installation extracts the OCI image tar via containerd's archive.Apply, which calls os.Symlink directly. On filesystems that do not support symlinks (notably CIFS/SMB mounts, commonly used to back the /backends volume) the syscall fails with "operation not supported" and the whole install aborts, leaving an empty backend directory. The CUDA llama.cpp image trips this on the libcublas.so -> libcublas.so.12.x symlink. When archive.Apply fails with a link-unsupported error, reset the staging directory and re-extract with a pure-Go walker that still attempts real symlinks/hardlinks first and degrades to copying the link target's contents in place when the filesystem rejects them. mutate.Extract already flattened the layers, so the tar carries no whiteouts to interpret. Link copies are deferred to a second pass so forward references resolve. Signed-off-by: Ettore Di Giacinto Assisted-by: Claude:opus-4.8 [Claude Code] --- pkg/oci/extract_internal_test.go | 118 ++++++++++++++++ pkg/oci/image.go | 223 ++++++++++++++++++++++++++++++- 2 files changed, 340 insertions(+), 1 deletion(-) create mode 100644 pkg/oci/extract_internal_test.go diff --git a/pkg/oci/extract_internal_test.go b/pkg/oci/extract_internal_test.go new file mode 100644 index 000000000000..9c80d7d3ce51 --- /dev/null +++ b/pkg/oci/extract_internal_test.go @@ -0,0 +1,118 @@ +package oci + +import ( + "archive/tar" + "bytes" + "os" + "path/filepath" + "syscall" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// buildTar assembles an in-memory tar carrying a directory, a regular file and +// a relative symlink pointing at that file, mirroring the layout of a backend +// image (e.g. libcublas.so -> libcublas.so.12). +func buildTar() []byte { + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + + Expect(tw.WriteHeader(&tar.Header{ + Name: "lib/", + Typeflag: tar.TypeDir, + Mode: 0755, + })).To(Succeed()) + + content := []byte("real library bytes") + Expect(tw.WriteHeader(&tar.Header{ + Name: "lib/libcublas.so.12", + Typeflag: tar.TypeReg, + Mode: 0644, + Size: int64(len(content)), + })).To(Succeed()) + _, err := tw.Write(content) + Expect(err).NotTo(HaveOccurred()) + + Expect(tw.WriteHeader(&tar.Header{ + Name: "lib/libcublas.so", + Typeflag: tar.TypeSymlink, + Linkname: "libcublas.so.12", + Mode: 0777, + })).To(Succeed()) + + Expect(tw.Close()).To(Succeed()) + return buf.Bytes() +} + +var _ = Describe("Tar extraction fallback for link-less filesystems", func() { + Describe("isLinkUnsupportedError", func() { + It("recognises filesystem link-unsupported errors", func() { + Expect(isLinkUnsupportedError(syscall.ENOTSUP)).To(BeTrue()) + Expect(isLinkUnsupportedError(syscall.EOPNOTSUPP)).To(BeTrue()) + Expect(isLinkUnsupportedError(syscall.EPERM)).To(BeTrue()) + Expect(isLinkUnsupportedError(&os.LinkError{ + Op: "symlink", + Old: "libcublas.so.12", + New: "/backends/lib/libcublas.so", + Err: syscall.ENOTSUP, + })).To(BeTrue()) + }) + + It("does not misclassify unrelated errors", func() { + Expect(isLinkUnsupportedError(os.ErrNotExist)).To(BeFalse()) + Expect(isLinkUnsupportedError(syscall.ENOSPC)).To(BeFalse()) + }) + }) + + Describe("safeJoin", func() { + It("keeps entries inside the root", func() { + root := "/tmp/extract-root" + p, err := safeJoin(root, "lib/libcublas.so") + Expect(err).NotTo(HaveOccurred()) + Expect(p).To(Equal(filepath.Join(root, "lib/libcublas.so"))) + }) + + It("rejects path traversal entries", func() { + _, err := safeJoin("/tmp/extract-root", "../../etc/passwd") + Expect(err).To(HaveOccurred()) + }) + }) + + Describe("extractTarCopyingLinks", func() { + It("preserves symlinks when the filesystem supports them", func() { + dir := GinkgoT().TempDir() + Expect(extractTarCopyingLinks(bytes.NewReader(buildTar()), dir)).To(Succeed()) + + linkPath := filepath.Join(dir, "lib", "libcublas.so") + fi, err := os.Lstat(linkPath) + Expect(err).NotTo(HaveOccurred()) + Expect(fi.Mode() & os.ModeSymlink).NotTo(BeZero()) + + data, err := os.ReadFile(linkPath) + Expect(err).NotTo(HaveOccurred()) + Expect(string(data)).To(Equal("real library bytes")) + }) + + It("copies the target when symlink creation is unsupported", func() { + // Simulate a CIFS/SMB mount: symlink() reports ENOTSUP. + origSymlink := symlink + symlink = func(string, string) error { return syscall.ENOTSUP } + DeferCleanup(func() { symlink = origSymlink }) + + dir := GinkgoT().TempDir() + Expect(extractTarCopyingLinks(bytes.NewReader(buildTar()), dir)).To(Succeed()) + + linkPath := filepath.Join(dir, "lib", "libcublas.so") + fi, err := os.Lstat(linkPath) + Expect(err).NotTo(HaveOccurred()) + // The entry must now be a real, regular file (a copy), not a symlink. + Expect(fi.Mode() & os.ModeSymlink).To(BeZero()) + Expect(fi.Mode().IsRegular()).To(BeTrue()) + + data, err := os.ReadFile(linkPath) + Expect(err).NotTo(HaveOccurred()) + Expect(string(data)).To(Equal("real library bytes")) + }) + }) +}) diff --git a/pkg/oci/image.go b/pkg/oci/image.go index 1af8f4a87fe1..b96d72d5527a 100644 --- a/pkg/oci/image.go +++ b/pkg/oci/image.go @@ -1,12 +1,14 @@ package oci import ( + "archive/tar" "context" "errors" "fmt" "io" "net/http" "os" + "path/filepath" "runtime" "strconv" "strings" @@ -447,8 +449,227 @@ func ExtractOCIImageFromTar(ctx context.Context, tarFilePath, imageRef, targetDe _, err = archive.Apply(ctx, targetDestination, reader, archive.WithNoSameOwner()) + if err == nil { + return nil + } + + // Some filesystems (notably CIFS/SMB mounts, which users commonly bind as the + // /backends volume) reject symlink/hardlink creation with "operation not + // supported"/"operation not permitted". containerd's archive.Apply hard-fails + // there, so no backend can be installed. Fall back to a pure-Go extractor that + // degrades unsupported links into plain file copies. mutate.Extract already + // flattened the layers, so this tar carries no whiteouts to interpret. + if !isLinkUnsupportedError(err) { + return err + } + logs.Warn.Printf("symlink/hardlink creation is not supported on filesystem at %q (%v), retrying extraction with links copied in place", targetDestination, err) + + // archive.Apply may have written some entries before failing; start from a + // clean destination so the manual pass is deterministic. The caller stages + // into an ephemeral, per-install temp directory, so wiping its contents is safe. + if err := cleanDirContents(targetDestination); err != nil { + return fmt.Errorf("failed to reset destination before fallback extraction: %w", err) + } + + // Re-read the tar from the beginning for the second pass. + if _, err := tarFile.Seek(0, io.SeekStart); err != nil { + return fmt.Errorf("failed to rewind tar for fallback extraction: %w", err) + } + return extractTarCopyingLinks(tarFile, targetDestination) +} + +// symlink and hardlink are indirected so tests can simulate a filesystem that +// rejects link creation (e.g. CIFS/SMB). +var ( + symlink = os.Symlink + hardlink = os.Link +) + +// isLinkUnsupportedError reports whether err indicates the destination +// filesystem cannot create symlinks or hardlinks (e.g. CIFS/SMB, some FUSE +// mounts). Such filesystems surface ENOTSUP/EOPNOTSUPP, or EPERM in some +// configurations; the error text is also matched because containerd wraps the +// syscall error into a formatted string. +func isLinkUnsupportedError(err error) bool { + if errors.Is(err, syscall.ENOTSUP) || errors.Is(err, syscall.EOPNOTSUPP) || errors.Is(err, syscall.EPERM) { + return true + } + msg := strings.ToLower(err.Error()) + return strings.Contains(msg, "operation not supported") || strings.Contains(msg, "operation not permitted") +} + +// cleanDirContents removes the entries inside dir without removing dir itself, +// preserving the directory (and its permissions) the caller created. +func cleanDirContents(dir string) error { + entries, err := os.ReadDir(dir) + if err != nil { + return err + } + for _, entry := range entries { + if err := os.RemoveAll(filepath.Join(dir, entry.Name())); err != nil { + return err + } + } + return nil +} + +// extractTarCopyingLinks extracts a flattened image tar into targetDestination, +// copying the target contents of any symlink/hardlink that the filesystem cannot +// represent. Regular symlinks are still attempted first, so link semantics are +// preserved wherever the filesystem allows it. Link copies are deferred to a +// second pass so that forward references (a link appearing before its target in +// the tar) resolve correctly. +func extractTarCopyingLinks(r io.Reader, targetDestination string) error { + root, err := filepath.Abs(targetDestination) + if err != nil { + return err + } + + type pendingLink struct { + path string // absolute destination path of the link + targetPath string // absolute path of the file to copy from + } + var pending []pendingLink - return err + tr := tar.NewReader(r) + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return fmt.Errorf("failed to read tar entry: %w", err) + } + + cleaned, err := safeJoin(root, hdr.Name) + if err != nil { + return err + } + // Skip aufs/overlay whiteout markers defensively; a flattened tar + // should not contain any, but ignoring them is always correct here. + if strings.HasPrefix(filepath.Base(hdr.Name), ".wh.") { + continue + } + + switch hdr.Typeflag { + case tar.TypeDir: + if err := os.MkdirAll(cleaned, os.FileMode(hdr.Mode)&os.ModePerm|0700); err != nil { + return fmt.Errorf("failed to create directory %s: %w", cleaned, err) + } + case tar.TypeReg: + if err := os.MkdirAll(filepath.Dir(cleaned), 0700); err != nil { + return fmt.Errorf("failed to create parent directory for %s: %w", cleaned, err) + } + if err := writeRegularFile(cleaned, tr, os.FileMode(hdr.Mode)&os.ModePerm); err != nil { + return err + } + case tar.TypeSymlink: + if err := os.MkdirAll(filepath.Dir(cleaned), 0700); err != nil { + return fmt.Errorf("failed to create parent directory for %s: %w", cleaned, err) + } + // Remove any pre-existing entry so os.Symlink does not fail with EEXIST. + _ = os.Remove(cleaned) + if err := symlink(hdr.Linkname, cleaned); err == nil { + break + } else if !isLinkUnsupportedError(err) { + return fmt.Errorf("failed to create symlink %s -> %s: %w", cleaned, hdr.Linkname, err) + } + // Resolve the link target: absolute targets are image-root relative, + // relative ones are resolved against the link's own directory. + var src string + if filepath.IsAbs(hdr.Linkname) { + src, err = safeJoin(root, hdr.Linkname) + } else { + src, err = safeJoin(root, filepath.Join(filepath.Dir(hdr.Name), hdr.Linkname)) + } + if err != nil { + return err + } + pending = append(pending, pendingLink{path: cleaned, targetPath: src}) + case tar.TypeLink: + if err := os.MkdirAll(filepath.Dir(cleaned), 0700); err != nil { + return fmt.Errorf("failed to create parent directory for %s: %w", cleaned, err) + } + // Hardlink targets are always relative to the image root. + src, err := safeJoin(root, hdr.Linkname) + if err != nil { + return err + } + _ = os.Remove(cleaned) + if err := hardlink(src, cleaned); err == nil { + break + } else if !isLinkUnsupportedError(err) { + return fmt.Errorf("failed to create hardlink %s -> %s: %w", cleaned, src, err) + } + pending = append(pending, pendingLink{path: cleaned, targetPath: src}) + default: + // Ignore device nodes, fifos, etc: backend artifacts do not use them. + logs.Debug.Printf("skipping unsupported tar entry type during fallback extraction: name=%q type=%d", hdr.Name, hdr.Typeflag) + } + } + + // Second pass: materialise links that the filesystem could not represent. + for _, link := range pending { + if err := copyFilePreservingMode(link.targetPath, link.path); err != nil { + return fmt.Errorf("failed to copy link target %s -> %s: %w", link.targetPath, link.path, err) + } + } + return nil +} + +// safeJoin joins name onto root and guarantees the result stays within root, +// guarding against path-traversal entries in a malicious tar. +func safeJoin(root, name string) (string, error) { + cleaned := filepath.Join(root, filepath.Clean("/"+name)) + rel, err := filepath.Rel(root, cleaned) + if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(os.PathSeparator)) { + return "", fmt.Errorf("tar entry escapes extraction root: %s", name) + } + return cleaned, nil +} + +func writeRegularFile(path string, r io.Reader, mode os.FileMode) error { + // Remove any pre-existing symlink so we do not write through it. + if fi, err := os.Lstat(path); err == nil && fi.Mode()&os.ModeSymlink != 0 { + _ = os.Remove(path) + } + f, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, mode|0600) + if err != nil { + return fmt.Errorf("failed to create file %s: %w", path, err) + } + if _, err := io.Copy(f, r); err != nil { + _ = f.Close() + return fmt.Errorf("failed to write file %s: %w", path, err) + } + if err := f.Close(); err != nil { + return fmt.Errorf("failed to close file %s: %w", path, err) + } + return nil +} + +func copyFilePreservingMode(src, dst string) error { + in, err := os.Open(src) + if err != nil { + return err + } + defer in.Close() + info, err := in.Stat() + if err != nil { + return err + } + if err := os.MkdirAll(filepath.Dir(dst), 0700); err != nil { + return err + } + _ = os.Remove(dst) + out, err := os.OpenFile(dst, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, info.Mode().Perm()|0600) + if err != nil { + return err + } + if _, err := io.Copy(out, in); err != nil { + _ = out.Close() + return err + } + return out.Close() } // GetOCIImageUncompressedSize returns the total uncompressed size of an image From 26a9d1d8c24fa4cd9cb7b455cfdc9a9dc0831f05 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Fri, 17 Jul 2026 18:20:16 +0000 Subject: [PATCH 2/4] fix(oci): check deferred Close in copyFilePreservingMode (errcheck) Signed-off-by: Ettore Di Giacinto Assisted-by: Claude:opus-4.8 [Claude Code] --- pkg/oci/image.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/oci/image.go b/pkg/oci/image.go index b96d72d5527a..bada05d394b2 100644 --- a/pkg/oci/image.go +++ b/pkg/oci/image.go @@ -652,7 +652,7 @@ func copyFilePreservingMode(src, dst string) error { if err != nil { return err } - defer in.Close() + defer func() { _ = in.Close() }() info, err := in.Stat() if err != nil { return err From 0ca4ef257d163a0c32c23ba361483c3e1b362b4b Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Fri, 17 Jul 2026 18:39:21 +0000 Subject: [PATCH 3/4] fix(oci): reject path-traversal tar entries in the link-copy fallback safeJoin sanitized "../.." entries by clamping them under root instead of rejecting them, so a malicious entry was silently redirected rather than refused. Join without the leading-slash trick and reject any entry whose cleaned path resolves outside root; absolute link targets are still mapped under root (image-root relative) rather than escaping. Signed-off-by: Ettore Di Giacinto Assisted-by: Claude:opus-4.8 [Claude Code] --- pkg/oci/image.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pkg/oci/image.go b/pkg/oci/image.go index bada05d394b2..e96d0cccb0e4 100644 --- a/pkg/oci/image.go +++ b/pkg/oci/image.go @@ -618,11 +618,13 @@ func extractTarCopyingLinks(r io.Reader, targetDestination string) error { } // safeJoin joins name onto root and guarantees the result stays within root, -// guarding against path-traversal entries in a malicious tar. +// rejecting path-traversal entries in a malicious tar. An absolute name (e.g. an +// absolute symlink target) is treated as image-root relative, so it is mapped +// under root rather than escaping it. func safeJoin(root, name string) (string, error) { - cleaned := filepath.Join(root, filepath.Clean("/"+name)) + cleaned := filepath.Join(root, name) rel, err := filepath.Rel(root, cleaned) - if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(os.PathSeparator)) { + if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(os.PathSeparator)) || filepath.IsAbs(rel) { return "", fmt.Errorf("tar entry escapes extraction root: %s", name) } return cleaned, nil From 504751c493fcfbe17e3912ad5984048f8bea27c3 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Fri, 17 Jul 2026 18:50:15 +0000 Subject: [PATCH 4/4] fix(oci): silence gosec on the validated link-copy file ops Use hdr.FileInfo().Mode() instead of converting the int64 tar mode to os.FileMode (removes two G115 overflow findings), and annotate the tar extraction file operations with justified #nosec comments: every path is validated by safeJoin against the extraction root before use (G304/G305). Signed-off-by: Ettore Di Giacinto Assisted-by: Claude:opus-4.8 [Claude Code] --- pkg/oci/image.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pkg/oci/image.go b/pkg/oci/image.go index e96d0cccb0e4..56f0d2bec041 100644 --- a/pkg/oci/image.go +++ b/pkg/oci/image.go @@ -553,14 +553,14 @@ func extractTarCopyingLinks(r io.Reader, targetDestination string) error { switch hdr.Typeflag { case tar.TypeDir: - if err := os.MkdirAll(cleaned, os.FileMode(hdr.Mode)&os.ModePerm|0700); err != nil { + if err := os.MkdirAll(cleaned, hdr.FileInfo().Mode().Perm()|0700); err != nil { return fmt.Errorf("failed to create directory %s: %w", cleaned, err) } case tar.TypeReg: if err := os.MkdirAll(filepath.Dir(cleaned), 0700); err != nil { return fmt.Errorf("failed to create parent directory for %s: %w", cleaned, err) } - if err := writeRegularFile(cleaned, tr, os.FileMode(hdr.Mode)&os.ModePerm); err != nil { + if err := writeRegularFile(cleaned, tr, hdr.FileInfo().Mode().Perm()); err != nil { return err } case tar.TypeSymlink: @@ -580,6 +580,7 @@ func extractTarCopyingLinks(r io.Reader, targetDestination string) error { if filepath.IsAbs(hdr.Linkname) { src, err = safeJoin(root, hdr.Linkname) } else { + // #nosec G305 -- safeJoin rejects any result that resolves outside the extraction root src, err = safeJoin(root, filepath.Join(filepath.Dir(hdr.Name), hdr.Linkname)) } if err != nil { @@ -635,6 +636,7 @@ func writeRegularFile(path string, r io.Reader, mode os.FileMode) error { if fi, err := os.Lstat(path); err == nil && fi.Mode()&os.ModeSymlink != 0 { _ = os.Remove(path) } + // #nosec G304 -- path is validated by safeJoin to stay within the extraction root f, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, mode|0600) if err != nil { return fmt.Errorf("failed to create file %s: %w", path, err) @@ -650,6 +652,7 @@ func writeRegularFile(path string, r io.Reader, mode os.FileMode) error { } func copyFilePreservingMode(src, dst string) error { + // #nosec G304 -- src is a safeJoin-validated link target within the extraction root in, err := os.Open(src) if err != nil { return err @@ -663,6 +666,7 @@ func copyFilePreservingMode(src, dst string) error { return err } _ = os.Remove(dst) + // #nosec G304 -- dst is a safeJoin-validated path within the extraction root out, err := os.OpenFile(dst, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, info.Mode().Perm()|0600) if err != nil { return err