From c08f68c7f9ae21e98ae47261a6917bceafc51776 Mon Sep 17 00:00:00 2001 From: Chun-Hung Tseng Date: Sat, 5 Sep 2026 10:10:09 +0200 Subject: [PATCH 1/5] Remove replaceFile's per-branch cleanup calls Defer the temporary file's close and removal in place of the per-path cleanup calls, and sync the directory through syncDirectory after the rename. --- cmd/oci/store.go | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/cmd/oci/store.go b/cmd/oci/store.go index cde7e3db..9302dd04 100644 --- a/cmd/oci/store.go +++ b/cmd/oci/store.go @@ -152,40 +152,27 @@ func replaceFile(ctx context.Context, dir, name string, content []byte, mode os. if err != nil { return err } - cleanup := func() { _ = os.Remove(tmpPath) } + defer os.Remove(tmpPath) + defer f.Close() if _, err := f.Write(content); err != nil { - f.Close() - cleanup() return err } if err := ctx.Err(); err != nil { - f.Close() - cleanup() return err } if err := f.Sync(); err != nil { - f.Close() - cleanup() return err } if err := f.Close(); err != nil { - cleanup() return err } if err := ctx.Err(); err != nil { - cleanup() return err } if err := os.Rename(tmpPath, filepath.Join(dir, name)); err != nil { - cleanup() - return err - } - d, err := os.Open(dir) - if err != nil { return err } - defer d.Close() - return d.Sync() + return syncDirectory(dir) } func (s *store) rootIndex() (v1.IndexManifest, error) { From 7075d89662134c5189d58a41d7082c3b3db6bb90 Mon Sep 17 00:00:00 2001 From: Chun-Hung Tseng Date: Wed, 16 Sep 2026 22:41:26 +0800 Subject: [PATCH 2/5] Extract checkLayout, checkJSON, and nestedIndex ensureLayoutLocked, ensureJSONFile, and pinLocked keep their behavior. The unpack read path needs the same marker, legacy-layout, JSON, and per-reference index checks without the repair steps around them. --- cmd/oci/store.go | 71 +++++++++++++++++++++++++++++++++--------------- 1 file changed, 49 insertions(+), 22 deletions(-) diff --git a/cmd/oci/store.go b/cmd/oci/store.go index 9302dd04..44e6fb72 100644 --- a/cmd/oci/store.go +++ b/cmd/oci/store.go @@ -8,6 +8,7 @@ import ( "context" "encoding/hex" "encoding/json" + "errors" "fmt" "io" "os" @@ -29,6 +30,8 @@ const ( refNameAnnotation = "org.opencontainers.image.ref.name" ) +var errNoMarker = errors.New("no store format marker") + type store struct { root string } @@ -53,6 +56,25 @@ func openStore(root string) (*store, error) { func (s *store) lockPath() string { return filepath.Join(s.root, metadataLockName) } +// checkLayout validates the store format without writing. errNoMarker means the +// directory carries no marker, which pull may create and a reader must refuse. +func (s *store) checkLayout() error { + marker, err := os.ReadFile(filepath.Join(s.root, markerName)) + if os.IsNotExist(err) { + if _, legacyErr := os.Lstat(filepath.Join(s.root, "refs.json")); legacyErr == nil { + return fmt.Errorf("store: legacy refs.json layout; remove the store and pull again") + } + return errNoMarker + } + if err != nil { + return err + } + if string(marker) != markerContents { + return fmt.Errorf("store: unsupported format marker %q", strings.TrimSpace(string(marker))) + } + return nil +} + func (s *store) withLock(ctx context.Context, fn func() error) error { l, err := acquireFlock(ctx, s.lockPath()) if err != nil { @@ -79,11 +101,8 @@ func (s *store) ensureLayoutLocked(ctx context.Context) error { if err := ctx.Err(); err != nil { return err } - marker, err := os.ReadFile(filepath.Join(s.root, markerName)) - if os.IsNotExist(err) { - if _, legacyErr := os.Lstat(filepath.Join(s.root, "refs.json")); legacyErr == nil { - return fmt.Errorf("store: legacy refs.json layout; remove the store and pull again") - } + err := s.checkLayout() + if errors.Is(err, errNoMarker) { entries, readErr := os.ReadDir(s.root) if readErr != nil { return readErr @@ -102,13 +121,9 @@ func (s *store) ensureLayoutLocked(ctx context.Context) error { if err := replaceFile(ctx, s.root, markerName, []byte(markerContents), 0o600); err != nil { return err } - marker = []byte(markerContents) } else if err != nil { return err } - if string(marker) != markerContents { - return fmt.Errorf("store: unsupported format marker %q", strings.TrimSpace(string(marker))) - } if err := ensureJSONFile(ctx, filepath.Join(s.root, "oci-layout"), []byte("{\"imageLayoutVersion\":\"1.0.0\"}\n")); err != nil { return err } @@ -135,9 +150,13 @@ func ensureJSONFile(ctx context.Context, path string, initial []byte) error { if err := ctx.Err(); err != nil { return err } + return checkJSON(filepath.Base(path), b) +} + +func checkJSON(name string, b []byte) error { var value any if err := json.Unmarshal(b, &value); err != nil { - return fmt.Errorf("store: corrupt %s: %w", filepath.Base(path), err) + return fmt.Errorf("store: corrupt %s: %w", name, err) } return nil } @@ -404,18 +423,9 @@ func (s *store) pinLocked(ctx context.Context, ref string, platform ocispec.Plat if err := ctx.Err(); err != nil { return err } - nested := v1.IndexManifest{SchemaVersion: 2, MediaType: types.OCIImageIndex} - for _, desc := range index.Manifests { - if desc.Annotations[refNameAnnotation] != ref { - continue - } - b, err := s.blobBytes(desc.Digest) - if err != nil { - return err - } - if err := json.Unmarshal(b, &nested); err != nil { - return fmt.Errorf("store: parse index for %s: %w", ref, err) - } + nested, err := s.nestedIndex(index, ref) + if err != nil { + return err } kept := nested.Manifests[:0] for _, desc := range nested.Manifests { @@ -462,3 +472,20 @@ func platformKey(platform *v1.Platform) string { } return platform.String() } + +func (s *store) nestedIndex(index v1.IndexManifest, name string) (v1.IndexManifest, error) { + nested := v1.IndexManifest{SchemaVersion: 2, MediaType: types.OCIImageIndex} + for _, desc := range index.Manifests { + if desc.Annotations[refNameAnnotation] != name { + continue + } + b, err := s.blobBytes(desc.Digest) + if err != nil { + return nested, err + } + if err := json.Unmarshal(b, &nested); err != nil { + return nested, fmt.Errorf("store: parse index for %s: %w", name, err) + } + } + return nested, nil +} From e2a1dcd2ac3f920c2ac7e2bbd214b6d5c511fc93 Mon Sep 17 00:00:00 2001 From: Chun-Hung Tseng Date: Wed, 16 Sep 2026 22:41:27 +0800 Subject: [PATCH 3/5] Add a store read path and rootfs cache Resolve a reference and platform to a manifest digest through digestFor, load that manifest through manifestFor, and derive the directory its rootfs unpacks to through cacheDir. Cache directories are keyed by manifest digest under a per-kind sha256 directory, and a symlink at either level is refused. openStoreForRead applies pull's format checks, oci-layout included, and creates or repairs nothing, so a mistyped --store leaves nothing behind. refuseRootfsInStore compares directory identity up the ancestor chain, so --rootfs can neither name a path inside the store, case aliases included, nor contain the store. --- cmd/oci/store.go | 179 +++++++++++++++++++++++++++++++++++++++++- cmd/oci/store_test.go | 130 ++++++++++++++++++++++++++++++ 2 files changed, 308 insertions(+), 1 deletion(-) diff --git a/cmd/oci/store.go b/cmd/oci/store.go index 44e6fb72..e13d6653 100644 --- a/cmd/oci/store.go +++ b/cmd/oci/store.go @@ -30,7 +30,10 @@ const ( refNameAnnotation = "org.opencontainers.image.ref.name" ) -var errNoMarker = errors.New("no store format marker") +var ( + errNoMarker = errors.New("no store format marker") + errNotPulled = errors.New("not pulled") +) type store struct { root string @@ -56,6 +59,37 @@ func openStore(root string) (*store, error) { func (s *store) lockPath() string { return filepath.Join(s.root, metadataLockName) } +// openStoreForRead creates and repairs nothing, so a mistyped --store leaves +// nothing behind. +func openStoreForRead(root string) (*store, error) { + root = filepath.Clean(root) + fi, err := os.Stat(root) + if os.IsNotExist(err) { + return nil, fmt.Errorf("store: %s does not exist", root) + } + if err != nil { + return nil, err + } + if !fi.IsDir() { + return nil, fmt.Errorf("store: %s is not a directory", root) + } + s := &store{root: root} + if err := s.checkLayout(); err != nil { + if errors.Is(err, errNoMarker) { + return nil, fmt.Errorf("store: %s is not an elfuse OCI store", root) + } + return nil, err + } + if b, err := os.ReadFile(filepath.Join(root, "oci-layout")); err == nil { + if err := checkJSON("oci-layout", b); err != nil { + return nil, err + } + } else if !os.IsNotExist(err) { + return nil, err + } + return s, nil +} + // checkLayout validates the store format without writing. errNoMarker means the // directory carries no marker, which pull may create and a reader must refuse. func (s *store) checkLayout() error { @@ -75,6 +109,98 @@ func (s *store) checkLayout() error { return nil } +const cacheRootfs = "rootfs" + +func (s *store) cacheBase(kind string) string { + return filepath.Join(s.root, kind, "sha256") +} + +func digestHex(dgst string) (string, error) { + d, err := v1.NewHash(dgst) + if err != nil || d.Algorithm != "sha256" { + return "", fmt.Errorf("store: unsupported digest %q for a cache key", dgst) + } + return d.Hex, nil +} + +func (s *store) cacheDir(kind, dgst string) (string, error) { + h, err := digestHex(dgst) + if err != nil { + return "", err + } + for _, p := range []string{filepath.Join(s.root, kind), s.cacheBase(kind)} { + if err := rejectSymlink(p); err != nil { + return "", err + } + } + return filepath.Join(s.cacheBase(kind), h), nil +} + +func rejectSymlink(path string) error { + fi, err := os.Lstat(path) + if os.IsNotExist(err) { + return nil + } + if err != nil { + return err + } + if fi.Mode()&os.ModeSymlink != 0 { + return fmt.Errorf("%s is a symlink; refusing to use it as a cache directory", path) + } + return nil +} + +func insideStore(storeRoot, path string) bool { + abs := resolvedAbs(path) + absStore := resolvedAbs(storeRoot) + if abs == "" || absStore == "" { + return true + } + storeInfo, err := os.Stat(absStore) + if err != nil { + return true + } + // Filesystem identity catches case aliases on APFS. A tail that + // cannot be stat'ed is checked through its ancestors. + for candidate := abs; ; candidate = filepath.Dir(candidate) { + if info, err := os.Stat(candidate); err == nil && os.SameFile(storeInfo, info) { + return true + } + if filepath.Dir(candidate) == candidate { + return false + } + } +} + +func refuseRootfsInStore(storeRoot, rootfs string) error { + if rootfs != "" && insideStore(storeRoot, rootfs) { + return fmt.Errorf("unpack: --rootfs %s is inside the store; drop --rootfs for the managed cache", rootfs) + } + if _, err := os.Stat(rootfs); err == nil && insideStore(rootfs, storeRoot) { + return fmt.Errorf("unpack: --rootfs %s contains the store", rootfs) + } + return nil +} + +func resolvedAbs(path string) string { + abs, err := filepath.Abs(path) + if err != nil { + return "" + } + rest := "" + for p := abs; ; { + if resolved, err := filepath.EvalSymlinks(p); err == nil { + return filepath.Join(resolved, rest) + } + parent := filepath.Dir(p) + if parent == p { + return abs + } + rest = filepath.Join(filepath.Base(p), rest) + p = parent + } +} + func (s *store) withLock(ctx context.Context, fn func() error) error { l, err := acquireFlock(ctx, s.lockPath()) if err != nil { @@ -489,3 +615,54 @@ func (s *store) nestedIndex(index v1.IndexManifest, name string) (v1.IndexManife } return nested, nil } + +func (s *store) digestFor(ref string, platform ocispec.Platform) (string, error) { + parsed, err := normalizeRef(ref) + if err != nil { + return "", err + } + index, err := s.rootIndex() + if os.IsNotExist(err) { + return "", notPulledError(ref, platform) + } + if err != nil { + return "", err + } + nested, err := s.nestedIndex(index, parsed.Name()) + if err != nil { + return "", err + } + for _, child := range nested.Manifests { + if samePlatform(child.Platform, platform) { + return child.Digest.String(), nil + } + } + return "", notPulledError(ref, platform) +} + +func notPulledError(ref string, platform ocispec.Platform) error { + p := platformString(platform) + return fmt.Errorf("store: %q %w for %s (run elfuse-oci pull --platform %s %s first)", ref, errNotPulled, p, p, ref) +} + +func (s *store) manifestFor(ctx context.Context, digest string) (ocispec.Manifest, error) { + var manifest ocispec.Manifest + if err := ctx.Err(); err != nil { + return manifest, err + } + hash, err := v1.NewHash(digest) + if err != nil { + return manifest, fmt.Errorf("store: manifest %s: %w", digest, err) + } + b, err := s.blobBytes(hash) + if err != nil { + return manifest, err + } + if err := json.Unmarshal(b, &manifest); err != nil { + return manifest, fmt.Errorf("store: parse manifest %s: %w", digest, err) + } + if manifest.SchemaVersion != 2 || manifest.Config.Digest == "" { + return manifest, fmt.Errorf("store: invalid manifest %s", digest) + } + return manifest, nil +} diff --git a/cmd/oci/store_test.go b/cmd/oci/store_test.go index 86fa6be6..5ff38eb1 100644 --- a/cmd/oci/store_test.go +++ b/cmd/oci/store_test.go @@ -11,6 +11,7 @@ import ( "io" "os" "path/filepath" + "strings" "sync" "testing" @@ -98,6 +99,44 @@ func TestRootIndexNamesNestedIndex(t *testing.T) { } } +func TestDigestForErrorKinds(t *testing.T) { + s := tempStore(t) + _, err := s.digestFor("absent:1", defaultPlatform) + if !errors.Is(err, errNotPulled) || !strings.Contains(err.Error(), "elfuse-oci pull") { + t.Fatalf("missing image error = %v", err) + } + if err := os.WriteFile(filepath.Join(s.root, "index.json"), []byte("{"), 0o600); err != nil { + t.Fatal(err) + } + if _, err := s.digestFor("absent:1", defaultPlatform); err == nil || errors.Is(err, errNotPulled) { + t.Fatalf("corrupt index error = %v", err) + } +} + +func TestManifestRoundTrip(t *testing.T) { + s, digest := storeWithImage(t, "fix:1", testImage{}) + manifest, err := s.manifestFor(context.Background(), digest) + if err != nil { + t.Fatal(err) + } + if len(manifest.Layers) != 1 || manifest.Config.MediaType != ocispec.MediaTypeImageConfig { + t.Fatalf("manifest = %+v", manifest) + } +} + +func TestManifestForRejectsInvalidManifest(t *testing.T) { + s := tempStore(t) + for _, body := range [][]byte{ + []byte(`{}`), + []byte(`{"schemaVersion":1,"config":{"digest":"sha256:` + strings.Repeat("0", 64) + `"}}`), + } { + desc := pushBlob(t, s, types.OCIManifestSchema1, body) + if _, err := s.manifestFor(context.Background(), desc.Digest.String()); err == nil { + t.Fatalf("manifest %s must fail validation", body) + } + } +} + func TestPinConcurrentWritersKeepAllEntries(t *testing.T) { s := tempStore(t) digest := pushTestImage(t, s, testImage{}) @@ -330,3 +369,94 @@ func TestWithLockRefusesExpiredContext(t *testing.T) { t.Fatalf("error = %v, ran = %v", err, ran) } } + +func TestCacheDirRejectsSymlinkedParent(t *testing.T) { + good := "sha256:" + strings.Repeat("a", 64) + for _, rel := range []string{cacheRootfs, filepath.Join(cacheRootfs, "sha256")} { + s := tempStore(t) + p := filepath.Join(s.root, rel) + os.MkdirAll(filepath.Dir(p), 0o755) + if err := os.Symlink(t.TempDir(), p); err != nil { + t.Fatal(err) + } + _, err := s.cacheDir(cacheRootfs, good) + if err == nil || !strings.Contains(err.Error(), "symlink") { + t.Errorf("%s: err = %v, want a symlink refusal", rel, err) + } + } +} + +func TestCacheDirRejectsOddDigests(t *testing.T) { + s := tempStore(t) + for _, bad := range []string{"sha512:" + strings.Repeat("a", 128), "sha256:short", "zzz", + "sha256:" + strings.Repeat("g", 64)} { + if _, err := s.cacheDir(cacheRootfs, bad); err == nil { + t.Errorf("digest %q must be rejected as a cache key", bad) + } + } +} + +func TestRefuseRootfsInStore(t *testing.T) { + s := tempStore(t) + for _, rootfs := range []string{s.root, filepath.Join(s.root, "rootfs", "x")} { + if err := refuseRootfsInStore(s.root, rootfs); err == nil || + !strings.Contains(err.Error(), "inside the store") { + t.Errorf("%s: err = %v, want a refusal", rootfs, err) + } + } + if err := refuseRootfsInStore(s.root, filepath.Dir(s.root)); err == nil || + !strings.Contains(err.Error(), "contains the store") { + t.Errorf("store parent: err = %v, want a refusal", err) + } + file := filepath.Join(t.TempDir(), "file") + if err := os.WriteFile(file, nil, 0o600); err != nil { + t.Fatal(err) + } + for _, rootfs := range []string{"", t.TempDir(), filepath.Join(file, "child")} { + if err := refuseRootfsInStore(s.root, rootfs); err != nil { + t.Errorf("%q: err = %v, want acceptance", rootfs, err) + } + } +} + +// A store that pull would refuse is refused for reading, and a mistyped path +// is not created. +func TestOpenStoreForReadRefusals(t *testing.T) { + legacy := t.TempDir() + if err := os.WriteFile(filepath.Join(legacy, "refs.json"), []byte("{}"), 0o600); err != nil { + t.Fatal(err) + } + if _, err := openStoreForRead(legacy); err == nil || !strings.Contains(err.Error(), "legacy refs.json") { + t.Errorf("legacy store: err = %v, want the legacy refusal", err) + } + + bare := t.TempDir() + if err := os.WriteFile(filepath.Join(bare, "index.json"), []byte("{}"), 0o600); err != nil { + t.Fatal(err) + } + if _, err := openStoreForRead(bare); err == nil || !strings.Contains(err.Error(), "not an elfuse OCI store") { + t.Errorf("marker-less store: err = %v, want a format refusal", err) + } + + s := tempStore(t) + if err := os.WriteFile(filepath.Join(s.root, "oci-layout"), []byte("{"), 0o600); err != nil { + t.Fatal(err) + } + if _, err := openStoreForRead(s.root); err == nil || !strings.Contains(err.Error(), "corrupt oci-layout") { + t.Errorf("malformed oci-layout: err = %v, want a format refusal", err) + } + if err := os.WriteFile(filepath.Join(s.root, markerName), []byte("2\n"), 0o600); err != nil { + t.Fatal(err) + } + if _, err := openStoreForRead(s.root); err == nil || !strings.Contains(err.Error(), "unsupported format marker") { + t.Errorf("newer marker: err = %v, want a format refusal", err) + } + + missing := filepath.Join(t.TempDir(), "absent") + if _, err := openStoreForRead(missing); err == nil { + t.Fatal("a missing store must fail") + } + if _, err := os.Lstat(missing); !os.IsNotExist(err) { + t.Error("opening for read must not create the store directory") + } +} From f4fa70d679e7e5ecfc36bdf2af03195e544d1be5 Mon Sep 17 00:00:00 2001 From: Chun-Hung Tseng Date: Wed, 2 Sep 2026 17:22:39 +0200 Subject: [PATCH 4/5] Add unpack for stored OCI images Without --rootfs the rootfs is published under the store by manifest digest through a rename, and only that content-addressed entry may reuse another unpack's tree when the rename is lost; a caller-named rootfs is merged in place or staged and renamed, and a lost rename is an error. ensurePrivateDir sets the kind directory's mode too, so the blob and rootfs caches share it. moby/go-archive owns layer application, whiteouts, decompression, containment, and metadata. Each header is rewritten after the previous entry has been applied, so parent symlinks already on disk resolve. Devices, FIFOs, and hardlinks to them become whiteouts. Absolute symlink targets are rebased relative to the link, and a hardlink to such a symlink is rebased at its own location. A .. above the root clamps there, in parent paths and in rebased targets. Special mode bits are cleared because ownership is never applied. Directories stay accessible until all layers finish, then their modes are restored, on failure too, skipping a record whose path a case alias in a later layer replaced. Staging trees are removed without following symlinks or failing on restrictive directory modes. Cancellation is checked on the decompressed stream, so an unpigz child's exit status cannot replace it. --- cmd/oci/common.go | 9 + cmd/oci/helpers_test.go | 85 ++++- cmd/oci/main.go | 3 +- cmd/oci/main_command_test.go | 20 +- cmd/oci/store.go | 29 +- cmd/oci/tarfilter.go | 328 +++++++++++++++++++ cmd/oci/tarfilter_test.go | 433 ++++++++++++++++++++++++ cmd/oci/unpack.go | 278 ++++++++++++++++ cmd/oci/unpack_test.go | 618 +++++++++++++++++++++++++++++++++++ go.mod | 9 +- go.sum | 22 +- 11 files changed, 1812 insertions(+), 22 deletions(-) create mode 100644 cmd/oci/tarfilter.go create mode 100644 cmd/oci/tarfilter_test.go create mode 100644 cmd/oci/unpack.go create mode 100644 cmd/oci/unpack_test.go diff --git a/cmd/oci/common.go b/cmd/oci/common.go index 40379b53..9a20d71f 100644 --- a/cmd/oci/common.go +++ b/cmd/oci/common.go @@ -94,3 +94,12 @@ func (cf *commonFlags) openStore() (*store, ocispec.Platform, error) { s, err := openStore(root) return s, platform, err } + +func (cf *commonFlags) openStoreForRead() (*store, ocispec.Platform, error) { + root, platform, err := cf.values() + if err != nil { + return nil, ocispec.Platform{}, err + } + s, err := openStoreForRead(root) + return s, platform, err +} diff --git a/cmd/oci/helpers_test.go b/cmd/oci/helpers_test.go index 627109fb..0ac0054c 100644 --- a/cmd/oci/helpers_test.go +++ b/cmd/oci/helpers_test.go @@ -10,10 +10,12 @@ import ( "context" "encoding/json" "io" + "math/rand" "os" "path/filepath" "strings" "testing" + "time" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/types" @@ -21,8 +23,13 @@ import ( ) type tarEntry struct { - Name string - Body string + Name string + Body string + Link string + Mode int64 + Type byte + Major int64 + ModTime time.Time } func buildLayerTar(t *testing.T, entries []tarEntry) []byte { @@ -30,7 +37,32 @@ func buildLayerTar(t *testing.T, entries []tarEntry) []byte { var b bytes.Buffer tw := tar.NewWriter(&b) for _, e := range entries { - hdr := &tar.Header{Name: e.Name, Mode: 0o644, Size: int64(len(e.Body)), Typeflag: tar.TypeReg} + hdr := &tar.Header{Name: e.Name, Mode: e.Mode, Size: int64(len(e.Body)), Typeflag: tar.TypeReg, ModTime: e.ModTime} + if hdr.Mode == 0 { + hdr.Mode = 0o644 + } + // tar.Writer rounds ModTime to whole seconds unless Format is set, and + // only PAX keeps the fraction. + if e.ModTime.Nanosecond() != 0 { + hdr.Format = tar.FormatPAX + } + switch { + case e.Type != 0: + hdr.Typeflag = e.Type + hdr.Size = 0 + hdr.Linkname = e.Link + hdr.Devmajor = e.Major + case e.Link != "": + hdr.Typeflag = tar.TypeSymlink + hdr.Linkname = e.Link + hdr.Size = 0 + case e.Name[len(e.Name)-1] == '/': + hdr.Typeflag = tar.TypeDir + if e.Mode == 0 { + hdr.Mode = 0o755 + } + hdr.Size = 0 + } if err := tw.WriteHeader(hdr); err != nil { t.Fatal(err) } @@ -47,7 +79,10 @@ func buildLayerTar(t *testing.T, entries []tarEntry) []byte { func gzipBytes(t *testing.T, b []byte) []byte { t.Helper() var z bytes.Buffer - zw := gzip.NewWriter(&z) + zw, err := gzip.NewWriterLevel(&z, gzip.BestSpeed) + if err != nil { + t.Fatal(err) + } if _, err := zw.Write(b); err != nil { t.Fatal(err) } @@ -220,3 +255,45 @@ func mustContain(t *testing.T, got string, wants ...string) { } } } + +func manifestOf(t *testing.T, s *store, digest string) ocispec.Manifest { + t.Helper() + manifest, err := s.manifestFor(context.Background(), digest) + if err != nil { + t.Fatal(err) + } + return manifest +} + +func runCaptured(t *testing.T, args ...string) (string, error) { + t.Helper() + var err error + _, stderr := captureOutput(t, func() { err = run(args) }) + return stderr, err +} + +func unpackFresh(t *testing.T, s *store, digest string) string { + t.Helper() + dest := filepath.Join(t.TempDir(), "rootfs") + if err := unpackFreshTo(t, s, digest, dest, false); err != nil { + t.Fatal(err) + } + return dest +} + +func unpackFreshTo(t *testing.T, s *store, digest, dest string, cached bool) (err error) { + t.Helper() + t.Cleanup(func() { _ = removeRootfsTree(dest) }) + captureOutput(t, func() { + err = unpackImageFresh(context.Background(), s, manifestOf(t, s, digest), dest, cached) + }) + return err +} + +// incompressibleBody returns seeded random bytes, which gzip cannot shrink. +func incompressibleBody(seed, n int) string { + r := rand.New(rand.NewSource(int64(seed))) + b := make([]byte, n) + r.Read(b) + return string(b) +} diff --git a/cmd/oci/main.go b/cmd/oci/main.go index 6177803e..80c8fc86 100644 --- a/cmd/oci/main.go +++ b/cmd/oci/main.go @@ -21,7 +21,8 @@ func main() { } type cli struct { - Pull pullCommand `cmd:"" help:"Pull an image into the local store"` + Pull pullCommand `cmd:"" help:"Pull an image into the local store"` + Unpack unpackCommand `cmd:"" help:"Unpack a stored image into a rootfs"` } func newParser(stdout, stderr io.Writer, target *cli) (*kong.Kong, error) { diff --git a/cmd/oci/main_command_test.go b/cmd/oci/main_command_test.go index 3df634b3..700945df 100644 --- a/cmd/oci/main_command_test.go +++ b/cmd/oci/main_command_test.go @@ -15,7 +15,7 @@ func TestUsageAndErrors(t *testing.T) { if err == nil { t.Fatal("missing command must fail") } - mustContain(t, stdout, "Usage: elfuse-oci ", "pull") + mustContain(t, stdout, "Usage: elfuse-oci ", "pull", "unpack") if stderr != "" { t.Fatalf("parse error wrote to stderr: %q", stderr) } @@ -24,18 +24,20 @@ func TestUsageAndErrors(t *testing.T) { if err == nil || !strings.Contains(err.Error(), "unexpected argument bogus") { t.Fatalf("unknown command error = %v", err) } - mustContain(t, stdout, "Usage: elfuse-oci ", "pull") + mustContain(t, stdout, "Usage: elfuse-oci ", "pull", "unpack") if stderr != "" { t.Fatalf("unknown command wrote to stderr: %q", stderr) } - stdout, stderr = captureOutput(t, func() { err = run([]string{"pull", "--nope", "x"}) }) - if err == nil || !strings.Contains(err.Error(), "unknown flag --nope") { - t.Fatalf("unknown flag error = %v", err) - } - mustContain(t, stdout, "Usage: elfuse-oci pull", "--platform", "--store", "--timeout") - if stderr != "" { - t.Fatalf("unknown flag wrote to stderr: %q", stderr) + for cmd, flag := range map[string]string{"pull": "--timeout", "unpack": "--rootfs"} { + stdout, stderr = captureOutput(t, func() { err = run([]string{cmd, "--nope", "x"}) }) + if err == nil || !strings.Contains(err.Error(), "unknown flag --nope") { + t.Fatalf("%s: unknown flag error = %v", cmd, err) + } + mustContain(t, stdout, "Usage: elfuse-oci "+cmd, "--platform", "--store", flag) + if stderr != "" { + t.Fatalf("%s: unknown flag wrote to stderr: %q", cmd, stderr) + } } } diff --git a/cmd/oci/store.go b/cmd/oci/store.go index e13d6653..7879ef65 100644 --- a/cmd/oci/store.go +++ b/cmd/oci/store.go @@ -355,11 +355,18 @@ func (r contextReader) Read(p []byte) (int, error) { return r.r.Read(p) } +// ensurePrivateDir creates a per-algorithm cache directory and sets the +// store's mode on it and on its kind directory. func ensurePrivateDir(path string) error { if err := os.MkdirAll(path, 0o700); err != nil { return err } - return os.Chmod(path, 0o700) + for _, p := range []string{path, filepath.Dir(path)} { + if err := os.Chmod(p, 0o700); err != nil { + return err + } + } + return nil } func syncDirectory(path string) error { @@ -374,6 +381,14 @@ func syncDirectory(path string) error { return err } +func (s *store) blob(hash v1.Hash) (io.ReadCloser, error) { + r, err := layout.Path(s.root).Blob(hash) + if err != nil { + return nil, fmt.Errorf("store: read blob %s: %w", hash, err) + } + return r, nil +} + func (s *store) writeBlob(ctx context.Context, desc v1.Descriptor, r io.ReadCloser) error { defer r.Close() if err := ctx.Err(); err != nil { @@ -383,9 +398,6 @@ func (s *store) writeBlob(ctx context.Context, desc v1.Descriptor, r io.ReadClos if err := ensurePrivateDir(dir); err != nil { return err } - if err := os.Chmod(filepath.Join(s.root, "blobs"), 0o700); err != nil { - return err - } path := filepath.Join(dir, desc.Digest.Hex) if f, err := os.Open(path); err == nil { fi, statErr := f.Stat() @@ -666,3 +678,12 @@ func (s *store) manifestFor(ctx context.Context, digest string) (ocispec.Manifes } return manifest, nil } + +func (s *store) loadRef(ctx context.Context, ref string, platform ocispec.Platform) (string, ocispec.Manifest, error) { + d, err := s.digestFor(ref, platform) + if err != nil { + return "", ocispec.Manifest{}, err + } + m, err := s.manifestFor(ctx, d) + return d, m, err +} diff --git a/cmd/oci/tarfilter.go b/cmd/oci/tarfilter.go new file mode 100644 index 00000000..1b07dc87 --- /dev/null +++ b/cmd/oci/tarfilter.go @@ -0,0 +1,328 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "archive/tar" + "bytes" + "errors" + "fmt" + "io" + "os" + "path" + "path/filepath" + "sort" + "strings" + "syscall" + + "github.com/moby/go-archive" +) + +const tarSpecialBits = 0o4000 | 0o2000 | 0o1000 +const filterCopyBufferSize = 32 * 1024 + +type layerRecord struct { + kind byte + mode os.FileMode + target string + layer int +} + +type layerPolicy struct { + root *os.Root + records map[string]layerRecord + layer int + warnedNode bool + warnedBits bool +} + +func newLayerPolicy(dest string) (*layerPolicy, error) { + root, err := os.OpenRoot(dest) + if err != nil { + return nil, err + } + p := &layerPolicy{root: root, records: make(map[string]layerRecord)} + err = walkRootfsDirs(root, ".", func(name string, info os.FileInfo) error { + p.records[name] = layerRecord{kind: tar.TypeDir, mode: info.Mode()} + return root.Chmod(name, info.Mode()|0o700) + }) + if err != nil { + return nil, errors.Join(err, p.Close()) + } + return p, nil +} + +func (p *layerPolicy) Close() error { + var dirs []string + for name, record := range p.records { + if record.kind == tar.TypeDir { + dirs = append(dirs, name) + } + } + // Descendants must remain reachable until their modes are restored. + sort.Slice(dirs, func(i, j int) bool { return len(dirs[i]) > len(dirs[j]) }) + var result error + for _, name := range dirs { + info, err := p.root.Lstat(name) + if os.IsNotExist(err) || errors.Is(err, syscall.ENOTDIR) { + continue + } + if err == nil && info.IsDir() { + err = p.root.Chmod(name, p.records[name].mode) + } + result = errors.Join(result, err) + } + return errors.Join(result, p.root.Close()) +} + +func (p *layerPolicy) filter(hdr *tar.Header) error { + name, err := p.entryPath(hdr.Name) + if err != nil { + return err + } + if name == "." { + return nil // go-archive ignores headers for the extraction root. + } + hdr.Name = name + base := path.Base(name) + if base == archive.WhiteoutOpaqueDir { + p.forget(path.Dir(name), true) + return nil + } + if strings.HasPrefix(base, archive.WhiteoutPrefix) { + p.forget(path.Join(path.Dir(name), strings.TrimPrefix(base, archive.WhiteoutPrefix)), false) + return nil + } + + record := layerRecord{kind: hdr.Typeflag} + if hdr.Typeflag == tar.TypeLink { + target, err := p.entryPath(hdr.Linkname) + if err != nil { + return err + } + record = p.records[target] + if record.kind != tar.TypeSymlink { + if info, err := p.root.Lstat(target); err == nil && info.Mode()&os.ModeSymlink != 0 { + link, err := p.root.Readlink(target) + if err != nil { + return err + } + if path.IsAbs(link) { + record = layerRecord{kind: tar.TypeSymlink, target: link} + } + } + } + hdr.Linkname = target + if record.kind == tar.TypeSymlink { + hdr.Typeflag = tar.TypeSymlink + hdr.Linkname = record.target + } + } + if hdr.Typeflag != tar.TypeDir { + p.forget(name, false) + } + switch record.kind { + case tar.TypeChar, tar.TypeBlock, tar.TypeFifo: + p.records[name] = layerRecord{kind: record.kind, layer: p.layer} + if !p.warnedNode { + p.warnedNode = true + fmt.Fprintf(os.Stderr, "elfuse-oci: unpack: dropping device and FIFO entries (first: %q)\n", hdr.Name) + } + hdr.Name = path.Join(path.Dir(name), archive.WhiteoutPrefix+base) + hdr.Typeflag, hdr.Linkname, hdr.Size, hdr.Devmajor, hdr.Devminor = tar.TypeReg, "", 0, 0, 0 + return nil + } + if hdr.Mode&tarSpecialBits != 0 { + hdr.Mode &^= tarSpecialBits + if !p.warnedBits { + p.warnedBits = true + fmt.Fprintf(os.Stderr, "elfuse-oci: unpack: clearing special permission bits (first: %q)\n", hdr.Name) + } + } + switch hdr.Typeflag { + case tar.TypeDir: + p.records[name] = layerRecord{kind: tar.TypeDir, mode: os.FileMode(hdr.Mode).Perm(), layer: p.layer} + hdr.Mode |= 0o700 + case tar.TypeSymlink: + if path.IsAbs(hdr.Linkname) { + p.records[name] = layerRecord{kind: tar.TypeSymlink, target: hdr.Linkname, layer: p.layer} + if hdr.Linkname, err = p.relativeTarget(path.Dir(name), hdr.Linkname); err != nil { + return err + } + } + } + return nil +} + +func (p *layerPolicy) forget(name string, opaque bool) { + if !opaque { + info, err := p.root.Lstat(name) + if os.IsNotExist(err) || errors.Is(err, syscall.ENOTDIR) || err == nil && !info.IsDir() { + delete(p.records, name) + return + } + } + prefix := name + "/" + if name == "." { + prefix = "" + } + for key, record := range p.records { + if opaque && (key == name || record.layer == p.layer) { + continue + } + if key == name || strings.HasPrefix(key, prefix) { + delete(p.records, key) + } + } +} + +// Entry operations replace or link the final component itself. +func (p *layerPolicy) entryPath(name string) (string, error) { + name = path.Clean(strings.TrimLeft(name, "/")) + if name == "." { + return name, nil + } + if !filepath.IsLocal(name) { + return "", fmt.Errorf("invalid entry path %q", name) + } + dir, err := p.resolveDirectory(path.Dir(name)) + if err != nil { + return "", err + } + return path.Join(dir, path.Base(name)), nil +} + +func (p *layerPolicy) resolveDirectory(dir string) (string, error) { + var resolved []string + pending := strings.Split(dir, "/") + links := 0 + for len(pending) != 0 { + part := pending[0] + pending = pending[1:] + switch part { + case "", ".": + continue + case "..": + // A symlink target may climb above the root; clamp there as the + // kernel does. Entry names were checked by entryPath. + if len(resolved) != 0 { + resolved = resolved[:len(resolved)-1] + } + continue + } + candidate := path.Join(strings.Join(resolved, "/"), part) + info, err := p.root.Lstat(candidate) + if os.IsNotExist(err) || err == nil && info.Mode()&os.ModeSymlink == 0 { + resolved = append(resolved, part) + continue + } + if err != nil { + return "", err + } + links++ + if links > 40 { + return "", fmt.Errorf("resolve %q: %w", dir, syscall.ELOOP) + } + target, err := p.root.Readlink(candidate) + if err != nil { + return "", err + } + if path.IsAbs(target) { + resolved = nil + } + pending = append(strings.Split(target, "/"), pending...) + } + return path.Join(".", strings.Join(resolved, "/")), nil +} + +func (p *layerPolicy) relativeTarget(dir, target string) (string, error) { + prefix := "" + if dir != "." { + prefix = strings.Repeat("../", strings.Count(dir, "/")+1) + } + // Keep target traversal intact: a/../b may cross a symlink at a. Drop only + // a .. whose parent resolves to the root. + var kept []string + for _, part := range strings.Split(strings.TrimLeft(target, "/"), "/") { + if part == ".." { + parent, err := p.resolveDirectory(strings.Join(kept, "/")) + if err != nil { + return "", err + } + if parent == "." { + continue + } + } + kept = append(kept, part) + } + target = strings.Join(kept, "/") + if target == "" { + target = "." + } + return prefix + target, nil +} + +// go-archive requests the next header only after applying the previous one. +type filteredLayer struct { + tr *tar.Reader + tw *tar.Writer + pending bytes.Buffer + inBody bool + err error + policy *layerPolicy + buf [filterCopyBufferSize]byte +} + +func filterLayer(src io.Reader, policy *layerPolicy) io.Reader { + policy.layer++ + r := &filteredLayer{tr: tar.NewReader(src), policy: policy} + r.tw = tar.NewWriter(&r.pending) + return r +} + +func (r *filteredLayer) Read(dst []byte) (int, error) { + if len(dst) == 0 { + return 0, nil + } + if r.pending.Len() != 0 { + return r.pending.Read(dst) + } + if r.err != nil { + return 0, r.err + } + if r.inBody { + // archive/tar bounds each body and yields none for header-only types. + n, err := r.tr.Read(r.buf[:]) + if n != 0 { + _, r.err = r.tw.Write(r.buf[:n]) + } + if err == io.EOF { + r.inBody = false + } else if err != nil { + r.err = err + } + } else { + hdr, err := r.tr.Next() + switch { + case err == io.EOF: + r.err = r.tw.Close() + if r.err == nil { + r.err = io.EOF + } + case err != nil: + r.err = err + default: + if r.err = r.policy.filter(hdr); r.err == nil { + // PAX carries rewritten names and sub-second times that USTAR cannot. + hdr.Format = tar.FormatPAX + r.err = r.tw.WriteHeader(hdr) + r.inBody = true + } + } + } + if r.pending.Len() != 0 { + return r.pending.Read(dst) + } + return 0, r.err +} diff --git a/cmd/oci/tarfilter_test.go b/cmd/oci/tarfilter_test.go new file mode 100644 index 00000000..3777d5a2 --- /dev/null +++ b/cmd/oci/tarfilter_test.go @@ -0,0 +1,433 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "archive/tar" + "bytes" + "context" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/moby/go-archive" +) + +// longCertPath is long enough that a rewritten target outgrows a USTAR header. +const longCertPath = "usr/share/ca-certificates/mozilla/Autoridad_de_Certificacion_Firmaprofesional_CIF_A62634068.crt" + +func filterEntries(t *testing.T, entries []tarEntry) map[string]*tar.Header { + t.Helper() + return filterEntriesWith(t, testLayerPolicy(t), entries) +} + +func filterEntriesWith(t *testing.T, policy *layerPolicy, entries []tarEntry) map[string]*tar.Header { + t.Helper() + var filtered bytes.Buffer + stream := filterLayer(bytes.NewReader(buildLayerTar(t, entries)), policy) + if _, err := archive.ApplyUncompressedLayer(policy.root.Name(), io.TeeReader(stream, &filtered), unpackOptions()); err != nil { + t.Fatal(err) + } + got := map[string]*tar.Header{} + for _, hdr := range readHeaders(t, &filtered) { + got[hdr.Name] = hdr + } + return got +} + +func readHeaders(t *testing.T, r io.Reader) []*tar.Header { + t.Helper() + tr := tar.NewReader(r) + var headers []*tar.Header + for { + hdr, err := tr.Next() + if err == io.EOF { + return headers + } + if err != nil { + t.Fatalf("after %d headers: %v", len(headers), err) + } + headers = append(headers, hdr) + } +} + +func TestFilterDropsUnsupportedNodes(t *testing.T) { + for _, c := range []struct { + name string + layers [][]tarEntry + want map[string]bool + }{ + {name: "devices and FIFOs become whiteouts", layers: [][]tarEntry{{ + {Name: "keep", Body: "x"}, + {Name: "dev-null", Type: tar.TypeChar, Major: 1 << 21}, + {Name: "disk", Type: tar.TypeBlock, Major: 8}, + {Name: "pipe", Type: tar.TypeFifo}, + }}, want: map[string]bool{ + "keep": true, "dev-null": false, "disk": false, "pipe": false, + ".wh.dev-null": true, ".wh.disk": true, ".wh.pipe": true, + }}, + {name: "hardlinks to dropped nodes become whiteouts", layers: [][]tarEntry{{ + {Name: "dev/null", Type: tar.TypeChar, Major: 1}, + {Name: "dev/alias", Link: "dev/null", Type: tar.TypeLink}, + {Name: "/dev/zero", Type: tar.TypeChar, Major: 1}, + {Name: "dev/zero-alias", Link: "dev/zero", Type: tar.TypeLink}, + {Name: "dev/zero-abs", Link: "/dev/zero", Type: tar.TypeLink}, + {Name: "keep", Body: "x"}, + {Name: "keep-alias", Link: "keep", Type: tar.TypeLink}, + }}, want: map[string]bool{ + "dev/null": false, "dev/alias": false, "dev/.wh.alias": true, + "/dev/zero": false, "dev/zero-alias": false, "dev/zero-abs": false, + "dev/.wh.zero-alias": true, "dev/.wh.zero-abs": true, + "keep": true, "keep-alias": true, + }}, + {name: "drops carry across layers until the path returns", layers: [][]tarEntry{ + {{Name: "dev/null", Type: tar.TypeChar, Major: 1}}, + {{Name: "dev/alias", Link: "dev/null", Type: tar.TypeLink}}, + {{Name: "dev/null", Body: "x"}, {Name: "dev/alias2", Link: "dev/null", Type: tar.TypeLink}}, + }, want: map[string]bool{ + "dev/null": true, "dev/alias2": true, + }}, + } { + t.Run(c.name, func(t *testing.T) { + policy := testLayerPolicy(t) + var got map[string]*tar.Header + captureOutput(t, func() { + for _, layer := range c.layers { + got = filterEntriesWith(t, policy, layer) + } + }) + for name, want := range c.want { + if (got[name] != nil) != want { + t.Errorf("%s: present=%v, want %v", name, got[name] != nil, want) + } + } + for name, hdr := range got { + if !strings.Contains(name, ".wh.") { + continue + } + if hdr.Typeflag != tar.TypeReg || hdr.Size != 0 { + t.Errorf("%s: type %c size %d, want an empty regular file", name, hdr.Typeflag, hdr.Size) + } + } + }) + } +} + +func TestFilterWarnsOnceAboutDroppedNodes(t *testing.T) { + _, stderr := captureOutput(t, func() { + filterEntries(t, []tarEntry{ + {Name: "dev-null", Type: tar.TypeChar, Major: 1}, + {Name: "pipe", Type: tar.TypeFifo}, + }) + }) + mustContain(t, stderr, "dropping device and FIFO entries", "dev-null") + if strings.Count(stderr, "dropping device") != 1 { + t.Errorf("warning repeated: %q", stderr) + } +} + +// Ownership is never applied, so a setuid or setgid bit would carry the +// unpacking user's ids. +func TestFilterClearsSpecialBits(t *testing.T) { + var got map[string]*tar.Header + _, stderr := captureOutput(t, func() { + got = filterEntries(t, []tarEntry{ + {Name: "wall", Body: "x", Mode: 0o2755}, + {Name: "sudoish", Body: "x", Mode: 0o4755}, + {Name: "tmpdir/", Mode: 0o1777}, + }) + }) + for name, want := range map[string]int64{"wall": 0o755, "sudoish": 0o755, "tmpdir": 0o777} { + if got[name] == nil || got[name].Mode != want { + t.Errorf("%s: mode = %o, want %o", name, got[name].Mode, want) + } + } + mustContain(t, stderr, "clearing special permission bits") +} + +func TestFilterRewritesAbsoluteSymlinks(t *testing.T) { + got := filterEntries(t, []tarEntry{ + {Name: "usr/bin/sh", Link: "/bin/busybox"}, + {Name: "bin", Link: "/usr/bin"}, + {Name: "loop", Link: "/"}, + {Name: "up", Link: "/../etc/x"}, + {Name: "etc/up", Link: "/../../etc/x"}, + {Name: "cross", Link: "/usr/../../etc/x"}, + {Name: "etc/rel", Link: "../keep"}, + {Name: "etc/ssl/certs/cert.pem", Link: "/" + longCertPath}, + {Name: "usr/lib/"}, + {Name: "lib", Link: "usr/lib"}, + {Name: "lib/bar", Link: "/usr/lib/foo"}, + }) + for name, want := range map[string]string{ + "usr/bin/sh": "../../bin/busybox", + "bin": "usr/bin", + "loop": ".", + "up": "etc/x", + "etc/up": "../etc/x", + "cross": "usr/../etc/x", + "etc/rel": "../keep", + "etc/ssl/certs/cert.pem": "../../../" + longCertPath, + "usr/lib/bar": "../../usr/lib/foo", + } { + if got[name] == nil || got[name].Linkname != want { + t.Errorf("%s: linkname = %q, want %q", name, got[name].Linkname, want) + } + } +} + +// A later entry must not resolve through a symlink that a whiteout or a +// replacement removed. +func TestFilterResolvesPastRemovedSymlinks(t *testing.T) { + for _, c := range []struct { + name string + second []tarEntry + link string + want string + }{ + {name: "plain whiteout", second: []tarEntry{ + {Name: ".wh.lib"}, + {Name: "lib/foo/bar", Link: "/etc/x"}, + }, link: "lib/foo/bar", want: "../../etc/x"}, + {name: "root opaque marker", second: []tarEntry{ + {Name: ".wh..wh..opq"}, + {Name: "lib/foo/bar", Link: "/etc/x"}, + }, link: "lib/foo/bar", want: "../../etc/x"}, + {name: "replaced by a file", second: []tarEntry{ + {Name: "lib", Body: "now a file"}, + {Name: "lib/"}, + {Name: "lib/foo/bar", Link: "/etc/x"}, + }, link: "lib/foo/bar", want: "../../etc/x"}, + } { + t.Run(c.name, func(t *testing.T) { + policy := testLayerPolicy(t) + filterEntriesWith(t, policy, []tarEntry{ + {Name: "usr/lib/arm64/"}, + {Name: "lib/foo", Link: "/usr/lib/arm64"}, + }) + got := filterEntriesWith(t, policy, c.second) + if got[c.link] == nil || got[c.link].Linkname != c.want { + t.Fatalf("%s: linkname = %v, want %q", c.link, got[c.link], c.want) + } + }) + } +} + +func TestFilterLayerKeepsSubSecondTimestamps(t *testing.T) { + stamp := time.Unix(1700000000, 700000000) + hdr := filterEntries(t, []tarEntry{{Name: "etc/foo", Body: "x", ModTime: stamp}})["etc/foo"] + if hdr == nil || !hdr.ModTime.Equal(stamp) { + t.Fatalf("header = %v, want mtime %v", hdr, stamp) + } +} + +func testLayerPolicy(t *testing.T) *layerPolicy { + t.Helper() + root := t.TempDir() + p, err := newLayerPolicy(root) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := p.Close(); err != nil { + t.Error(err) + } + if err := removeRootfsTree(root); err != nil { + t.Error(err) + } + }) + return p +} + +// bigEntries builds n incompressible files of size bytes each; 2 MiB of them +// outgrow what go-archive or an unpigz pipe reads ahead. +func bigEntries(n, size int) []tarEntry { + var entries []tarEntry + for i := range n { + entries = append(entries, tarEntry{Name: fmt.Sprintf("f%04d", i), Body: incompressibleBody(i, size)}) + } + return entries +} + +type countingReader struct { + r io.Reader + n int +} + +func (c *countingReader) Read(p []byte) (int, error) { + n, err := c.r.Read(p) + c.n += n + return n, err +} + +// An extraction failure must stop reading the remaining compressed payload. +func TestApplyLayerReportsMidStreamFailure(t *testing.T) { + entries := append([]tarEntry{{Name: "bad", Link: "missing", Type: tar.TypeLink}}, bigEntries(32, 64*1024)...) + raw := gzipBytes(t, buildLayerTar(t, entries)) + src := &countingReader{r: bytes.NewReader(raw)} + err := applyLayer(context.Background(), src, testLayerPolicy(t)) + if err == nil { + t.Fatal("a hardlink to a missing target must fail the layer") + } + if src.n >= len(raw) { + t.Fatalf("read %d of %d bytes after the failure", src.n, len(raw)) + } +} + +type cancelOnReadReader struct { + r io.Reader + cancel context.CancelFunc + after int +} + +func (c *cancelOnReadReader) Read(p []byte) (int, error) { + n, err := c.r.Read(p) + if c.after -= n; c.after <= 0 { + c.cancel() + } + return n, err +} + +func TestApplyLayerStopsOnCancellationMidLayer(t *testing.T) { + raw := gzipBytes(t, buildLayerTar(t, bigEntries(32, 64*1024))) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + src := &cancelOnReadReader{r: bytes.NewReader(raw), cancel: cancel, after: len(raw) / 2} + err := applyLayer(ctx, src, testLayerPolicy(t)) + if err == nil { + t.Fatal("a cancelled context must abort the layer") + } + if !strings.Contains(err.Error(), context.Canceled.Error()) { + t.Fatalf("err = %v, want context.Canceled", err) + } +} + +func TestFilterLayerReadsEntriesOnDemand(t *testing.T) { + p := testLayerPolicy(t) + root := p.root.Name() + raw := buildLayerTar(t, []tarEntry{ + {Name: "parent/"}, + {Name: "parent/link", Link: "/target"}, + }) + tr := tar.NewReader(filterLayer(bytes.NewReader(raw), p)) + if _, err := tr.Next(); err != nil { + t.Fatal(err) + } + // Both bodies are empty, so the second header must be read after this change. + if err := os.MkdirAll(filepath.Join(root, "real", "dir"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.Symlink("real/dir", filepath.Join(root, "parent")); err != nil { + t.Fatal(err) + } + hdr, err := tr.Next() + if err != nil || hdr.Name != "real/dir/link" || hdr.Linkname != "../../target" { + t.Fatalf("second header = %v, %v", hdr, err) + } +} + +func TestFilterLayerStreamsPayloadAndPadding(t *testing.T) { + for _, size := range []int{0, 1, 511, 512, 513, filterCopyBufferSize * 3} { + t.Run(fmt.Sprint(size), func(t *testing.T) { + body := incompressibleBody(42, size) + raw := buildLayerTar(t, []tarEntry{{Name: "payload", Body: body}, {Name: "after", Body: "after"}}) + tr := tar.NewReader(filterLayer(bytes.NewReader(raw), testLayerPolicy(t))) + for _, want := range []string{body, "after"} { + if _, err := tr.Next(); err != nil { + t.Fatal(err) + } + got, err := io.ReadAll(tr) + if err != nil || string(got) != want { + t.Fatalf("payload: length %d, %v", len(got), err) + } + } + if _, err := tr.Next(); err != io.EOF { + t.Fatalf("end of stream = %v", err) + } + }) + } +} + +func TestFilterLayerRejectsTruncatedPayload(t *testing.T) { + raw := buildLayerTar(t, []tarEntry{{Name: "file", Body: strings.Repeat("x", 1024)}}) + _, err := io.Copy(io.Discard, filterLayer(bytes.NewReader(raw[:600]), testLayerPolicy(t))) + if !errors.Is(err, io.ErrUnexpectedEOF) { + t.Fatalf("truncated payload: %v", err) + } +} + +func filteredNames(t *testing.T, raw []byte) []string { + t.Helper() + var names []string + for _, hdr := range readHeaders(t, filterLayer(bytes.NewReader(raw), testLayerPolicy(t))) { + names = append(names, hdr.Name) + } + return names +} + +// archive/tar ignores the size field of a header-only entry on both sides, so +// a directory or hardlink written with one carries no body to forward. +func TestFilterLayerIgnoresHeaderOnlySize(t *testing.T) { + var raw bytes.Buffer + tw := tar.NewWriter(&raw) + for _, hdr := range []*tar.Header{ + {Name: "etc/", Typeflag: tar.TypeDir, Mode: 0o755, Size: 4096}, + {Name: "etc/orig", Typeflag: tar.TypeReg, Mode: 0o644, Size: 1}, + {Name: "etc/alias", Typeflag: tar.TypeLink, Linkname: "etc/orig", Mode: 0o644, Size: 1}, + } { + if err := tw.WriteHeader(hdr); err != nil { + t.Fatal(err) + } + if hdr.Typeflag == tar.TypeReg { + if _, err := tw.Write([]byte("x")); err != nil { + t.Fatal(err) + } + } + } + if err := tw.Close(); err != nil { + t.Fatal(err) + } + if got := filteredNames(t, raw.Bytes()); len(got) != 3 { + t.Fatalf("filtered = %v", got) + } +} + +// tar.Writer refuses a regular file named with a trailing slash; tar.Reader +// accepts one. +func TestFilterLayerCleansTrailingSlashOnRegularFile(t *testing.T) { + var raw bytes.Buffer + tw := tar.NewWriter(&raw) + if err := tw.WriteHeader(&tar.Header{Name: "etc/foo", Typeflag: tar.TypeReg, Mode: 0o644, Size: 1}); err != nil { + t.Fatal(err) + } + if _, err := tw.Write([]byte("x")); err != nil { + t.Fatal(err) + } + if err := tw.Close(); err != nil { + t.Fatal(err) + } + b := raw.Bytes() + copy(b, "etc/foo/\x00") + sum := 0 + for i, c := range b[:512] { + if i >= 148 && i < 156 { + sum += ' ' + } else { + sum += int(c) + } + } + copy(b[148:], fmt.Sprintf("%06o\x00 ", sum)) + hdr, err := tar.NewReader(bytes.NewReader(b)).Next() + if err != nil || hdr.Name != "etc/foo/" || hdr.Typeflag != tar.TypeReg { + t.Fatalf("fixture = %+v, %v", hdr, err) + } + if got := filteredNames(t, b); len(got) != 1 || got[0] != "etc/foo" { + t.Fatalf("filtered = %v", got) + } +} diff --git a/cmd/oci/unpack.go b/cmd/oci/unpack.go new file mode 100644 index 00000000..a16d0bc9 --- /dev/null +++ b/cmd/oci/unpack.go @@ -0,0 +1,278 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "time" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/moby/go-archive" + "github.com/moby/go-archive/compression" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" +) + +// staleRootfsTempAge is the age past which an unpack into the cache removes a +// staging tree. Unpacks take no store lock, so a shorter age could remove a +// tree another unpack is still filling. +const staleRootfsTempAge = 24 * time.Hour + +type unpackCommand struct { + commonFlags + Rootfs string `help:"Unpack into this directory instead of the managed cache" type:"path" default:""` + Ref string `arg:"" name:"ref" help:"Stored image reference"` +} + +func (c *unpackCommand) Run() error { + s, platform, err := c.commonFlags.openStoreForRead() + if err != nil { + return err + } + if err := refuseRootfsInStore(s.root, c.Rootfs); err != nil { + return err + } + ctx := context.Background() + digest, manifest, err := s.loadRef(ctx, c.Ref, platform) + if err != nil { + return err + } + if c.Rootfs != "" { + err = unpackImage(ctx, s, c.Ref, manifest, c.Rootfs) + } else { + var dest string + if dest, err = s.cacheDir(cacheRootfs, digest); err == nil { + err = ensureRootfs(ctx, s, c.Ref, manifest, dest) + } + } + if err != nil { + return err + } + fmt.Fprintf(os.Stderr, "Unpacked %s\n", c.Ref) + return nil +} + +func ensureRootfs(ctx context.Context, s *store, ref string, manifest ocispec.Manifest, dest string) error { + published, err := existingDirectory(dest) + if err != nil { + return err + } + if published { + fmt.Fprintf(os.Stderr, "Already unpacked %s -> %s\n", ref, dest) + return nil + } + fmt.Fprintf(os.Stderr, "Unpacking %s -> %s\n", ref, dest) + return unpackImageFresh(ctx, s, manifest, dest, true) +} + +func unpackImage(ctx context.Context, s *store, ref string, manifest ocispec.Manifest, dest string) error { + // dest may be a symlink to the directory; classify its target. + dest = resolvedAbs(dest) + fmt.Fprintf(os.Stderr, "Unpacking %s -> %s\n", ref, dest) + exists, err := existingDirectory(dest) + if err != nil { + return err + } + if exists { + return unpackInto(ctx, s, manifest, dest) + } + return unpackImageFresh(ctx, s, manifest, dest, false) +} + +func existingDirectory(path string) (bool, error) { + fi, err := os.Lstat(path) + if os.IsNotExist(err) { + return false, nil + } + if err != nil { + return false, err + } + if fi.IsDir() { + return true, nil + } + kind := "file" + if fi.Mode()&os.ModeSymlink != 0 { + kind = "symlink" + } + return false, fmt.Errorf("%s is a %s, want a directory", path, kind) +} + +// A cached entry takes the store's private mode; a directory the caller named +// does not. +func unpackImageFresh(ctx context.Context, s *store, manifest ocispec.Manifest, dest string, cached bool) (err error) { + dest = filepath.Clean(dest) + parent := filepath.Dir(dest) + mode := os.FileMode(0o755) + if cached { + mode = 0o700 + if err := ensurePrivateDir(parent); err != nil { + return err + } + if err := sweepStaleRootfsTemps(parent); err != nil { + return err + } + } else if err := os.MkdirAll(parent, mode); err != nil { + return err + } + tmp, err := os.MkdirTemp(parent, rootfsTempPrefix(filepath.Base(dest))) + if err != nil { + return err + } + defer func() { err = errors.Join(err, removeRootfsTree(tmp)) }() + if err := os.Chmod(tmp, mode); err != nil { + return err + } + if err := unpackInto(ctx, s, manifest, tmp); err != nil { + return err + } + if err := os.Rename(tmp, dest); err != nil { + // Only a content-addressed cache entry can lose this race benignly, + // because any completed tree there holds the same image. + if cached { + if published, _ := existingDirectory(dest); published { + return nil + } + } + return err + } + return syncDirectory(parent) +} + +func rootfsTempPrefix(base string) string { + return "." + base + ".tmp-" +} + +func isRootfsTemp(name string) bool { + return strings.HasPrefix(name, ".") && strings.Contains(name, ".tmp-") +} + +func sweepStaleRootfsTemps(dir string) error { + entries, err := os.ReadDir(dir) + if err != nil { + return err + } + for _, entry := range entries { + if !entry.IsDir() || !isRootfsTemp(entry.Name()) { + continue + } + fi, err := entry.Info() + if err != nil || time.Since(fi.ModTime()) < staleRootfsTempAge { + continue + } + if err := removeRootfsTree(filepath.Join(dir, entry.Name())); err != nil { + return err + } + } + return nil +} + +func unpackInto(ctx context.Context, s *store, manifest ocispec.Manifest, dest string) (err error) { + if err := ctx.Err(); err != nil { + return err + } + policy, err := newLayerPolicy(dest) + if err != nil { + return err + } + defer func() { err = errors.Join(err, policy.Close()) }() + for i, layer := range manifest.Layers { + if err := applyStoredLayer(ctx, s, policy, layer); err != nil { + return fmt.Errorf("unpack: layer %d (%s): %w", i, layer.Digest, err) + } + } + return nil +} + +func applyStoredLayer(ctx context.Context, s *store, policy *layerPolicy, layer ocispec.Descriptor) (err error) { + if err := ctx.Err(); err != nil { + return err + } + hash, err := v1.NewHash(layer.Digest.String()) + if err != nil { + return err + } + blob, err := s.blob(hash) + if err != nil { + return err + } + defer func() { err = errors.Join(err, blob.Close()) }() + return applyLayer(ctx, blob, policy) +} + +func unpackOptions() *archive.TarOptions { + return &archive.TarOptions{NoLchown: true, BestEffortXattrs: true} +} + +// Cancellation is checked on the decompressed stream, so an unpigz child's +// exit status cannot replace it. +func applyLayer(ctx context.Context, blob io.Reader, policy *layerPolicy) (err error) { + decompressed, err := compression.DecompressStream(blob) + if err != nil { + return err + } + defer func() { err = errors.Join(err, decompressed.Close()) }() + layer := filterLayer(contextReader{ctx: ctx, r: decompressed}, policy) + _, err = archive.ApplyUncompressedLayer(policy.root.Name(), layer, unpackOptions()) + return err +} + +// walkRootfsDirs visits directories before their children and never follows +// symlinks, so permissions can be relaxed before reading a directory. +func walkRootfsDirs(root *os.Root, name string, visit func(string, os.FileInfo) error) error { + info, err := root.Lstat(name) + if err != nil { + return err + } + if !info.IsDir() { + return nil + } + if err := visit(name, info); err != nil { + return err + } + dir, err := root.Open(name) + if err != nil { + return err + } + names, err := dir.Readdirnames(-1) + if closeErr := dir.Close(); err == nil { + err = closeErr + } + if err != nil { + return err + } + for _, child := range names { + if err := walkRootfsDirs(root, filepath.Join(name, child), visit); err != nil { + return err + } + } + return nil +} + +func removeRootfsTree(name string) error { + // Open the parent so a symlink at name is removed as a link. + parent, err := os.OpenRoot(filepath.Dir(name)) + if os.IsNotExist(err) { + return nil + } + if err != nil { + return err + } + defer parent.Close() + base := filepath.Base(name) + err = walkRootfsDirs(parent, base, func(rel string, info os.FileInfo) error { + return parent.Chmod(rel, info.Mode().Perm()|0o700) + }) + if os.IsNotExist(err) { + return nil + } + if err != nil { + return err + } + return parent.RemoveAll(base) +} diff --git a/cmd/oci/unpack_test.go b/cmd/oci/unpack_test.go new file mode 100644 index 00000000..2fe9a8b5 --- /dev/null +++ b/cmd/oci/unpack_test.go @@ -0,0 +1,618 @@ +// Copyright 2026 elfuse contributors +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "archive/tar" + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestUnpackAppliesWhiteoutsAcrossLayers(t *testing.T) { + s, d := storeWithImage(t, "wh:1", testImage{layers: [][]tarEntry{ + {{Name: "a/"}, {Name: "a/keep", Body: "k"}, {Name: "a/gone", Body: "g"}, + {Name: "a/sub/"}, {Name: "a/sub/old", Body: "o"}, {Name: "a/dev", Body: "d"}}, + {{Name: "a/.wh.gone"}, {Name: "a/sub/.wh..wh..opq"}, {Name: "a/sub/new", Body: "n"}, + {Name: "a/dev", Type: tar.TypeChar, Major: 1}}, + {{Name: "a/devlink", Link: "a/dev", Type: tar.TypeLink}}, + }}) + dest := unpackFresh(t, s, d) + for p, want := range map[string]bool{ + "a/keep": true, "a/gone": false, + "a/sub/old": false, "a/sub/new": true, + "a/.wh.gone": false, + "a/dev": false, "a/.wh.dev": false, "a/devlink": false, + } { + _, err := os.Lstat(filepath.Join(dest, p)) + if want != (err == nil) { + t.Errorf("%s: present=%v, want %v", p, err == nil, want) + } + } + b, err := os.ReadFile(filepath.Join(dest, "a/sub/new")) + if err != nil || string(b) != "n" { + t.Fatalf("a/sub/new = %q, %v", b, err) + } +} + +// A hardlink to a dropped device must whiteout its own path, or a file the +// image meant to replace survives from the layer below. +func TestUnpackHardlinkToDroppedDeviceRemovesLowerFile(t *testing.T) { + s, d := storeWithImage(t, "hlwh:1", testImage{layers: [][]tarEntry{ + {{Name: "dev/"}, {Name: "dev/alias", Body: "stale"}}, + {{Name: "dev/null", Type: tar.TypeChar, Major: 1}, + {Name: "dev/alias", Link: "dev/null", Type: tar.TypeLink}}, + }}) + dest := unpackFresh(t, s, d) + if _, err := os.Lstat(filepath.Join(dest, "dev/alias")); !os.IsNotExist(err) { + b, _ := os.ReadFile(filepath.Join(dest, "dev/alias")) + t.Fatalf("dev/alias survived as %q (%v); the image replaced it with a device", b, err) + } +} + +func TestUnpackFreshFixtures(t *testing.T) { + for _, c := range []struct { + name string + layers [][]tarEntry + file string + want string + suffix string + }{ + {name: "symlink under a symlinked parent resolves", layers: [][]tarEntry{ + {{Name: "usr/"}, {Name: "usr/lib/"}, {Name: "usr/lib/foo", Body: "foo"}, + {Name: "lib", Link: "usr/lib"}, {Name: "lib/bar", Link: "/usr/lib/foo"}}, + }, file: "usr/lib/bar", want: "foo"}, + {name: "dotdot after a symlinked parent resolves", layers: [][]tarEntry{ + {{Name: "usr/"}, {Name: "usr/lib/"}, {Name: "usr/lib/foo", Body: "foo"}, + {Name: "lib", Link: "usr/lib"}, {Name: "link", Link: "/lib/../../usr/lib/foo"}}, + }, file: "link", want: "foo"}, + {name: "trailing separator", layers: [][]tarEntry{{{Name: "f", Body: "x"}}}, + file: "f", want: "x", suffix: string(filepath.Separator)}, + } { + t.Run(c.name, func(t *testing.T) { + s, d := storeWithImage(t, "fix:1", testImage{layers: c.layers}) + dest := filepath.Join(t.TempDir(), "rootfs") + c.suffix + if err := unpackFreshTo(t, s, d, dest, false); err != nil { + t.Fatal(err) + } + b, err := os.ReadFile(filepath.Join(dest, c.file)) + if err != nil || string(b) != c.want { + t.Fatalf("%s = %q, %v; want %q", c.file, b, err, c.want) + } + }) + } +} + +func TestUnpackHardlinkSharesInode(t *testing.T) { + s, d := storeWithImage(t, "hl:1", testImage{layers: [][]tarEntry{ + {{Name: "orig", Body: "x"}, {Name: "alias", Link: "orig", Type: tar.TypeLink}}, + }}) + dest := unpackFresh(t, s, d) + a, err := os.Stat(filepath.Join(dest, "orig")) + if err != nil { + t.Fatal(err) + } + b, err := os.Stat(filepath.Join(dest, "alias")) + if err != nil { + t.Fatal(err) + } + if !os.SameFile(a, b) { + t.Fatal("hardlink must share the inode") + } +} + +func TestUnpackFreshCleansUpStaging(t *testing.T) { + for _, c := range []struct { + name string + plant func(t *testing.T, s *store, digest, dest string) + wantEntries int + }{ + {name: "corrupt layer", wantEntries: 0, plant: func(t *testing.T, s *store, digest, dest string) { + m := manifestOf(t, s, digest) + blob := filepath.Join(s.root, "blobs", "sha256", m.Layers[0].Digest.Hex()) + if err := os.Chmod(blob, 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(blob, []byte("not gzip"), 0o644); err != nil { + t.Fatal(err) + } + }}, + {name: "regular file at dest", wantEntries: 1, plant: func(t *testing.T, s *store, digest, dest string) { + if err := os.WriteFile(dest, nil, 0o644); err != nil { + t.Fatal(err) + } + }}, + {name: "dangling symlink at dest", wantEntries: 1, plant: func(t *testing.T, s *store, digest, dest string) { + if err := os.Symlink(filepath.Join(t.TempDir(), "gone"), dest); err != nil { + t.Fatal(err) + } + }}, + } { + t.Run(c.name, func(t *testing.T) { + s, d := storeWithImage(t, "bad:1", testImage{}) + parent := t.TempDir() + dest := filepath.Join(parent, "rootfs") + c.plant(t, s, d, dest) + if err := unpackFreshTo(t, s, d, dest, false); err == nil { + t.Fatal("unpack must fail") + } + entries, readErr := os.ReadDir(parent) + if readErr != nil { + t.Fatal(readErr) + } + if len(entries) != c.wantEntries { + t.Fatalf("parent holds %v, want %d entries", entries, c.wantEntries) + } + }) + } +} + +func TestUnpackFreshLostRenameRace(t *testing.T) { + for _, c := range []struct { + name string + inStore bool + wantErr bool + }{ + {name: "managed cache reuses the winner", inStore: true, wantErr: false}, + {name: "named rootfs reports the failure", inStore: false, wantErr: true}, + } { + t.Run(c.name, func(t *testing.T) { + s, d := storeWithImage(t, "race:1", testImage{layers: [][]tarEntry{ + {{Name: "f", Body: "x"}}, + }}) + dest := filepath.Join(t.TempDir(), "rootfs") + if c.inStore { + var err error + if dest, err = s.cacheDir(cacheRootfs, d); err != nil { + t.Fatal(err) + } + } + // An existing directory makes the rename fail the way a peer that + // published first would. + if err := os.MkdirAll(filepath.Join(dest, "occupied"), 0o755); err != nil { + t.Fatal(err) + } + parent := filepath.Dir(dest) + if err := unpackFreshTo(t, s, d, dest, c.inStore); (err != nil) != c.wantErr { + t.Fatalf("err = %v, wantErr = %v", err, c.wantErr) + } + entries, readErr := os.ReadDir(parent) + if readErr != nil { + t.Fatal(readErr) + } + if len(entries) != 1 { + t.Fatalf("staging leftovers: %v", entries) + } + }) + } +} + +func TestCmdUnpackStoreCacheAndAlreadyUnpacked(t *testing.T) { + s, d := storeWithImage(t, "cache:1", testImage{layers: [][]tarEntry{ + {{Name: "etc/"}, {Name: "etc/os-release", Body: "ID=fixture"}}, + }}) + stderr, err := runCaptured(t, "unpack", "--store", s.root, "--rootfs", "", "cache:1") + if err != nil { + t.Fatal(err) + } + dest, err := s.cacheDir(cacheRootfs, d) + if err != nil { + t.Fatal(err) + } + mustContain(t, stderr, "Unpacking cache:1 -> "+dest, "Unpacked cache:1") + b, err := os.ReadFile(filepath.Join(dest, "etc/os-release")) + if err != nil || string(b) != "ID=fixture" { + t.Fatalf("cache content = %q, %v", b, err) + } + + stderr, err = runCaptured(t, "unpack", "--store", s.root, "cache:1") + if err != nil { + t.Fatal(err) + } + mustContain(t, stderr, "Already unpacked") +} + +func TestUnpackCachePrivateModeAndTempSweep(t *testing.T) { + s, _ := storeWithImage(t, "modes:1", testImage{layers: [][]tarEntry{{{Name: "f", Body: "x"}}}}) + base := s.cacheBase(cacheRootfs) + if err := os.MkdirAll(base, 0o755); err != nil { + t.Fatal(err) + } + stale := filepath.Join(base, ".deadbeef.tmp-1") + fresh := filepath.Join(base, ".deadbeef.tmp-2") + for _, d := range []string{stale, fresh} { + if err := os.MkdirAll(d, 0o755); err != nil { + t.Fatal(err) + } + } + old := time.Now().Add(-2 * staleRootfsTempAge) + if err := os.Chtimes(stale, old, old); err != nil { + t.Fatal(err) + } + if _, err := runCaptured(t, "unpack", "--store", s.root, "modes:1"); err != nil { + t.Fatal(err) + } + if _, err := os.Lstat(stale); !os.IsNotExist(err) { + t.Errorf("abandoned staging tree survived: %v", err) + } + if _, err := os.Lstat(fresh); err != nil { + t.Errorf("a recent staging tree must be left alone: %v", err) + } + for _, dir := range []string{filepath.Join(s.root, cacheRootfs), base} { + fi, err := os.Stat(dir) + if err != nil { + t.Fatal(err) + } + if fi.Mode().Perm() != 0o700 { + t.Errorf("%s mode = %o, want 700", dir, fi.Mode().Perm()) + } + } +} + +func TestCmdUnpackExplicitRootfsMerges(t *testing.T) { + s, _ := storeWithImage(t, "merge:1", testImage{layers: [][]tarEntry{ + {{Name: "fromimage", Body: "i"}}, + }}) + dest := t.TempDir() + if err := os.WriteFile(filepath.Join(dest, "user-file"), []byte("mine"), 0o644); err != nil { + t.Fatal(err) + } + if _, err := runCaptured(t, "unpack", "--store", s.root, "--rootfs", dest, "merge:1"); err != nil { + t.Fatal(err) + } + for f, want := range map[string]string{"user-file": "mine", "fromimage": "i"} { + b, err := os.ReadFile(filepath.Join(dest, f)) + if err != nil || string(b) != want { + t.Fatalf("%s = %q, %v", f, b, err) + } + } +} + +func TestCmdUnpackExplicitRootfsFollowsSymlink(t *testing.T) { + s, _ := storeWithImage(t, "link:1", testImage{layers: [][]tarEntry{ + {{Name: "fromimage", Body: "i"}}, + }}) + real := filepath.Join(t.TempDir(), "real") + if err := os.Mkdir(real, 0o755); err != nil { + t.Fatal(err) + } + link := filepath.Join(t.TempDir(), "link") + if err := os.Symlink(real, link); err != nil { + t.Fatal(err) + } + if _, err := runCaptured(t, "unpack", "--store", s.root, "--rootfs", link, "link:1"); err != nil { + t.Fatalf("unpack into a symlinked directory = %v", err) + } + if b, err := os.ReadFile(filepath.Join(real, "fromimage")); err != nil || string(b) != "i" { + t.Fatalf("fromimage = %q, %v", b, err) + } +} + +func TestUnpackImageRefusesNonDirectory(t *testing.T) { + s, _ := storeWithImage(t, "demo:1", testImage{}) + file := filepath.Join(t.TempDir(), "not-a-dir") + if err := os.WriteFile(file, []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + _, m, err := s.loadRef(context.Background(), "demo:1", defaultPlatform) + if err != nil { + t.Fatal(err) + } + err = unpackImage(context.Background(), s, "demo:1", m, file) + if err == nil || !strings.Contains(err.Error(), "want a directory") { + t.Fatalf("unpack into a regular file = %v, want a not-a-directory refusal", err) + } + if b, readErr := os.ReadFile(file); readErr != nil || string(b) != "x" { + t.Fatalf("planted file = %q, %v; refusal must not touch it", b, readErr) + } +} + +func TestUnpackMergeResolvesPreexistingSymlinkedParent(t *testing.T) { + s, _ := storeWithImage(t, "usrmerge:2", testImage{layers: [][]tarEntry{ + {{Name: "usr/"}, {Name: "usr/lib/"}, {Name: "usr/lib/foo", Body: "foo"}, + {Name: "lib64", Link: "lib"}, {Name: "lib64/x", Link: "/usr/lib/foo"}}, + }}) + dest := t.TempDir() + if err := os.MkdirAll(filepath.Join(dest, "usr", "lib"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.Symlink("usr/lib", filepath.Join(dest, "lib")); err != nil { + t.Fatal(err) + } + if _, err := runCaptured(t, "unpack", "--store", s.root, "--rootfs", dest, "usrmerge:2"); err != nil { + t.Fatal(err) + } + b, err := os.ReadFile(filepath.Join(dest, "usr", "lib", "x")) + if err != nil || string(b) != "foo" { + t.Fatalf("usr/lib/x = %q, %v; want it to resolve to foo", b, err) + } +} + +func TestUnpackAppliesFilteredHeaders(t *testing.T) { + s, d := storeWithImage(t, "filt:1", testImage{layers: [][]tarEntry{ + {{Name: "dev/"}, {Name: "dev/null", Type: tar.TypeChar, Major: 1}, + {Name: "dev/alias", Link: "dev/null", Type: tar.TypeLink}, + {Name: "bin/"}, {Name: "bin/busybox", Body: "x", Mode: 0o2755}, + {Name: "bin/sh", Link: "/bin/busybox"}}, + }}) + dest := unpackFresh(t, s, d) + for _, absent := range []string{"dev/null", "dev/alias"} { + if _, err := os.Lstat(filepath.Join(dest, absent)); err == nil { + t.Errorf("%s: must not be extracted", absent) + } + } + if target, err := os.Readlink(filepath.Join(dest, "bin/sh")); err != nil || target != "../bin/busybox" { + t.Errorf("bin/sh -> %q, %v; want ../bin/busybox", target, err) + } + fi, err := os.Stat(filepath.Join(dest, "bin/busybox")) + if err != nil { + t.Fatal(err) + } + if fi.Mode()&os.ModeSetgid != 0 { + t.Errorf("bin/busybox mode %v keeps setgid", fi.Mode()) + } +} + +func TestCmdUnpackDoesNotCreateStore(t *testing.T) { + missing := filepath.Join(t.TempDir(), "typo") + if _, err := runCaptured(t, "unpack", "--store", missing, "demo:1"); err == nil { + t.Fatal("a missing store must fail") + } + if _, statErr := os.Lstat(missing); !os.IsNotExist(statErr) { + t.Error("unpack must not create the store directory") + } +} + +func TestUnpackIntoStopsOnCancelledContext(t *testing.T) { + s, d := storeWithImage(t, "cancel:1", testImage{layers: [][]tarEntry{{{Name: "f", Body: "x"}}}}) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + dest := t.TempDir() + err := unpackInto(ctx, s, manifestOf(t, s, d), dest) + if err == nil || !strings.Contains(err.Error(), context.Canceled.Error()) { + t.Fatalf("err = %v, want context.Canceled", err) + } +} + +// With unpigz on PATH, the child's exit status must not replace +// context.Canceled. +func TestUnpackIntoCancelsMidLayer(t *testing.T) { + s, d := storeWithImage(t, "cancelmid:1", testImage{layers: [][]tarEntry{bigEntries(400, 32*1024)}}) + for _, shim := range []bool{false, true} { + t.Run(fmt.Sprintf("unpigz=%v", shim), func(t *testing.T) { + if shim { + prependPath(t, filepath.Dir(writeShellStub(t, "unpigz", "exec gunzip \"$@\"\n"))) + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + dest := t.TempDir() + stop := make(chan struct{}) + defer close(stop) + go func() { + for { + select { + case <-stop: + return + default: + } + if _, err := os.Lstat(filepath.Join(dest, "f0000")); err == nil { + cancel() + return + } + time.Sleep(time.Millisecond) + } + }() + err := unpackInto(ctx, s, manifestOf(t, s, d), dest) + if !errors.Is(err, context.Canceled) { + t.Fatalf("err = %v, want context.Canceled", err) + } + }) + } +} + +func TestUnpackLayerPathTransitions(t *testing.T) { + for _, c := range []struct { + name string + layers [][]tarEntry + files map[string]string + links []string + absent []string + }{ + {name: "read-only directory", layers: [][]tarEntry{ + {{Name: "ro/", Mode: 0o555}, {Name: "ro/f", Body: "first"}}, + {{Name: "ro/second", Body: "second"}}, + }, files: map[string]string{"ro/f": "first", "ro/second": "second"}}, + {name: "replace symlinked parent", layers: [][]tarEntry{ + {{Name: "usr/lib/"}, {Name: "usr/lib/foo", Body: "foo"}, {Name: "lib", Link: "usr/lib"}}, + {{Name: "lib/"}, {Name: "lib/bar", Link: "/usr/lib/foo"}}, + }, files: map[string]string{"lib/bar": "foo"}}, + {name: "hardlink to symlink to dropped node", layers: [][]tarEntry{ + {{Name: "node", Type: tar.TypeFifo}, {Name: "link", Link: "node"}, + {Name: "alias", Type: tar.TypeLink, Link: "link"}}, + }, links: []string{"link", "alias"}, absent: []string{"node"}}, + {name: "replace symlink with dropped node", layers: [][]tarEntry{ + {{Name: "target", Body: "kept"}, {Name: "node", Link: "target"}}, + {{Name: "node", Type: tar.TypeFifo}, {Name: "alias", Type: tar.TypeLink, Link: "node"}}, + }, files: map[string]string{"target": "kept"}, absent: []string{"node", "alias"}}, + {name: "replace dropped node through parent alias", layers: [][]tarEntry{ + {{Name: "dev/"}, {Name: "alias", Link: "dev"}, {Name: "dev/null", Type: tar.TypeFifo}}, + {{Name: "alias/null", Body: "restored"}, {Name: "copy", Type: tar.TypeLink, Link: "dev/null"}}, + }, files: map[string]string{"copy": "restored"}}, + {name: "absolute target parent traversal", layers: [][]tarEntry{ + {{Name: "a/b/"}, {Name: "l", Link: "a/b"}, {Name: "target", Body: "wrong"}, + {Name: "a/target", Body: "right"}, {Name: "link", Link: "/l/../target"}}, + }, files: map[string]string{"link": "right"}}, + {name: "hardlinked absolute symlink aliases", layers: [][]tarEntry{ + {{Name: "target", Body: "right"}, {Name: "link", Link: "/target"}, + {Name: "dir/alias", Type: tar.TypeLink, Link: "link"}}, + {{Name: "deep/dir/alias", Type: tar.TypeLink, Link: "dir/alias"}}, + }, files: map[string]string{"link": "right", "dir/alias": "right", "deep/dir/alias": "right"}}, + {name: "file replaces a directory case alias", layers: [][]tarEntry{ + {{Name: "A/"}, {Name: "A/sub/"}}, + {{Name: "a", Body: "x"}}, + }, files: map[string]string{"a": "x"}}, + {name: "parent symlink above the root clamps", layers: [][]tarEntry{ + {{Name: "top", Link: "../.."}, {Name: "top/f", Body: "x"}, + {Name: "a/"}, {Name: "a/up", Link: "../../../out"}, {Name: "a/up/g", Body: "y"}}, + }, files: map[string]string{"f": "x", "out/g": "y"}}, + } { + t.Run(c.name, func(t *testing.T) { + s, d := storeWithImage(t, "paths:1", testImage{layers: c.layers}) + dest := unpackFresh(t, s, d) + for name, want := range c.files { + got, err := os.ReadFile(filepath.Join(dest, name)) + if err != nil || string(got) != want { + t.Errorf("%s = %q, %v; want %q", name, got, err, want) + } + } + for _, name := range c.links { + if fi, err := os.Lstat(filepath.Join(dest, name)); err != nil || fi.Mode()&os.ModeSymlink == 0 { + t.Errorf("%s = %v, %v; want symlink", name, fi, err) + } + } + for _, name := range c.absent { + if _, err := os.Lstat(filepath.Join(dest, name)); !os.IsNotExist(err) { + t.Errorf("%s survived: %v", name, err) + } + } + if c.name == "read-only directory" { + fi, err := os.Stat(filepath.Join(dest, "ro")) + if err != nil || fi.Mode().Perm() != 0o555 { + t.Errorf("ro = %v, %v; want mode 0555", fi, err) + } + } + }) + } +} + +func TestUnpackRestoresDirectoryModesAfterFailure(t *testing.T) { + s, digest := storeWithImage(t, "modes:1", testImage{layers: [][]tarEntry{{ + {Name: "ro/new", Body: "new"}, + {Name: "closed/", Mode: 0o4000}, + {Name: "closed/file", Body: "file"}, + {Name: "bad", Type: tar.TypeLink, Link: "missing"}, + }}}) + dest := t.TempDir() + t.Cleanup(func() { _ = removeRootfsTree(dest) }) + ro := filepath.Join(dest, "ro") + if err := os.Mkdir(ro, 0o500); err != nil { + t.Fatal(err) + } + err := unpackInto(context.Background(), s, manifestOf(t, s, digest), dest) + if err == nil { + t.Fatal("missing hardlink target must fail") + } + for name, mode := range map[string]os.FileMode{"ro": 0o500, "closed": 0} { + fi, err := os.Stat(filepath.Join(dest, name)) + if err != nil || fi.Mode().Perm() != mode { + t.Errorf("%s: %v, %v; want mode %o", name, fi, err, mode) + } + } + if got, err := os.ReadFile(filepath.Join(ro, "new")); err != nil || string(got) != "new" { + t.Fatalf("read-only parent: %q, %v", got, err) + } +} + +func TestRemoveRootfsTreePreservesSymlinkTargets(t *testing.T) { + root, outside := t.TempDir(), t.TempDir() + if err := os.WriteFile(filepath.Join(outside, "kept"), []byte("kept"), 0o444); err != nil { + t.Fatal(err) + } + if err := os.Mkdir(filepath.Join(root, "closed"), 0o700); err != nil { + t.Fatal(err) + } + if err := os.Symlink(outside, filepath.Join(root, "closed", "link")); err != nil { + t.Fatal(err) + } + if err := os.Chmod(filepath.Join(root, "closed"), 0); err != nil { + t.Fatal(err) + } + if err := removeRootfsTree(root); err != nil { + t.Fatal(err) + } + if _, err := os.Lstat(root); !os.IsNotExist(err) { + t.Fatalf("root survived: %v", err) + } + if got, err := os.ReadFile(filepath.Join(outside, "kept")); err != nil || string(got) != "kept" { + t.Fatalf("symlink target: %q, %v", got, err) + } +} + +func TestUnpackOpaqueKeepsCurrentLayerPolicy(t *testing.T) { + s, d := storeWithImage(t, "opaque:1", testImage{layers: [][]tarEntry{ + {{Name: "target", Body: "right"}, {Name: "d/"}, {Name: "d/old", Body: "old"}}, + {{Name: "d/", Mode: 0o555}, {Name: "d/link", Link: "/target"}, + {Name: "d/node", Type: tar.TypeFifo}, {Name: "d/.wh..wh..opq"}, + {Name: "d/sub/alias", Type: tar.TypeLink, Link: "d/link"}, + {Name: "d/dropped", Type: tar.TypeLink, Link: "d/node"}}, + }}) + dest := unpackFresh(t, s, d) + if got, err := os.ReadFile(filepath.Join(dest, "d/sub/alias")); err != nil || string(got) != "right" { + t.Fatalf("hardlinked symlink: %q, %v", got, err) + } + for _, name := range []string{"d/old", "d/node", "d/dropped"} { + if _, err := os.Lstat(filepath.Join(dest, name)); !os.IsNotExist(err) { + t.Errorf("%s survived: %v", name, err) + } + } +} + +func TestUnpackRefusesEscapingPaths(t *testing.T) { + for _, entries := range [][]tarEntry{ + {{Name: "../outside", Body: "bad"}}, + {{Name: "hardlink", Type: tar.TypeLink, Link: "../outside"}}, + {{Name: "cycle", Link: "cycle"}, {Name: "cycle/file", Body: "bad"}}, + } { + s, d := storeWithImage(t, "escape:1", testImage{layers: [][]tarEntry{entries}}) + dest := filepath.Join(t.TempDir(), "rootfs") + if err := unpackFreshTo(t, s, d, dest, false); err == nil { + t.Errorf("accepted %v", entries) + } + if _, err := os.Lstat(dest); !os.IsNotExist(err) { + t.Errorf("published failed tree: %v", err) + } + } +} + +func TestUnpackRefusesStoreCaseAliases(t *testing.T) { + s, err := openStore(filepath.Join(t.TempDir(), "store")) + if err != nil { + t.Fatal(err) + } + if err := s.ensureLayout(context.Background()); err != nil { + t.Fatal(err) + } + alias := filepath.Join(filepath.Dir(s.root), "STORE") + original, err := os.Stat(s.root) + if err != nil { + t.Fatal(err) + } + other, err := os.Stat(alias) + if os.IsNotExist(err) { + t.Skip("requires a case-insensitive filesystem") + } + if err != nil || !os.SameFile(original, other) { + t.Fatalf("case alias: %v", err) + } + digest := pushTestImage(t, s, testImage{layers: [][]tarEntry{{{Name: ".wh.index.json"}}}}) + pinImage(t, s, "demo:1", defaultPlatform, digest) + before, err := os.ReadFile(filepath.Join(s.root, "index.json")) + if err != nil { + t.Fatal(err) + } + for _, dest := range []string{alias, filepath.Join(alias, "missing", "child")} { + if _, err := runCaptured(t, "unpack", "--store", s.root, "--rootfs", dest, "demo:1"); err == nil || !strings.Contains(err.Error(), "inside the store") { + t.Errorf("rootfs %s: %v", dest, err) + } + } + after, err := os.ReadFile(filepath.Join(s.root, "index.json")) + if err != nil || string(before) != string(after) { + t.Fatalf("store index changed: %v", err) + } +} diff --git a/go.mod b/go.mod index 8f24b08a..c4244352 100644 --- a/go.mod +++ b/go.mod @@ -5,16 +5,21 @@ go 1.25.0 require ( github.com/alecthomas/kong v1.16.1 github.com/google/go-containerregistry v0.21.7 + github.com/moby/go-archive v0.3.3 github.com/opencontainers/image-spec v1.1.1 ) require ( + github.com/containerd/log v0.1.0 // indirect github.com/docker/cli v29.5.3+incompatible // indirect github.com/docker/docker-credential-helpers v0.9.3 // indirect - github.com/klauspost/compress v1.18.6 // indirect + github.com/klauspost/compress v1.18.7 // indirect + github.com/moby/patternmatcher v0.6.1 // indirect + github.com/moby/sys/sequential v0.7.0 // indirect + github.com/moby/sys/user v0.4.1 // indirect + github.com/moby/sys/userns v0.1.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/sirupsen/logrus v1.9.4 // indirect golang.org/x/sync v0.21.0 // indirect golang.org/x/sys v0.46.0 // indirect - gotest.tools/v3 v3.5.2 // indirect ) diff --git a/go.sum b/go.sum index 1c61142e..303f1f7c 100644 --- a/go.sum +++ b/go.sum @@ -1,9 +1,13 @@ +github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= github.com/alecthomas/assert/v2 v2.11.0 h1:2Q9r3ki8+JYXvGsDyBXwH3LcJ+WK5D0gc5E8vS6K3D0= github.com/alecthomas/assert/v2 v2.11.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k= github.com/alecthomas/kong v1.16.1 h1:ixhCt93XkJ98kGposQ54+bl0IK6XwqB40AsMynU7Z8E= github.com/alecthomas/kong v1.16.1/go.mod h1:wrlbXem1CWqUV5Vbmss5ISYhsVPkBb1Yo7YKJghju2I= github.com/alecthomas/repr v0.5.2 h1:SU73FTI9D1P5UNtvseffFSGmdNci/O6RsqzeXJtP0Qs= github.com/alecthomas/repr v0.5.2/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= +github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= +github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/docker/cli v29.5.3+incompatible h1:nbEFfz774vBwQ5KRYv7c/AghjReqnGISvrRhzjV0evs= @@ -16,8 +20,22 @@ github.com/google/go-containerregistry v0.21.7 h1:/vPFuVXDjtFREsVArW+0h1CIl5urnO github.com/google/go-containerregistry v0.21.7/go.mod h1:kjSbt7/zMsKLWfnHrIvKvhXHUw91jbe9DNjPPJ32gXE= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= -github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao= -github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/compress v1.18.7 h1:aUyZsS4kH3QTKurYhAOwAHxllVPnOthb3vPfnF1Ehjw= +github.com/klauspost/compress v1.18.7/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/moby/go-archive v0.3.3 h1:OxxR9paxsluYi+zDUEXTTaIxtkK3viymW+Ka7vRhhME= +github.com/moby/go-archive v0.3.3/go.mod h1:Npdv43fFqlhZW7Xo8fbm3ZMYFvAGNviUPqX21VERbcE= +github.com/moby/patternmatcher v0.6.1 h1:qlhtafmr6kgMIJjKJMDmMWq7WLkKIo23hsrpR3x084U= +github.com/moby/patternmatcher v0.6.1/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= +github.com/moby/sys/mount v0.3.5 h1:eS3fsZTjHaBihwjp4/+5Z3jxqLXYsbwxqpVSfFv3M00= +github.com/moby/sys/mount v0.3.5/go.mod h1:WUQDO+/uCiCIkIztx8SrwIDVn2dtMFRBebRhpDFT71M= +github.com/moby/sys/mountinfo v0.7.2 h1:1shs6aH5s4o5H2zQLn796ADW1wMrIwHsyJ2v9KouLrg= +github.com/moby/sys/mountinfo v0.7.2/go.mod h1:1YOa8w8Ih7uW0wALDUgT1dTTSBrZ+HiBLGws92L2RU4= +github.com/moby/sys/sequential v0.7.0 h1:ASQNGNROJSuOO6LL6bPHbKvuZu6NU8P4ldPWk31zj/8= +github.com/moby/sys/sequential v0.7.0/go.mod h1:NfSTAp6V3fw4tmkD62PEcOKeZKquXT8VKCkf7aVR79o= +github.com/moby/sys/user v0.4.1 h1:RgjRlaDKi/Xmyrz4t8lyzXT6v2ooFeO/7xtchmhVWE0= +github.com/moby/sys/user v0.4.1/go.mod h1:E9QsW5WRe1kUAf7kW8hXKwu1uhsZEAdPLYHYSDudF4Y= +github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g= +github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= From 2a39064f971f3d2873df8fead0f0489f0be59977 Mon Sep 17 00:00:00 2001 From: Chun-Hung Tseng Date: Wed, 2 Sep 2026 17:22:54 +0200 Subject: [PATCH 5/5] Document unpacking OCI images Describe the rootfs cache, --rootfs, what moby/go-archive handles and what elfuse rewrites first, and the case-sensitive volume a real rootfs needs. --- README.md | 6 +++--- docs/oci-images.md | 45 +++++++++++++++++++++++++++++++++++++++------ docs/testing.md | 5 +++-- docs/usage.md | 19 ++++++++++++++----- 4 files changed, 59 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index a9d80967..4cd00964 100644 --- a/README.md +++ b/README.md @@ -46,8 +46,8 @@ linker resolved against an external sysroot via `--sysroot`. ## OCI Images -`elfuse-oci` is a separate Go binary that pulls OCI images into a local -OCI image layout. It does not add container isolation. See +`elfuse-oci` is a separate Go binary that pulls and unpacks OCI images. It +does not add container isolation. See [docs/usage.md](docs/usage.md#oci-images) and [docs/oci-images.md](docs/oci-images.md). @@ -159,7 +159,7 @@ The build signs `build/elfuse` before use. Override the signing identity with `make check` flow, the QEMU and Rosetta cross-check matrices, and fixture handling. - [docs/oci-images.md](docs/oci-images.md): the `elfuse-oci` store, - pull behavior, and validation. + pull and unpack behavior, and validation. - [docs/filenames.md](docs/filenames.md): how a guest filename becomes a name on disk and back: case folding and normalization on the sysroot volume, the escape encoding, and the length limits both systems impose. diff --git a/docs/oci-images.md b/docs/oci-images.md index 5a59fbb8..2a3e9d84 100644 --- a/docs/oci-images.md +++ b/docs/oci-images.md @@ -8,8 +8,8 @@ requiring a registry or a container daemon. This is separate from the OCI runtime specification, which describes how a container is started. `elfuse-oci` is a separate Go command that pulls images into such a local -layout. At this point in the stack it does not unpack or run them, and it does -not add namespaces, cgroups, or other container isolation. Command syntax is in +layout and unpacks their filesystems. It does not run images or add namespaces, +cgroups, or other container isolation. Command syntax is in [usage.md](usage.md#oci-images). ## Store @@ -23,6 +23,7 @@ The store is an [OCI image layout](https://github.com/opencontainers/image-spec/ oci-layout index.json blobs// + rootfs/sha256/ ``` The marker records the elfuse store format. `oci-layout`, `index.json`, and @@ -48,8 +49,10 @@ The store and blob directories use mode 0700. Metadata and the lock use 0600, and immutable blobs use 0400, so another local user cannot read a private image through default permissions. -The store is a cache. A directory containing `refs.json` has an incompatible -format and is rejected; remove it and pull the image again. +The store is a cache. A directory that carries no format marker but does hold +`refs.json` has an incompatible format and is rejected; remove it and pull the +image again. `unpack` applies the same format checks as `pull` but never +creates or repairs the store. A failed pull can leave unreferenced blobs, and pulling a moved tag can leave the old blobs unreferenced. This version has no pruning command. To reclaim @@ -72,11 +75,41 @@ pinned as baseline `amd64`. The pull timeout covers the registry request, store publication, and the wait for another writer. The default value, zero, does not set a deadline. +## Unpack + +Without `--rootfs`, `unpack` caches the rootfs under the store by manifest +digest. It extracts into a sibling temporary directory and renames the +completed tree into place, so concurrent unpacks may duplicate work but only a +completed tree is published. The cache takes the store's 0700 mode, a symlink +at a cache path is rejected, and the next unpack that extracts into the cache +removes staging trees older than a day. A cached tree is reused as is; remove +it to unpack again. + +`--rootfs DIR` applies the image to `DIR`, following a symlink to it. An +existing directory is updated in place; an absent one is staged and renamed, +and losing that rename is an error. A destination inside the store, or one +containing it, is rejected. + +Layers are applied in manifest order by `moby/go-archive`, which handles +whiteouts, hardlinks, path containment, file metadata, and decompression. +Ownership is not applied, and unsupported extended attributes do not fail +extraction. elfuse rewrites each entry before go-archive sees it: + +- Device and FIFO entries, and hardlinks to them, become whiteouts, so no + lower-layer file survives at their paths. +- Absolute symlink targets are rewritten relative to the link's on-disk + parent, the rewrite `symlinkat` applies inside a sysroot + ([filenames.md](filenames.md#symlink-targets)); a hardlink to such a + symlink is rebased at its own location. +- Directory modes are restored after all layers are applied, so a read-only + directory still receives later entries. +- Setuid, setgid, and sticky bits are cleared. + ## Validation The offline tests create manifests and layers in temporary stores. They cover reference normalization, exact platform selection, index structure, blob validation, credential-helper resolution, concurrent pulls, stale temporary files, private permissions, legacy-store refusal, lock cancellation, CLI -parsing, and the race detector. Set `ELFUSE_OCI_NETTEST=1` to add a Docker Hub -round trip. +parsing, layer application, cache publication, and the race detector. Set +`ELFUSE_OCI_NETTEST=1` to add a Docker Hub round trip. diff --git a/docs/testing.md b/docs/testing.md index c356ae0a..d83a1732 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -559,5 +559,6 @@ make oci-test ELFUSE_OCI_NETTEST=1 make oci-test ``` -The default suite constructs image data in temporary stores and does not use a -registry. `ELFUSE_OCI_NETTEST=1` adds a pull from Docker Hub. +The default suite constructs image data in temporary stores and covers the +CLI, store, and unpack without a registry. `ELFUSE_OCI_NETTEST=1` adds a pull +from Docker Hub. diff --git a/docs/usage.md b/docs/usage.md index c569e032..076c649d 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -325,8 +325,8 @@ Off by default, and useful when a guest misbehaves rather than in normal use: ## OCI Images -`build/elfuse-oci` is separate from the C runtime. It pulls images into a local -OCI image layout and does not unpack them. +`build/elfuse-oci` is separate from the C runtime. It pulls images into an OCI +image layout and unpacks their filesystems for `elfuse --sysroot`. ### Build @@ -343,26 +343,35 @@ does not require Go. ```sh build/elfuse-oci pull debian:stable-slim +build/elfuse-oci unpack debian:stable-slim --rootfs ~/debian-rootfs +build/elfuse --sysroot ~/debian-rootfs /bin/sh ``` +A default APFS volume folds case, which a Linux rootfs does not expect. For real +use, attach a case-sensitive APFS sparsebundle at the `--rootfs` directory before +unpacking. `--create-sysroot` does not work here: it mounts a new volume over +that directory. + ### Commands | Command | Meaning | |---------|---------| | `pull ` | Fetch one platform of an image into the store | +| `unpack ` | Apply a stored image to a rootfs directory | | `help`, `version` | Print help or the elfuse-oci version | An abbreviated reference receives the Docker Hub registry, the `library` repository when needed, and the `latest` tag when no tag is present. Digest -references are accepted. Pull options may appear before or after ``. +references are accepted. Options may appear before or after ``. ### Flags | Option | Commands | Meaning | |--------|----------|---------| -| `--store DIR` | `pull` | Store directory; default `$ELFUSE_OCI_STORE`, then `~/.local/share/elfuse/oci` | -| `--platform OS/ARCH[/VARIANT]` | `pull` | Target `linux/arm64` or `linux/amd64`; default `linux/arm64` | +| `--store DIR` | `pull`, `unpack` | Store directory; default `$ELFUSE_OCI_STORE`, then `~/.local/share/elfuse/oci` | +| `--platform OS/ARCH[/VARIANT]` | `pull`, `unpack` | Target `linux/arm64` or `linux/amd64`; default `linux/arm64` | | `--timeout DURATION` | `pull` | Bound the pull and lock wait; zero sets no deadline | +| `--rootfs DIR` | `unpack` | Unpack into `DIR`; otherwise use the managed cache | ### Environment