feat(push): support --all-tags to push all tags

cmd/nerdctl/image/image_push.go

@@ -47,6 +47,8 @@ func PushCommand() *cobra.Command {
 	cmd.Flags().Bool("all-platforms", false, "Push content for all platforms")
 	// #endregion
 
+	cmd.Flags().BoolP("all-tags", "a", false, "Push all tags of an image to the repository")
+
 	cmd.Flags().Bool("estargz", false, "Convert the image into eStargz")
 	cmd.Flags().Bool("ipfs-ensure-image", true, "Ensure the entire contents of the image is locally available before push")
 	cmd.Flags().String("ipfs-address", "", "multiaddr of IPFS API (default uses $IPFS_PATH env variable if defined or local directory ~/.ipfs)")
@@ -85,6 +87,10 @@ func pushOptions(cmd *cobra.Command) (types.ImagePushOptions, error) {
 	if err != nil {
 		return types.ImagePushOptions{}, err
 	}
+	allTags, err := cmd.Flags().GetBool("all-tags")
+	if err != nil {
+		return types.ImagePushOptions{}, err
+	}
 	estargz, err := cmd.Flags().GetBool("estargz")
 	if err != nil {
 		return types.ImagePushOptions{}, err
@@ -119,6 +125,7 @@ func pushOptions(cmd *cobra.Command) (types.ImagePushOptions, error) {
 		SociOptions:                    sociOptions,
 		Platforms:                      platform,
 		AllPlatforms:                   allPlatforms,
+		AllTags:                        allTags,
 		Estargz:                        estargz,
 		IpfsEnsureImage:                ipfsEnsureImage,
 		IpfsAddress:                    ipfsAddress,

cmd/nerdctl/image/image_push_linux_test.go

@@ -17,9 +17,12 @@
 package image
 
 import (
+	"encoding/json"
 	"errors"
 	"fmt"
 	"net/http"
+	"slices"
+	"strings"
 	"testing"
 
 	"gotest.tools/v3/assert"
@@ -278,7 +281,151 @@ func TestPush(t *testing.T) {
 				},
 				Expected: test.Expects(0, nil, nil),
 			},
+			{
+				Description: "all tags",
+				Require:     require.Not(nerdtest.Docker),
+				Setup: func(data test.Data, helpers test.Helpers) {
+					helpers.Ensure("pull", "--quiet", testutil.CommonImage)
+					testImageRepo := fmt.Sprintf("%s:%d/%s",
+						registryNoAuthHTTPRandom.IP.String(), registryNoAuthHTTPRandom.Port, data.Identifier())
+					data.Labels().Set("testImageRepo", testImageRepo)
+					helpers.Ensure("tag", testutil.CommonImage, testImageRepo+":v1")
+					helpers.Ensure("tag", testutil.CommonImage, testImageRepo+":v2")
+				},
+				Cleanup: func(data test.Data, helpers test.Helpers) {
+					if data.Labels().Get("testImageRepo") != "" {
+						helpers.Anyhow("rmi", "-f", data.Labels().Get("testImageRepo")+":v1")
+						helpers.Anyhow("rmi", "-f", data.Labels().Get("testImageRepo")+":v2")
+					}
+				},
+				Command: func(data test.Data, helpers test.Helpers) test.TestableCommand {
+					return helpers.Command("push", "--insecure-registry", "--all-tags", data.Labels().Get("testImageRepo"))
+				},
+				Expected: func(data test.Data, helpers test.Helpers) *test.Expected {
+					return &test.Expected{
+						Output: func(stdout string, t tig.T) {
+							assertRegistryHasTags(t, registryNoAuthHTTPRandom, data.Identifier(), "v1", "v2")
+						},
+					}
+				},
+			},
+			{
+				Description: "all tags, with a tag",
+				Require:     require.Not(nerdtest.Docker),
+				Setup: func(data test.Data, helpers test.Helpers) {
+					helpers.Ensure("pull", "--quiet", testutil.CommonImage)
+					testImageRef := fmt.Sprintf("%s:%d/%s:v1",
+						registryNoAuthHTTPRandom.IP.String(), registryNoAuthHTTPRandom.Port, data.Identifier())
+					data.Labels().Set("testImageRef", testImageRef)
+					helpers.Ensure("tag", testutil.CommonImage, testImageRef)
+				},
+				Cleanup: func(data test.Data, helpers test.Helpers) {
+					if data.Labels().Get("testImageRef") != "" {
+						helpers.Anyhow("rmi", "-f", data.Labels().Get("testImageRef"))
+					}
+				},
+				Command: func(data test.Data, helpers test.Helpers) test.TestableCommand {
+					return helpers.Command("push", "--insecure-registry", "--all-tags", data.Labels().Get("testImageRef"))
+				},
+				Expected: test.Expects(1, []error{errors.New("tag can't be used with --all-tags/-a")}, nil),
+			},
+			{
+				Description: "all tags, no local tag",
+				Require:     require.Not(nerdtest.Docker),
+				Command: func(data test.Data, helpers test.Helpers) test.TestableCommand {
+					testImageRepo := fmt.Sprintf("%s:%d/%s",
+						registryNoAuthHTTPRandom.IP.String(), registryNoAuthHTTPRandom.Port, data.Identifier())
+					return helpers.Command("push", "--insecure-registry", "--all-tags", testImageRepo)
+				},
+				Expected: test.Expects(1, []error{errors.New("an image does not exist locally with the tag")}, nil),
+			},
+			{
+				Description: "all tags, soci",
+				Require: require.All(
+					nerdtest.Soci,
+					require.Not(nerdtest.Docker),
+				),
+				Setup: func(data test.Data, helpers test.Helpers) {
+					helpers.Ensure("pull", "--quiet", testutil.UbuntuImage)
+					testImageRepo := fmt.Sprintf("%s:%d/%s",
+						registryNoAuthHTTPRandom.IP.String(), registryNoAuthHTTPRandom.Port, data.Identifier())
+					data.Labels().Set("testImageRepo", testImageRepo)
+					helpers.Ensure("tag", testutil.UbuntuImage, testImageRepo+":v1")
+					helpers.Ensure("tag", testutil.UbuntuImage, testImageRepo+":v2")
+				},
+				Cleanup: func(data test.Data, helpers test.Helpers) {
+					if data.Labels().Get("testImageRepo") != "" {
+						helpers.Anyhow("rmi", "-f", data.Labels().Get("testImageRepo")+":v1")
+						helpers.Anyhow("rmi", "-f", data.Labels().Get("testImageRepo")+":v2")
+					}
+				},
+				Command: func(data test.Data, helpers test.Helpers) test.TestableCommand {
+					return helpers.Command("push", "--snapshotter=soci", "--insecure-registry", "--all-tags", "--soci-span-size=2097152", "--soci-min-layer-size=20971520", data.Labels().Get("testImageRepo"))
+				},
+				Expected: func(data test.Data, helpers test.Helpers) *test.Expected {
+					return &test.Expected{
+						Output: func(stdout string, t tig.T) {
+							assertRegistryHasTags(t, registryNoAuthHTTPRandom, data.Identifier(), "v1", "v2")
+							assertRegistrySociIndex(t, registryNoAuthHTTPRandom, data.Identifier())
+						},
+					}
+				},
+			},
 		},
 	}
 	testCase.Run(t)
 }
+
+// referrersFallbackTagPrefix starts the tag under which the distribution spec has a registry
+// without the referrers API keep the artifacts referring to a manifest ("sha256-<hex>").
+const referrersFallbackTagPrefix = "sha256-"
+
+// registryTags returns the tags the registry lists for the repository `repo`.
+func registryTags(t tig.T, reg *registry.Server, repo string) []string {
+	t.Helper()
+
+	tagsURL := fmt.Sprintf("http://%s:%d/v2/%s/tags/list", reg.IP.String(), reg.Port, repo)
+	resp, err := http.Get(tagsURL)
+	assert.NilError(t, err, "error making http request")
+	defer func() {
+		if resp.Body != nil {
+			_ = resp.Body.Close()
+		}
+	}()
+	assert.Equal(t, resp.StatusCode, http.StatusOK, "tag list should be available")
+
+	var tagList struct {
+		Name string   `json:"name"`
+		Tags []string `json:"tags"`
+	}
+	assert.NilError(t, json.NewDecoder(resp.Body).Decode(&tagList), "error decoding the tag list")
+
+	return tagList.Tags
+}
+
+// assertRegistryHasTags verifies the registry lists every tag of `want` for the repository `repo`.
+// The listing legitimately holds more than the pushed tags: a SOCI index adds a referrers fallback
+// tag, and a re-run of the test hits a repository the previous run already populated.
+func assertRegistryHasTags(t tig.T, reg *registry.Server, repo string, want ...string) {
+	t.Helper()
+
+	tags := registryTags(t, reg, repo)
+	for _, tag := range want {
+		assert.Assert(t, slices.Contains(tags, tag), "expected tag %q in %v", tag, tags)
+	}
+}
+
+// assertRegistrySociIndex verifies a SOCI index was pushed to the repository `repo`.
+//
+// The test registry is distribution 2.x, which predates the referrers API, so SOCI attaches its
+// index through the referrers fallback tag. A push without SOCI never creates such a tag, so its
+// presence is what tells the index apart from the tags of the image itself.
+func assertRegistrySociIndex(t tig.T, reg *registry.Server, repo string) {
+	t.Helper()
+
+	tags := registryTags(t, reg, repo)
+	found := slices.ContainsFunc(tags, func(tag string) bool {
+		return strings.HasPrefix(tag, referrersFallbackTagPrefix)
+	})
+	assert.Assert(t, found, "expected a SOCI index referrers tag in %v", tags)
+}

docs/command-reference.md

@@ -899,6 +899,7 @@ Flags:
 
 - :nerd_face: `--platform=(amd64|arm64|...)`: Push content for a specific platform
 - :nerd_face: `--all-platforms`: Push content for all platforms
+- :whale: `-a, --all-tags`: Push all tags of an image to the repository. `NAME` must not contain a tag.
 - :nerd_face: `--sign`: Sign the image (none|cosign|notation). See [`./cosign.md`](./cosign.md) and [`./notation.md`](./notation.md) for details.
 - :nerd_face: `--cosign-key`: Path to the private key file, KMS, URI or Kubernetes Secret for `--sign=cosign`
 - :nerd_face: `--notation-key-name`: Signing key name for a key previously added to notation's key list for `--sign=notation`
@@ -908,7 +909,7 @@ Flags:
 - :nerd_face: `--soci-span-size`: Span size in bytes that soci index uses to segment layer data. Default is 4 MiB.
 - :nerd_face: `--soci-min-layer-size`: Minimum layer size in bytes to build zTOC for. Smaller layers won't have zTOC and not lazy pulled. Default is 10 MiB.
 
-Unimplemented `docker push` flags: `--all-tags`, `--disable-content-trust` (default true)
+Unimplemented `docker push` flags: `--disable-content-trust` (default true)
 
 ### :whale: nerdctl load
 

pkg/api/types/image_types.go

@@ -205,6 +205,8 @@ type ImagePushOptions struct {
 	Platforms []string
 	// AllPlatforms convert content for all platforms
 	AllPlatforms bool
+	// AllTags push all the tags of the repository named by the reference
+	AllTags bool
 
 	// Estargz convert image to sStargz
 	Estargz bool

pkg/cmd/image/push.go

@@ -24,6 +24,9 @@ import (
 	"net/http"
 	"os"
 	"path/filepath"
+	"regexp"
+	"slices"
+	"strings"
 
 	"github.com/opencontainers/go-digest"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
@@ -57,8 +60,72 @@ import (
 	"github.com/containerd/nerdctl/v2/pkg/snapshotterutil"
 )
 
+const (
+	// Suffixes of the temporary images push creates for itself before uploading them.
+	tmpReducedPlatformSuffix = "-tmp-reduced-platform"
+	tmpEsgzSuffix            = "-tmp-esgz"
+)
+
 // Push pushes an image specified by `rawRef`.
+// With options.AllTags, `rawRef` must be a bare repository name, and every local tag of that
+// repository is pushed.
 func Push(ctx context.Context, client *containerd.Client, rawRef string, options types.ImagePushOptions) error {
+	if !options.AllTags {
+		return pushSingle(ctx, client, rawRef, options, false)
+	}
+
+	parsedReference, err := referenceutil.Parse(rawRef)
+	if err != nil {
+		return err
+	}
+	// ExplicitTag, not Tag: Parse normalizes a bare repository name to ":latest".
+	if parsedReference.ExplicitTag != "" || parsedReference.Digest != "" {
+		return errors.New("tag can't be used with --all-tags/-a")
+	}
+	if parsedReference.Protocol != "" {
+		return fmt.Errorf("--all-tags is not supported for %q references", parsedReference.Protocol)
+	}
+
+	imgs, err := localTags(ctx, client, parsedReference.Name())
+	if err != nil {
+		return err
+	}
+	if len(imgs) == 0 {
+		return fmt.Errorf("an image does not exist locally with the tag: %s", parsedReference.Name())
+	}
+
+	// A SOCI index is attached to the image manifest rather than to the tag, so it only needs to be
+	// built once per distinct target. Doing it per tag makes every tag overwrite the index pushed by
+	// the previous one: https://github.com/containerd/nerdctl/issues/3751
+	indexed := make(map[digest.Digest]struct{}, len(imgs))
+	for _, img := range imgs {
+		_, alreadyIndexed := indexed[img.Target.Digest]
+		if err = pushSingle(ctx, client, img.Name, options, alreadyIndexed); err != nil {
+			return err
+		}
+		indexed[img.Target.Digest] = struct{}{}
+	}
+	return nil
+}
+
+// localTags returns the local images tagged under the repository `name`, sorted by name.
+func localTags(ctx context.Context, client *containerd.Client, name string) ([]images.Image, error) {
+	imgs, err := client.ImageService().List(ctx, fmt.Sprintf("name~=^%s:", regexp.QuoteMeta(name)))
+	if err != nil {
+		return nil, err
+	}
+	// Drop the temporary images push creates for itself, which an interrupted push may have left behind.
+	imgs = slices.DeleteFunc(imgs, func(img images.Image) bool {
+		return strings.HasSuffix(img.Name, tmpReducedPlatformSuffix) || strings.HasSuffix(img.Name, tmpEsgzSuffix)
+	})
+	// ImageService().List does not guarantee an order, and the order decides which tag gets indexed.
+	slices.SortFunc(imgs, func(a, b images.Image) int {
+		return strings.Compare(a.Name, b.Name)
+	})
+	return imgs, nil
+}
+
+func pushSingle(ctx context.Context, client *containerd.Client, rawRef string, options types.ImagePushOptions, alreadyIndexed bool) error {
 	parsedReference, err := referenceutil.Parse(rawRef)
 	if err != nil {
 		return err
@@ -120,7 +187,7 @@ func Push(ctx context.Context, client *containerd.Client, rawRef string, options
 	}
 	pushRef := ref
 	if !options.AllPlatforms {
-		pushRef = ref + "-tmp-reduced-platform"
+		pushRef = ref + tmpReducedPlatformSuffix
 		// Push fails with "400 Bad Request" when the manifest is multi-platform but we do not locally have multi-platform blobs.
 		// So we create a tmp reduced-platform image to avoid the error.
 		// Ensure all the layers are here: https://github.com/containerd/nerdctl/issues/3425
@@ -140,7 +207,7 @@ func Push(ctx context.Context, client *containerd.Client, rawRef string, options
 	}
 
 	if options.Estargz {
-		pushRef = ref + "-tmp-esgz"
+		pushRef = ref + tmpEsgzSuffix
 		esgzImg, err := nerdconverter.Convert(ctx, client, pushRef, ref, converter.WithPlatform(platMC), converter.WithLayerConvertFunc(eStargzConvertFunc()))
 		if err != nil {
 			return fmt.Errorf("failed to convert to eStargz: %v", err)
@@ -185,7 +252,7 @@ func Push(ctx context.Context, client *containerd.Client, rawRef string, options
 		options.SignOptions); err != nil {
 		return err
 	}
-	if options.GOptions.Snapshotter == "soci" {
+	if options.GOptions.Snapshotter == "soci" && !alreadyIndexed {
 		if err = snapshotterutil.CreateSociIndexV1(ref, options.GOptions, options.AllPlatforms, options.Platforms, options.SociOptions); err != nil {
 			return err
 		}
@@ -281,11 +348,16 @@ func pushImageWithLocal(ctx context.Context, client *containerd.Client, parsedRe
 		if options.GOptions.InsecureRegistry {
 			log.G(ctx).WithError(err).Warnf("server %q does not seem to support HTTPS, falling back to plain HTTP", refDomain)
 			dOpts = append(dOpts, dockerconfigresolver.WithPlainHTTP(true))
-			resolver, err = dockerconfigresolver.New(ctx, refDomain, dOpts...)
+			// Rebuild the resolver rather than calling dockerconfigresolver.New, which would fall
+			// back to the process-wide dockerconfigresolver.PushTracker. That tracker is keyed by
+			// digest, not by reference, so a second push of an already-pushed digest short-circuits
+			// with ErrAlreadyExists and its tag is never written to the registry.
+			ho, err = dockerconfigresolver.NewHostOptions(ctx, refDomain, dOpts...)
 			if err != nil {
 				return err
 			}
-			return pushFunc(resolver)
+			resolverOpts.Hosts = dockerconfig.ConfigureHosts(ctx, *ho)
+			return pushFunc(docker.NewResolver(resolverOpts))
 		}
 		log.G(ctx).WithError(err).Errorf("server %q does not seem to support HTTPS", refDomain)
 		log.G(ctx).Info("Hint: you may want to try --insecure-registry to allow plain HTTP (if you are in a trusted network)")