diff --git a/cmd/registry-replacer/main.go b/cmd/registry-replacer/main.go index 48e2bb4cb1b..b8d3c7c0060 100644 --- a/cmd/registry-replacer/main.go +++ b/cmd/registry-replacer/main.go @@ -383,7 +383,7 @@ func replacer( } func ensureReplacement(image *api.ProjectDirectoryImageBuildStepConfiguration, dockerfile []byte) ([]cidockerfile.OrgRepoTag, error) { - toReplace := cidockerfile.ExtractRegistryReferences(dockerfile, "") + toReplace := cidockerfile.ExtractRegistryReferences(dockerfile) var result []cidockerfile.OrgRepoTag for _, toReplace := range toReplace { orgRepoTag, err := cidockerfile.OrgRepoTagFromPullString(toReplace) diff --git a/pkg/defaults/defaults.go b/pkg/defaults/defaults.go index 9a8a5594a95..6827a969af3 100644 --- a/pkg/defaults/defaults.go +++ b/pkg/defaults/defaults.go @@ -980,7 +980,7 @@ func readDockerfileForImage(image api.ProjectDirectoryImageBuildStepConfiguratio // required InputImageTagStepConfiguration steps to import them, as well as an updated // image configuration with the detected inputs added func processDetectedBaseImages(baseImages map[string]api.ImageStreamTagReference, image api.ProjectDirectoryImageBuildStepConfiguration, details dockerfileDetails) ([]api.StepConfiguration, api.ProjectDirectoryImageBuildStepConfiguration) { - detectedBaseImages := dockerfile.DetectInputsFromDockerfile(details.content, image.Inputs, image.From) + detectedBaseImages := dockerfile.DetectInputsFromDockerfile(details.content, image.Inputs, image.From, baseImages) if len(detectedBaseImages) == 0 { return nil, image } diff --git a/pkg/dockerfile/extract.go b/pkg/dockerfile/extract.go index d90fd229c65..9991e8fcb74 100644 --- a/pkg/dockerfile/extract.go +++ b/pkg/dockerfile/extract.go @@ -24,10 +24,9 @@ func (ort OrgRepoTag) String() string { } // ExtractRegistryReferences finds all registry.ci.openshift.org and quay-proxy.ci.openshift.org references in the Dockerfile -func ExtractRegistryReferences(dockerfile []byte, from api.PipelineImageStreamTagReference) []string { +func ExtractRegistryReferences(dockerfile []byte) []string { var refs []string seen := sets.Set[string]{} - lastFromRef := "" for _, line := range bytes.Split(dockerfile, []byte("\n")) { upper := bytes.ToUpper(line) @@ -40,25 +39,11 @@ func ExtractRegistryReferences(dockerfile []byte, from api.PipelineImageStreamTa continue } ref := string(match) - if bytes.HasPrefix(upper, []byte("FROM")) { - lastFromRef = ref - } - if !seen.Has(ref) { refs = append(refs, ref) seen.Insert(ref) } } - if from != "" { - // If from is specified, remove the last detected FROM ref, it will be replaced - var newRefs []string - for _, ref := range refs { - if ref != lastFromRef { - newRefs = append(newRefs, ref) - } - } - refs = newRefs - } return refs } diff --git a/pkg/dockerfile/inputs.go b/pkg/dockerfile/inputs.go index 2e700d47ece..a345dd5f738 100644 --- a/pkg/dockerfile/inputs.go +++ b/pkg/dockerfile/inputs.go @@ -9,11 +9,15 @@ import ( // DetectInputsFromDockerfile parses a Dockerfile and detects registry references that need to be added as base images // Returns a map of base image names to ImageStreamTagReferences // The ImageStreamTagReference.As field contains the original registry reference from the Dockerfile -func DetectInputsFromDockerfile(dockerfile []byte, existingInputs map[string]api.ImageBuildInputs, from api.PipelineImageStreamTagReference) map[string]api.ImageStreamTagReference { - registryRefs := ExtractRegistryReferences(dockerfile, from) - baseImages := make(map[string]api.ImageStreamTagReference) +func DetectInputsFromDockerfile(dockerfile []byte, existingInputs map[string]api.ImageBuildInputs, from api.PipelineImageStreamTagReference, baseImages map[string]api.ImageStreamTagReference) map[string]api.ImageStreamTagReference { + registryRefs := ExtractRegistryReferences(dockerfile) + detected := make(map[string]api.ImageStreamTagReference) for _, ref := range registryRefs { + if from != "" && matchesFromBaseImage(ref, from, baseImages) { + logrus.WithField("reference", ref).WithField("from", from).Debug("Skipping Dockerfile input already provided by image from") + continue + } if HasManualReplacementFor(existingInputs, ref) { logrus.WithField("reference", ref).Debug("Skipping Dockerfile inputs detection: manual replacement exists") continue @@ -24,7 +28,7 @@ func DetectInputsFromDockerfile(dockerfile []byte, existingInputs map[string]api continue } baseImageKey := orgRepoTag.String() - baseImages[baseImageKey] = api.ImageStreamTagReference{ + detected[baseImageKey] = api.ImageStreamTagReference{ Namespace: orgRepoTag.Org, Name: orgRepoTag.Repo, Tag: orgRepoTag.Tag, @@ -32,5 +36,20 @@ func DetectInputsFromDockerfile(dockerfile []byte, existingInputs map[string]api } } - return baseImages + return detected +} + +func matchesFromBaseImage(ref string, from api.PipelineImageStreamTagReference, baseImages map[string]api.ImageStreamTagReference) bool { + if from == "" || baseImages == nil { + return false + } + base, ok := baseImages[string(from)] + if !ok { + return false + } + orgRepoTag, err := OrgRepoTagFromPullString(ref) + if err != nil { + return false + } + return orgRepoTag.Org == base.Namespace && orgRepoTag.Repo == base.Name && orgRepoTag.Tag == base.Tag } diff --git a/pkg/dockerfile/inputs_test.go b/pkg/dockerfile/inputs_test.go index 9110fbde9ff..e5508072f9f 100644 --- a/pkg/dockerfile/inputs_test.go +++ b/pkg/dockerfile/inputs_test.go @@ -13,6 +13,7 @@ func TestDetectInputsFromDockerfile(t *testing.T) { name string dockerfile string existingInputs map[string]api.ImageBuildInputs + baseImages map[string]api.ImageStreamTagReference expected map[string]api.ImageStreamTagReference from api.PipelineImageStreamTagReference }{ @@ -142,20 +143,26 @@ FROM registry.ci.openshift.org/ocp/4.19:base AS runtime }, }, { - name: "from: is specified - should exclude the last and only detected FROM ref", + name: "from matches base image - skip duplicate", dockerfile: `FROM registry.ci.openshift.org/ocp/4.19:base RUN echo "hello" `, - from: "src", + from: "src", + baseImages: map[string]api.ImageStreamTagReference{ + "src": {Namespace: "ocp", Name: "4.19", Tag: "base"}, + }, expected: map[string]api.ImageStreamTagReference{}, }, { - name: "from: is specified - should exclude the last detected FROM ref", + name: "from matches only its base image in multi-stage", dockerfile: `FROM registry.ci.openshift.org/ocp/4.18:base AS builder FROM registry.ci.openshift.org/ocp/4.19:base RUN echo "hello" `, from: "src", + baseImages: map[string]api.ImageStreamTagReference{ + "src": {Namespace: "ocp", Name: "4.19", Tag: "base"}, + }, expected: map[string]api.ImageStreamTagReference{ "ocp_4.18_base": { Namespace: "ocp", @@ -166,13 +173,35 @@ RUN echo "hello" }, }, { - name: "from: is specified - should exclude the last detected FROM ref with COPY", + name: "multi-stage cli with unrelated from", + dockerfile: `FROM registry.ci.openshift.org/ocp/4.14:cli AS cli +FROM quay.io/centos/centos:stream9 +COPY --from=cli /usr/bin/oc /usr/bin/ +`, + from: "stream9", + baseImages: map[string]api.ImageStreamTagReference{ + "stream9": {Namespace: "openshift", Name: "centos", Tag: "stream9"}, + }, + expected: map[string]api.ImageStreamTagReference{ + "ocp_4.14_cli": { + Namespace: "ocp", + Name: "4.14", + Tag: "cli", + As: "registry.ci.openshift.org/ocp/4.14:cli", + }, + }, + }, + { + name: "from matches only its base image with COPY", dockerfile: `FROM registry.ci.openshift.org/ocp/4.18:base AS builder FROM registry.ci.openshift.org/openshift/release:rhel-9-release-golang-1.24-openshift-4.21 COPY --from=registry.ci.openshift.org/ocp/4.19:base /something /somewhere RUN echo "hello" `, from: "src", + baseImages: map[string]api.ImageStreamTagReference{ + "src": {Namespace: "ocp", Name: "4.19", Tag: "base"}, + }, expected: map[string]api.ImageStreamTagReference{ "ocp_4.18_base": { Namespace: "ocp", @@ -180,11 +209,11 @@ RUN echo "hello" Tag: "base", As: "registry.ci.openshift.org/ocp/4.18:base", }, - "ocp_4.19_base": { - Namespace: "ocp", - Name: "4.19", - Tag: "base", - As: "registry.ci.openshift.org/ocp/4.19:base", + "openshift_release_rhel-9-release-golang-1.24-openshift-4.21": { + Namespace: "openshift", + Name: "release", + Tag: "rhel-9-release-golang-1.24-openshift-4.21", + As: "registry.ci.openshift.org/openshift/release:rhel-9-release-golang-1.24-openshift-4.21", }, }, }, @@ -192,7 +221,7 @@ RUN echo "hello" for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - result := DetectInputsFromDockerfile([]byte(tc.dockerfile), tc.existingInputs, tc.from) + result := DetectInputsFromDockerfile([]byte(tc.dockerfile), tc.existingInputs, tc.from, tc.baseImages) if diff := cmp.Diff(tc.expected, result); diff != "" { t.Errorf("result differs from expected:\n%s", diff) diff --git a/pkg/steps/input_image_tag.go b/pkg/steps/input_image_tag.go index 1e009d8e5fc..de83d55ca4b 100644 --- a/pkg/steps/input_image_tag.go +++ b/pkg/steps/input_image_tag.go @@ -61,9 +61,12 @@ func (s *inputImageTagStep) Inputs() (api.InputDefinition, error) { } s.imageName = from.Image.Name } else { - imageName := api.QuayImageReference(s.config.BaseImage) - logrus.Debugf("Resolved %s to %s.", s.config.BaseImage.ISTagName(), imageName) - s.imageName = imageName + _, _, pullSpec, err := s.resolveOfficialImport(context.TODO()) + if err != nil { + return nil, err + } + s.imageName = pullSpec + logrus.Debugf("Resolved %s to %s.", s.config.BaseImage.ISTagName(), s.imageName) } return api.InputDefinition{s.imageName}, nil @@ -80,25 +83,31 @@ func (s *inputImageTagStep) run(ctx context.Context) error { return fmt.Errorf("could not resolve inputs for image tag step: %w", err) } - var objectReferenceName string + var from *coreapi.ObjectReference + var refPolicy imagev1.TagReferencePolicyType + var sourcePullSpec string + if s.config.ExternalImage != nil { - externalPullSpec := externalImageReference(s.config) - logrus.Infof("Tagging %s into %s:%s.", externalPullSpec, api.PipelineImageStream, s.config.To) - objectReferenceName = externalPullSpec - } else { + sourcePullSpec = externalImageReference(s.config) + logrus.Infof("Tagging %s into %s:%s.", sourcePullSpec, api.PipelineImageStream, s.config.To) + from = &coreapi.ObjectReference{Kind: "DockerImage", Name: sourcePullSpec} + refPolicy = imagev1.SourceTagReferencePolicy + } else if api.IsCreatedForClusterBotJob(s.config.BaseImage.Namespace) { logrus.Infof("Tagging %s into %s:%s.", s.config.BaseImage.ISTagName(), api.PipelineImageStream, s.config.To) - objectReferenceName = api.QuayImageReference(s.config.BaseImage) - } - from := &coreapi.ObjectReference{ - Kind: "DockerImage", - Name: objectReferenceName, - } - if api.IsCreatedForClusterBotJob(s.config.BaseImage.Namespace) { from = &coreapi.ObjectReference{ Kind: "ImageStreamImage", Name: fmt.Sprintf("%s@%s", s.config.BaseImage.Name, s.imageName), Namespace: s.config.BaseImage.Namespace, } + sourcePullSpec = s.imageName + refPolicy = imagev1.SourceTagReferencePolicy + } else { + logrus.Infof("Tagging %s into %s:%s.", s.config.BaseImage.ISTagName(), api.PipelineImageStream, s.config.To) + var err error + from, refPolicy, sourcePullSpec, err = s.resolveOfficialImport(ctx) + if err != nil { + return err + } } ist := &imagev1.ImageStreamTag{ @@ -107,10 +116,8 @@ func (s *inputImageTagStep) run(ctx context.Context) error { Namespace: s.jobSpec.Namespace(), }, Tag: &imagev1.TagReference{ - ReferencePolicy: imagev1.TagReferencePolicy{ - Type: imagev1.SourceTagReferencePolicy, - }, - From: from, + ReferencePolicy: imagev1.TagReferencePolicy{Type: refPolicy}, + From: from, ImportPolicy: imagev1.TagImportPolicy{ ImportMode: imagev1.ImportModePreserveOriginal, }, @@ -137,7 +144,7 @@ func (s *inputImageTagStep) run(ctx context.Context) error { ImageStreamName: api.PipelineImageStream, TagName: string(s.config.To), FullTagName: s.jobSpec.Namespace() + "/" + api.PipelineImageStream + ":" + string(s.config.To), - SourceImage: objectReferenceName, + SourceImage: sourcePullSpec, SourceImageKind: from.Kind, StartTime: startTime, CompletionTime: time.Now(), @@ -147,6 +154,22 @@ func (s *inputImageTagStep) run(ctx context.Context) error { return nil } +func (s *inputImageTagStep) resolveOfficialImport(ctx context.Context) (*coreapi.ObjectReference, imagev1.TagReferencePolicyType, string, error) { + from, ok, err := utils.ResolveOfficialInputFrom(ctx, s.client, s.jobSpec.Namespace(), s.config.BaseImage) + if err != nil { + return nil, imagev1.SourceTagReferencePolicy, "", err + } + if !ok { + pullSpec := api.QuayImageReference(s.config.BaseImage) + return &coreapi.ObjectReference{Kind: "DockerImage", Name: pullSpec}, imagev1.SourceTagReferencePolicy, pullSpec, nil + } + pullSpec := from.Name + if from.Kind != "DockerImage" && from.Namespace != "" { + pullSpec = fmt.Sprintf("%s/%s", from.Namespace, from.Name) + } + return from, imagev1.LocalTagReferencePolicy, pullSpec, nil +} + // waitForTagInSpec waits for the tag on the image stream are to show in spec func waitForTagInSpec(ctx context.Context, client ctrlruntimeclient.WithWatch, ns, name, tag string, timeout time.Duration) error { obj := &imagev1.ImageStream{} diff --git a/pkg/steps/input_image_tag_test.go b/pkg/steps/input_image_tag_test.go index 453a9eeda2f..3fafa89189b 100644 --- a/pkg/steps/input_image_tag_test.go +++ b/pkg/steps/input_image_tag_test.go @@ -136,6 +136,178 @@ func TestInputImageTagStep(t *testing.T) { } } +func TestInputImageTagStepOfficialSpec(t *testing.T) { + specPullSpec := "quay-proxy.ci.openshift.org/openshift/ci@sha256:deadbeef" + baseImage := api.ImageStreamTagReference{ + Namespace: "ocp", + Name: "4.22", + Tag: "hyperkube", + } + config := api.InputImageTagStepConfiguration{ + InputImage: api.InputImage{ + To: "ocp_4_22_hyperkube", + BaseImage: baseImage, + }, + } + client := loggingclient.New(fakectrlruntimeclient.NewClientBuilder().WithRuntimeObjects( + &imagev1.ImageStream{ + ObjectMeta: metav1.ObjectMeta{Namespace: "ocp", Name: "4.22"}, + Spec: imagev1.ImageStreamSpec{Tags: []imagev1.TagReference{{ + Name: "hyperkube", + From: &corev1.ObjectReference{Kind: "DockerImage", Name: specPullSpec}, + }}}, + }, + &imagev1.ImageStream{ + ObjectMeta: metav1.ObjectMeta{Namespace: "target-namespace", Name: api.PipelineImageStream}, + Spec: imagev1.ImageStreamSpec{ + LookupPolicy: imagev1.ImageLookupPolicy{Local: true}, + Tags: []imagev1.TagReference{{Name: "ocp_4_22_hyperkube"}}, + }, + Status: imagev1.ImageStreamStatus{ + PublicDockerImageRepository: "some-reg/target-namespace/pipeline", + Tags: []imagev1.NamedTagEventList{{ + Tag: "ocp_4_22_hyperkube", + Items: []imagev1.TagEvent{{Image: "sha256:47e2f82dbede8ff990e6e240f82d78830e7558f7b30df7bd8c0693992018b1e3"}}, + }}, + }, + }).Build(), nil) + jobspec := &api.JobSpec{} + jobspec.SetNamespace("target-namespace") + iits := InputImageTagStep(&config, client, jobspec) + specification := stepExpectation{ + name: "[input:ocp_4_22_hyperkube]", + requires: nil, + creates: []api.StepLink{api.InternalImageLink("ocp_4_22_hyperkube")}, + inputs: inputsExpectation{values: api.InputDefinition{specPullSpec}, err: false}, + } + examineStep(t, iits, specification) + executeStep(t, iits, executionExpectation{ + prerun: doneExpectation{value: false, err: false}, + runError: false, + postrun: doneExpectation{value: true, err: false}, + }) + expectedImageStreamTag := &imagev1.ImageStreamTag{ + ObjectMeta: metav1.ObjectMeta{Name: "pipeline:ocp_4_22_hyperkube", Namespace: jobspec.Namespace(), ResourceVersion: "1"}, + Tag: &imagev1.TagReference{ + From: &corev1.ObjectReference{Kind: "DockerImage", Name: specPullSpec}, + ImportPolicy: imagev1.TagImportPolicy{ImportMode: imagev1.ImportModePreserveOriginal}, + ReferencePolicy: imagev1.TagReferencePolicy{Type: imagev1.LocalTagReferencePolicy}, + }, + } + targetImageStreamTag := &imagev1.ImageStreamTag{} + if err := client.Get(context.Background(), ctrlruntimeclient.ObjectKey{Namespace: jobspec.Namespace(), Name: "pipeline:ocp_4_22_hyperkube"}, targetImageStreamTag); err != nil { + t.Fatalf("failed to get pipeline tag: %v", err) + } + if !equality.Semantic.DeepEqual(expectedImageStreamTag, targetImageStreamTag) { + t.Errorf("unexpected pipeline tag:\n%s", diff.ObjectReflectDiff(expectedImageStreamTag, targetImageStreamTag)) + } +} + +func TestInputImageTagStepLegacyStream(t *testing.T) { + baseImage := api.ImageStreamTagReference{Namespace: "ocp", Name: "5.0", Tag: "cli"} + config := api.InputImageTagStepConfiguration{ + InputImage: api.InputImage{To: "cli", BaseImage: baseImage}, + } + quayRef := api.QuayImageReference(baseImage) + client := loggingclient.New(fakectrlruntimeclient.NewClientBuilder().WithRuntimeObjects( + &imagev1.ImageStream{ + ObjectMeta: metav1.ObjectMeta{Namespace: "target-namespace", Name: api.PipelineImageStream}, + Spec: imagev1.ImageStreamSpec{ + LookupPolicy: imagev1.ImageLookupPolicy{Local: true}, + Tags: []imagev1.TagReference{{Name: "cli"}}, + }, + Status: imagev1.ImageStreamStatus{ + PublicDockerImageRepository: "some-reg/target-namespace/pipeline", + Tags: []imagev1.NamedTagEventList{{ + Tag: "cli", + Items: []imagev1.TagEvent{{Image: "sha256:47e2f82dbede8ff990e6e240f82d78830e7558f7b30df7bd8c0693992018b1e3"}}, + }}, + }, + }).Build(), nil) + jobspec := &api.JobSpec{} + jobspec.SetNamespace("target-namespace") + iits := InputImageTagStep(&config, client, jobspec) + examineStep(t, iits, stepExpectation{ + name: "[input:cli]", + creates: []api.StepLink{api.InternalImageLink("cli")}, + inputs: inputsExpectation{values: api.InputDefinition{quayRef}, err: false}, + }) + executeStep(t, iits, executionExpectation{ + prerun: doneExpectation{value: false, err: false}, runError: false, postrun: doneExpectation{value: true, err: false}, + }) + expected := &imagev1.ImageStreamTag{ + ObjectMeta: metav1.ObjectMeta{Name: "pipeline:cli", Namespace: jobspec.Namespace(), ResourceVersion: "1"}, + Tag: &imagev1.TagReference{ + From: &corev1.ObjectReference{Kind: "DockerImage", Name: quayRef}, + ImportPolicy: imagev1.TagImportPolicy{ImportMode: imagev1.ImportModePreserveOriginal}, + ReferencePolicy: imagev1.TagReferencePolicy{Type: imagev1.SourceTagReferencePolicy}, + }, + } + got := &imagev1.ImageStreamTag{} + if err := client.Get(context.Background(), ctrlruntimeclient.ObjectKey{Namespace: jobspec.Namespace(), Name: "pipeline:cli"}, got); err != nil { + t.Fatalf("get pipeline tag: %v", err) + } + if !equality.Semantic.DeepEqual(expected, got) { + t.Errorf("unexpected tag:\n%s", diff.ObjectReflectDiff(expected, got)) + } +} + +func TestInputImageTagStepStableFirst(t *testing.T) { + baseImage := api.ImageStreamTagReference{Namespace: "ocp", Name: "4.22", Tag: "cli"} + config := api.InputImageTagStepConfiguration{ + InputImage: api.InputImage{To: "ocp_4_22_cli", BaseImage: baseImage}, + } + client := loggingclient.New(fakectrlruntimeclient.NewClientBuilder().WithRuntimeObjects( + &imagev1.ImageStream{ + ObjectMeta: metav1.ObjectMeta{Namespace: "target-namespace", Name: api.StableImageStream}, + Spec: imagev1.ImageStreamSpec{Tags: []imagev1.TagReference{{Name: "cli"}}}, + Status: imagev1.ImageStreamStatus{ + PublicDockerImageRepository: "registry/target-namespace/stable", + Tags: []imagev1.NamedTagEventList{{ + Tag: "cli", + Items: []imagev1.TagEvent{{Image: "sha256:1111"}}, + }}, + }, + }, + &imagev1.ImageStream{ + ObjectMeta: metav1.ObjectMeta{Namespace: "target-namespace", Name: api.PipelineImageStream}, + Spec: imagev1.ImageStreamSpec{ + LookupPolicy: imagev1.ImageLookupPolicy{Local: true}, + Tags: []imagev1.TagReference{{Name: "ocp_4_22_cli"}}, + }, + Status: imagev1.ImageStreamStatus{ + PublicDockerImageRepository: "registry/target-namespace/pipeline", + Tags: []imagev1.NamedTagEventList{{ + Tag: "ocp_4_22_cli", + Items: []imagev1.TagEvent{{Image: "sha256:2222"}}, + }}, + }, + }).Build(), nil) + jobspec := &api.JobSpec{} + jobspec.SetNamespace("target-namespace") + iits := InputImageTagStep(&config, client, jobspec) + executeStep(t, iits, executionExpectation{ + prerun: doneExpectation{value: false, err: false}, runError: false, postrun: doneExpectation{value: true, err: false}, + }) + expected := &imagev1.ImageStreamTag{ + ObjectMeta: metav1.ObjectMeta{Name: "pipeline:ocp_4_22_cli", Namespace: jobspec.Namespace(), ResourceVersion: "1"}, + Tag: &imagev1.TagReference{ + From: &corev1.ObjectReference{ + Kind: "ImageStreamTag", Name: "stable:cli", Namespace: jobspec.Namespace(), + }, + ImportPolicy: imagev1.TagImportPolicy{ImportMode: imagev1.ImportModePreserveOriginal}, + ReferencePolicy: imagev1.TagReferencePolicy{Type: imagev1.LocalTagReferencePolicy}, + }, + } + got := &imagev1.ImageStreamTag{} + if err := client.Get(context.Background(), ctrlruntimeclient.ObjectKey{Namespace: jobspec.Namespace(), Name: "pipeline:ocp_4_22_cli"}, got); err != nil { + t.Fatalf("get pipeline tag: %v", err) + } + if !equality.Semantic.DeepEqual(expected, got) { + t.Errorf("unexpected tag:\n%s", diff.ObjectReflectDiff(expected, got)) + } +} + func TestInputImageTagStepExternal(t *testing.T) { config := api.InputImageTagStepConfiguration{ InputImage: api.InputImage{ diff --git a/pkg/steps/release/promote.go b/pkg/steps/release/promote.go index 575d4b46f27..66c7a6f2af6 100644 --- a/pkg/steps/release/promote.go +++ b/pkg/steps/release/promote.go @@ -242,14 +242,9 @@ func getTagCommand(tagSpecs []string, loglevel int) string { loglevel, strings.Join(tagSpecs, " ")) } -// quayProxyTagFromISKey derives the quay-proxy image tag from an IS tag key of the form -// "namespace/streamname-quay:tag", which is the format produced by getQuayProxyTarget when -// ImageStreamTagReference.Name is non-empty (the standard ocp promotion case). -// Example: "ocp/4.21-quay:ovn-kubernetes" → "quay-proxy.ci.openshift.org/openshift/ci:ocp_4.21_ovn-kubernetes". -// -// The stream segment must end with "-quay"; we split namespace/stream from tag using the last ":" in -// the segment after "/". Using strings.Index(rest, "-quay:") is wrong when the promotion Name (stream -// base before "-quay") itself contains "-quay", e.g. "something-quay-operator-quay:component". +// quayProxyTagFromISKey derives the quay-proxy floating tag from an IS tag key. +// Handles "namespace/stream-quay:tag" (4.23+) and consolidated "ocp/4.13:tag" (4.11–4.22). +// Example: "ocp/4.13:cli" → "quay-proxy.ci.openshift.org/openshift/ci:ocp_4.13_cli". func quayProxyTagFromISKey(isTagKey string) (string, bool) { slashIdx := strings.Index(isTagKey, "/") if slashIdx == -1 { @@ -267,10 +262,14 @@ func quayProxyTagFromISKey(isTagKey string) (string, bool) { return "", false } const quayStreamSuffix = "-quay" - if !strings.HasSuffix(streamPart, quayStreamSuffix) { + var streamName string + if strings.HasSuffix(streamPart, quayStreamSuffix) { + streamName = strings.TrimSuffix(streamPart, quayStreamSuffix) + } else if api.ConsolidatedQuayPromotionVersion(streamPart) { + streamName = streamPart + } else { return "", false } - streamName := strings.TrimSuffix(streamPart, quayStreamSuffix) if streamName == "" { return "", false } @@ -287,7 +286,26 @@ func promotionCLIImageInfoFilterOS(nodeArchitectures []string) string { return "linux/amd64" } -const quayPromotionDigestTagAttempts = 5 +const ( + quayPromotionDigestTagAttempts = 5 + quayPromotionMirrorAttempts = 5 +) + +// getMirrorRetryShell mirrors images with retries and fails the promotion pod if all attempts fail. +func getMirrorRetryShell(registryConfig string, images []string, loglevel int) string { + mirrorCmd := getMirrorCommand(registryConfig, images, loglevel) + n := quayPromotionMirrorAttempts + return fmt.Sprintf(`for r in {1..%d}; do + echo Mirror attempt $r + if %s; then break; fi + if [ "${r}" -eq %d ]; then + exit 1 + fi + backoff=$(($RANDOM %% 120))s + echo Sleeping randomized $backoff before retry + sleep $backoff +done`, n, mirrorCmd, n) +} // getResolveAndTagRetryShell resolves quay-proxy digest via oc image info and oc tags the IST; retries when QCI moves after mirror. func getResolveAndTagRetryShell(registryConfig, quayProxyTag, isTag string, loglevel int, filterByOS string) string { @@ -328,10 +346,8 @@ func getPromotionPod(imageMirrorTarget map[string]string, timeStr string, namesp var images []string var pruneImages []string var tags []string - // resolveAndTagPairs holds [quayProxyTag, isTag] for concrete *-quay IS targets in a - // quay promotion step. These are handled post-mirror via oc image info + oc tag so - // that spec.from always carries the exact QCI manifest digest rather than a floating tag. - // Non-release namespaces use oc tag directly. + // resolveAndTagPairs holds [quayProxyTag, isTag] for official ocp IS targets (consolidated + // ocp/4.x:tag and legacy *-quay). Resolved post-mirror via oc image info + oc tag. var resolveAndTagPairs [][2]string isQuayStep := name == api.PromotionQuayStepName @@ -373,29 +389,25 @@ func getPromotionPod(imageMirrorTarget map[string]string, timeStr string, namesp // Generate mirror commands if there are images to mirror if len(images) > 0 { - mirrorCommand := fmt.Sprintf("for r in {1..5}; do echo Mirror attempt $r; %s && break; backoff=$(($RANDOM %% 120))s; echo Sleeping randomized $backoff before retry; sleep $backoff; done", getMirrorCommand(registryConfig, images, 2)) - commands = append(commands, mirrorCommand) + commands = append(commands, getMirrorRetryShell(registryConfig, images, 2)) } - // Generate tag/resolve-and-tag commands. - if isQuayStep && (len(tags) > 0 || len(resolveAndTagPairs) > 0) { - // Quay promotion: run under set +e so one failure does not block the rest. - tagCommands := []string{"set +e"} - - // Template-based IS tags (contain ${component}): batch-then-individual retry, unchanged. - if len(tags) > 0 { - singleCmd := fmt.Sprintf(retryLoopTemplate, 2, `"Tag attempt $r (all together)"`, getTagCommand(tags, 2), ":") - tagCommands = append(tagCommands, singleCmd) - for _, tagPair := range tags { - individualCmd := fmt.Sprintf(retryLoopTemplate, 3, `"Tag attempt $r (individual)"`, getTagCommand([]string{tagPair}, 2), retryLoopWithBackoff) - tagCommands = append(tagCommands, individualCmd) - } - } - - // Concrete *-quay IS targets: resolve QCI digest post-mirror, then tag. + if isQuayStep { for _, pair := range resolveAndTagPairs { quayProxyTag, isTag := pair[0], pair[1] - tagCommands = append(tagCommands, getResolveAndTagRetryShell(registryConfig, quayProxyTag, isTag, 2, promotionCLIImageInfoFilterOS(nodeArchitectures))) + commands = append(commands, getResolveAndTagRetryShell(registryConfig, quayProxyTag, isTag, 2, promotionCLIImageInfoFilterOS(nodeArchitectures))) + } + } + + // Non-official IS tags (ci/ci-quay, ${component} templates) keep best-effort batch tagging. + if isQuayStep && len(tags) > 0 { + tagCommands := []string{"set +e"} + + singleCmd := fmt.Sprintf(retryLoopTemplate, 2, `"Tag attempt $r (all together)"`, getTagCommand(tags, 2), ":") + tagCommands = append(tagCommands, singleCmd) + for _, tagPair := range tags { + individualCmd := fmt.Sprintf(retryLoopTemplate, 3, `"Tag attempt $r (individual)"`, getTagCommand([]string{tagPair}, 2), retryLoopWithBackoff) + tagCommands = append(tagCommands, individualCmd) } tagCommands = append(tagCommands, "set -e") diff --git a/pkg/steps/release/promote_test.go b/pkg/steps/release/promote_test.go index c3e0ac30fab..aea35cb3059 100644 --- a/pkg/steps/release/promote_test.go +++ b/pkg/steps/release/promote_test.go @@ -1105,6 +1105,22 @@ func TestGetPublicImageReference(t *testing.T) { } } +func TestGetMirrorRetryShell(t *testing.T) { + regcfg := "/etc/push-secret/.dockerconfigjson" + got := getMirrorRetryShell(regcfg, []string{"src=dst"}, 2) + for _, sub := range []string{ + "for r in {1..5}", + "Mirror attempt $r", + "oc image mirror", + `[ "${r}" -eq 5 ]`, + "exit 1", + } { + if !strings.Contains(got, sub) { + t.Fatalf("missing substring %q in:\n%s", sub, got) + } + } +} + func TestGetResolveAndTagRetryShell(t *testing.T) { regcfg := "/etc/push-secret/.dockerconfigjson" proxyTag := "quay-proxy.ci.openshift.org/openshift/ci:ocp_4.21_ovn-kubernetes" @@ -1165,8 +1181,20 @@ func TestQuayProxyTagFromISKey(t *testing.T) { wantOK: false, }, { - name: "no -quay: suffix", + name: "consolidated ocp stream", + isTagKey: "ocp/4.13:secondary-scheduler-operator", + wantTag: "quay-proxy.ci.openshift.org/openshift/ci:ocp_4.13_secondary-scheduler-operator", + wantOK: true, + }, + { + name: "consolidated 4.21 stream", isTagKey: "ocp/4.21:ovn-kubernetes", + wantTag: "quay-proxy.ci.openshift.org/openshift/ci:ocp_4.21_ovn-kubernetes", + wantOK: true, + }, + { + name: "non-consolidated stream without -quay", + isTagKey: "ocp/4.23:ovn-kubernetes", wantOK: false, }, } diff --git a/pkg/steps/release/snapshot.go b/pkg/steps/release/snapshot.go index 0789c546993..2f6a7d0f812 100644 --- a/pkg/steps/release/snapshot.go +++ b/pkg/steps/release/snapshot.go @@ -82,22 +82,38 @@ func snapshotStream(ctx context.Context, client loggingclient.LoggingClient, sou } snapshot.ObjectMeta.Annotations[api.ReleaseConfigAnnotation] = value } + var source *imagev1.ImageStream + if !api.IsCreatedForClusterBotJob(sourceNamespace) { + source = &imagev1.ImageStream{} + if err := client.Get(ctx, ctrlruntimeclient.ObjectKey{Namespace: sourceNamespace, Name: sourceName}, source); err != nil { + return nil, fmt.Errorf("could not resolve source imagestream %s/%s for release %s: %w", sourceNamespace, sourceName, targetRelease, err) + } + } for _, tag := range integratedStream.Tags { from := &coreapi.ObjectReference{ Kind: "DockerImage", Name: api.QuayImageReference(api.ImageStreamTagReference{Namespace: sourceNamespace, Name: sourceName, Tag: tag}), } - // a special case for cluster-bot if api.IsCreatedForClusterBotJob(sourceNamespace) { - source := &imagev1.ImageStream{} - if err := client.Get(ctx, ctrlruntimeclient.ObjectKey{Namespace: sourceNamespace, Name: sourceName}, source); err != nil { - return nil, fmt.Errorf("could not resolve source imagestream %s/%s for release %s: %w", sourceNamespace, sourceName, targetRelease, err) + if source == nil { + source = &imagev1.ImageStream{} + if err := client.Get(ctx, ctrlruntimeclient.ObjectKey{Namespace: sourceNamespace, Name: sourceName}, source); err != nil { + return nil, fmt.Errorf("could not resolve source imagestream %s/%s for release %s: %w", sourceNamespace, sourceName, targetRelease, err) + } } if valid, _ := utils.FindStatusTag(source, tag); valid != nil { from = valid } else { continue } + } else if api.ConsolidatedQuayPromotionVersion(sourceName) && source != nil { + from = utils.OfficialImageTagFrom(source, api.ImageStreamTagReference{ + Namespace: sourceNamespace, + Name: sourceName, + Tag: tag, + }) + } else if ref := utils.FindSpecTag(source, tag); ref != nil && ref.Kind == "DockerImage" && ref.Name != "" { + from.Name = ref.Name } if refPolicy == nil { sourcePolicy := imagev1.SourceTagReferencePolicy diff --git a/pkg/steps/release/testdata/zz_fixture_TestGetPromotionPod_basic_case.yaml b/pkg/steps/release/testdata/zz_fixture_TestGetPromotionPod_basic_case.yaml index 9cb190a91b0..8dd802dd685 100644 --- a/pkg/steps/release/testdata/zz_fixture_TestGetPromotionPod_basic_case.yaml +++ b/pkg/steps/release/testdata/zz_fixture_TestGetPromotionPod_basic_case.yaml @@ -7,11 +7,17 @@ metadata: spec: containers: - args: - - for r in {1..5}; do echo Mirror attempt $r; oc image mirror --loglevel=2 --keep-manifest-list - --registry-config=/etc/push-secret/.dockerconfigjson --max-per-registry=10 docker-registry.default.svc:5000/ci-op-y2n8rsh3/pipeline@sha256:afd71aa3cbbf7d2e00cd8696747b2abf164700147723c657919c20b13d13ec62=registry.ci.openshift.org/ci/applyconfig:latest - docker-registry.default.svc:5000/ci-op-y2n8rsh3/pipeline@sha256:bbb=registry.ci.openshift.org/ci/bin:latest - && break; backoff=$(($RANDOM % 120))s; echo Sleeping randomized $backoff before - retry; sleep $backoff; done + - |- + for r in {1..5}; do + echo Mirror attempt $r + if oc image mirror --loglevel=2 --keep-manifest-list --registry-config=/etc/push-secret/.dockerconfigjson --max-per-registry=10 docker-registry.default.svc:5000/ci-op-y2n8rsh3/pipeline@sha256:afd71aa3cbbf7d2e00cd8696747b2abf164700147723c657919c20b13d13ec62=registry.ci.openshift.org/ci/applyconfig:latest docker-registry.default.svc:5000/ci-op-y2n8rsh3/pipeline@sha256:bbb=registry.ci.openshift.org/ci/bin:latest; then break; fi + if [ "${r}" -eq 5 ]; then + exit 1 + fi + backoff=$(($RANDOM % 120))s + echo Sleeping randomized $backoff before retry + sleep $backoff + done command: - /bin/sh - -c diff --git a/pkg/steps/release/testdata/zz_fixture_TestGetPromotionPod_basic_case__arm64_only.yaml b/pkg/steps/release/testdata/zz_fixture_TestGetPromotionPod_basic_case__arm64_only.yaml index f9972c14740..1cf9f9c0029 100644 --- a/pkg/steps/release/testdata/zz_fixture_TestGetPromotionPod_basic_case__arm64_only.yaml +++ b/pkg/steps/release/testdata/zz_fixture_TestGetPromotionPod_basic_case__arm64_only.yaml @@ -7,11 +7,17 @@ metadata: spec: containers: - args: - - for r in {1..5}; do echo Mirror attempt $r; oc image mirror --loglevel=2 --keep-manifest-list - --registry-config=/etc/push-secret/.dockerconfigjson --max-per-registry=10 docker-registry.default.svc:5000/ci-op-y2n8rsh3/pipeline@sha256:afd71aa3cbbf7d2e00cd8696747b2abf164700147723c657919c20b13d13ec62=registry.ci.openshift.org/ci/applyconfig:latest - docker-registry.default.svc:5000/ci-op-y2n8rsh3/pipeline@sha256:bbb=registry.ci.openshift.org/ci/bin:latest - && break; backoff=$(($RANDOM % 120))s; echo Sleeping randomized $backoff before - retry; sleep $backoff; done + - |- + for r in {1..5}; do + echo Mirror attempt $r + if oc image mirror --loglevel=2 --keep-manifest-list --registry-config=/etc/push-secret/.dockerconfigjson --max-per-registry=10 docker-registry.default.svc:5000/ci-op-y2n8rsh3/pipeline@sha256:afd71aa3cbbf7d2e00cd8696747b2abf164700147723c657919c20b13d13ec62=registry.ci.openshift.org/ci/applyconfig:latest docker-registry.default.svc:5000/ci-op-y2n8rsh3/pipeline@sha256:bbb=registry.ci.openshift.org/ci/bin:latest; then break; fi + if [ "${r}" -eq 5 ]; then + exit 1 + fi + backoff=$(($RANDOM % 120))s + echo Sleeping randomized $backoff before retry + sleep $backoff + done command: - /bin/sh - -c diff --git a/pkg/steps/release/testdata/zz_fixture_TestGetPromotionPod_basic_case__multi_architecture.yaml b/pkg/steps/release/testdata/zz_fixture_TestGetPromotionPod_basic_case__multi_architecture.yaml index 9cb190a91b0..8dd802dd685 100644 --- a/pkg/steps/release/testdata/zz_fixture_TestGetPromotionPod_basic_case__multi_architecture.yaml +++ b/pkg/steps/release/testdata/zz_fixture_TestGetPromotionPod_basic_case__multi_architecture.yaml @@ -7,11 +7,17 @@ metadata: spec: containers: - args: - - for r in {1..5}; do echo Mirror attempt $r; oc image mirror --loglevel=2 --keep-manifest-list - --registry-config=/etc/push-secret/.dockerconfigjson --max-per-registry=10 docker-registry.default.svc:5000/ci-op-y2n8rsh3/pipeline@sha256:afd71aa3cbbf7d2e00cd8696747b2abf164700147723c657919c20b13d13ec62=registry.ci.openshift.org/ci/applyconfig:latest - docker-registry.default.svc:5000/ci-op-y2n8rsh3/pipeline@sha256:bbb=registry.ci.openshift.org/ci/bin:latest - && break; backoff=$(($RANDOM % 120))s; echo Sleeping randomized $backoff before - retry; sleep $backoff; done + - |- + for r in {1..5}; do + echo Mirror attempt $r + if oc image mirror --loglevel=2 --keep-manifest-list --registry-config=/etc/push-secret/.dockerconfigjson --max-per-registry=10 docker-registry.default.svc:5000/ci-op-y2n8rsh3/pipeline@sha256:afd71aa3cbbf7d2e00cd8696747b2abf164700147723c657919c20b13d13ec62=registry.ci.openshift.org/ci/applyconfig:latest docker-registry.default.svc:5000/ci-op-y2n8rsh3/pipeline@sha256:bbb=registry.ci.openshift.org/ci/bin:latest; then break; fi + if [ "${r}" -eq 5 ]; then + exit 1 + fi + backoff=$(($RANDOM % 120))s + echo Sleeping randomized $backoff before retry + sleep $backoff + done command: - /bin/sh - -c diff --git a/pkg/steps/release/testdata/zz_fixture_TestGetPromotionPod_promotion_quay.yaml b/pkg/steps/release/testdata/zz_fixture_TestGetPromotionPod_promotion_quay.yaml index bca45821dbe..d9b7fb6ca61 100644 --- a/pkg/steps/release/testdata/zz_fixture_TestGetPromotionPod_promotion_quay.yaml +++ b/pkg/steps/release/testdata/zz_fixture_TestGetPromotionPod_promotion_quay.yaml @@ -9,7 +9,16 @@ spec: - args: - |- oc image mirror --loglevel=2 --keep-manifest-list --registry-config=/etc/push-secret/.dockerconfigjson --max-per-registry=10 quay.io/openshift/ci:ci_a_latest=quay.io/openshift/ci:20240603235401_prune_ci_a_latest quay.io/openshift/ci:ci_c_latest=quay.io/openshift/ci:20240603235401_prune_ci_c_latest || true - for r in {1..5}; do echo Mirror attempt $r; oc image mirror --loglevel=2 --keep-manifest-list --registry-config=/etc/push-secret/.dockerconfigjson --max-per-registry=10 registry.build02.ci.openshift.org/ci-op-y2n8rsh3/pipeline@sha256:bbb=quay.io/openshift/ci:ci_a_latest registry.build02.ci.openshift.org/ci-op-y2n8rsh3/pipeline@sha256:ddd=quay.io/openshift/ci:ci_c_latest && break; backoff=$(($RANDOM % 120))s; echo Sleeping randomized $backoff before retry; sleep $backoff; done + for r in {1..5}; do + echo Mirror attempt $r + if oc image mirror --loglevel=2 --keep-manifest-list --registry-config=/etc/push-secret/.dockerconfigjson --max-per-registry=10 registry.build02.ci.openshift.org/ci-op-y2n8rsh3/pipeline@sha256:bbb=quay.io/openshift/ci:ci_a_latest registry.build02.ci.openshift.org/ci-op-y2n8rsh3/pipeline@sha256:ddd=quay.io/openshift/ci:ci_c_latest; then break; fi + if [ "${r}" -eq 5 ]; then + exit 1 + fi + backoff=$(($RANDOM % 120))s + echo Sleeping randomized $backoff before retry + sleep $backoff + done set +e for r in {1..2}; do echo "Tag attempt $r (all together)"; oc tag --source=docker --loglevel=2 --reference-policy='source' --import-mode='PreserveOriginal' --reference quay-proxy.ci.openshift.org/openshift/ci@sha256:ddd ci/${component}-quay:c quay-proxy.ci.openshift.org/openshift/ci@sha256:bbb ci/ci-quay:${component} && break; :; done for r in {1..3}; do echo "Tag attempt $r (individual)"; oc tag --source=docker --loglevel=2 --reference-policy='source' --import-mode='PreserveOriginal' --reference quay-proxy.ci.openshift.org/openshift/ci@sha256:ddd ci/${component}-quay:c && break; backoff=$(($RANDOM % 120))s; echo Sleeping randomized $backoff before retry; sleep $backoff; done diff --git a/pkg/steps/release/testdata/zz_fixture_TestGetPromotionPod_promotion_quay_4.12.yaml b/pkg/steps/release/testdata/zz_fixture_TestGetPromotionPod_promotion_quay_4.12.yaml index 155ab340e83..3f23851b2fc 100644 --- a/pkg/steps/release/testdata/zz_fixture_TestGetPromotionPod_promotion_quay_4.12.yaml +++ b/pkg/steps/release/testdata/zz_fixture_TestGetPromotionPod_promotion_quay_4.12.yaml @@ -7,14 +7,45 @@ metadata: spec: containers: - args: - - |- + - | oc image mirror --loglevel=2 --keep-manifest-list --registry-config=/etc/push-secret/.dockerconfigjson --max-per-registry=10 quay.io/openshift/ci:ocp_4.12_ovn-kubernetes=quay.io/openshift/ci:20240603235401_prune_ovn-kubernetes quay.io/openshift/ci:ocp_4.12_ovn-kubernetes-base=quay.io/openshift/ci:20240603235401_prune_ovn-kubernetes-base || true - for r in {1..5}; do echo Mirror attempt $r; oc image mirror --loglevel=2 --keep-manifest-list --registry-config=/etc/push-secret/.dockerconfigjson --max-per-registry=10 registry.build02.ci.openshift.org/ci-op-y2n8rsh3/pipeline@sha256:aaa=quay.io/openshift/ci:ocp_4.12_ovn-kubernetes registry.build02.ci.openshift.org/ci-op-y2n8rsh3/pipeline@sha256:bbb=quay.io/openshift/ci:ocp_4.12_ovn-kubernetes-base && break; backoff=$(($RANDOM % 120))s; echo Sleeping randomized $backoff before retry; sleep $backoff; done - set +e - for r in {1..2}; do echo "Tag attempt $r (all together)"; oc tag --source=docker --loglevel=2 --reference-policy='source' --import-mode='PreserveOriginal' --reference quay-proxy.ci.openshift.org/openshift/ci@sha256:aaa ocp/4.12:ovn-kubernetes quay-proxy.ci.openshift.org/openshift/ci@sha256:bbb ocp/4.12:ovn-kubernetes-base && break; :; done - for r in {1..3}; do echo "Tag attempt $r (individual)"; oc tag --source=docker --loglevel=2 --reference-policy='source' --import-mode='PreserveOriginal' --reference quay-proxy.ci.openshift.org/openshift/ci@sha256:aaa ocp/4.12:ovn-kubernetes && break; backoff=$(($RANDOM % 120))s; echo Sleeping randomized $backoff before retry; sleep $backoff; done - for r in {1..3}; do echo "Tag attempt $r (individual)"; oc tag --source=docker --loglevel=2 --reference-policy='source' --import-mode='PreserveOriginal' --reference quay-proxy.ci.openshift.org/openshift/ci@sha256:bbb ocp/4.12:ovn-kubernetes-base && break; backoff=$(($RANDOM % 120))s; echo Sleeping randomized $backoff before retry; sleep $backoff; done - set -e + for r in {1..5}; do + echo Mirror attempt $r + if oc image mirror --loglevel=2 --keep-manifest-list --registry-config=/etc/push-secret/.dockerconfigjson --max-per-registry=10 registry.build02.ci.openshift.org/ci-op-y2n8rsh3/pipeline@sha256:aaa=quay.io/openshift/ci:ocp_4.12_ovn-kubernetes registry.build02.ci.openshift.org/ci-op-y2n8rsh3/pipeline@sha256:bbb=quay.io/openshift/ci:ocp_4.12_ovn-kubernetes-base; then break; fi + if [ "${r}" -eq 5 ]; then + exit 1 + fi + backoff=$(($RANDOM % 120))s + echo Sleeping randomized $backoff before retry + sleep $backoff + done + for r in {1..5}; do + _digest=$(oc image info --registry-config=/etc/push-secret/.dockerconfigjson --filter-by-os=linux/amd64 quay-proxy.ci.openshift.org/openshift/ci:ocp_4.12_ovn-kubernetes | sed -n '/^Digest:[[:space:]]/s/^Digest:[[:space:]]*//p' | head -n1) + if [ -n "${_digest}" ] && oc tag --source=docker --loglevel=2 --reference-policy='source' --import-mode='PreserveOriginal' --reference quay-proxy.ci.openshift.org/openshift/ci@${_digest} ocp/4.12:ovn-kubernetes; then + break + fi + echo "promotion-quay: digest-tag failed for ocp/4.12:ovn-kubernetes attempt ${r}/5 (QCI digest may have moved after mirror)" >&2 + if [ "${r}" -eq 5 ]; then + exit 1 + fi + echo "promotion-quay: retrying digest-tag for ocp/4.12:ovn-kubernetes (attempt $((r+1))/5 after randomized backoff)" >&2 + backoff=$(($RANDOM % 120))s + sleep "${backoff}" + done + + for r in {1..5}; do + _digest=$(oc image info --registry-config=/etc/push-secret/.dockerconfigjson --filter-by-os=linux/amd64 quay-proxy.ci.openshift.org/openshift/ci:ocp_4.12_ovn-kubernetes-base | sed -n '/^Digest:[[:space:]]/s/^Digest:[[:space:]]*//p' | head -n1) + if [ -n "${_digest}" ] && oc tag --source=docker --loglevel=2 --reference-policy='source' --import-mode='PreserveOriginal' --reference quay-proxy.ci.openshift.org/openshift/ci@${_digest} ocp/4.12:ovn-kubernetes-base; then + break + fi + echo "promotion-quay: digest-tag failed for ocp/4.12:ovn-kubernetes-base attempt ${r}/5 (QCI digest may have moved after mirror)" >&2 + if [ "${r}" -eq 5 ]; then + exit 1 + fi + echo "promotion-quay: retrying digest-tag for ocp/4.12:ovn-kubernetes-base (attempt $((r+1))/5 after randomized backoff)" >&2 + backoff=$(($RANDOM % 120))s + sleep "${backoff}" + done command: - /bin/sh - -c diff --git a/pkg/steps/release/testdata/zz_fixture_TestGetPromotionPod_promotion_quay_multiple_tags.yaml b/pkg/steps/release/testdata/zz_fixture_TestGetPromotionPod_promotion_quay_multiple_tags.yaml index ecfd4ffe18b..e221b1e4ff9 100644 --- a/pkg/steps/release/testdata/zz_fixture_TestGetPromotionPod_promotion_quay_multiple_tags.yaml +++ b/pkg/steps/release/testdata/zz_fixture_TestGetPromotionPod_promotion_quay_multiple_tags.yaml @@ -7,15 +7,59 @@ metadata: spec: containers: - args: - - |- + - | oc image mirror --loglevel=2 --keep-manifest-list --registry-config=/etc/push-secret/.dockerconfigjson --max-per-registry=10 quay.io/openshift/ci:ocp_4.21_ovn-kubernetes=quay.io/openshift/ci:20240603235401_prune_ovn-kubernetes quay.io/openshift/ci:ocp_4.21_ovn-kubernetes-base=quay.io/openshift/ci:20240603235401_prune_ovn-kubernetes-base quay.io/openshift/ci:ocp_4.21_ovn-kubernetes-microshift=quay.io/openshift/ci:20240603235401_prune_ovn-kubernetes-microshift || true - for r in {1..5}; do echo Mirror attempt $r; oc image mirror --loglevel=2 --keep-manifest-list --registry-config=/etc/push-secret/.dockerconfigjson --max-per-registry=10 registry.build02.ci.openshift.org/ci-op-y2n8rsh3/pipeline@sha256:aaa=quay.io/openshift/ci:ocp_4.21_ovn-kubernetes registry.build02.ci.openshift.org/ci-op-y2n8rsh3/pipeline@sha256:bbb=quay.io/openshift/ci:ocp_4.21_ovn-kubernetes-base registry.build02.ci.openshift.org/ci-op-y2n8rsh3/pipeline@sha256:ccc=quay.io/openshift/ci:ocp_4.21_ovn-kubernetes-microshift && break; backoff=$(($RANDOM % 120))s; echo Sleeping randomized $backoff before retry; sleep $backoff; done - set +e - for r in {1..2}; do echo "Tag attempt $r (all together)"; oc tag --source=docker --loglevel=2 --reference-policy='source' --import-mode='PreserveOriginal' --reference quay-proxy.ci.openshift.org/openshift/ci@sha256:aaa ocp/4.21:ovn-kubernetes quay-proxy.ci.openshift.org/openshift/ci@sha256:bbb ocp/4.21:ovn-kubernetes-base quay-proxy.ci.openshift.org/openshift/ci@sha256:ccc ocp/4.21:ovn-kubernetes-microshift && break; :; done - for r in {1..3}; do echo "Tag attempt $r (individual)"; oc tag --source=docker --loglevel=2 --reference-policy='source' --import-mode='PreserveOriginal' --reference quay-proxy.ci.openshift.org/openshift/ci@sha256:aaa ocp/4.21:ovn-kubernetes && break; backoff=$(($RANDOM % 120))s; echo Sleeping randomized $backoff before retry; sleep $backoff; done - for r in {1..3}; do echo "Tag attempt $r (individual)"; oc tag --source=docker --loglevel=2 --reference-policy='source' --import-mode='PreserveOriginal' --reference quay-proxy.ci.openshift.org/openshift/ci@sha256:bbb ocp/4.21:ovn-kubernetes-base && break; backoff=$(($RANDOM % 120))s; echo Sleeping randomized $backoff before retry; sleep $backoff; done - for r in {1..3}; do echo "Tag attempt $r (individual)"; oc tag --source=docker --loglevel=2 --reference-policy='source' --import-mode='PreserveOriginal' --reference quay-proxy.ci.openshift.org/openshift/ci@sha256:ccc ocp/4.21:ovn-kubernetes-microshift && break; backoff=$(($RANDOM % 120))s; echo Sleeping randomized $backoff before retry; sleep $backoff; done - set -e + for r in {1..5}; do + echo Mirror attempt $r + if oc image mirror --loglevel=2 --keep-manifest-list --registry-config=/etc/push-secret/.dockerconfigjson --max-per-registry=10 registry.build02.ci.openshift.org/ci-op-y2n8rsh3/pipeline@sha256:aaa=quay.io/openshift/ci:ocp_4.21_ovn-kubernetes registry.build02.ci.openshift.org/ci-op-y2n8rsh3/pipeline@sha256:bbb=quay.io/openshift/ci:ocp_4.21_ovn-kubernetes-base registry.build02.ci.openshift.org/ci-op-y2n8rsh3/pipeline@sha256:ccc=quay.io/openshift/ci:ocp_4.21_ovn-kubernetes-microshift; then break; fi + if [ "${r}" -eq 5 ]; then + exit 1 + fi + backoff=$(($RANDOM % 120))s + echo Sleeping randomized $backoff before retry + sleep $backoff + done + for r in {1..5}; do + _digest=$(oc image info --registry-config=/etc/push-secret/.dockerconfigjson --filter-by-os=linux/amd64 quay-proxy.ci.openshift.org/openshift/ci:ocp_4.21_ovn-kubernetes | sed -n '/^Digest:[[:space:]]/s/^Digest:[[:space:]]*//p' | head -n1) + if [ -n "${_digest}" ] && oc tag --source=docker --loglevel=2 --reference-policy='source' --import-mode='PreserveOriginal' --reference quay-proxy.ci.openshift.org/openshift/ci@${_digest} ocp/4.21:ovn-kubernetes; then + break + fi + echo "promotion-quay: digest-tag failed for ocp/4.21:ovn-kubernetes attempt ${r}/5 (QCI digest may have moved after mirror)" >&2 + if [ "${r}" -eq 5 ]; then + exit 1 + fi + echo "promotion-quay: retrying digest-tag for ocp/4.21:ovn-kubernetes (attempt $((r+1))/5 after randomized backoff)" >&2 + backoff=$(($RANDOM % 120))s + sleep "${backoff}" + done + + for r in {1..5}; do + _digest=$(oc image info --registry-config=/etc/push-secret/.dockerconfigjson --filter-by-os=linux/amd64 quay-proxy.ci.openshift.org/openshift/ci:ocp_4.21_ovn-kubernetes-base | sed -n '/^Digest:[[:space:]]/s/^Digest:[[:space:]]*//p' | head -n1) + if [ -n "${_digest}" ] && oc tag --source=docker --loglevel=2 --reference-policy='source' --import-mode='PreserveOriginal' --reference quay-proxy.ci.openshift.org/openshift/ci@${_digest} ocp/4.21:ovn-kubernetes-base; then + break + fi + echo "promotion-quay: digest-tag failed for ocp/4.21:ovn-kubernetes-base attempt ${r}/5 (QCI digest may have moved after mirror)" >&2 + if [ "${r}" -eq 5 ]; then + exit 1 + fi + echo "promotion-quay: retrying digest-tag for ocp/4.21:ovn-kubernetes-base (attempt $((r+1))/5 after randomized backoff)" >&2 + backoff=$(($RANDOM % 120))s + sleep "${backoff}" + done + + for r in {1..5}; do + _digest=$(oc image info --registry-config=/etc/push-secret/.dockerconfigjson --filter-by-os=linux/amd64 quay-proxy.ci.openshift.org/openshift/ci:ocp_4.21_ovn-kubernetes-microshift | sed -n '/^Digest:[[:space:]]/s/^Digest:[[:space:]]*//p' | head -n1) + if [ -n "${_digest}" ] && oc tag --source=docker --loglevel=2 --reference-policy='source' --import-mode='PreserveOriginal' --reference quay-proxy.ci.openshift.org/openshift/ci@${_digest} ocp/4.21:ovn-kubernetes-microshift; then + break + fi + echo "promotion-quay: digest-tag failed for ocp/4.21:ovn-kubernetes-microshift attempt ${r}/5 (QCI digest may have moved after mirror)" >&2 + if [ "${r}" -eq 5 ]; then + exit 1 + fi + echo "promotion-quay: retrying digest-tag for ocp/4.21:ovn-kubernetes-microshift (attempt $((r+1))/5 after randomized backoff)" >&2 + backoff=$(($RANDOM % 120))s + sleep "${backoff}" + done command: - /bin/sh - -c diff --git a/pkg/steps/release/testdata/zz_fixture_TestGetPromotionPod_promotion_quay_non_release_namespace.yaml b/pkg/steps/release/testdata/zz_fixture_TestGetPromotionPod_promotion_quay_non_release_namespace.yaml index 1c496a5cf7e..5b798cda565 100644 --- a/pkg/steps/release/testdata/zz_fixture_TestGetPromotionPod_promotion_quay_non_release_namespace.yaml +++ b/pkg/steps/release/testdata/zz_fixture_TestGetPromotionPod_promotion_quay_non_release_namespace.yaml @@ -8,11 +8,33 @@ spec: containers: - args: - |- - for r in {1..5}; do echo Mirror attempt $r; oc image mirror --loglevel=2 --keep-manifest-list --registry-config=/etc/push-secret/.dockerconfigjson --max-per-registry=10 registry.build02.ci.openshift.org/ci-op-y2n8rsh3/pipeline@sha256:bbb=quay.io/openshift/ci:ci_ci_sanitize-prow-jobs registry.build02.ci.openshift.org/ci-op-y2n8rsh3/pipeline@sha256:aaa=quay.io/openshift/ci:ocp_4.21_ovn-kubernetes && break; backoff=$(($RANDOM % 120))s; echo Sleeping randomized $backoff before retry; sleep $backoff; done + for r in {1..5}; do + echo Mirror attempt $r + if oc image mirror --loglevel=2 --keep-manifest-list --registry-config=/etc/push-secret/.dockerconfigjson --max-per-registry=10 registry.build02.ci.openshift.org/ci-op-y2n8rsh3/pipeline@sha256:bbb=quay.io/openshift/ci:ci_ci_sanitize-prow-jobs registry.build02.ci.openshift.org/ci-op-y2n8rsh3/pipeline@sha256:aaa=quay.io/openshift/ci:ocp_4.21_ovn-kubernetes; then break; fi + if [ "${r}" -eq 5 ]; then + exit 1 + fi + backoff=$(($RANDOM % 120))s + echo Sleeping randomized $backoff before retry + sleep $backoff + done + for r in {1..5}; do + _digest=$(oc image info --registry-config=/etc/push-secret/.dockerconfigjson --filter-by-os=linux/amd64 quay-proxy.ci.openshift.org/openshift/ci:ocp_4.21_ovn-kubernetes | sed -n '/^Digest:[[:space:]]/s/^Digest:[[:space:]]*//p' | head -n1) + if [ -n "${_digest}" ] && oc tag --source=docker --loglevel=2 --reference-policy='source' --import-mode='PreserveOriginal' --reference quay-proxy.ci.openshift.org/openshift/ci@${_digest} ocp/4.21:ovn-kubernetes; then + break + fi + echo "promotion-quay: digest-tag failed for ocp/4.21:ovn-kubernetes attempt ${r}/5 (QCI digest may have moved after mirror)" >&2 + if [ "${r}" -eq 5 ]; then + exit 1 + fi + echo "promotion-quay: retrying digest-tag for ocp/4.21:ovn-kubernetes (attempt $((r+1))/5 after randomized backoff)" >&2 + backoff=$(($RANDOM % 120))s + sleep "${backoff}" + done + set +e - for r in {1..2}; do echo "Tag attempt $r (all together)"; oc tag --source=docker --loglevel=2 --reference-policy='source' --import-mode='PreserveOriginal' --reference quay-proxy.ci.openshift.org/openshift/ci@sha256:bbb ci/ci-quay:sanitize-prow-jobs quay-proxy.ci.openshift.org/openshift/ci@sha256:aaa ocp/4.21:ovn-kubernetes && break; :; done + for r in {1..2}; do echo "Tag attempt $r (all together)"; oc tag --source=docker --loglevel=2 --reference-policy='source' --import-mode='PreserveOriginal' --reference quay-proxy.ci.openshift.org/openshift/ci@sha256:bbb ci/ci-quay:sanitize-prow-jobs && break; :; done for r in {1..3}; do echo "Tag attempt $r (individual)"; oc tag --source=docker --loglevel=2 --reference-policy='source' --import-mode='PreserveOriginal' --reference quay-proxy.ci.openshift.org/openshift/ci@sha256:bbb ci/ci-quay:sanitize-prow-jobs && break; backoff=$(($RANDOM % 120))s; echo Sleeping randomized $backoff before retry; sleep $backoff; done - for r in {1..3}; do echo "Tag attempt $r (individual)"; oc tag --source=docker --loglevel=2 --reference-policy='source' --import-mode='PreserveOriginal' --reference quay-proxy.ci.openshift.org/openshift/ci@sha256:aaa ocp/4.21:ovn-kubernetes && break; backoff=$(($RANDOM % 120))s; echo Sleeping randomized $backoff before retry; sleep $backoff; done set -e command: - /bin/sh diff --git a/pkg/steps/utils/image.go b/pkg/steps/utils/image.go index 7fc80cef62b..4b10c3b987f 100644 --- a/pkg/steps/utils/image.go +++ b/pkg/steps/utils/image.go @@ -43,14 +43,24 @@ func ImageDigestFor(client ctrlruntimeclient.Client, namespace func() string, na if len(image) > 0 { return fmt.Sprintf("%s@%s", registry, image), nil } - if ref == nil && findSpecTag(is, tag) == nil { + if ref == nil && !hasSpecTag(is, tag) { return "", fmt.Errorf("image stream %q has no tag %q in spec or status", name, tag) } return fmt.Sprintf("%s:%s", registry, tag), nil } } -func findSpecTag(is *imagev1.ImageStream, tag string) *coreapi.ObjectReference { +func hasSpecTag(is *imagev1.ImageStream, tag string) bool { + for _, t := range is.Spec.Tags { + if t.Name == tag { + return true + } + } + return false +} + +// FindSpecTag returns the spec tag's From reference when present. +func FindSpecTag(is *imagev1.ImageStream, tag string) *coreapi.ObjectReference { for _, t := range is.Spec.Tags { if t.Name != tag { continue @@ -60,6 +70,44 @@ func findSpecTag(is *imagev1.ImageStream, tag string) *coreapi.ObjectReference { return nil } +// OfficialImageTagFrom returns an import source from spec, then status, then quay-proxy. +func OfficialImageTagFrom(source *imagev1.ImageStream, base api.ImageStreamTagReference) *coreapi.ObjectReference { + if source != nil { + if ref := FindSpecTag(source, base.Tag); ref != nil && ref.Name != "" { + return ref + } + if ref, _ := FindStatusTag(source, base.Tag); ref != nil && ref.Name != "" { + return ref + } + } + return &coreapi.ObjectReference{Kind: "DockerImage", Name: api.QuayImageReference(base)} +} + +// ResolveOfficialInputFrom resolves consolidated ocp inputs: stable in job ns, then spec/status/quay on source IS. +// When ok is false, callers use QuayImageReference with Source policy (e.g. 4.23, 5.0). +func ResolveOfficialInputFrom(ctx context.Context, client ctrlruntimeclient.Client, jobNamespace string, base api.ImageStreamTagReference) (*coreapi.ObjectReference, bool, error) { + if !api.ConsolidatedQuayPromotionVersion(base.Name) || !api.RefersToOfficialImage(base.Namespace, api.WithoutOKD) { + return nil, false, nil + } + stable := &imagev1.ImageStream{} + if err := client.Get(ctx, ctrlruntimeclient.ObjectKey{Namespace: jobNamespace, Name: api.StableImageStream}, stable); err == nil { + if _, exists, _ := util.ResolvePullSpec(stable, base.Tag, true); exists { + return &coreapi.ObjectReference{ + Kind: "ImageStreamTag", + Namespace: jobNamespace, + Name: fmt.Sprintf("%s:%s", api.StableImageStream, base.Tag), + }, true, nil + } + } else if !kerrors.IsNotFound(err) { + return nil, false, fmt.Errorf("get stable imagestream in %s: %w", jobNamespace, err) + } + source := &imagev1.ImageStream{} + if err := client.Get(ctx, ctrlruntimeclient.ObjectKey{Namespace: base.Namespace, Name: base.Name}, source); err != nil && !kerrors.IsNotFound(err) { + return nil, false, fmt.Errorf("get source imagestream %s: %w", base.ISTagName(), err) + } + return OfficialImageTagFrom(source, base), true, nil +} + // FindStatusTag returns an object reference to a tag if // it exists in the ImageStream's Spec func FindStatusTag(is *imagev1.ImageStream, tag string) (*coreapi.ObjectReference, string) { diff --git a/pkg/steps/utils/image_test.go b/pkg/steps/utils/image_test.go index 0da6e7da9c8..4777b2ed4d1 100644 --- a/pkg/steps/utils/image_test.go +++ b/pkg/steps/utils/image_test.go @@ -11,6 +11,7 @@ import ( coreapi "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/kubernetes/scheme" ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" @@ -18,6 +19,7 @@ import ( imagev1 "github.com/openshift/api/image/v1" + "github.com/openshift/ci-tools/pkg/api" "github.com/openshift/ci-tools/pkg/testhelper" ) @@ -27,6 +29,84 @@ func init() { } } +func TestResolveOfficialInputFrom(t *testing.T) { + specPull := "quay-proxy.ci.openshift.org/openshift/ci@sha256:abc" + base := api.ImageStreamTagReference{Namespace: "ocp", Name: "4.22", Tag: "hyperkube"} + tests := []struct { + name string + base api.ImageStreamTagReference + objects []runtime.Object + wantOK bool + wantFrom *coreapi.ObjectReference + }{ + {name: "non-consolidated", base: api.ImageStreamTagReference{Namespace: "ocp", Name: "5.0", Tag: "cli"}, wantOK: false}, + { + name: "spec docker", + base: base, + objects: []runtime.Object{&imagev1.ImageStream{ + ObjectMeta: metav1.ObjectMeta{Namespace: "ocp", Name: "4.22"}, + Spec: imagev1.ImageStreamSpec{Tags: []imagev1.TagReference{{ + Name: "hyperkube", + From: &coreapi.ObjectReference{Kind: "DockerImage", Name: specPull}, + }}}, + }}, + wantOK: true, + wantFrom: &coreapi.ObjectReference{Kind: "DockerImage", Name: specPull}, + }, + { + name: "spec image stream image", + base: api.ImageStreamTagReference{Namespace: "ocp", Name: "4.22", Tag: "cli"}, + objects: []runtime.Object{&imagev1.ImageStream{ + ObjectMeta: metav1.ObjectMeta{Namespace: "ocp", Name: "4.22"}, + Spec: imagev1.ImageStreamSpec{Tags: []imagev1.TagReference{{ + Name: "cli", + From: &coreapi.ObjectReference{Kind: "ImageStreamImage", Name: "4.22@sha256:deadbeef", Namespace: "ocp"}, + }}}, + }}, + wantOK: true, + wantFrom: &coreapi.ObjectReference{Kind: "ImageStreamImage", Name: "4.22@sha256:deadbeef", Namespace: "ocp"}, + }, + { + name: "quay fallback", + base: base, + wantOK: true, + wantFrom: &coreapi.ObjectReference{Kind: "DockerImage", Name: api.QuayImageReference(base)}, + }, + { + name: "stable first", + base: api.ImageStreamTagReference{Namespace: "ocp", Name: "4.22", Tag: "cli"}, + objects: []runtime.Object{&imagev1.ImageStream{ + ObjectMeta: metav1.ObjectMeta{Namespace: "job-ns", Name: api.StableImageStream}, + Spec: imagev1.ImageStreamSpec{Tags: []imagev1.TagReference{{Name: "cli"}}}, + Status: imagev1.ImageStreamStatus{ + PublicDockerImageRepository: "registry/job-ns/stable", + Tags: []imagev1.NamedTagEventList{{Tag: "cli", Items: []imagev1.TagEvent{{Image: "sha256:1111"}}}}, + }, + }}, + wantOK: true, + wantFrom: &coreapi.ObjectReference{Kind: "ImageStreamTag", Name: "stable:cli", Namespace: "job-ns"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + client := fakectrlruntimeclient.NewClientBuilder().WithScheme(scheme.Scheme).WithRuntimeObjects(tt.objects...).Build() + from, ok, err := ResolveOfficialInputFrom(context.Background(), client, "job-ns", tt.base) + if err != nil { + t.Fatalf("ResolveOfficialInputFrom() error = %v", err) + } + if ok != tt.wantOK { + t.Fatalf("ok = %v, want %v", ok, tt.wantOK) + } + if !ok { + return + } + if from.Kind != tt.wantFrom.Kind || from.Name != tt.wantFrom.Name || from.Namespace != tt.wantFrom.Namespace { + t.Fatalf("from = %+v, want %+v", from, tt.wantFrom) + } + }) + } +} + func TestReimportTag(t *testing.T) { var testCases = []struct { name string @@ -460,3 +540,37 @@ func TestGetEvaluator(t *testing.T) { } } } + +func TestImageDigestForSpecTagWithoutFrom(t *testing.T) { + is := &imagev1.ImageStream{ + ObjectMeta: metav1.ObjectMeta{Namespace: "ns", Name: "pipeline"}, + Spec: imagev1.ImageStreamSpec{ + Tags: []imagev1.TagReference{{Name: "pending"}}, + }, + Status: imagev1.ImageStreamStatus{ + PublicDockerImageRepository: "registry/ns/pipeline", + }, + } + client := fakectrlruntimeclient.NewClientBuilder().WithScheme(scheme.Scheme).WithRuntimeObjects(is).Build() + got, err := ImageDigestFor(client, func() string { return "ns" }, "pipeline", "pending")() + if err != nil { + t.Fatalf("ImageDigestFor() error = %v", err) + } + if got != "registry/ns/pipeline:pending" { + t.Fatalf("ImageDigestFor() = %q, want %q", got, "registry/ns/pipeline:pending") + } +} + +func TestImageDigestForMissingTag(t *testing.T) { + is := &imagev1.ImageStream{ + ObjectMeta: metav1.ObjectMeta{Namespace: "ns", Name: "pipeline"}, + Status: imagev1.ImageStreamStatus{ + PublicDockerImageRepository: "registry/ns/pipeline", + }, + } + client := fakectrlruntimeclient.NewClientBuilder().WithScheme(scheme.Scheme).WithRuntimeObjects(is).Build() + _, err := ImageDigestFor(client, func() string { return "ns" }, "pipeline", "missing")() + if err == nil { + t.Fatal("ImageDigestFor() expected error for missing tag") + } +}