[WIP] refactor(ci): migrate HyperShift calico private workflow from cucushift to hypershift step registry

ci-operator/config/openshift/hypershift/openshift-hypershift-release-4.19__periodics.yaml

@@ -208,22 +208,7 @@ tests:
     env:
       BASE_DOMAIN: qe.devcluster.openshift.com
       HYPERSHIFT_GUEST_INFRA_OCP_ACCOUNT: "true"
-      TEST_ARGS: --disable-monitor=apiserver-incluster-availability,service-type-load-balancer-availability
-      TEST_SKIPS: The default cluster RBAC policy should have correct RBAC rules\|
-        Cluster scoped load balancer healthcheck port and path should be 10256/healthz\|
-        Prometheus \[apigroup:image.openshift.io\] when installed on the cluster should
-        provide named network metrics\| Unidling \[apigroup:apps.openshift.io\]\[apigroup:route.openshift.io\]
-        should work with UDP\| Unidling with Deployments \[apigroup:route.openshift.io\]
-        should work with TCP (when fully idled)\| Unidling \[apigroup:apps.openshift.io\]\[apigroup:route.openshift.io\]
-        should work with TCP (when fully idled)\| Unidling with Deployments \[apigroup:route.openshift.io\]
-        should work with UDP\|pod should not start for sysctls not on whitelist \[apigroup:k8s.cni.cncf.io\]
-        net.ipv4.conf.IFNAME.arp_filter\| pod should not start for sysctls not on
-        whitelist \[apigroup:k8s.cni.cncf.io\] net.ipv4.conf.all.send_redirects\|
-        sysctl allowlist update should start a pod with custom sysctl only when the
-        sysctl is added to whitelist\|Ensure HTTPRoute object is created
-    test:
-    - chain: hypershift-conformance
-    workflow: cucushift-installer-rehearse-aws-ipi-ovn-hypershift-private-guest-calico
+    workflow: hypershift-aws-conformance-calico-private
 - as: e2e-kubevirt-metal-conformance-calico
   capabilities:
   - intranet

ci-operator/config/openshift/hypershift/openshift-hypershift-release-4.20__periodics.yaml

@@ -220,22 +220,7 @@ tests:
     env:
       BASE_DOMAIN: qe.devcluster.openshift.com
       HYPERSHIFT_GUEST_INFRA_OCP_ACCOUNT: "true"
-      TEST_ARGS: --disable-monitor=apiserver-incluster-availability,service-type-load-balancer-availability
-      TEST_SKIPS: The default cluster RBAC policy should have correct RBAC rules\|
-        Cluster scoped load balancer healthcheck port and path should be 10256/healthz\|
-        Prometheus \[apigroup:image.openshift.io\] when installed on the cluster should
-        provide named network metrics\| Unidling \[apigroup:apps.openshift.io\]\[apigroup:route.openshift.io\]
-        should work with UDP\| Unidling with Deployments \[apigroup:route.openshift.io\]
-        should work with TCP (when fully idled)\| Unidling \[apigroup:apps.openshift.io\]\[apigroup:route.openshift.io\]
-        should work with TCP (when fully idled)\| Unidling with Deployments \[apigroup:route.openshift.io\]
-        should work with UDP\|pod should not start for sysctls not on whitelist \[apigroup:k8s.cni.cncf.io\]
-        net.ipv4.conf.IFNAME.arp_filter\| pod should not start for sysctls not on
-        whitelist \[apigroup:k8s.cni.cncf.io\] net.ipv4.conf.all.send_redirects\|
-        sysctl allowlist update should start a pod with custom sysctl only when the
-        sysctl is added to whitelist\|Ensure HTTPRoute object is created
-    test:
-    - chain: hypershift-conformance
-    workflow: cucushift-installer-rehearse-aws-ipi-ovn-hypershift-private-guest-calico
+    workflow: hypershift-aws-conformance-calico-private
 - as: e2e-kubevirt-metal-conformance-calico
   capabilities:
   - intranet

ci-operator/config/openshift/hypershift/openshift-hypershift-release-4.21__periodics.yaml

@@ -296,22 +296,7 @@ tests:
     env:
       BASE_DOMAIN: qe.devcluster.openshift.com
       HYPERSHIFT_GUEST_INFRA_OCP_ACCOUNT: "true"
-      TEST_ARGS: --disable-monitor=apiserver-incluster-availability,service-type-load-balancer-availability
-      TEST_SKIPS: The default cluster RBAC policy should have correct RBAC rules\|
-        Cluster scoped load balancer healthcheck port and path should be 10256/healthz\|
-        Prometheus \[apigroup:image.openshift.io\] when installed on the cluster should
-        provide named network metrics\| Unidling \[apigroup:apps.openshift.io\]\[apigroup:route.openshift.io\]
-        should work with UDP\| Unidling with Deployments \[apigroup:route.openshift.io\]
-        should work with TCP (when fully idled)\| Unidling \[apigroup:apps.openshift.io\]\[apigroup:route.openshift.io\]
-        should work with TCP (when fully idled)\| Unidling with Deployments \[apigroup:route.openshift.io\]
-        should work with UDP\|pod should not start for sysctls not on whitelist \[apigroup:k8s.cni.cncf.io\]
-        net.ipv4.conf.IFNAME.arp_filter\| pod should not start for sysctls not on
-        whitelist \[apigroup:k8s.cni.cncf.io\] net.ipv4.conf.all.send_redirects\|
-        sysctl allowlist update should start a pod with custom sysctl only when the
-        sysctl is added to whitelist\|Ensure HTTPRoute object is created
-    test:
-    - chain: hypershift-conformance
-    workflow: cucushift-installer-rehearse-aws-ipi-ovn-hypershift-private-guest-calico
+    workflow: hypershift-aws-conformance-calico-private
 - as: e2e-kubevirt-metal-conformance-calico
   capabilities:
   - intranet

ci-operator/config/openshift/hypershift/openshift-hypershift-release-4.22__periodics.yaml

@@ -237,6 +237,37 @@ tests:
   steps:
     cluster_profile: openshift-org-aws
     workflow: hypershift-kubevirt-csi-e2e
+- as: e2e-aws-conformance-calico
+  minimum_interval: 168h
+  steps:
+    cluster_profile: hypershift-aws
+    workflow: hypershift-aws-conformance-calico
+- as: e2e-aws-conformance-calico-private
+  minimum_interval: 168h
+  steps:
+    cluster_profile: aws-qe
+    env:
+      BASE_DOMAIN: qe.devcluster.openshift.com
+      HYPERSHIFT_GUEST_INFRA_OCP_ACCOUNT: "true"
+    workflow: hypershift-aws-conformance-calico-private
+- as: e2e-kubevirt-metal-conformance-calico
+  capabilities:
+  - intranet
+  minimum_interval: 168h
+  steps:
+    cluster_profile: equinix-ocp-hcp
+    env:
+      KONFLUX_DEPLOY_CATALOG_SOURCE: "true"
+      KONFLUX_DEPLOY_OPERATORS: "true"
+      KONFLUX_DEPLOY_SUBSCRIPTION: "false"
+      LOCAL_STORAGE_OPERATOR_SUB_SOURCE: local-storage-konflux
+      LVM_OPERATOR_SUB_CHANNEL: stable-4.22
+      LVM_OPERATOR_SUB_SOURCE: lvm-catalogsource
+      METALLB_OPERATOR_SUB_SOURCE: metallb-konflux
+      ODF_OPERATOR_SUB_CHANNEL: stable-4.21
+      ODF_OPERATOR_SUB_SOURCE: redhat-operators-v4-21
+      REDHAT_OPERATORS_INDEX_TAG: v4.21
+    workflow: hypershift-kubevirt-baremetalds-conformance-calico
 - as: e2e-azure-aks-ovn-conformance
   cron: 0 */2 * * *
   steps:

ci-operator/jobs/openshift/hypershift/openshift-hypershift-release-4.22-periodics.yaml

@@ -418,6 +418,172 @@ periodics:
     - name: result-aggregator
       secret:
         secretName: result-aggregator
+- agent: kubernetes
+  cluster: build01
+  decorate: true
+  decoration_config:
+    skip_cloning: true
+  extra_refs:
+  - base_ref: release-4.22
+    org: openshift
+    repo: hypershift
+  labels:
+    ci-operator.openshift.io/cloud: hypershift-aws
+    ci-operator.openshift.io/cloud-cluster-profile: hypershift-aws
+    ci-operator.openshift.io/variant: periodics
+    ci.openshift.io/generator: prowgen
+    job-release: "4.22"
+    pj-rehearse.openshift.io/can-be-rehearsed: "true"
+  minimum_interval: 168h
+  name: periodic-ci-openshift-hypershift-release-4.22-periodics-e2e-aws-conformance-calico
+  spec:
+    containers:
+    - args:
+      - --gcs-upload-secret=/secrets/gcs/service-account.json
+      - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson
+      - --lease-server-credentials-file=/etc/boskos/credentials
+      - --report-credentials-file=/etc/report/credentials
+      - --secret-dir=/secrets/ci-pull-credentials
+      - --target=e2e-aws-conformance-calico
+      - --variant=periodics
+      command:
+      - ci-operator
+      env:
+      - name: HTTP_SERVER_IP
+        valueFrom:
+          fieldRef:
+            fieldPath: status.podIP
+      image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest
+      imagePullPolicy: Always
+      name: ""
+      ports:
+      - containerPort: 8080
+        name: http
+      resources:
+        requests:
+          cpu: 10m
+      volumeMounts:
+      - mountPath: /etc/boskos
+        name: boskos
+        readOnly: true
+      - mountPath: /secrets/ci-pull-credentials
+        name: ci-pull-credentials
+        readOnly: true
+      - mountPath: /secrets/gcs
+        name: gcs-credentials
+        readOnly: true
+      - mountPath: /secrets/manifest-tool
+        name: manifest-tool-local-pusher
+        readOnly: true
+      - mountPath: /etc/pull-secret
+        name: pull-secret
+        readOnly: true
+      - mountPath: /etc/report
+        name: result-aggregator
+        readOnly: true
+    serviceAccountName: ci-operator
+    volumes:
+    - name: boskos
+      secret:
+        items:
+        - key: credentials
+          path: credentials
+        secretName: boskos-credentials
+    - name: ci-pull-credentials
+      secret:
+        secretName: ci-pull-credentials
+    - name: manifest-tool-local-pusher
+      secret:
+        secretName: manifest-tool-local-pusher
+    - name: pull-secret
+      secret:
+        secretName: registry-pull-credentials
+    - name: result-aggregator
+      secret:
+        secretName: result-aggregator
+- agent: kubernetes
+  cluster: build01
+  decorate: true
+  decoration_config:
+    skip_cloning: true
+  extra_refs:
+  - base_ref: release-4.22
+    org: openshift
+    repo: hypershift
+  labels:
+    ci-operator.openshift.io/cloud: aws
+    ci-operator.openshift.io/cloud-cluster-profile: aws-qe
+    ci-operator.openshift.io/variant: periodics
+    ci.openshift.io/generator: prowgen
+    job-release: "4.22"
+    pj-rehearse.openshift.io/can-be-rehearsed: "true"
+  minimum_interval: 168h
+  name: periodic-ci-openshift-hypershift-release-4.22-periodics-e2e-aws-conformance-calico-private
+  spec:
+    containers:
+    - args:
+      - --gcs-upload-secret=/secrets/gcs/service-account.json
+      - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson
+      - --lease-server-credentials-file=/etc/boskos/credentials
+      - --report-credentials-file=/etc/report/credentials
+      - --secret-dir=/secrets/ci-pull-credentials
+      - --target=e2e-aws-conformance-calico-private
+      - --variant=periodics
+      command:
+      - ci-operator
+      env:
+      - name: HTTP_SERVER_IP
+        valueFrom:
+          fieldRef:
+            fieldPath: status.podIP
+      image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest
+      imagePullPolicy: Always
+      name: ""
+      ports:
+      - containerPort: 8080
+        name: http
+      resources:
+        requests:
+          cpu: 10m
+      volumeMounts:
+      - mountPath: /etc/boskos
+        name: boskos
+        readOnly: true
+      - mountPath: /secrets/ci-pull-credentials
+        name: ci-pull-credentials
+        readOnly: true
+      - mountPath: /secrets/gcs
+        name: gcs-credentials
+        readOnly: true
+      - mountPath: /secrets/manifest-tool
+        name: manifest-tool-local-pusher
+        readOnly: true
+      - mountPath: /etc/pull-secret
+        name: pull-secret
+        readOnly: true
+      - mountPath: /etc/report
+        name: result-aggregator
+        readOnly: true
+    serviceAccountName: ci-operator
+    volumes:
+    - name: boskos
+      secret:
+        items:
+        - key: credentials
+          path: credentials
+        secretName: boskos-credentials
+    - name: ci-pull-credentials
+      secret:
+        secretName: ci-pull-credentials
+    - name: manifest-tool-local-pusher
+      secret:
+        secretName: manifest-tool-local-pusher
+    - name: pull-secret
+      secret:
+        secretName: registry-pull-credentials
+    - name: result-aggregator
+      secret:
+        secretName: result-aggregator
 - agent: kubernetes
   cluster: build01
   decorate: true
@@ -1660,6 +1826,90 @@ periodics:
     - name: result-aggregator
       secret:
         secretName: result-aggregator
+- agent: kubernetes
+  cluster: build11
+  decorate: true
+  decoration_config:
+    skip_cloning: true
+  extra_refs:
+  - base_ref: release-4.22
+    org: openshift
+    repo: hypershift
+  labels:
+    capability/intranet: intranet
+    ci-operator.openshift.io/cloud: equinix-ocp-metal
+    ci-operator.openshift.io/cloud-cluster-profile: equinix-ocp-hcp
+    ci-operator.openshift.io/variant: periodics
+    ci.openshift.io/generator: prowgen
+    job-release: "4.22"
+    pj-rehearse.openshift.io/can-be-rehearsed: "true"
+  minimum_interval: 168h
+  name: periodic-ci-openshift-hypershift-release-4.22-periodics-e2e-kubevirt-metal-conformance-calico
+  spec:
+    containers:
+    - args:
+      - --gcs-upload-secret=/secrets/gcs/service-account.json
+      - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson
+      - --lease-server-credentials-file=/etc/boskos/credentials
+      - --report-credentials-file=/etc/report/credentials
+      - --secret-dir=/secrets/ci-pull-credentials
+      - --target=e2e-kubevirt-metal-conformance-calico
+      - --variant=periodics
+      command:
+      - ci-operator
+      env:
+      - name: HTTP_SERVER_IP
+        valueFrom:
+          fieldRef:
+            fieldPath: status.podIP
+      image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest
+      imagePullPolicy: Always
+      name: ""
+      ports:
+      - containerPort: 8080
+        name: http
+      resources:
+        requests:
+          cpu: 10m
+      volumeMounts:
+      - mountPath: /etc/boskos
+        name: boskos
+        readOnly: true
+      - mountPath: /secrets/ci-pull-credentials
+        name: ci-pull-credentials
+        readOnly: true
+      - mountPath: /secrets/gcs
+        name: gcs-credentials
+        readOnly: true
+      - mountPath: /secrets/manifest-tool
+        name: manifest-tool-local-pusher
+        readOnly: true
+      - mountPath: /etc/pull-secret
+        name: pull-secret
+        readOnly: true
+      - mountPath: /etc/report
+        name: result-aggregator
+        readOnly: true
+    serviceAccountName: ci-operator
+    volumes:
+    - name: boskos
+      secret:
+        items:
+        - key: credentials
+          path: credentials
+        secretName: boskos-credentials
+    - name: ci-pull-credentials
+      secret:
+        secretName: ci-pull-credentials
+    - name: manifest-tool-local-pusher
+      secret:
+        secretName: manifest-tool-local-pusher
+    - name: pull-secret
+      secret:
+        secretName: registry-pull-credentials
+    - name: result-aggregator
+      secret:
+        secretName: result-aggregator
 - agent: kubernetes
   cluster: build01
   cron: 0 10 * * *

ci-operator/step-registry/cucushift/hypershift-extended/enable-qe/pull-secret/cucushift-hypershift-extended-enable-qe-pull-secret-commands.sh

@@ -135,6 +135,18 @@ rm /tmp/global-pull-secret.json
 echo "{\"spec\":{\"pullSecret\":{\"name\":\"$CLUSTER_NAME-pull-secret-new\"}}}" > /tmp/patch.json
 oc patch hostedclusters -n "$HYPERSHIFT_NAMESPACE" "$CLUSTER_NAME" --type=merge -p="$(cat /tmp/patch.json)"
 
+# Patching the HostedCluster pullSecret triggers a MachineDeployment rolling update
+# (new ignition/user-data). Wait for the rollout to complete before proceeding,
+# otherwise conformance tests will run on a cluster with nodes being replaced.
+echo "Waiting for MachineDeployment rollouts"
+MD_NAMESPACE="${HYPERSHIFT_NAMESPACE}-${CLUSTER_NAME}"
+timeout 5m bash -c 'until oc get machinedeployments -n "'"${MD_NAMESPACE}"'" -l "cluster.x-k8s.io/cluster-name='"${CLUSTER_NAME}"'" --no-headers 2>/dev/null | grep -q .; do sleep 10; done'
+for md in $(oc get machinedeployments -n "${MD_NAMESPACE}" -l "cluster.x-k8s.io/cluster-name=${CLUSTER_NAME}" -o jsonpath='{.items[*].metadata.name}'); do
+  oc wait machinedeployment "${md}" -n "${MD_NAMESPACE}" --for=condition=RollingOut=True --timeout=5m
+  echo "Waiting for MachineDeployment ${md} to finish rolling out..."
+  oc wait machinedeployment "${md}" -n "${MD_NAMESPACE}" --for=condition=RollingOut=False --timeout=45m
+done
+
 echo "check day-2 pull-secret update"
 export KUBECONFIG="${SHARED_DIR}/nested_kubeconfig"
 RETRIES=45