diff --git a/test/extended/networking/network_segmentation.go b/test/extended/networking/network_segmentation.go index c010f7265617..f8b3f647fc17 100644 --- a/test/extended/networking/network_segmentation.go +++ b/test/extended/networking/network_segmentation.go @@ -22,17 +22,13 @@ import ( kubeauthorizationv1 "k8s.io/api/authorization/v1" v1 "k8s.io/api/core/v1" - kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/rand" clientset "k8s.io/client-go/kubernetes" - "k8s.io/kubectl/pkg/util/podutils" "k8s.io/kubernetes/test/e2e/framework" e2ekubectl "k8s.io/kubernetes/test/e2e/framework/kubectl" frameworkpod "k8s.io/kubernetes/test/e2e/framework/pod" - admissionapi "k8s.io/pod-security-admission/api" utilnet "k8s.io/utils/net" "k8s.io/utils/pointer" @@ -63,1228 +59,6 @@ const podReadyPollInterval = 6 * time.Second const serverConnectPollTimeout = 30 * time.Second const serverConnectPollInterval = 1 * time.Second -var _ = Describe("[sig-network][OCPFeatureGate:NetworkSegmentation][Feature:UserDefinedPrimaryNetworks]", func() { - // TODO: so far, only the isolation tests actually require this PSA ... Feels wrong to run everything priviliged. - // I've tried to have multiple kubeframeworks (from multiple OCs) running (with different project names) but - // it didn't work. - // disable automatic namespace creation, we need to add the required UDN label - oc := exutil.NewCLIWithoutNamespace("network-segmentation-e2e") - f := oc.KubeFramework() - f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged - - InOVNKubernetesContext(func() { - const ( - nodeHostnameKey = "kubernetes.io/hostname" - port = 9000 - defaultPort = 8080 - userDefinedNetworkIPv4Subnet = "203.203.0.0/16" - userDefinedNetworkIPv6Subnet = "2014:100:200::0/60" - nadName = "gryffindor" - - udnCrReadyTimeout = 60 * time.Second - ) - - var ( - cs clientset.Interface - nadClient nadclient.K8sCniCncfIoV1Interface - ) - - BeforeEach(func() { - cs = f.ClientSet - - var err error - nadClient, err = nadclient.NewForConfig(f.ClientConfig()) - Expect(err).NotTo(HaveOccurred()) - }) - - DescribeTableSubtree("created using", - func(createNetworkFn func(c *networkAttachmentConfigParams) error) { - - DescribeTable( - "can perform east/west traffic between nodes", - func( - netConfig *networkAttachmentConfigParams, - clientPodConfig podConfiguration, - serverPodConfig podConfiguration, - ) { - var err error - l := map[string]string{ - "e2e-framework": f.BaseName, - } - if netConfig.role == "primary" { - l[RequiredUDNNamespaceLabel] = "" - } - ns, err := f.CreateNamespace(context.TODO(), f.BaseName, l) - Expect(err).NotTo(HaveOccurred()) - err = udnWaitForOpenShift(oc, ns.Name) - Expect(err).NotTo(HaveOccurred()) - f.Namespace = ns - - netConfig.namespace = f.Namespace.Name - // correctCIDRFamily makes use of the ginkgo framework so it needs to be in the testcase - netConfig.cidr = correctCIDRFamily(oc, userDefinedNetworkIPv4Subnet, userDefinedNetworkIPv6Subnet) - workerNodes, err := getWorkerNodesOrdered(cs) - Expect(err).NotTo(HaveOccurred()) - Expect(len(workerNodes)).To(BeNumerically(">=", 1)) - - clientPodConfig.namespace = f.Namespace.Name - clientPodConfig.nodeSelector = map[string]string{nodeHostnameKey: workerNodes[0].Name} - serverPodConfig.namespace = f.Namespace.Name - serverPodConfig.nodeSelector = map[string]string{nodeHostnameKey: workerNodes[len(workerNodes)-1].Name} - - By("creating the network") - netConfig.namespace = f.Namespace.Name - Expect(createNetworkFn(netConfig)).To(Succeed()) - - By("creating client/server pods") - runUDNPod(cs, f.Namespace.Name, serverPodConfig, nil) - runUDNPod(cs, f.Namespace.Name, clientPodConfig, nil) - - var serverIP string - for i, cidr := range strings.Split(netConfig.cidr, ",") { - if cidr != "" { - By("asserting the server pod has an IP from the configured range") - serverIP, err = podIPsForUserDefinedPrimaryNetwork( - cs, - f.Namespace.Name, - serverPodConfig.name, - namespacedName(f.Namespace.Name, netConfig.name), - i, - ) - Expect(err).NotTo(HaveOccurred()) - const netPrefixLengthPerNode = 24 - By(fmt.Sprintf("asserting the server pod IP %v is from the configured range %v/%v", serverIP, cidr, netPrefixLengthPerNode)) - subnet, err := getNetCIDRSubnet(cidr) - Expect(err).NotTo(HaveOccurred()) - Expect(inRange(subnet, serverIP)).To(Succeed()) - } - - By("asserting the *client* pod can contact the server pod exposed endpoint") - namespacePodShouldReach(oc, f.Namespace.Name, clientPodConfig.name, formatHostAndPort(net.ParseIP(serverIP), port)) - } - }, - Entry( - "for two pods connected over a L2 primary UDN", - &networkAttachmentConfigParams{ - name: nadName, - topology: "layer2", - role: "primary", - }, - *podConfig( - "client-pod", - ), - *podConfig("server-pod", withCommand(func() []string { - return httpServerContainerCmd(port) - })), - ), - Entry( - "two pods connected over a L3 primary UDN", - &networkAttachmentConfigParams{ - name: nadName, - topology: "layer3", - role: "primary", - }, - *podConfig( - "client-pod", - ), - *podConfig("server-pod", withCommand(func() []string { - return httpServerContainerCmd(port) - })), - ), - ) - - DescribeTable( - "is isolated from the default network", - func( - netConfigParams *networkAttachmentConfigParams, - udnPodConfig podConfiguration, - ) { - l := map[string]string{ - "e2e-framework": f.BaseName, - } - if netConfigParams.role == "primary" { - l[RequiredUDNNamespaceLabel] = "" - } - ns, err := f.CreateNamespace(context.TODO(), f.BaseName, l) - Expect(err).NotTo(HaveOccurred()) - err = udnWaitForOpenShift(oc, ns.Name) - Expect(err).NotTo(HaveOccurred()) - f.Namespace = ns - By("Creating second namespace for default network pods") - defaultNetNamespace := f.Namespace.Name + "-default" - _, err = cs.CoreV1().Namespaces().Create(context.Background(), &v1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: defaultNetNamespace, - }, - }, metav1.CreateOptions{}) - Expect(err).NotTo(HaveOccurred()) - defer func() { - Expect(cs.CoreV1().Namespaces().Delete(context.Background(), defaultNetNamespace, metav1.DeleteOptions{})).To(Succeed()) - }() - - By("creating the network") - netConfigParams.namespace = f.Namespace.Name - // correctCIDRFamily makes use of the ginkgo framework so it needs to be in the testcase - netConfigParams.cidr = correctCIDRFamily(oc, userDefinedNetworkIPv4Subnet, userDefinedNetworkIPv6Subnet) - Expect(createNetworkFn(netConfigParams)).To(Succeed()) - Expect(err).NotTo(HaveOccurred()) - - udnPodConfig.namespace = f.Namespace.Name - - udnPod := runUDNPod(cs, f.Namespace.Name, udnPodConfig, func(pod *v1.Pod) { - pod.Spec.Containers[0].ReadinessProbe = &v1.Probe{ - ProbeHandler: v1.ProbeHandler{ - HTTPGet: &v1.HTTPGetAction{ - Path: "/healthz", - Port: intstr.FromInt32(port), - }, - }, - InitialDelaySeconds: 5, - PeriodSeconds: 1, - // FIXME: On OCP we have seen readiness probe failures happening for the UDN pod which - // causes immediate container restarts - the first readiness probe failure usually happens because - // connection gets reset by the pod since normally a liveness probe fails first causing a - // restart that also causes the readiness probes to start failing. - // Hence increase the failure threshold to 3 tries. - FailureThreshold: 3, - TimeoutSeconds: 3, - } - pod.Spec.Containers[0].LivenessProbe = &v1.Probe{ - ProbeHandler: v1.ProbeHandler{ - HTTPGet: &v1.HTTPGetAction{ - Path: "/healthz", - Port: intstr.FromInt32(port), - }, - }, - InitialDelaySeconds: 5, - PeriodSeconds: 1, - // FIXME: On OCP we have seen liveness probe failures happening for the UDN pod which - // causes immediate container restarts. Hence increase the failure threshold to 3 tries - // TBD: We unfortunately don't know why the 1st liveness probe timesout - once we know the - // why we could bring this back to 1 even though 1 is still aggressive. - FailureThreshold: 3, - // FIXME: On OCP, we have seen this flake in the CI; example: - // Pod event: Type=Warning Reason=Unhealthy Message=Liveness probe failed: Get "http://[fd01:0:0:5::2ed]:9000/healthz": - // context deadline exceeded (Client.Timeout exceeded while awaiting headers) LastTimestamp=2025-01-21 15:16:43 +0000 UTC Count=1 - // Pod event: Type=Normal Reason=Killing Message=Container agnhost-container failed liveness probe, will be restarted - // LastTimestamp=2025-01-21 15:16:43 +0000 UTC Count=1 - // Pod event: Type=Warning Reason=Unhealthy Message=Readiness probe failed: Get "http://[fd01:0:0:5::2ed]:9000/healthz": - // context deadline exceeded (Client.Timeout exceeded while awaiting headers) LastTimestamp=2025-01-21 15:16:43 +0000 UTC Count=1 - // Pod event: Type=Warning Reason=Unhealthy Message=Readiness probe failed: Get "http://[fd01:0:0:5::2ed]:9000/healthz": - // read tcp [fd01:0:0:5::2]:33400->[fd01:0:0:5::2ed]:9000: read: connection reset by peer LastTimestamp=2025-01-21 15:16:43 +0000 UTC Count=1 - // While we don't know why 1second wasn't enough to receive the headers for the liveness probe - // it is clear the TCP conn is getting established but 1second is not enough to complete the probe. - // Let's increase the timeout to 3seconds till we understand what causes the 1st probe failure. - TimeoutSeconds: 3, - } - pod.Spec.Containers[0].StartupProbe = &v1.Probe{ - ProbeHandler: v1.ProbeHandler{ - HTTPGet: &v1.HTTPGetAction{ - Path: "/healthz", - Port: intstr.FromInt32(port), - }, - }, - InitialDelaySeconds: 5, - PeriodSeconds: 1, - FailureThreshold: 3, - // FIXME: Figure out why it sometimes takes more than 3seconds for the healthcheck to complete - TimeoutSeconds: 3, - } - // add NET_ADMIN to change pod routes - pod.Spec.Containers[0].SecurityContext = &v1.SecurityContext{ - Capabilities: &v1.Capabilities{ - Add: []v1.Capability{"NET_ADMIN"}, - }, - } - }) - - const podGenerateName = "udn-test-pod-" - By("creating default network pod") - defaultPod := frameworkpod.CreateExecPodOrFail( - context.Background(), - f.ClientSet, - defaultNetNamespace, - podGenerateName, - func(pod *v1.Pod) { - pod.Spec.Containers[0].Args = []string{"netexec"} - setRuntimeDefaultPSA(pod) - }, - ) - - By("creating default network client pod") - defaultClientPod := frameworkpod.CreateExecPodOrFail( - context.Background(), - f.ClientSet, - defaultNetNamespace, - podGenerateName, - func(pod *v1.Pod) { - setRuntimeDefaultPSA(pod) - }, - ) - - udnIPv4, udnIPv6, err := podIPsForDefaultNetwork( - cs, - f.Namespace.Name, - udnPod.GetName(), - ) - Expect(err).NotTo(HaveOccurred()) - - for _, destIP := range []string{udnIPv4, udnIPv6} { - if destIP == "" { - continue - } - // positive case for UDN pod is a successful healthcheck, checked later - By("checking the default network pod can't reach UDN pod on IP " + destIP) - Consistently(func() bool { - return connectToServer(podConfiguration{namespace: defaultPod.Namespace, name: defaultPod.Name}, destIP, port) != nil - }, serverConnectPollTimeout, serverConnectPollInterval).Should(BeTrue()) - } - - defaultIPv4, defaultIPv6, err := podIPsForDefaultNetwork( - cs, - defaultPod.Namespace, - defaultPod.Name, - ) - Expect(err).NotTo(HaveOccurred()) - - for _, destIP := range []string{defaultIPv4, defaultIPv6} { - if destIP == "" { - continue - } - By("checking the default network client pod can reach default pod on IP " + destIP) - Eventually(func() bool { - return connectToServer(podConfiguration{namespace: defaultClientPod.Namespace, name: defaultClientPod.Name}, destIP, defaultPort) == nil - }, serverConnectPollTimeout, serverConnectPollInterval).Should(BeTrue()) - By("checking the UDN pod can't reach the default network pod on IP " + destIP) - Consistently(func() bool { - return connectToServer(udnPodConfig, destIP, defaultPort) != nil - }, serverConnectPollTimeout, serverConnectPollInterval).Should(BeTrue()) - } - - // connectivity check is run every second + 1sec initialDelay - // By this time we have spent at least 20 seconds doing the above consistently checks - udnPod, err = cs.CoreV1().Pods(udnPod.Namespace).Get(context.Background(), udnPod.Name, metav1.GetOptions{}) - Expect(err).NotTo(HaveOccurred()) - Expect(udnPod.Status.ContainerStatuses[0].RestartCount).To(Equal(int32(0))) - - By("asserting healthcheck works (kubelet can access the UDN pod)") - // The pod should be ready - Expect(podutils.IsPodReady(udnPod)).To(BeTrue()) - - // TODO - //By("checking non-kubelet default network host process can't reach the UDN pod") - - By("asserting UDN pod can't reach host via default network interface") - // Now try to reach the host from the UDN pod - defaultPodHostIP := udnPod.Status.HostIPs - for _, hostIP := range defaultPodHostIP { - By("checking the UDN pod can't reach the host on IP " + hostIP.IP) - ping := "ping" - if utilnet.IsIPv6String(hostIP.IP) { - ping = "ping6" - } - Consistently(func() bool { - _, err := e2ekubectl.RunKubectl(udnPod.Namespace, "exec", udnPod.Name, "--", - ping, "-I", "eth0", "-c", "1", "-W", "1", hostIP.IP, - ) - return err == nil - }, 4*time.Second, 1*time.Second).Should(BeFalse()) - } - - By("asserting UDN pod can reach the kapi service in the default network") - // Use the service name to get test the DNS access - Consistently(func() bool { - _, err := e2ekubectl.RunKubectl( - udnPodConfig.namespace, - "exec", - udnPodConfig.name, - "--", - "curl", - "--connect-timeout", - // FIXME: We have seen in OCP CI that it can take two seconds or maybe more - // for a single curl to succeed. Example: - // STEP: asserting UDN pod can reach the kapi service in the default network @ 01/20/25 00:38:42.32 - // I0120 00:38:42.320808 70120 builder.go:121] Running '/usr/bin/kubectl - // --server=https://api.ci-op-bkg2qwwq-4edbf.XXXXXXXXXXXXXXXXXXXXXX:6443 --kubeconfig=/tmp/kubeconfig-1734723086 - // --namespace=e2e-test-network-segmentation-e2e-kzdw7 exec udn-pod -- curl --connect-timeout 2 --insecure https://kubernetes.default/healthz' - // I0120 00:38:44.108334 70120 builder.go:146] stderr: " % Total % Received % Xferd Average Speed Time Time Time Current\n Dload Upload Total Spent Left Speed\n\r 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0\r100 2 100 2 0 0 9 0 --:--:-- --:--:-- --:--:-- 9\r100 2 100 2 0 0 9 0 --:--:-- --:--:-- --:--:-- 9\n" - // I0120 00:38:44.108415 70120 builder.go:147] stdout: "ok" --> 2 seconds later - // I0120 00:38:45.109237 70120 builder.go:121] Running '/usr/bin/kubectl - // --server=https://api.ci-op-bkg2qwwq-4edbf.XXXXXXXXXXXXXXXXXXXXXX:6443 --kubeconfig=/tmp/kubeconfig-1734723086 - // --namespace=e2e-test-network-segmentation-e2e-kzdw7 exec udn-pod -- curl --connect-timeout 2 --insecure https://kubernetes.default/healthz' - // I0120 00:38:48.460089 70120 builder.go:135] rc: 28 - // around the same time we have observed OVS issues like: - // Jan 20 00:38:45.329999 ci-op-bkg2qwwq-4edbf-xv8kb-worker-b-flqxd ovs-vswitchd[1094]: ovs|03661|timeval|WARN|context switches: 0 voluntary, 695 involuntary - // Jan 20 00:38:45.329967 ci-op-bkg2qwwq-4edbf-xv8kb-worker-b-flqxd ovs-vswitchd[1094]: ovs|03660|timeval|WARN|Unreasonably long 1730ms poll interval (32ms user, 903ms system) - // which might need more investigation. Bumping the timeout to 5seconds can help with this - // but we need to figure out what exactly is causing random timeouts in CI when trying to reach kapi-server - // sometimes we have also seen more than 2seconds being taken for the timeout which also needs to be investigated: - // I0118 13:35:50.419638 87083 builder.go:121] Running '/usr/bin/kubectl - // --server=https://api.ostest.test.metalkube.org:6443 --kubeconfig=/tmp/secret/kubeconfig - // --namespace=e2e-test-network-segmentation-e2e-d4fzk exec udn-pod -- curl --connect-timeout 2 --insecure https://kubernetes.default/healthz' - // I0118 13:35:54.093268 87083 builder.go:135] rc: 28 --> takes close to 4seconds? - "5", - "--insecure", - "https://kubernetes.default/healthz") - return err == nil - }, 15*time.Second, 3*time.Second).Should(BeTrue()) - - By("asserting UDN pod can't reach default services via default network interface") - // route setup is already done, get kapi IPs - kapi, err := cs.CoreV1().Services("default").Get(context.Background(), "kubernetes", metav1.GetOptions{}) - Expect(err).NotTo(HaveOccurred()) - for _, kapiIP := range kapi.Spec.ClusterIPs { - By("checking the UDN pod can't reach kapi service on IP " + kapiIP) - Consistently(func() bool { - _, err := e2ekubectl.RunKubectl( - udnPodConfig.namespace, - "exec", - udnPodConfig.name, - "--", - "curl", - "--connect-timeout", - "2", - "--interface", - "eth0", - "--insecure", - fmt.Sprintf("https://%s/healthz", kapiIP)) - return err != nil - }, 5*time.Second, 1*time.Second).Should(BeTrue()) - } - }, - Entry( - "with L2 primary UDN", - &networkAttachmentConfigParams{ - name: nadName, - topology: "layer2", - role: "primary", - }, - *podConfig("udn-pod", withCommand(func() []string { - return httpServerContainerCmd(port) - })), - ), - Entry( - "with L3 primary UDN", - &networkAttachmentConfigParams{ - name: nadName, - topology: "layer3", - role: "primary", - }, - *podConfig("udn-pod", withCommand(func() []string { - return httpServerContainerCmd(port) - })), - ), - ) - DescribeTable( - "isolates overlapping CIDRs", - func( - topology string, - numberOfPods int, - userDefinedv4Subnet string, - userDefinedv6Subnet string, - - ) { - l := map[string]string{ - "e2e-framework": f.BaseName, - RequiredUDNNamespaceLabel: "", - } - ns, err := f.CreateNamespace(context.TODO(), f.BaseName, l) - Expect(err).NotTo(HaveOccurred()) - err = udnWaitForOpenShift(oc, ns.Name) - Expect(err).NotTo(HaveOccurred()) - f.Namespace = ns - red := "red" - blue := "blue" - - namespaceRed := f.Namespace.Name + "-" + red - namespaceBlue := f.Namespace.Name + "-" + blue - - for _, namespace := range []string{namespaceRed, namespaceBlue} { - By("Creating namespace " + namespace) - _, err := cs.CoreV1().Namespaces().Create(context.Background(), &v1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: namespace, - Labels: l, - }, - }, metav1.CreateOptions{}) - Expect(err).NotTo(HaveOccurred()) - defer func() { - By("Removing namespace " + namespace) - Expect(cs.CoreV1().Namespaces().Delete( - context.Background(), - namespace, - metav1.DeleteOptions{}, - )).To(Succeed()) - }() - } - networkNamespaceMap := map[string]string{namespaceRed: red, namespaceBlue: blue} - for namespace, network := range networkNamespaceMap { - By("creating the network " + network + " in namespace " + namespace) - netConfig := &networkAttachmentConfigParams{ - topology: topology, - cidr: correctCIDRFamily(oc, userDefinedv4Subnet, userDefinedv6Subnet), - role: "primary", - namespace: namespace, - name: network, - } - - Expect(createNetworkFn(netConfig)).To(Succeed()) - // update the name because createNetworkFn may mutate the netConfig.name - // for cluster scope objects (i.g.: CUDN cases) to enable parallel testing. - networkNamespaceMap[namespace] = netConfig.name - - } - red = networkNamespaceMap[namespaceRed] - blue = networkNamespaceMap[namespaceBlue] - - workerNodes, err := getWorkerNodesOrdered(cs) - Expect(err).NotTo(HaveOccurred()) - pods := []*v1.Pod{} - redIPs := map[string]bool{} - blueIPs := map[string]bool{} - podIPs := []string{} - bluePort := int(9091) - redPort := int(9092) - for namespace, network := range networkNamespaceMap { - for i := 0; i < numberOfPods; i++ { - httpServerPort := redPort - if network != red { - httpServerPort = bluePort - } - podConfig := *podConfig( - fmt.Sprintf("%s-pod-%d", network, i), - withCommand(func() []string { - return httpServerContainerCmd(uint16(httpServerPort)) - }), - ) - podConfig.namespace = namespace - //ensure testing accross nodes - if i%2 == 0 { - podConfig.nodeSelector = map[string]string{nodeHostnameKey: workerNodes[0].Name} - - } else { - - podConfig.nodeSelector = map[string]string{nodeHostnameKey: workerNodes[len(workerNodes)-1].Name} - } - By("creating pod " + podConfig.name + " in " + podConfig.namespace) - pod := runUDNPod( - cs, - podConfig.namespace, - podConfig, - func(pod *v1.Pod) { - setRuntimeDefaultPSA(pod) - }) - pods = append(pods, pod) - podIP, err := podIPsForUserDefinedPrimaryNetwork( - cs, - pod.Namespace, - pod.Name, - namespacedName(namespace, network), - 0, - ) - Expect(err).NotTo(HaveOccurred()) - podIPs = append(podIPs, podIP) - if network == red { - redIPs[podIP] = true - } else { - blueIPs[podIP] = true - } - } - } - - By("ensuring pods only communicate with pods in their network") - for _, pod := range pods { - isRedPod := strings.Contains(pod.Name, red) - expectedHostname := red - if !isRedPod { - expectedHostname = blue - } - for _, ip := range podIPs { - isRedIP := redIPs[ip] - httpServerPort := redPort - if !isRedIP { - httpServerPort = bluePort - } - sameNetwork := isRedPod == isRedIP - if !sameNetwork { - _, err := connectToServerWithPath(pod.Namespace, pod.Name, ip, "/hostname", httpServerPort) - Expect(err).Should(HaveOccurred(), "should isolate from different networks") - } else { - Eventually(func(g Gomega) { - result, err := connectToServerWithPath(pod.Namespace, pod.Name, ip, "/hostname", httpServerPort) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(result).To(ContainSubstring(expectedHostname)) - }). - WithTimeout(serverConnectPollTimeout). - WithPolling(serverConnectPollInterval). - Should(Succeed(), "should not isolate from same network") - } - } - } - }, - // can completely fill the L2 topology because it does not depend on the size of the clusters hostsubnet - Entry( - "with L2 primary UDN", - "layer2", - 4, - "203.203.0.0/29", - "2014:100:200::0/125", - ), - // limit the number of pods to 5 - Entry( - "with L3 primary UDN", - "layer3", - 5, - userDefinedNetworkIPv4Subnet, - userDefinedNetworkIPv6Subnet, - ), - ) - }, - Entry("NetworkAttachmentDefinitions", func(c *networkAttachmentConfigParams) error { - netConfig := newNetworkAttachmentConfig(*c) - nad := generateNAD(netConfig) - _, err := nadClient.NetworkAttachmentDefinitions(c.namespace).Create(context.Background(), nad, metav1.CreateOptions{}) - return err - }), - Entry("UserDefinedNetwork", func(c *networkAttachmentConfigParams) error { - udnManifest := generateUserDefinedNetworkManifest(c) - cleanup, err := createManifest(c.namespace, udnManifest) - DeferCleanup(cleanup) - Eventually(userDefinedNetworkReadyFunc(oc.AdminDynamicClient(), c.namespace, c.name), udnCrReadyTimeout, time.Second).Should(Succeed()) - return err - }), - Entry("ClusterUserDefinedNetwork", func(c *networkAttachmentConfigParams) error { - cudnName := randomNetworkMetaName() - c.name = cudnName - cudnManifest := generateClusterUserDefinedNetworkManifest(c) - cleanup, err := createManifest("", cudnManifest) - DeferCleanup(func() { - cleanup() - By(fmt.Sprintf("delete pods in %s namespace to unblock CUDN CR & associate NAD deletion", c.namespace)) - Expect(cs.CoreV1().Pods(c.namespace).DeleteCollection(context.Background(), metav1.DeleteOptions{}, metav1.ListOptions{})).To(Succeed()) - _, err := e2ekubectl.RunKubectl("", "delete", "clusteruserdefinednetwork", cudnName, "--wait", fmt.Sprintf("--timeout=%ds", 120)) - Expect(err).NotTo(HaveOccurred()) - }) - Eventually(clusterUserDefinedNetworkReadyFunc(oc.AdminDynamicClient(), c.name), udnCrReadyTimeout, time.Second).Should(Succeed()) - return err - }), - ) - - Context("UserDefinedNetwork CRD controller", func() { - const ( - testUdnName = "test-net" - userDefinedNetworkResource = "userdefinednetwork" - ) - - BeforeEach(func() { - namespace, err := f.CreateNamespace(context.TODO(), f.BaseName, map[string]string{ - "e2e-framework": f.BaseName, - }) - Expect(err).NotTo(HaveOccurred()) - err = udnWaitForOpenShift(oc, namespace.Name) - Expect(err).NotTo(HaveOccurred()) - f.Namespace = namespace - - By("create tests UserDefinedNetwork") - cleanup, err := createManifest(f.Namespace.Name, newUserDefinedNetworkManifest(testUdnName)) - DeferCleanup(cleanup) - Expect(err).NotTo(HaveOccurred()) - Eventually(userDefinedNetworkReadyFunc(oc.AdminDynamicClient(), f.Namespace.Name, testUdnName), udnCrReadyTimeout, time.Second).Should(Succeed()) - }) - - It("should create NetworkAttachmentDefinition according to spec", func() { - udnUidRaw, err := e2ekubectl.RunKubectl(f.Namespace.Name, "get", userDefinedNetworkResource, testUdnName, "-o", "jsonpath='{.metadata.uid}'") - Expect(err).NotTo(HaveOccurred(), "should get the UserDefinedNetwork UID") - testUdnUID := strings.Trim(udnUidRaw, "'") - - By("verify a NetworkAttachmentDefinition is created according to spec") - assertNetAttachDefManifest(nadClient, f.Namespace.Name, testUdnName, testUdnUID) - }) - - It("should delete NetworkAttachmentDefinition when UserDefinedNetwork is deleted", func() { - By("delete UserDefinedNetwork") - _, err := e2ekubectl.RunKubectl(f.Namespace.Name, "delete", userDefinedNetworkResource, testUdnName) - Expect(err).NotTo(HaveOccurred()) - - By("verify a NetworkAttachmentDefinition has been deleted") - Eventually(func() bool { - _, err := nadClient.NetworkAttachmentDefinitions(f.Namespace.Name).Get(context.Background(), testUdnName, metav1.GetOptions{}) - return err != nil && kerrors.IsNotFound(err) - }, time.Second*3, time.Second*1).Should(BeTrue(), - "NetworkAttachmentDefinition should be deleted following UserDefinedNetwork deletion") - }) - - Context("pod connected to UserDefinedNetwork", func() { - const testPodName = "test-pod-udn" - - var ( - udnInUseDeleteTimeout = 65 * time.Second - deleteNetworkTimeout = 5 * time.Second - deleteNetworkInterval = 1 * time.Second - ) - - BeforeEach(func() { - By("create pod") - networkAttachments := []nadapi.NetworkSelectionElement{ - {Name: testUdnName, Namespace: f.Namespace.Name}, - } - cfg := podConfig(testPodName, withNetworkAttachment(networkAttachments)) - cfg.namespace = f.Namespace.Name - runUDNPod(cs, f.Namespace.Name, *cfg, nil) - }) - - It("cannot be deleted when being used", func() { - By("verify UserDefinedNetwork cannot be deleted") - cmd := e2ekubectl.NewKubectlCommand(f.Namespace.Name, "delete", userDefinedNetworkResource, testUdnName) - cmd.WithTimeout(time.NewTimer(deleteNetworkTimeout).C) - _, err := cmd.Exec() - Expect(err).To(HaveOccurred(), - "should fail to delete UserDefinedNetwork when used") - - By("verify UserDefinedNetwork associated NetworkAttachmentDefinition cannot be deleted") - Eventually(func() error { - ctx, cancel := context.WithTimeout(context.Background(), deleteNetworkTimeout) - defer cancel() - _ = nadClient.NetworkAttachmentDefinitions(f.Namespace.Name).Delete(ctx, testUdnName, metav1.DeleteOptions{}) - _, err := nadClient.NetworkAttachmentDefinitions(f.Namespace.Name).Get(ctx, testUdnName, metav1.GetOptions{}) - return err - }, udnInUseDeleteTimeout, deleteNetworkInterval).ShouldNot(HaveOccurred(), - "should fail to delete UserDefinedNetwork associated NetworkAttachmentDefinition when used") - - By("verify UserDefinedNetwork status reports consuming pod") - err = validateUDNStatusReportsConsumers(oc.AdminDynamicClient(), f.Namespace.Name, testUdnName, testPodName) - Expect(err).ToNot(HaveOccurred()) - - By("delete test pod") - err = cs.CoreV1().Pods(f.Namespace.Name).Delete(context.Background(), testPodName, metav1.DeleteOptions{}) - Expect(err).ToNot(HaveOccurred()) - - By("verify UserDefinedNetwork has been deleted") - Eventually(func() error { - _, err := e2ekubectl.RunKubectl(f.Namespace.Name, "get", userDefinedNetworkResource, testUdnName) - return err - }, udnInUseDeleteTimeout, deleteNetworkInterval).Should(HaveOccurred(), - "UserDefinedNetwork should be deleted following test pod deletion") - - By("verify UserDefinedNetwork associated NetworkAttachmentDefinition has been deleted") - Eventually(func() bool { - _, err := nadClient.NetworkAttachmentDefinitions(f.Namespace.Name).Get(context.Background(), testUdnName, metav1.GetOptions{}) - return err != nil && kerrors.IsNotFound(err) - }, deleteNetworkTimeout, deleteNetworkInterval).Should(BeTrue(), - "NetworkAttachmentDefinition should be deleted following UserDefinedNetwork deletion") - }) - }) - }) - - It("when primary network exist, UserDefinedNetwork status should report not-ready", func() { - const ( - primaryNadName = "cluster-primary-net" - primaryUdnName = "primary-net" - ) - - l := map[string]string{ - "e2e-framework": f.BaseName, - RequiredUDNNamespaceLabel: "", - } - ns, err := f.CreateNamespace(context.TODO(), f.BaseName, l) - Expect(err).NotTo(HaveOccurred()) - err = udnWaitForOpenShift(oc, ns.Name) - Expect(err).NotTo(HaveOccurred()) - f.Namespace = ns - - By("create primary network NetworkAttachmentDefinition") - primaryNetNad := generateNAD(newNetworkAttachmentConfig(networkAttachmentConfigParams{ - role: "primary", - topology: "layer3", - name: primaryNadName, - networkName: primaryNadName, - cidr: correctCIDRFamily(oc, userDefinedNetworkIPv4Subnet, userDefinedNetworkIPv6Subnet), - })) - _, err = nadClient.NetworkAttachmentDefinitions(f.Namespace.Name).Create(context.Background(), primaryNetNad, metav1.CreateOptions{}) - Expect(err).NotTo(HaveOccurred()) - - By("create primary network UserDefinedNetwork") - cleanup, err := createManifest(f.Namespace.Name, newPrimaryUserDefinedNetworkManifest(oc, primaryUdnName)) - DeferCleanup(cleanup) - Expect(err).NotTo(HaveOccurred()) - - expectedMessage := fmt.Sprintf("primary network already exist in namespace %q: %q", f.Namespace.Name, primaryNadName) - Eventually(func(g Gomega) []metav1.Condition { - conditionsJSON, err := e2ekubectl.RunKubectl(f.Namespace.Name, "get", "userdefinednetwork", primaryUdnName, "-o", "jsonpath={.status.conditions}") - g.Expect(err).NotTo(HaveOccurred()) - var actualConditions []metav1.Condition - g.Expect(json.Unmarshal([]byte(conditionsJSON), &actualConditions)).To(Succeed()) - return normalizeConditions(actualConditions) - }, 5*time.Second, 1*time.Second).Should(SatisfyAny( - ConsistOf(metav1.Condition{ - Type: "NetworkCreated", - Status: metav1.ConditionFalse, - Reason: "SyncError", - Message: expectedMessage, - }), - ConsistOf(metav1.Condition{ - Type: "NetworkReady", - Status: metav1.ConditionFalse, - Reason: "SyncError", - Message: expectedMessage, - }), - )) - }) - - Context("ClusterUserDefinedNetwork CRD Controller", func() { - const clusterUserDefinedNetworkResource = "clusteruserdefinednetwork" - - var testTenantNamespaces []string - var defaultNetNamespace *v1.Namespace - - BeforeEach(func() { - namespace, err := f.CreateNamespace(context.TODO(), f.BaseName, map[string]string{ - "e2e-framework": f.BaseName, - RequiredUDNNamespaceLabel: "", - }) - f.Namespace = namespace - Expect(err).NotTo(HaveOccurred()) - err = udnWaitForOpenShift(oc, namespace.Name) - Expect(err).NotTo(HaveOccurred()) - testTenantNamespaces = []string{ - f.Namespace.Name + "blue", - f.Namespace.Name + "red", - } - - By("Creating test tenants namespaces") - for _, nsName := range testTenantNamespaces { - _, err := cs.CoreV1().Namespaces().Create(context.Background(), &v1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: nsName, - Labels: map[string]string{RequiredUDNNamespaceLabel: ""}, - }}, metav1.CreateOptions{}) - Expect(err).NotTo(HaveOccurred()) - DeferCleanup(func() error { - err := cs.CoreV1().Namespaces().Delete(context.Background(), nsName, metav1.DeleteOptions{}) - return err - }) - } - // default cluster network namespace, for use when only testing secondary UDNs/NADs - defaultNetNamespace = &v1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: f.Namespace.Name + "-default", - }, - } - f.AddNamespacesToDelete(defaultNetNamespace) - _, err = cs.CoreV1().Namespaces().Create(context.Background(), defaultNetNamespace, metav1.CreateOptions{}) - Expect(err).NotTo(HaveOccurred()) - testTenantNamespaces = append(testTenantNamespaces, defaultNetNamespace.Name) - }) - - var testClusterUdnName string - - BeforeEach(func() { - testClusterUdnName = randomNetworkMetaName() - By("create test CR") - cleanup, err := createManifest("", newClusterUDNManifest(testClusterUdnName, testTenantNamespaces...)) - DeferCleanup(func() error { - cleanup() - _, _ = e2ekubectl.RunKubectl("", "delete", clusterUserDefinedNetworkResource, testClusterUdnName) - Eventually(func() error { - _, err := e2ekubectl.RunKubectl("", "get", clusterUserDefinedNetworkResource, testClusterUdnName) - return err - }, 1*time.Minute, 3*time.Second).Should(MatchError(ContainSubstring(fmt.Sprintf("clusteruserdefinednetworks.k8s.ovn.org %q not found", testClusterUdnName)))) - return nil - }) - Expect(err).NotTo(HaveOccurred()) - Eventually(clusterUserDefinedNetworkReadyFunc(oc.AdminDynamicClient(), testClusterUdnName), udnCrReadyTimeout, time.Second).Should(Succeed()) - }) - - It("should create NAD according to spec in each target namespace and report active namespaces", func() { - Eventually( - validateClusterUDNStatusReportsActiveNamespacesFunc(oc.AdminDynamicClient(), testClusterUdnName, testTenantNamespaces...), - 1*time.Minute, 3*time.Second).Should(Succeed()) - - udnUidRaw, err := e2ekubectl.RunKubectl("", "get", clusterUserDefinedNetworkResource, testClusterUdnName, "-o", "jsonpath='{.metadata.uid}'") - Expect(err).NotTo(HaveOccurred(), "should get the ClsuterUserDefinedNetwork UID") - testUdnUID := strings.Trim(udnUidRaw, "'") - - By("verify a NetworkAttachmentDefinition is created according to spec") - for _, testNsName := range testTenantNamespaces { - assertClusterNADManifest(nadClient, testNsName, testClusterUdnName, testUdnUID) - } - }) - - It("when CR is deleted, should delete all managed NAD in each target namespace", func() { - By("delete test CR") - _, err := e2ekubectl.RunKubectl("", "delete", clusterUserDefinedNetworkResource, testClusterUdnName) - Expect(err).NotTo(HaveOccurred()) - - for _, nsName := range testTenantNamespaces { - By(fmt.Sprintf("verify a NAD has been deleted from namesapce %q", nsName)) - Eventually(func() bool { - _, err := nadClient.NetworkAttachmentDefinitions(nsName).Get(context.Background(), testClusterUdnName, metav1.GetOptions{}) - return err != nil && kerrors.IsNotFound(err) - }, time.Second*3, time.Second*1).Should(BeTrue(), - "NADs in target namespaces should be deleted following ClusterUserDefinedNetwork deletion") - } - }) - - It("should create NAD in new created namespaces that apply to namespace-selector", func() { - testNewNs := f.Namespace.Name + "green" - - By("add new target namespace to CR namespace-selector") - patch := fmt.Sprintf(`[{"op": "add", "path": "./spec/namespaceSelector/matchExpressions/0/values/-", "value": "%s"}]`, testNewNs) - _, err := e2ekubectl.RunKubectl("", "patch", clusterUserDefinedNetworkResource, testClusterUdnName, "--type=json", "-p="+patch) - Expect(err).NotTo(HaveOccurred()) - Eventually(clusterUserDefinedNetworkReadyFunc(oc.AdminDynamicClient(), testClusterUdnName), udnCrReadyTimeout, time.Second).Should(Succeed()) - Eventually( - validateClusterUDNStatusReportsActiveNamespacesFunc(oc.AdminDynamicClient(), testClusterUdnName, testTenantNamespaces...), - 1*time.Minute, 3*time.Second).Should(Succeed()) - - By("create the new target namespace") - _, err = cs.CoreV1().Namespaces().Create(context.Background(), &v1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: testNewNs, - Labels: map[string]string{RequiredUDNNamespaceLabel: ""}, - }}, metav1.CreateOptions{}) - Expect(err).NotTo(HaveOccurred()) - DeferCleanup(func() error { - err := cs.CoreV1().Namespaces().Delete(context.Background(), testNewNs, metav1.DeleteOptions{}) - return err - }) - - expectedActiveNamespaces := append(testTenantNamespaces, testNewNs) - Eventually( - validateClusterUDNStatusReportsActiveNamespacesFunc(oc.AdminDynamicClient(), testClusterUdnName, expectedActiveNamespaces...), - 1*time.Minute, 3*time.Second).Should(Succeed()) - - udnUidRaw, err := e2ekubectl.RunKubectl("", "get", clusterUserDefinedNetworkResource, testClusterUdnName, "-o", "jsonpath='{.metadata.uid}'") - Expect(err).NotTo(HaveOccurred(), "should get the ClsuterUserDefinedNetwork UID") - testUdnUID := strings.Trim(udnUidRaw, "'") - - By("verify a NAD exist in new namespace according to spec") - assertClusterNADManifest(nadClient, testNewNs, testClusterUdnName, testUdnUID) - }) - - When("namespace-selector is mutated", func() { - It("should create NAD in namespaces that apply to mutated namespace-selector", func() { - testNewNs := f.Namespace.Name + "green" - - By("create new namespace") - _, err := cs.CoreV1().Namespaces().Create(context.Background(), &v1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: testNewNs, - Labels: map[string]string{RequiredUDNNamespaceLabel: ""}, - }}, metav1.CreateOptions{}) - Expect(err).NotTo(HaveOccurred()) - DeferCleanup(func() error { - err := cs.CoreV1().Namespaces().Delete(context.Background(), testNewNs, metav1.DeleteOptions{}) - return err - }) - - By("add new namespace to CR namespace-selector") - patch := fmt.Sprintf(`[{"op": "add", "path": "./spec/namespaceSelector/matchExpressions/0/values/-", "value": "%s"}]`, testNewNs) - _, err = e2ekubectl.RunKubectl("", "patch", clusterUserDefinedNetworkResource, testClusterUdnName, "--type=json", "-p="+patch) - Expect(err).NotTo(HaveOccurred()) - - By("verify status reports the new added namespace as active") - expectedActiveNs := append(testTenantNamespaces, testNewNs) - Eventually( - validateClusterUDNStatusReportsActiveNamespacesFunc(oc.AdminDynamicClient(), testClusterUdnName, expectedActiveNs...), - 1*time.Minute, 3*time.Second).Should(Succeed()) - - By("verify a NAD is created in new target namespace according to spec") - udnUidRaw, err := e2ekubectl.RunKubectl("", "get", clusterUserDefinedNetworkResource, testClusterUdnName, "-o", "jsonpath='{.metadata.uid}'") - Expect(err).NotTo(HaveOccurred(), "should get the ClusterUserDefinedNetwork UID") - testUdnUID := strings.Trim(udnUidRaw, "'") - assertClusterNADManifest(nadClient, testNewNs, testClusterUdnName, testUdnUID) - }) - - It("should delete managed NAD in namespaces that no longer apply to namespace-selector", func() { - By("remove one active namespace from CR namespace-selector") - activeTenantNs := testTenantNamespaces[1] - patch := fmt.Sprintf(`[{"op": "replace", "path": "./spec/namespaceSelector/matchExpressions/0/values", "value": [%q]}]`, activeTenantNs) - _, err := e2ekubectl.RunKubectl("", "patch", clusterUserDefinedNetworkResource, testClusterUdnName, "--type=json", "-p="+patch) - Expect(err).NotTo(HaveOccurred()) - - By("verify status reports remained target namespaces only as active") - expectedActiveNs := []string{activeTenantNs} - Eventually( - validateClusterUDNStatusReportsActiveNamespacesFunc(oc.AdminDynamicClient(), testClusterUdnName, expectedActiveNs...), - 1*time.Minute, 3*time.Second).Should(Succeed()) - - removedTenantNs := testTenantNamespaces[0] - By("verify managed NAD not exist in removed target namespace") - Eventually(func() bool { - _, err := nadClient.NetworkAttachmentDefinitions(removedTenantNs).Get(context.Background(), testClusterUdnName, metav1.GetOptions{}) - return err != nil && kerrors.IsNotFound(err) - }, time.Second*300, time.Second*1).Should(BeTrue(), - "NAD in target namespaces should be deleted following CR namespace-selector mutation") - }) - }) - - Context("pod connected to ClusterUserDefinedNetwork", func() { - const testPodName = "test-pod-cluster-udn" - - var ( - udnInUseDeleteTimeout = 65 * time.Second - deleteNetworkTimeout = 5 * time.Second - deleteNetworkInterval = 1 * time.Second - - inUseNetTestTenantNamespace string - ) - - BeforeEach(func() { - inUseNetTestTenantNamespace = defaultNetNamespace.Name - - By("create pod in one of the test tenant namespaces") - networkAttachments := []nadapi.NetworkSelectionElement{ - {Name: testClusterUdnName, Namespace: inUseNetTestTenantNamespace}, - } - cfg := podConfig(testPodName, withNetworkAttachment(networkAttachments)) - cfg.namespace = inUseNetTestTenantNamespace - runUDNPod(cs, inUseNetTestTenantNamespace, *cfg, setRuntimeDefaultPSA) - }) - - It("CR & managed NADs cannot be deleted when being used", func() { - By("verify CR cannot be deleted") - cmd := e2ekubectl.NewKubectlCommand("", "delete", clusterUserDefinedNetworkResource, testClusterUdnName) - cmd.WithTimeout(time.NewTimer(deleteNetworkTimeout).C) - _, err := cmd.Exec() - Expect(err).To(HaveOccurred(), "should fail to delete ClusterUserDefinedNetwork when used") - - By("verify CR associate NAD cannot be deleted") - Eventually(func() error { - ctx, cancel := context.WithTimeout(context.Background(), deleteNetworkTimeout) - defer cancel() - _ = nadClient.NetworkAttachmentDefinitions(inUseNetTestTenantNamespace).Delete(ctx, testClusterUdnName, metav1.DeleteOptions{}) - _, err := nadClient.NetworkAttachmentDefinitions(inUseNetTestTenantNamespace).Get(ctx, testClusterUdnName, metav1.GetOptions{}) - return err - }, udnInUseDeleteTimeout, deleteNetworkInterval).ShouldNot(HaveOccurred(), - "should fail to delete UserDefinedNetwork associated NetworkAttachmentDefinition when used") - - By("verify CR status reports consuming pod") - err = validateClusterUDNStatusReportConsumers(oc.AdminDynamicClient(), testClusterUdnName, inUseNetTestTenantNamespace, testPodName) - Expect(err).NotTo(HaveOccurred()) - - By("delete test pod") - err = cs.CoreV1().Pods(inUseNetTestTenantNamespace).Delete(context.Background(), testPodName, metav1.DeleteOptions{}) - Expect(err).ToNot(HaveOccurred()) - - By("verify CR is gone") - Eventually(func() error { - _, err := e2ekubectl.RunKubectl("", "get", clusterUserDefinedNetworkResource, testClusterUdnName) - return err - }, udnInUseDeleteTimeout, deleteNetworkInterval).Should(HaveOccurred(), - "ClusterUserDefinedNetwork should be deleted following test pod deletion") - - By("verify CR associate NADs are gone") - for _, nsName := range testTenantNamespaces { - Eventually(func() bool { - _, err := nadClient.NetworkAttachmentDefinitions(nsName).Get(context.Background(), testClusterUdnName, metav1.GetOptions{}) - return err != nil && kerrors.IsNotFound(err) - }, deleteNetworkTimeout, deleteNetworkInterval).Should(BeTrue(), - "NADs in target namespaces should be deleted following ClusterUserDefinedNetwork deletion") - } - }) - }) - }) - - It("when primary network exist, ClusterUserDefinedNetwork status should report not-ready", func() { - namespace, err := f.CreateNamespace(context.TODO(), f.BaseName, map[string]string{ - "e2e-framework": f.BaseName, - RequiredUDNNamespaceLabel: "", - }) - Expect(err).NotTo(HaveOccurred()) - err = udnWaitForOpenShift(oc, namespace.Name) - Expect(err).NotTo(HaveOccurred()) - f.Namespace = namespace - testTenantNamespaces := []string{ - f.Namespace.Name + "blue", - f.Namespace.Name + "red", - } - By("Creating test tenants namespaces") - for _, nsName := range testTenantNamespaces { - _, err := cs.CoreV1().Namespaces().Create(context.Background(), &v1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: nsName, - Labels: map[string]string{RequiredUDNNamespaceLabel: ""}, - }}, metav1.CreateOptions{}) - Expect(err).NotTo(HaveOccurred()) - DeferCleanup(func() error { - err := cs.CoreV1().Namespaces().Delete(context.Background(), nsName, metav1.DeleteOptions{}) - return err - }) - } - - By("create primary network NAD in one of the tenant namespaces") - const primaryNadName = "some-primary-net" - primaryNetTenantNs := testTenantNamespaces[0] - primaryNetNad := generateNAD(newNetworkAttachmentConfig(networkAttachmentConfigParams{ - role: "primary", - topology: "layer3", - name: primaryNadName, - networkName: primaryNadName, - cidr: correctCIDRFamily(oc, userDefinedNetworkIPv4Subnet, userDefinedNetworkIPv6Subnet), - })) - _, err = nadClient.NetworkAttachmentDefinitions(primaryNetTenantNs).Create(context.Background(), primaryNetNad, metav1.CreateOptions{}) - Expect(err).NotTo(HaveOccurred()) - - By("create primary Cluster UDN CR") - cudnName := randomNetworkMetaName() - cleanup, err := createManifest(f.Namespace.Name, newPrimaryClusterUDNManifest(oc, cudnName, testTenantNamespaces...)) - Expect(err).NotTo(HaveOccurred()) - DeferCleanup(func() { - cleanup() - _, err := e2ekubectl.RunKubectl("", "delete", "clusteruserdefinednetwork", cudnName, "--wait", fmt.Sprintf("--timeout=%ds", 60)) - Expect(err).NotTo(HaveOccurred()) - }) - - expectedMessage := fmt.Sprintf("primary network already exist in namespace %q: %q", primaryNetTenantNs, primaryNadName) - Eventually(func(g Gomega) []metav1.Condition { - conditionsJSON, err := e2ekubectl.RunKubectl(f.Namespace.Name, "get", "clusteruserdefinednetwork", cudnName, "-o", "jsonpath={.status.conditions}") - g.Expect(err).NotTo(HaveOccurred()) - var actualConditions []metav1.Condition - g.Expect(json.Unmarshal([]byte(conditionsJSON), &actualConditions)).To(Succeed()) - return normalizeConditions(actualConditions) - }, 5*time.Second, 1*time.Second).Should(SatisfyAny( - ContainElement(metav1.Condition{ - Type: "NetworkReady", - Status: metav1.ConditionFalse, - Reason: "NetworkAttachmentDefinitionSyncError", - Message: expectedMessage, - }), - ContainElement(metav1.Condition{ - Type: "NetworkCreated", - Status: metav1.ConditionFalse, - Reason: "NetworkAttachmentDefinitionSyncError", - Message: expectedMessage, - }), - )) - }) - - Context("UDN Pod", func() { - const ( - testUdnName = "test-net" - testPodName = "test-pod-udn" - ) - - var udnPod *v1.Pod - - BeforeEach(func() { - l := map[string]string{ - "e2e-framework": f.BaseName, - RequiredUDNNamespaceLabel: "", - } - ns, err := f.CreateNamespace(context.TODO(), f.BaseName, l) - Expect(err).NotTo(HaveOccurred()) - err = udnWaitForOpenShift(oc, ns.Name) - Expect(err).NotTo(HaveOccurred()) - f.Namespace = ns - By("create tests UserDefinedNetwork") - cleanup, err := createManifest(f.Namespace.Name, newPrimaryUserDefinedNetworkManifest(oc, testUdnName)) - DeferCleanup(cleanup) - Expect(err).NotTo(HaveOccurred()) - Eventually(userDefinedNetworkReadyFunc(oc.AdminDynamicClient(), f.Namespace.Name, testUdnName), udnCrReadyTimeout, time.Second).Should(Succeed()) - By("create UDN pod") - cfg := podConfig(testPodName, withCommand(func() []string { - return httpServerContainerCmd(port) - })) - cfg.namespace = f.Namespace.Name - udnPod = runUDNPod(cs, f.Namespace.Name, *cfg, nil) - }) - - It("should react to k8s.ovn.org/open-default-ports annotations changes", func() { - By("Creating second namespace for default network pod") - defaultNetNamespace := f.Namespace.Name + "-default" - _, err := cs.CoreV1().Namespaces().Create(context.Background(), &v1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: defaultNetNamespace, - }, - }, metav1.CreateOptions{}) - Expect(err).NotTo(HaveOccurred()) - defer func() { - Expect(cs.CoreV1().Namespaces().Delete(context.Background(), defaultNetNamespace, metav1.DeleteOptions{})).To(Succeed()) - }() - - By("creating default network client pod") - defaultClientPod := frameworkpod.CreateExecPodOrFail( - context.Background(), - f.ClientSet, - defaultNetNamespace, - "default-net-client-pod", - func(pod *v1.Pod) { - pod.Spec.Containers[0].Args = []string{"netexec"} - setRuntimeDefaultPSA(pod) - }, - ) - - udnIPv4, udnIPv6, err := podIPsForDefaultNetwork( - cs, - f.Namespace.Name, - udnPod.GetName(), - ) - Expect(err).NotTo(HaveOccurred()) - - By(fmt.Sprintf("verify default network client pod can't access UDN pod on port %d", port)) - for _, destIP := range []string{udnIPv4, udnIPv6} { - if destIP == "" { - continue - } - By("checking the default network pod can't reach UDN pod on IP " + destIP) - Consistently(func() bool { - return connectToServer(podConfiguration{namespace: defaultClientPod.Namespace, name: defaultClientPod.Name}, destIP, port) != nil - }, serverConnectPollTimeout, serverConnectPollInterval).Should(BeTrue()) - } - - By("Open UDN pod port") - udnPod.Annotations[openDefaultPortsAnnotation] = fmt.Sprintf( - `- protocol: tcp - port: %d`, port) - udnPod, err = cs.CoreV1().Pods(udnPod.Namespace).Update(context.Background(), udnPod, metav1.UpdateOptions{}) - Expect(err).NotTo(HaveOccurred()) - - By(fmt.Sprintf("verify default network client pod can access UDN pod on open port %d", port)) - for _, destIP := range []string{udnIPv4, udnIPv6} { - if destIP == "" { - continue - } - By("checking the default network pod can reach UDN pod on IP " + destIP) - Eventually(func() bool { - return connectToServer(podConfiguration{namespace: defaultClientPod.Namespace, name: defaultClientPod.Name}, destIP, port) == nil - }, serverConnectPollTimeout, serverConnectPollInterval).Should(BeTrue()) - } - - By("Update UDN pod port with the wrong syntax") - // this should clean up open ports and throw an event - udnPod.Annotations[openDefaultPortsAnnotation] = fmt.Sprintf( - `- protocol: ppp - port: %d`, port) - udnPod, err = cs.CoreV1().Pods(udnPod.Namespace).Update(context.Background(), udnPod, metav1.UpdateOptions{}) - Expect(err).NotTo(HaveOccurred()) - - By(fmt.Sprintf("verify default network client pod can't access UDN pod on port %d", port)) - for _, destIP := range []string{udnIPv4, udnIPv6} { - if destIP == "" { - continue - } - By("checking the default network pod can't reach UDN pod on IP " + destIP) - Eventually(func() bool { - return connectToServer(podConfiguration{namespace: defaultClientPod.Namespace, name: defaultClientPod.Name}, destIP, port) != nil - }, serverConnectPollTimeout, serverConnectPollInterval).Should(BeTrue()) - } - By("Verify syntax error is reported via event") - events, err := cs.CoreV1().Events(udnPod.Namespace).List(context.Background(), metav1.ListOptions{}) - Expect(err).NotTo(HaveOccurred()) - found := false - for _, event := range events.Items { - if event.Reason == "ErrorUpdatingResource" && strings.Contains(event.Message, "invalid protocol ppp") { - found = true - break - } - } - Expect(found).To(BeTrue(), "should have found an event for invalid protocol") - }) - }) - }) -}) - // randomNetworkMetaName return pseudo random name for network related objects (NAD,UDN,CUDN). // CUDN is cluster-scoped object, in case tests running in parallel, having random names avoids // conflicting with other tests. diff --git a/test/extended/networking/network_segmentation_policy.go b/test/extended/networking/network_segmentation_policy.go deleted file mode 100644 index 8927d7a1320f..000000000000 --- a/test/extended/networking/network_segmentation_policy.go +++ /dev/null @@ -1,342 +0,0 @@ -package networking - -import ( - "context" - "fmt" - "net" - "strings" - - nadclient "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/clientset/versioned/typed/k8s.cni.cncf.io/v1" - "github.com/onsi/ginkgo/v2" - "github.com/onsi/gomega" - exutil "github.com/openshift/origin/test/extended/util" - - v1 "k8s.io/api/core/v1" - knet "k8s.io/api/networking/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/rand" - clientset "k8s.io/client-go/kubernetes" - "k8s.io/kubernetes/test/e2e/framework" - admissionapi "k8s.io/pod-security-admission/api" -) - -var _ = ginkgo.Describe("[sig-network][OCPFeatureGate:NetworkSegmentation][Feature:UserDefinedPrimaryNetworks] Network Policies", func() { - defer ginkgo.GinkgoRecover() - - // disable automatic namespace creation, we need to add the required UDN label - oc := exutil.NewCLIWithoutNamespace("network-segmentation-policy-e2e") - f := oc.KubeFramework() - f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged - InOVNKubernetesContext(func() { - const ( - nodeHostnameKey = "kubernetes.io/hostname" - nadName = "tenant-red" - userDefinedNetworkIPv4Subnet = "203.203.0.0/16" - userDefinedNetworkIPv6Subnet = "2014:100:200::0/60" - port = 9000 - randomStringLength = 5 - nameSpaceYellowSuffix = "yellow" - namespaceBlueSuffix = "blue" - ) - - var ( - cs clientset.Interface - nadClient nadclient.K8sCniCncfIoV1Interface - allowServerPodLabel = map[string]string{"foo": "bar"} - denyServerPodLabel = map[string]string{"abc": "xyz"} - ) - - ginkgo.BeforeEach(func() { - cs = f.ClientSet - namespace, err := f.CreateNamespace(context.TODO(), f.BaseName, map[string]string{ - "e2e-framework": f.BaseName, - RequiredUDNNamespaceLabel: "", - }) - f.Namespace = namespace - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = udnWaitForOpenShift(oc, namespace.Name) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - nadClient, err = nadclient.NewForConfig(f.ClientConfig()) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - namespaceYellow := getNamespaceName(f, nameSpaceYellowSuffix) - namespaceBlue := getNamespaceName(f, namespaceBlueSuffix) - for _, namespace := range []string{namespaceYellow, namespaceBlue} { - ginkgo.By("Creating namespace " + namespace) - ns, err := cs.CoreV1().Namespaces().Create(context.Background(), &v1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: namespace, - Labels: map[string]string{RequiredUDNNamespaceLabel: ""}, - }, - }, metav1.CreateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - f.AddNamespacesToDelete(ns) - } - }) - - ginkgo.AfterEach(func() { - if ginkgo.CurrentSpecReport().Failed() { - exutil.DumpPodStatesInNamespace(f.Namespace.Name, oc) - exutil.DumpPodStatesInNamespace(getNamespaceName(f, nameSpaceYellowSuffix), oc) - exutil.DumpPodStatesInNamespace(getNamespaceName(f, namespaceBlueSuffix), oc) - } - }) - - ginkgo.DescribeTable( - "pods within namespace should be isolated when deny policy is present", - func( - topology string, - clientPodConfig podConfiguration, - serverPodConfig podConfiguration, - ) { - ginkgo.By("Creating the attachment configuration") - netConfig := newNetworkAttachmentConfig(networkAttachmentConfigParams{ - name: nadName, - topology: topology, - cidr: correctCIDRFamily(oc, userDefinedNetworkIPv4Subnet, userDefinedNetworkIPv6Subnet), - role: "primary", - }) - netConfig.namespace = f.Namespace.Name - _, err := nadClient.NetworkAttachmentDefinitions(f.Namespace.Name).Create( - context.Background(), - generateNAD(netConfig), - metav1.CreateOptions{}, - ) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - workerNodes, err := getWorkerNodesOrdered(cs) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(len(workerNodes)).To(gomega.BeNumerically(">=", 1)) - - ginkgo.By("creating client/server pods") - clientPodConfig.namespace = f.Namespace.Name - clientPodConfig.nodeSelector = map[string]string{nodeHostnameKey: workerNodes[0].Name} - serverPodConfig.namespace = f.Namespace.Name - serverPodConfig.nodeSelector = map[string]string{nodeHostnameKey: workerNodes[len(workerNodes)-1].Name} - runUDNPod(cs, f.Namespace.Name, serverPodConfig, nil) - runUDNPod(cs, f.Namespace.Name, clientPodConfig, nil) - - var serverIP string - for i, cidr := range strings.Split(netConfig.cidr, ",") { - if cidr != "" { - ginkgo.By("asserting the server pod has an IP from the configured range") - serverIP, err = podIPsForUserDefinedPrimaryNetwork( - cs, - f.Namespace.Name, - serverPodConfig.name, - namespacedName(f.Namespace.Name, netConfig.name), - i, - ) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - ginkgo.By(fmt.Sprintf("asserting the server pod IP %v is from the configured range %v", serverIP, cidr)) - subnet, err := getNetCIDRSubnet(cidr) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(inRange(subnet, serverIP)).To(gomega.Succeed()) - } - - ginkgo.By("asserting the *client* pod can contact the server pod exposed endpoint") - namespacePodShouldReach(oc, f.Namespace.Name, clientPodConfig.name, formatHostAndPort(net.ParseIP(serverIP), port)) - } - - ginkgo.By("creating a \"default deny\" network policy") - _, err = makeDenyAllPolicy(f, f.Namespace.Name) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("asserting the *client* pod can not contact the server pod exposed endpoint") - podShouldNotReach(oc, clientPodConfig.name, formatHostAndPort(net.ParseIP(serverIP), port)) - - }, - ginkgo.Entry( - "in L2 dualstack primary UDN", - "layer2", - *podConfig( - "client-pod", - ), - *podConfig( - "server-pod", - withCommand(func() []string { - return httpServerContainerCmd(port) - }), - ), - ), - ginkgo.Entry( - "in L3 dualstack primary UDN", - "layer3", - *podConfig( - "client-pod", - ), - *podConfig( - "server-pod", - withCommand(func() []string { - return httpServerContainerCmd(port) - }), - ), - ), - ) - - ginkgo.DescribeTable( - "allow ingress traffic to one pod from a particular namespace", - func( - topology string, - clientPodConfig podConfiguration, - allowServerPodConfig podConfiguration, - denyServerPodConfig podConfiguration, - ) { - - namespaceYellow := getNamespaceName(f, nameSpaceYellowSuffix) - namespaceBlue := getNamespaceName(f, namespaceBlueSuffix) - - nad := networkAttachmentConfigParams{ - topology: topology, - cidr: correctCIDRFamily(oc, userDefinedNetworkIPv4Subnet, userDefinedNetworkIPv6Subnet), - // Both yellow and blue namespaces are going to served by green network. - // Use random suffix for the network name to avoid race between tests. - networkName: fmt.Sprintf("%s-%s", "green", rand.String(randomStringLength)), - role: "primary", - } - - // Use random suffix in net conf name to avoid race between tests. - netConfName := fmt.Sprintf("sharednet-%s", rand.String(randomStringLength)) - for _, namespace := range []string{namespaceYellow, namespaceBlue} { - ginkgo.By("creating the attachment configuration for " + netConfName + " in namespace " + namespace) - netConfig := newNetworkAttachmentConfig(nad) - netConfig.namespace = namespace - netConfig.name = netConfName - - _, err := nadClient.NetworkAttachmentDefinitions(namespace).Create( - context.Background(), - generateNAD(netConfig), - metav1.CreateOptions{}, - ) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - - workerNodes, err := getWorkerNodesOrdered(cs) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(len(workerNodes)).To(gomega.BeNumerically(">=", 1)) - - ginkgo.By("creating client/server pods") - allowServerPodConfig.namespace = namespaceYellow - allowServerPodConfig.nodeSelector = map[string]string{nodeHostnameKey: workerNodes[len(workerNodes)-1].Name} - denyServerPodConfig.namespace = namespaceYellow - denyServerPodConfig.nodeSelector = map[string]string{nodeHostnameKey: workerNodes[len(workerNodes)-1].Name} - clientPodConfig.namespace = namespaceBlue - clientPodConfig.nodeSelector = map[string]string{nodeHostnameKey: workerNodes[0].Name} - runUDNPod(cs, namespaceYellow, allowServerPodConfig, func(pod *v1.Pod) { - setRuntimeDefaultPSA(pod) - }) - runUDNPod(cs, namespaceYellow, denyServerPodConfig, func(pod *v1.Pod) { - setRuntimeDefaultPSA(pod) - }) - runUDNPod(cs, namespaceBlue, clientPodConfig, func(pod *v1.Pod) { - setRuntimeDefaultPSA(pod) - }) - - ginkgo.By("asserting the server pods have an IP from the configured range") - var allowServerPodIP, denyServerPodIP string - for i, cidr := range strings.Split(nad.cidr, ",") { - if cidr != "" { - subnet, err := getNetCIDRSubnet(cidr) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - allowServerPodIP, err = podIPsForUserDefinedPrimaryNetwork(cs, namespaceYellow, allowServerPodConfig.name, - namespacedName(namespaceYellow, netConfName), i) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - ginkgo.By(fmt.Sprintf("asserting the allow server pod IP %v is from the configured range %v", allowServerPodIP, cidr)) - gomega.Expect(inRange(subnet, allowServerPodIP)).To(gomega.Succeed()) - denyServerPodIP, err = podIPsForUserDefinedPrimaryNetwork(cs, namespaceYellow, denyServerPodConfig.name, - namespacedName(namespaceYellow, netConfName), i) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - ginkgo.By(fmt.Sprintf("asserting the deny server pod IP %v is from the configured range %v", denyServerPodIP, cidr)) - gomega.Expect(inRange(subnet, denyServerPodIP)).To(gomega.Succeed()) - } - } - - ginkgo.By("asserting the *client* pod can contact the allow server pod exposed endpoint") - namespacePodShouldReach(oc, clientPodConfig.namespace, clientPodConfig.name, formatHostAndPort(net.ParseIP(allowServerPodIP), port)) - - ginkgo.By("asserting the *client* pod can contact the deny server pod exposed endpoint") - namespacePodShouldReach(oc, clientPodConfig.namespace, clientPodConfig.name, formatHostAndPort(net.ParseIP(denyServerPodIP), port)) - - ginkgo.By("creating a \"default deny\" network policy") - _, err = makeDenyAllPolicy(f, namespaceYellow) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("asserting the *client* pod can not contact the allow server pod exposed endpoint") - namespacePodShouldNotReach(oc, clientPodConfig.namespace, clientPodConfig.name, formatHostAndPort(net.ParseIP(allowServerPodIP), port)) - - ginkgo.By("asserting the *client* pod can not contact the deny server pod exposed endpoint") - namespacePodShouldNotReach(oc, clientPodConfig.namespace, clientPodConfig.name, formatHostAndPort(net.ParseIP(denyServerPodIP), port)) - - ginkgo.By("creating a \"allow-traffic-to-pod\" network policy") - _, err = allowTrafficToPodFromNamespacePolicy(f, namespaceYellow, namespaceBlue, "allow-traffic-to-pod", allowServerPodLabel) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("asserting the *client* pod can contact the allow server pod exposed endpoint") - namespacePodShouldReach(oc, clientPodConfig.namespace, clientPodConfig.name, formatHostAndPort(net.ParseIP(allowServerPodIP), port)) - - ginkgo.By("asserting the *client* pod can not contact deny server pod exposed endpoint") - namespacePodShouldNotReach(oc, clientPodConfig.namespace, clientPodConfig.name, formatHostAndPort(net.ParseIP(denyServerPodIP), port)) - }, - ginkgo.Entry( - "in L2 primary UDN", - "layer2", - *podConfig( - "client-pod", - ), - *podConfig( - "allow-server-pod", - withCommand(func() []string { - return httpServerContainerCmd(port) - }), - withLabels(allowServerPodLabel), - ), - *podConfig( - "deny-server-pod", - withCommand(func() []string { - return httpServerContainerCmd(port) - }), - withLabels(denyServerPodLabel), - ), - ), - ginkgo.Entry( - "in L3 primary UDN", - "layer3", - *podConfig( - "client-pod", - ), - *podConfig( - "allow-server-pod", - withCommand(func() []string { - return httpServerContainerCmd(port) - }), - withLabels(allowServerPodLabel), - ), - *podConfig( - "deny-server-pod", - withCommand(func() []string { - return httpServerContainerCmd(port) - }), - withLabels(denyServerPodLabel), - ), - )) - }) -}) - -func getNamespaceName(f *framework.Framework, nsSuffix string) string { - return fmt.Sprintf("%s-%s", f.Namespace.Name, nsSuffix) -} - -func allowTrafficToPodFromNamespacePolicy(f *framework.Framework, namespace, fromNamespace, policyName string, podLabel map[string]string) (*knet.NetworkPolicy, error) { - policy := &knet.NetworkPolicy{ - ObjectMeta: metav1.ObjectMeta{ - Name: policyName, - }, - Spec: knet.NetworkPolicySpec{ - PodSelector: metav1.LabelSelector{MatchLabels: podLabel}, - PolicyTypes: []knet.PolicyType{knet.PolicyTypeIngress}, - Ingress: []knet.NetworkPolicyIngressRule{{From: []knet.NetworkPolicyPeer{ - {NamespaceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"kubernetes.io/metadata.name": fromNamespace}}}}}}, - }, - } - return f.ClientSet.NetworkingV1().NetworkPolicies(namespace).Create(context.TODO(), policy, metav1.CreateOptions{}) -}