diff --git a/openshift/.gitignore b/openshift/.gitignore index 968e914a11..10935954f1 100644 --- a/openshift/.gitignore +++ b/openshift/.gitignore @@ -1,2 +1,3 @@ vendor/ bin/ +ovn-kubernetes-tests-ext diff --git a/openshift/cmd/ovn-kubernetes-tests-ext/main.go b/openshift/cmd/ovn-kubernetes-tests-ext/main.go index 9f90ff0a20..afad03597c 100644 --- a/openshift/cmd/ovn-kubernetes-tests-ext/main.go +++ b/openshift/cmd/ovn-kubernetes-tests-ext/main.go @@ -14,6 +14,9 @@ import ( "github.com/ovn-kubernetes/ovn-kubernetes/test/e2e/deploymentconfig" "github.com/ovn-kubernetes/ovn-kubernetes/test/e2e/infraprovider" + // import OTP (OpenShift Tests Private) migration tests + _ "github.com/ovn-kubernetes/ovn-kubernetes/openshift/test/otp" + "github.com/openshift-eng/openshift-tests-extension/pkg/cmd" "github.com/openshift-eng/openshift-tests-extension/pkg/extension" "github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests" @@ -42,6 +45,36 @@ const ( featureLabelNetworkSegmentation = "Feature:NetworkSegmentation" ) +// otpBlockingTests lists the substring patterns for OTP tests that should be blocking +// NOTE: Per reviewer feedback (anuragthehatter), all OTP tests are marked [informing] +// for initial rollout. This list is kept for future reference when tests graduate to blocking. +var otpBlockingTests = []string{ + // All OTP tests are currently [informing] - uncomment to promote to blocking: + // "should not expose API tokens in ovnkube-node logs", + // "should have secure permissions on CNI configuration files", + // "should handle large IPv6 exclude ranges without timeout", + // "should support Dummy CNI plugin with Multus", + // "should execute ovn-db-run-command script successfully", + // "should create healthy pod with single-stack gateway on dual-stack cluster", + // "should show aggregated status from all zones in AdminPolicyBasedExternalRoute", + // "should assign dual-stack IPs with Whereabouts IPAM", +} + +// isOTPBlocking checks if an OTP test should be marked as blocking +// Currently returns false for all tests (all OTP tests are [informing]) +func isOTPBlocking(name string) bool { + // All OTP tests marked as [informing] per reviewer feedback + return false + + // Original logic (commented out for now): + // for _, title := range otpBlockingTests { + // if strings.Contains(name, title) { + // return true + // } + // } + // return false +} + // shouldIncludeTest determines if a test should be included based on cluster capabilities // and test labels. When ocpInfra is nil (no cluster access), all tests are included. func shouldIncludeTest(spec *extensiontests.ExtensionTestSpec) bool { @@ -136,8 +169,12 @@ func main() { blockingTests := sets.New(test.BlockingTests...) specs.Walk(func(spec *extensiontests.ExtensionTestSpec) { - for _, label := range getTestExtensionLabels() { - spec.Labels.Insert(label) + isOTP := strings.Contains(spec.Name, "[OTP]") + + if !isOTP { + for _, label := range getTestExtensionLabels() { + spec.Labels.Insert(label) + } } // Exclude Network Segmentation tests on SingleReplica topology (e.g., MicroShift, SNO) @@ -155,9 +192,15 @@ func main() { spec.Labels.Insert(label) } - spec.Name = generatePrependedLabelsStr(spec.Labels) + " " + spec.Name // prepend ginkgo labels to test name + if !isOTP { + spec.Name = generatePrependedLabelsStr(spec.Labels) + " " + spec.Name // prepend ginkgo labels to test name + } switch { + case isOTP && isOTPBlocking(spec.Name): + spec.Lifecycle = extensiontests.LifecycleBlocking + case isOTP: + spec.Lifecycle = extensiontests.LifecycleInforming case informingTests.Has(spec.Name): spec.Lifecycle = extensiontests.LifecycleInforming case blockingTests.Has(spec.Name): diff --git a/openshift/test/e2e_test.go b/openshift/test/e2e_test.go index 4ec7d744cc..4024b9af05 100644 --- a/openshift/test/e2e_test.go +++ b/openshift/test/e2e_test.go @@ -4,6 +4,9 @@ import ( // import OVN-Kubernetes E2Es _ "github.com/ovn-kubernetes/ovn-kubernetes/test/e2e" + // import OTP (OpenShift Tests Private) migration tests + _ "github.com/ovn-kubernetes/ovn-kubernetes/openshift/test/otp" + // Ensure that logging flags are part of the command line. _ "k8s.io/component-base/logs/testinit" ) diff --git a/openshift/test/otp/multus.go b/openshift/test/otp/multus.go new file mode 100644 index 0000000000..5bde9d8ad4 --- /dev/null +++ b/openshift/test/otp/multus.go @@ -0,0 +1,792 @@ +package otp + +import ( + "bytes" + "context" + "fmt" + "regexp" + "strings" + "time" + + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/client-go/tools/remotecommand" +) + +var _ = g.Describe("[JIRA:Networking][OTP][sig-network] OTP Multus", func() { + var ( + clientset *kubernetes.Clientset + config *rest.Config + ctx context.Context + ) + + g.BeforeEach(func() { + ctx = context.Background() + + // Load kubeconfig + loadingRules := clientcmd.NewDefaultClientConfigLoadingRules() + configOverrides := &clientcmd.ConfigOverrides{} + kubeConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, configOverrides) + + var err error + config, err = kubeConfig.ClientConfig() + o.Expect(err).NotTo(o.HaveOccurred()) + + clientset, err = kubernetes.NewForConfig(config) + o.Expect(err).NotTo(o.HaveOccurred()) + }) + + // High-57589: Whereabouts CNI Timeout with Large Exclude Range + g.It("[OTP][informing][57589] should handle large IPv6 exclude ranges without timeout", func() { + const testNS = "test-whereabouts-57589" + + g.By("Creating test namespace") + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: testNS, + Labels: map[string]string{ + "pod-security.kubernetes.io/enforce": "privileged", + "pod-security.kubernetes.io/audit": "privileged", + "pod-security.kubernetes.io/warn": "privileged", + }, + }, + } + _, err := clientset.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + defer func() { + g.By("Cleaning up test namespace") + _ = clientset.CoreV1().Namespaces().Delete(ctx, testNS, metav1.DeleteOptions{}) + }() + + g.By("Creating NetworkAttachmentDefinition with large exclude range") + nadConfig := `{ + "cniVersion": "0.3.1", + "name": "bridge-net", + "type": "bridge", + "bridge": "test-br0", + "isGateway": false, + "ipMasq": false, + "ipam": { + "type": "whereabouts", + "range": "fd43:01f1:3daa:0baa::/64", + "exclude": [ "fd43:01f1:3daa:0baa::/100" ], + "log_file": "/tmp/whereabouts.log", + "log_level" : "debug" + } + }` + + err = createNAD(ctx, config, testNS, "nad-w-excludes", nadConfig) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Creating pod with secondary network") + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod", + Namespace: testNS, + Annotations: map[string]string{ + "k8s.v1.cni.cncf.io/networks": "nad-w-excludes", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "test", + Image: "registry.access.redhat.com/ubi8/ubi-minimal:latest", + Command: []string{"sleep", "3600"}, + }, + }, + }, + } + + _, err = clientset.CoreV1().Pods(testNS).Create(ctx, pod, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Waiting for pod to reach Running state (max 60s)") + // Pod should be Running within 60 seconds (test validates no timeout) + o.Eventually(func() corev1.PodPhase { + p, err := clientset.CoreV1().Pods(testNS).Get(ctx, "test-pod", metav1.GetOptions{}) + if err != nil { + return corev1.PodPending + } + return p.Status.Phase + }, 60, 5).Should(o.Equal(corev1.PodRunning), + "Pod did not reach Running state within 60s - Whereabouts may have timed out") + + g.By("Verifying secondary network attachment") + p, err := clientset.CoreV1().Pods(testNS).Get(ctx, "test-pod", metav1.GetOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + networkStatus, ok := p.Annotations["k8s.v1.cni.cncf.io/network-status"] + o.Expect(ok).To(o.BeTrue(), "Pod missing network-status annotation") + o.Expect(networkStatus).NotTo(o.BeEmpty()) + + // Verify at least 2 networks (primary + secondary) + networkCount := strings.Count(networkStatus, `"name"`) + o.Expect(networkCount).To(o.BeNumerically(">=", 2), + "Expected at least 2 networks, got %d", networkCount) + }) + + // Medium-76652: Dummy CNI Support + g.It("[OTP][informing][76652] should support Dummy CNI plugin with Multus", func() { + const testNS = "test-dummy-cni-76652" + + g.By("Creating test namespace") + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: testNS, + Labels: map[string]string{ + "pod-security.kubernetes.io/enforce": "privileged", + "pod-security.kubernetes.io/audit": "privileged", + "pod-security.kubernetes.io/warn": "privileged", + }, + }, + } + _, err := clientset.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + defer func() { + g.By("Cleaning up test namespace") + _ = clientset.CoreV1().Namespaces().Delete(ctx, testNS, metav1.DeleteOptions{}) + }() + + g.By("Creating NetworkAttachmentDefinition with dummy CNI and static IPAM") + dummyConfig := `{ + "cniVersion": "0.3.1", + "name": "dummy-net", + "type": "dummy", + "ipam": { + "type": "static", + "addresses": [ + { + "address": "10.10.10.2/24" + } + ] + } + }` + + err = createNAD(ctx, config, testNS, "dummy-net", dummyConfig) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Creating pod with dummy network attached") + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-dummy-pod", + Namespace: testNS, + Annotations: map[string]string{ + "k8s.v1.cni.cncf.io/networks": "dummy-net", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "test", + Image: "registry.access.redhat.com/ubi8/ubi-minimal:latest", + Command: []string{"sleep", "3600"}, + }, + }, + }, + } + + _, err = clientset.CoreV1().Pods(testNS).Create(ctx, pod, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Waiting for pod to reach Running state") + o.Eventually(func() corev1.PodPhase { + p, err := clientset.CoreV1().Pods(testNS).Get(ctx, "test-dummy-pod", metav1.GetOptions{}) + if err != nil { + return corev1.PodPending + } + return p.Status.Phase + }, 60, 5).Should(o.Equal(corev1.PodRunning), + "Pod did not reach Running state within 60s") + + g.By("Verifying dummy network interface is created") + p, err := clientset.CoreV1().Pods(testNS).Get(ctx, "test-dummy-pod", metav1.GetOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + networkStatus, ok := p.Annotations["k8s.v1.cni.cncf.io/network-status"] + o.Expect(ok).To(o.BeTrue(), "Pod missing network-status annotation") + o.Expect(networkStatus).NotTo(o.BeEmpty()) + + g.By("Validating dummy interface has correct IP and configuration") + // Network status should contain 2 interfaces: ovn-kubernetes (primary) + dummy-net (secondary) + o.Expect(networkStatus).To(o.ContainSubstring("ovn-kubernetes"), "Should have primary OVN network") + o.Expect(networkStatus).To(o.ContainSubstring("dummy-net"), "Should have dummy network") + o.Expect(networkStatus).To(o.ContainSubstring("10.10.10.2"), "Should have assigned dummy IP") + + // Verify we have at least 2 network interfaces + networkCount := strings.Count(networkStatus, `"name"`) + o.Expect(networkCount).To(o.BeNumerically(">=", 2), + "Expected at least 2 networks (primary + dummy), got %d", networkCount) + }) + + // Medium-66876: Support Dual Stack IP assignment for whereabouts CNI/IPAM + g.It("[OTP][informing][66876] should assign dual-stack IPs with Whereabouts IPAM", func() { + const testNS = "test-whereabouts-dualstack-66876" + + g.By("Creating test namespace") + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: testNS, + Labels: map[string]string{ + "pod-security.kubernetes.io/enforce": "privileged", + "pod-security.kubernetes.io/audit": "privileged", + "pod-security.kubernetes.io/warn": "privileged", + }, + }, + } + _, err := clientset.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + defer func() { + g.By("Cleaning up test namespace") + _ = clientset.CoreV1().Namespaces().Delete(ctx, testNS, metav1.DeleteOptions{}) + }() + + g.By("Creating NetworkAttachmentDefinition with dual-stack Whereabouts IPAM") + dualStackConfig := `{ + "cniVersion": "0.3.1", + "name": "whereabouts-dualstack", + "type": "macvlan", + "mode": "bridge", + "ipam": { + "type": "whereabouts", + "ipRanges": [ + { + "range": "192.168.10.0/24" + }, + { + "range": "fd00:dead:beef:10::/64" + } + ] + } + }` + + err = createNAD(ctx, config, testNS, "whereabouts-dualstack", dualStackConfig) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Creating deployment with 2 pods using pod affinity for same-node placement") + deployment := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod", + Namespace: testNS, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: int32Ptr(2), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "test-pod", + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": "test-pod", + }, + Annotations: map[string]string{ + "k8s.v1.cni.cncf.io/networks": "whereabouts-dualstack", + }, + }, + Spec: corev1.PodSpec{ + Affinity: &corev1.Affinity{ + PodAffinity: &corev1.PodAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "test-pod", + }, + }, + TopologyKey: "kubernetes.io/hostname", + }, + }, + }, + }, + Containers: []corev1.Container{ + { + Name: "test-pod", + Image: "registry.access.redhat.com/ubi9/python-39:latest", + Command: []string{"/bin/bash", "-c"}, + Args: []string{ + `cat > /tmp/server.py <<'PYEOF' +import http.server +import socketserver +import socket +PORT = 8080 +class DualStackTCPServer(socketserver.TCPServer): + address_family = socket.AF_INET6 + def __init__(self, server_address, RequestHandlerClass, bind_and_activate=True): + super().__init__(server_address, RequestHandlerClass, bind_and_activate=False) + self.socket.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 0) + if bind_and_activate: + self.server_bind() + self.server_activate() +class Handler(http.server.SimpleHTTPRequestHandler): + def do_GET(self): + self.send_response(200) + self.send_header('Content-type', 'text/plain') + self.end_headers() + self.wfile.write(b'whereabouts-dualstack-test-pod\n') +with DualStackTCPServer(("::", PORT), Handler) as httpd: + httpd.serve_forever() +PYEOF +python3 /tmp/server.py`, + }, + Ports: []corev1.ContainerPort{ + {ContainerPort: 8080}, + }, + }, + }, + }, + }, + }, + } + + _, err = clientset.AppsV1().Deployments(testNS).Create(ctx, deployment, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Waiting for both pods to reach Running state") + o.Eventually(func() int { + pods, err := clientset.CoreV1().Pods(testNS).List(ctx, metav1.ListOptions{ + LabelSelector: "app=test-pod", + }) + if err != nil { + return 0 + } + runningCount := 0 + for _, pod := range pods.Items { + if pod.Status.Phase == corev1.PodRunning { + runningCount++ + } + } + return runningCount + }, 120, 10).Should(o.Equal(2), "Both pods should reach Running state") + + g.By("Verifying pods have dual-stack IPs on secondary interface") + pods, err := clientset.CoreV1().Pods(testNS).List(ctx, metav1.ListOptions{ + LabelSelector: "app=test-pod", + }) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(pods.Items)).To(o.Equal(2), "Should have 2 pods") + + ipv4Assigned := 0 + ipv6Assigned := 0 + var podIPs []string + + for _, pod := range pods.Items { + networkStatus, ok := pod.Annotations["k8s.v1.cni.cncf.io/network-status"] + o.Expect(ok).To(o.BeTrue(), "Pod %s should have network-status annotation", pod.Name) + + // Check for dual-stack IPs in network status + hasIPv4 := strings.Contains(networkStatus, "192.168.10.") + hasIPv6 := strings.Contains(networkStatus, "fd00:dead:beef:10::") + + if hasIPv4 { + ipv4Assigned++ + } + if hasIPv6 { + ipv6Assigned++ + } + + o.Expect(hasIPv4 && hasIPv6).To(o.BeTrue(), + "Pod %s should have both IPv4 (192.168.10.x) and IPv6 (fd00:dead:beef:10::x) addresses", pod.Name) + + // Extract IPv4 for uniqueness check + if hasIPv4 { + ipv4Regex := regexp.MustCompile(`192\.168\.10\.\d+`) + matches := ipv4Regex.FindString(networkStatus) + if matches != "" { + podIPs = append(podIPs, matches) + } + } + } + + o.Expect(ipv4Assigned).To(o.Equal(2), "Both pods should have IPv4 addresses") + o.Expect(ipv6Assigned).To(o.Equal(2), "Both pods should have IPv6 addresses") + + g.By("Verifying dual-stack IP uniqueness") + o.Expect(len(podIPs)).To(o.BeNumerically(">=", 2), "Should have extracted at least 2 IPv4 addresses") + + if len(podIPs) >= 2 { + o.Expect(podIPs[0]).NotTo(o.Equal(podIPs[1]), "Pods should have different IPv4 addresses") + } + + g.By("Testing IPv4 connectivity between pods on the same node") + // Both pods are guaranteed to be on the same node via pod affinity + // Macvlan in bridge mode requires same-node for L2 connectivity + o.Expect(len(pods.Items)).To(o.Equal(2), "Should have exactly 2 pods") + + ipv4Regex := regexp.MustCompile(`192\.168\.10\.\d+`) + ipv6Regex := regexp.MustCompile(`fd00:dead:beef:10::[a-f0-9]+`) + + // Extract IPs from pod 0 and pod 1 + srcPod := pods.Items[0].Name + networkStatus1 := pods.Items[1].Annotations["k8s.v1.cni.cncf.io/network-status"] + + dstIPv4 := ipv4Regex.FindString(networkStatus1) + dstIPv6 := ipv6Regex.FindString(networkStatus1) + + o.Expect(dstIPv4).NotTo(o.BeEmpty(), "Pod 1 should have IPv4 address") + o.Expect(dstIPv6).NotTo(o.BeEmpty(), "Pod 1 should have IPv6 address") + + // Verify both pods are on the same node (should always be true due to affinity) + o.Expect(pods.Items[0].Spec.NodeName).To(o.Equal(pods.Items[1].Spec.NodeName), + "Both pods should be on the same node due to pod affinity") + + scheme := runtime.NewScheme() + err = corev1.AddToScheme(scheme) + o.Expect(err).NotTo(o.HaveOccurred()) + + // Test IPv4 connectivity + curlCmd := []string{"curl", "-s", "--connect-timeout", "5", fmt.Sprintf("http://%s:8080", dstIPv4)} + req := clientset.CoreV1().RESTClient().Post(). + Resource("pods"). + Name(srcPod). + Namespace(testNS). + SubResource("exec"). + VersionedParams(&corev1.PodExecOptions{ + Container: "test-pod", + Command: curlCmd, + Stdout: true, + Stderr: true, + }, runtime.NewParameterCodec(scheme)) + + exec, err := remotecommand.NewSPDYExecutor(config, "POST", req.URL()) + o.Expect(err).NotTo(o.HaveOccurred()) + + var stdout, stderr bytes.Buffer + err = exec.StreamWithContext(ctx, remotecommand.StreamOptions{ + Stdout: &stdout, + Stderr: &stderr, + }) + o.Expect(err).NotTo(o.HaveOccurred(), "IPv4 connectivity test failed: %s", stderr.String()) + o.Expect(stdout.String()).To(o.ContainSubstring("whereabouts-dualstack-test-pod"), + "IPv4 connectivity: Expected response from hello-sdn server") + + g.By("Testing IPv6 connectivity between pods on secondary network") + // Test IPv6 connectivity - curl requires brackets around IPv6 and -g flag + curlCmd = []string{"curl", "-s", "-6", "-g", "--connect-timeout", "5", fmt.Sprintf("http://[%s]:8080", dstIPv6)} + req = clientset.CoreV1().RESTClient().Post(). + Resource("pods"). + Name(srcPod). + Namespace(testNS). + SubResource("exec"). + VersionedParams(&corev1.PodExecOptions{ + Container: "test-pod", + Command: curlCmd, + Stdout: true, + Stderr: true, + }, runtime.NewParameterCodec(scheme)) + + exec, err = remotecommand.NewSPDYExecutor(config, "POST", req.URL()) + o.Expect(err).NotTo(o.HaveOccurred()) + + stdout.Reset() + stderr.Reset() + err = exec.StreamWithContext(ctx, remotecommand.StreamOptions{ + Stdout: &stdout, + Stderr: &stderr, + }) + o.Expect(err).NotTo(o.HaveOccurred(), "IPv6 connectivity test failed: %s", stderr.String()) + o.Expect(stdout.String()).To(o.ContainSubstring("whereabouts-dualstack-test-pod"), + "IPv6 connectivity: Expected response from hello-sdn server") + }) + + // OCP-69947: Macvlan pods send Unsolicited Neighbor Advertisements + // Note: Marked as informing due to timing sensitivity with tcpdump in automated environment + g.It("[OTP][informing][69947] should send Unsolicited Neighbor Advertisements when macvlan pod is created", func() { + testNS := "test-macvlan-na-69947" + + g.By("Creating test namespace") + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: testNS, + Labels: map[string]string{ + "pod-security.kubernetes.io/enforce": "privileged", + "pod-security.kubernetes.io/audit": "privileged", + "pod-security.kubernetes.io/warn": "privileged", + }, + }, + } + _, err := clientset.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + defer func() { + _ = clientset.CoreV1().Namespaces().Delete(ctx, testNS, metav1.DeleteOptions{}) + }() + + g.By("Creating NetworkAttachmentDefinition with dual-stack whereabouts IPAM") + nadConfig := `{ + "cniVersion": "0.3.1", + "name": "whereabouts-dualstack", + "type": "macvlan", + "mode": "bridge", + "ipam": { + "type": "whereabouts", + "ipRanges": [ + { + "range": "192.168.10.0/24" + }, + { + "range": "fd00:dead:beef:10::/64" + } + ] + } + }` + + err = createNAD(ctx, config, testNS, "whereabouts-dualstack", nadConfig) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Creating sniffer pod to capture ICMPv6 Neighbor Advertisements") + snifferPod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "sniff-pod", + Namespace: testNS, + Annotations: map[string]string{ + "k8s.v1.cni.cncf.io/networks": "whereabouts-dualstack", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "sniffer", + Image: "quay.io/openshifttest/hello-sdn@sha256:c89445416459e7adea9a5a416b3365ed3d74f2491beb904d61dc8d1eb89a72a4", + Command: []string{"/bin/sh", "-c"}, + Args: []string{ + // Start tcpdump to capture ICMPv6 Neighbor Advertisements on net1 + // Filter: icmp6 type 136 (Neighbor Advertisement) + `tcpdump -i net1 -n 'icmp6 and icmp6[0] = 136' -w /tmp/capture.pcap & + sleep 3600`, + }, + SecurityContext: &corev1.SecurityContext{ + Capabilities: &corev1.Capabilities{ + Add: []corev1.Capability{"NET_RAW", "NET_ADMIN"}, + }, + }, + }, + }, + }, + } + + _, err = clientset.CoreV1().Pods(testNS).Create(ctx, snifferPod, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + // Wait for sniffer pod to be running + o.Eventually(func() corev1.PodPhase { + pod, err := clientset.CoreV1().Pods(testNS).Get(ctx, "sniff-pod", metav1.GetOptions{}) + if err != nil { + return corev1.PodPending + } + return pod.Status.Phase + }, 60, 5).Should(o.Equal(corev1.PodRunning), "Sniffer pod should be running") + + // Give tcpdump time to start capturing + // tcpdump needs extra time after pod reaches Running to initialize and start listening + time.Sleep(20 * time.Second) + + g.By("Creating 6 test pods with macvlan secondary network") + rc := &corev1.ReplicationController{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod", + Namespace: testNS, + }, + Spec: corev1.ReplicationControllerSpec{ + Replicas: int32Ptr(6), + Selector: map[string]string{ + "name": "test-pod", + }, + Template: &corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "name": "test-pod", + }, + Annotations: map[string]string{ + "k8s.v1.cni.cncf.io/networks": "whereabouts-dualstack", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "test-pod", + Image: "quay.io/openshifttest/hello-sdn@sha256:c89445416459e7adea9a5a416b3365ed3d74f2491beb904d61dc8d1eb89a72a4", + Env: []corev1.EnvVar{ + { + Name: "RESPONSE", + Value: "Hello", + }, + }, + }, + }, + }, + }, + }, + } + + _, err = clientset.CoreV1().ReplicationControllers(testNS).Create(ctx, rc, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Waiting for all 6 test pods to reach Running state") + o.Eventually(func() int { + pods, err := clientset.CoreV1().Pods(testNS).List(ctx, metav1.ListOptions{ + LabelSelector: "name=test-pod", + }) + if err != nil { + return 0 + } + runningCount := 0 + for _, pod := range pods.Items { + if pod.Status.Phase == corev1.PodRunning { + runningCount++ + } + } + return runningCount + }, 120, 10).Should(o.Equal(6), "All 6 test pods should be running") + + // Wait additional time for Unsolicited Neighbor Advertisements to be sent + time.Sleep(15 * time.Second) + + g.By("Analyzing captured ICMPv6 Neighbor Advertisements") + // Stop tcpdump and read the capture file + scheme := runtime.NewScheme() + err = corev1.AddToScheme(scheme) + o.Expect(err).NotTo(o.HaveOccurred()) + + // Kill tcpdump process + killCmd := []string{"/bin/sh", "-c", "pkill tcpdump"} + req := clientset.CoreV1().RESTClient().Post(). + Resource("pods"). + Name("sniff-pod"). + Namespace(testNS). + SubResource("exec"). + VersionedParams(&corev1.PodExecOptions{ + Container: "sniffer", + Command: killCmd, + Stdout: true, + Stderr: true, + }, runtime.NewParameterCodec(scheme)) + + exec, err := remotecommand.NewSPDYExecutor(config, "POST", req.URL()) + o.Expect(err).NotTo(o.HaveOccurred()) + + var stdout, stderr bytes.Buffer + _ = exec.StreamWithContext(ctx, remotecommand.StreamOptions{ + Stdout: &stdout, + Stderr: &stderr, + }) + + // Wait for tcpdump to flush pcap file to disk + time.Sleep(5 * time.Second) + + // Read and analyze the pcap file using tcpdump + // Check for ICMPv6 NA packets with solicited flag = 0 (Unsolicited) + analyzeCmd := []string{"/bin/sh", "-c", + `tcpdump -r /tmp/capture.pcap -n 'icmp6 and icmp6[0] = 136' -v 2>/dev/null | grep "Neighbor Advertisement" | wc -l`} + + req = clientset.CoreV1().RESTClient().Post(). + Resource("pods"). + Name("sniff-pod"). + Namespace(testNS). + SubResource("exec"). + VersionedParams(&corev1.PodExecOptions{ + Container: "sniffer", + Command: analyzeCmd, + Stdout: true, + Stderr: true, + }, runtime.NewParameterCodec(scheme)) + + exec, err = remotecommand.NewSPDYExecutor(config, "POST", req.URL()) + o.Expect(err).NotTo(o.HaveOccurred()) + + stdout.Reset() + stderr.Reset() + err = exec.StreamWithContext(ctx, remotecommand.StreamOptions{ + Stdout: &stdout, + Stderr: &stderr, + }) + o.Expect(err).NotTo(o.HaveOccurred(), "Failed to analyze pcap: %s", stderr.String()) + + naCount := strings.TrimSpace(stdout.String()) + o.Expect(naCount).NotTo(o.Equal("0"), "Should have captured at least one ICMPv6 Neighbor Advertisement") + + g.By("Verifying Neighbor Advertisements are Unsolicited (solicited flag = 0)") + // Check that captured NAs have solicited flag = 0 + // In unsolicited NA, the destination is ff02::1 (all nodes multicast) + verifyCmd := []string{"/bin/sh", "-c", + `tcpdump -r /tmp/capture.pcap -n 'icmp6 and icmp6[0] = 136' 2>/dev/null | grep "ff02::1" | wc -l`} + + req = clientset.CoreV1().RESTClient().Post(). + Resource("pods"). + Name("sniff-pod"). + Namespace(testNS). + SubResource("exec"). + VersionedParams(&corev1.PodExecOptions{ + Container: "sniffer", + Command: verifyCmd, + Stdout: true, + Stderr: true, + }, runtime.NewParameterCodec(scheme)) + + exec, err = remotecommand.NewSPDYExecutor(config, "POST", req.URL()) + o.Expect(err).NotTo(o.HaveOccurred()) + + stdout.Reset() + stderr.Reset() + err = exec.StreamWithContext(ctx, remotecommand.StreamOptions{ + Stdout: &stdout, + Stderr: &stderr, + }) + o.Expect(err).NotTo(o.HaveOccurred(), "Failed to verify unsolicited NAs: %s", stderr.String()) + + unsolicitedCount := strings.TrimSpace(stdout.String()) + o.Expect(unsolicitedCount).NotTo(o.Equal("0"), + "Should have captured Unsolicited Neighbor Advertisements (destination ff02::1)") + }) +}) + +// createNAD creates a NetworkAttachmentDefinition +func createNAD(ctx context.Context, config *rest.Config, namespace, name, nadConfig string) error { + dynamicClient, err := dynamic.NewForConfig(config) + if err != nil { + return err + } + + nadGVR := schema.GroupVersionResource{ + Group: "k8s.cni.cncf.io", + Version: "v1", + Resource: "network-attachment-definitions", + } + + nad := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "k8s.cni.cncf.io/v1", + "kind": "NetworkAttachmentDefinition", + "metadata": map[string]interface{}{ + "name": name, + "namespace": namespace, + }, + "spec": map[string]interface{}{ + "config": nadConfig, + }, + }, + } + + _, err = dynamicClient.Resource(nadGVR).Namespace(namespace).Create(ctx, nad, metav1.CreateOptions{}) + return err +} + +// int32Ptr returns a pointer to an int32 +func int32Ptr(i int32) *int32 { + return &i +} diff --git a/openshift/test/otp/networking_tools.go b/openshift/test/otp/networking_tools.go new file mode 100644 index 0000000000..18bca9f385 --- /dev/null +++ b/openshift/test/otp/networking_tools.go @@ -0,0 +1,546 @@ +package otp + +import ( + "bytes" + "context" + "fmt" + "strings" + + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/dynamic" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/client-go/tools/remotecommand" +) + +var _ = g.Describe("[JIRA:Networking][OTP][sig-network] OTP Networking Tools", func() { + var ( + clientset *kubernetes.Clientset + config *rest.Config + ctx context.Context + ) + + g.BeforeEach(func() { + ctx = context.Background() + + // Load kubeconfig + loadingRules := clientcmd.NewDefaultClientConfigLoadingRules() + configOverrides := &clientcmd.ConfigOverrides{} + kubeConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, configOverrides) + + var err error + config, err = kubeConfig.ClientConfig() + o.Expect(err).NotTo(o.HaveOccurred()) + + clientset, err = kubernetes.NewForConfig(config) + o.Expect(err).NotTo(o.HaveOccurred()) + }) + + // Medium-55889: ovn-db-run-command Script Functionality + g.It("[OTP][informing][55889] should execute ovn-db-run-command script successfully", func() { + g.By("Finding an ovnkube-node pod with northd container") + pods, err := clientset.CoreV1().Pods("openshift-ovn-kubernetes").List(ctx, metav1.ListOptions{ + LabelSelector: "app=ovnkube-node", + }) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(pods.Items)).To(o.BeNumerically(">", 0), "Expected at least one ovnkube-node pod") + + nodePod := pods.Items[0].Name + + g.By("Testing ovn-nbctl command (equivalent to ovn-db-run-command)") + // Execute: ovn-nbctl show + // Note: ovn-db-run-command script may not exist in older versions + execCmd := []string{ + "ovn-nbctl", + "--no-leader-only", + "show", + } + + scheme := runtime.NewScheme() + err = corev1.AddToScheme(scheme) + o.Expect(err).NotTo(o.HaveOccurred()) + + req := clientset.CoreV1().RESTClient().Post(). + Resource("pods"). + Name(nodePod). + Namespace("openshift-ovn-kubernetes"). + SubResource("exec"). + VersionedParams(&corev1.PodExecOptions{ + Container: "northd", + Command: execCmd, + Stdout: true, + Stderr: true, + }, runtime.NewParameterCodec(scheme)) + + exec, err := remotecommand.NewSPDYExecutor(config, "POST", req.URL()) + o.Expect(err).NotTo(o.HaveOccurred()) + + var stdout, stderr bytes.Buffer + err = exec.StreamWithContext(ctx, remotecommand.StreamOptions{ + Stdout: &stdout, + Stderr: &stderr, + }) + o.Expect(err).NotTo(o.HaveOccurred(), "ovn-db-run-command execution failed: %s", stderr.String()) + + output := stdout.String() + g.By("Verifying command output contains expected OVN database content") + // The 'show' command should produce non-empty output showing OVN topology + o.Expect(output).NotTo(o.BeEmpty(), "ovn-nbctl produced no output") + + // Verify output looks like OVN Northbound DB content (contains typical elements) + hasValidContent := strings.Contains(output, "switch") || + strings.Contains(output, "router") || + strings.Contains(output, "port") || + strings.Contains(output, "Logical") || + strings.Contains(output, "join") + o.Expect(hasValidContent).To(o.BeTrue(), + "Output doesn't appear to be valid OVN database content: %s", output) + }) + + // Medium-67625: ovnkube-trace pod-to-pod + g.It("[OTP][informing][67625] should trace pod-to-pod traffic successfully", func() { + g.By("Finding ovnkube-node pods") + pods, err := clientset.CoreV1().Pods("openshift-ovn-kubernetes").List(ctx, metav1.ListOptions{ + LabelSelector: "app=ovnkube-node", + }) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(pods.Items)).To(o.BeNumerically(">=", 2), "Need at least 2 nodes for pod-to-pod test") + + g.By("Creating test namespace for trace pods") + const traceNS = "test-ovnkube-trace-67625" + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: traceNS, + Labels: map[string]string{ + "pod-security.kubernetes.io/enforce": "privileged", + "pod-security.kubernetes.io/audit": "privileged", + "pod-security.kubernetes.io/warn": "privileged", + }, + }, + } + _, err = clientset.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + defer func() { + _ = clientset.CoreV1().Namespaces().Delete(ctx, traceNS, metav1.DeleteOptions{}) + }() + + g.By("Creating source pod") + srcPod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "src-pod", + Namespace: traceNS, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "test", + Image: "registry.access.redhat.com/ubi8/ubi-minimal:latest", + Command: []string{"sleep", "3600"}, + }, + }, + }, + } + _, err = clientset.CoreV1().Pods(traceNS).Create(ctx, srcPod, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Creating destination pod") + dstPod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "dst-pod", + Namespace: traceNS, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "test", + Image: "registry.access.redhat.com/ubi8/ubi-minimal:latest", + Command: []string{"sleep", "3600"}, + }, + }, + }, + } + _, err = clientset.CoreV1().Pods(traceNS).Create(ctx, dstPod, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Waiting for pods to be Running") + o.Eventually(func() bool { + src, _ := clientset.CoreV1().Pods(traceNS).Get(ctx, "src-pod", metav1.GetOptions{}) + dst, _ := clientset.CoreV1().Pods(traceNS).Get(ctx, "dst-pod", metav1.GetOptions{}) + return src.Status.Phase == corev1.PodRunning && dst.Status.Phase == corev1.PodRunning + }, 60, 5).Should(o.BeTrue(), "Pods did not reach Running state") + + g.By("Running ovnkube-trace from src to dst pod") + output, err := runOVNKubeTrace(ctx, clientset, config, + traceNS, "src-pod", + traceNS, "dst-pod", + "tcp", "8080") + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Verifying trace output shows packet delivery") + o.Expect(output).To(o.ContainSubstring("output"), "Trace should show output action") + o.Expect(output).NotTo(o.ContainSubstring("drop"), "Trace should not show packet drops") + }) + + // Medium-67648: ovnkube-trace pod-to-hostnetworkpod + g.It("[OTP][informing][67648] should trace pod-to-hostnetworkpod traffic successfully", func() { + g.By("Creating test namespace") + const traceNS = "test-ovnkube-trace-67648" + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: traceNS, + Labels: map[string]string{ + "pod-security.kubernetes.io/enforce": "privileged", + "pod-security.kubernetes.io/audit": "privileged", + "pod-security.kubernetes.io/warn": "privileged", + }, + }, + } + _, err := clientset.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + defer func() { + _ = clientset.CoreV1().Namespaces().Delete(ctx, traceNS, metav1.DeleteOptions{}) + }() + + g.By("Creating source pod (regular overlay pod)") + srcPod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "src-pod", + Namespace: traceNS, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "test", + Image: "registry.access.redhat.com/ubi8/ubi-minimal:latest", + Command: []string{"sleep", "3600"}, + }, + }, + }, + } + _, err = clientset.CoreV1().Pods(traceNS).Create(ctx, srcPod, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Creating destination host-network pod") + dstPod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "dst-hostnet-pod", + Namespace: traceNS, + }, + Spec: corev1.PodSpec{ + HostNetwork: true, + Containers: []corev1.Container{ + { + Name: "test", + Image: "registry.access.redhat.com/ubi8/ubi-minimal:latest", + Command: []string{"sleep", "3600"}, + }, + }, + }, + } + _, err = clientset.CoreV1().Pods(traceNS).Create(ctx, dstPod, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Waiting for pods to be Running") + o.Eventually(func() bool { + src, _ := clientset.CoreV1().Pods(traceNS).Get(ctx, "src-pod", metav1.GetOptions{}) + dst, _ := clientset.CoreV1().Pods(traceNS).Get(ctx, "dst-hostnet-pod", metav1.GetOptions{}) + return src.Status.Phase == corev1.PodRunning && dst.Status.Phase == corev1.PodRunning + }, 60, 5).Should(o.BeTrue(), "Pods did not reach Running state") + + g.By("Running ovnkube-trace from overlay pod to host-network pod") + output, err := runOVNKubeTrace(ctx, clientset, config, + traceNS, "src-pod", + traceNS, "dst-hostnet-pod", + "tcp", "22") + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Verifying trace shows routing to host network") + // Trace should show packet reaching the node (might show different path than pod-to-pod) + o.Expect(output).NotTo(o.BeEmpty(), "Trace should produce output") + // Host-network traffic bypasses some OVN overlay, so just verify no hard drops + o.Expect(output).NotTo(o.ContainSubstring("policy drop"), "Should not be blocked by policy") + }) + + // Medium-45146: BZ 1986708 - Pod should be healthy when gw IP is single stack on dual stack cluster + g.It("[OTP][informing][45146] should create healthy pod with single-stack gateway on dual-stack cluster", func() { + const testNS = "test-single-stack-gw-45146" + + g.By("Creating test namespace") + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: testNS, + Labels: map[string]string{ + "pod-security.kubernetes.io/enforce": "privileged", + "pod-security.kubernetes.io/audit": "privileged", + "pod-security.kubernetes.io/warn": "privileged", + }, + }, + } + _, err := clientset.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + defer func() { + g.By("Cleaning up test namespace") + _ = clientset.CoreV1().Namespaces().Delete(ctx, testNS, metav1.DeleteOptions{}) + }() + + g.By("Creating pod with single-stack gateway routing annotations") + // This simulates a gateway pod with single-stack routing on a dual-stack cluster + // Testing BZ 1986708 - pod should remain healthy despite stack mismatch + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gw-single-stack-pod", + Namespace: testNS, + Annotations: map[string]string{ + "k8s.ovn.org/routing-namespaces": testNS, + "k8s.ovn.org/routing-network": "foo", + // Single-stack IPv4 network status on potentially dual-stack cluster + "k8s.v1.cni.cncf.io/network-status": `[{"name":"foo","interface":"net1","ips":["172.19.0.5"],"mac":"01:23:45:67:89:10"}]`, + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "test", + Image: "registry.access.redhat.com/ubi8/ubi-minimal:latest", + Command: []string{"sleep", "3600"}, + }, + }, + }, + } + + _, err = clientset.CoreV1().Pods(testNS).Create(ctx, pod, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Waiting for pod to reach Running state (validating BZ 1986708 fix)") + // The bug was that pods with single-stack gw on dual-stack clusters would fail + // This test ensures the pod becomes healthy + o.Eventually(func() corev1.PodPhase { + p, err := clientset.CoreV1().Pods(testNS).Get(ctx, "gw-single-stack-pod", metav1.GetOptions{}) + if err != nil { + return corev1.PodPending + } + return p.Status.Phase + }, 60, 5).Should(o.Equal(corev1.PodRunning), + "Pod with single-stack gateway should reach Running state on dual-stack cluster") + + g.By("Verifying pod is healthy with Ready condition") + p, err := clientset.CoreV1().Pods(testNS).Get(ctx, "gw-single-stack-pod", metav1.GetOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + // Check that pod has at least one container in Ready state + hasReadyContainer := false + for _, containerStatus := range p.Status.ContainerStatuses { + if containerStatus.Ready { + hasReadyContainer = true + break + } + } + o.Expect(hasReadyContainer).To(o.BeTrue(), + "Pod should have at least one ready container, validating health despite single-stack GW") + + g.By("Verifying routing annotations are preserved") + o.Expect(p.Annotations["k8s.ovn.org/routing-namespaces"]).To(o.Equal(testNS), + "Routing namespace annotation should be preserved") + o.Expect(p.Annotations["k8s.ovn.org/routing-network"]).To(o.Equal("foo"), + "Routing network annotation should be preserved") + }) + + // Medium-69761: Check apbexternalroute status when all zones reported success + g.It("[OTP][informing][69761] should show aggregated status from all zones in AdminPolicyBasedExternalRoute", func() { + const testNS = "test-apbexternalroute-69761" + + g.By("Creating test namespace") + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: testNS, + }, + } + _, err := clientset.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + defer func() { + g.By("Cleaning up test namespace and AdminPolicyBasedExternalRoute") + dynamicClient, _ := dynamic.NewForConfig(config) + apbrGVR := schema.GroupVersionResource{ + Group: "k8s.ovn.org", + Version: "v1", + Resource: "adminpolicybasedexternalroutes", + } + _ = dynamicClient.Resource(apbrGVR).Delete(ctx, "default-route-policy", metav1.DeleteOptions{}) + _ = clientset.CoreV1().Namespaces().Delete(ctx, testNS, metav1.DeleteOptions{}) + }() + + g.By("Creating AdminPolicyBasedExternalRoute with static next hops") + dynamicClient, err := dynamic.NewForConfig(config) + o.Expect(err).NotTo(o.HaveOccurred()) + + apbrGVR := schema.GroupVersionResource{ + Group: "k8s.ovn.org", + Version: "v1", + Resource: "adminpolicybasedexternalroutes", + } + + apbr := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "k8s.ovn.org/v1", + "kind": "AdminPolicyBasedExternalRoute", + "metadata": map[string]interface{}{ + "name": "default-route-policy", + }, + "spec": map[string]interface{}{ + "from": map[string]interface{}{ + "namespaceSelector": map[string]interface{}{ + "matchLabels": map[string]interface{}{ + "kubernetes.io/metadata.name": testNS, + }, + }, + }, + "nextHops": map[string]interface{}{ + "static": []interface{}{ + map[string]interface{}{"ip": "172.18.0.8"}, + map[string]interface{}{"ip": "172.18.0.9"}, + }, + }, + }, + }, + } + + _, err = dynamicClient.Resource(apbrGVR).Create(ctx, apbr, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Waiting for AdminPolicyBasedExternalRoute status to be populated") + var apbrStatus *unstructured.Unstructured + o.Eventually(func() bool { + apbrStatus, err = dynamicClient.Resource(apbrGVR).Get(ctx, "default-route-policy", metav1.GetOptions{}) + if err != nil { + return false + } + status, found, _ := unstructured.NestedMap(apbrStatus.Object, "status") + return found && len(status) > 0 + }, 120, 10).Should(o.BeTrue(), "AdminPolicyBasedExternalRoute status should be populated") + + g.By("Verifying status contains messages from all zones (nodes)") + status, found, err := unstructured.NestedMap(apbrStatus.Object, "status") + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(found).To(o.BeTrue(), "Status field should exist") + + messages, found, err := unstructured.NestedSlice(status, "messages") + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(found).To(o.BeTrue(), "Status.messages field should exist") + o.Expect(len(messages)).To(o.BeNumerically(">", 0), "Status.messages should contain at least one zone report") + + // Get node count to validate we have messages from nodes + nodes, err := clientset.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + nodeCount := len(nodes.Items) + + g.By(fmt.Sprintf("Verifying status.messages has entries from zones (cluster has %d nodes)", nodeCount)) + // Each message should be prefixed with zone name (node name by default) + // Format: ": configured external gateway IPs: 172.18.0.8,172.18.0.9" + for _, msg := range messages { + msgStr, ok := msg.(string) + o.Expect(ok).To(o.BeTrue(), "Each message should be a string") + o.Expect(msgStr).To(o.ContainSubstring("configured external gateway IPs"), + "Message should describe configured gateway IPs") + o.Expect(msgStr).To(o.MatchRegexp(`^[^:]+:`), + "Message should be prefixed with zone name followed by colon") + } + + g.By("Verifying status.status is Success when all zones report success") + statusValue, found, err := unstructured.NestedString(status, "status") + o.Expect(err).NotTo(o.HaveOccurred()) + if found { + // Status should be "Success" when all zones reported successfully + // or empty if not all zones have reported yet + o.Expect(statusValue).To(o.Or(o.Equal("Success"), o.BeEmpty()), + "Status should be 'Success' when all zones reported, or empty if still pending") + } + + g.By("Verifying external gateway IPs are configured in messages") + // Check that at least one message mentions the configured IPs + hasExpectedIPs := false + for _, msg := range messages { + msgStr := msg.(string) + if strings.Contains(msgStr, "172.18.0.8") && strings.Contains(msgStr, "172.18.0.9") { + hasExpectedIPs = true + break + } + } + o.Expect(hasExpectedIPs).To(o.BeTrue(), + "At least one zone should report the configured external gateway IPs (172.18.0.8, 172.18.0.9)") + }) +}) + +// runOVNKubeTrace executes ovnkube-trace in an ovnkube-node pod +func runOVNKubeTrace(ctx context.Context, clientset *kubernetes.Clientset, config *rest.Config, + srcNS, srcPod, dstNS, dstPod, protocol, port string) (string, error) { + + // Find an ovnkube-node pod + pods, err := clientset.CoreV1().Pods("openshift-ovn-kubernetes").List(ctx, metav1.ListOptions{ + LabelSelector: "app=ovnkube-node", + }) + if err != nil { + return "", err + } + if len(pods.Items) == 0 { + return "", fmt.Errorf("no ovnkube-node pods found in openshift-ovn-kubernetes namespace") + } + + nodePod := pods.Items[0].Name + + // Build ovnkube-trace command + execCmd := []string{ + "ovnkube-trace", + "-src-namespace", srcNS, + "-src", srcPod, + "-dst-namespace", dstNS, + "-dst", dstPod, + "-" + protocol, + "-dst-port", port, + "-loglevel", "2", + } + + scheme := runtime.NewScheme() + if err := corev1.AddToScheme(scheme); err != nil { + return "", err + } + + req := clientset.CoreV1().RESTClient().Post(). + Resource("pods"). + Name(nodePod). + Namespace("openshift-ovn-kubernetes"). + SubResource("exec"). + VersionedParams(&corev1.PodExecOptions{ + Container: "ovnkube-controller", + Command: execCmd, + Stdout: true, + Stderr: true, + }, runtime.NewParameterCodec(scheme)) + + exec, err := remotecommand.NewSPDYExecutor(config, "POST", req.URL()) + if err != nil { + return "", err + } + + var stdout, stderr bytes.Buffer + err = exec.StreamWithContext(ctx, remotecommand.StreamOptions{ + Stdout: &stdout, + Stderr: &stderr, + }) + if err != nil { + return stdout.String() + "\n" + stderr.String(), err + } + + return stdout.String(), nil +} diff --git a/openshift/test/otp/security.go b/openshift/test/otp/security.go new file mode 100644 index 0000000000..bac800a7ee --- /dev/null +++ b/openshift/test/otp/security.go @@ -0,0 +1,290 @@ +package otp + +import ( + "bytes" + "context" + "strings" + + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/client-go/tools/remotecommand" +) + +var _ = g.Describe("[JIRA:Networking][OTP][sig-network] OTP Security", func() { + var ( + clientset *kubernetes.Clientset + config *rest.Config + ctx context.Context + ) + + g.BeforeEach(func() { + ctx = context.Background() + + // Load kubeconfig + loadingRules := clientcmd.NewDefaultClientConfigLoadingRules() + configOverrides := &clientcmd.ConfigOverrides{} + kubeConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, configOverrides) + + var err error + config, err = kubeConfig.ClientConfig() + o.Expect(err).NotTo(o.HaveOccurred()) + + clientset, err = kubernetes.NewForConfig(config) + o.Expect(err).NotTo(o.HaveOccurred()) + }) + + // Medium-49216: API Token Logging Security + g.It("[OTP][informing][49216] should not expose API tokens in ovnkube-node logs", func() { + g.By("Getting all ovnkube-node pods") + pods, err := clientset.CoreV1().Pods("openshift-ovn-kubernetes").List(ctx, metav1.ListOptions{ + LabelSelector: "app=ovnkube-node", + }) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(pods.Items)).To(o.BeNumerically(">", 0), "Expected at least one ovnkube-node pod") + + g.By("Checking logs from each ovnkube-node pod for token exposure") + totalViolations := 0 + failedPods := []string{} + skippedPods := []string{} + + for _, pod := range pods.Items { + // Get logs from ovnkube-controller container + logOptions := &corev1.PodLogOptions{ + Container: "ovnkube-controller", + TailLines: int64Ptr(10000), + } + + req := clientset.CoreV1().Pods("openshift-ovn-kubernetes").GetLogs(pod.Name, logOptions) + logs, err := req.DoRaw(ctx) + + // If logs can't be retrieved, record and skip this pod + if err != nil { + g.GinkgoWriter.Printf("Warning: could not retrieve logs for pod %s: %v\n", pod.Name, err) + skippedPods = append(skippedPods, pod.Name) + continue + } + + logsStr := string(logs) + + // Search for sensitive patterns + patterns := []string{"api-token", "authorization", "bearer"} + podViolations := 0 + + for _, pattern := range patterns { + if strings.Contains(strings.ToLower(logsStr), pattern) { + // Filter out false positives (configuration field names without values) + lines := strings.Split(logsStr, "\n") + for _, line := range lines { + lowerLine := strings.ToLower(line) + if strings.Contains(lowerLine, pattern) { + // Check if it's just an empty field (e.g., "Token: " with no value) + if !strings.Contains(lowerLine, "token:") || + (strings.Contains(lowerLine, "token:") && !strings.Contains(lowerLine, "token: ")) { + // This might be an actual token + if strings.Contains(lowerLine, "eyj") || // JWT tokens start with "eyJ" + strings.Contains(lowerLine, "bearer ") { + podViolations++ + break + } + } + } + } + } + } + + if podViolations > 0 { + totalViolations += podViolations + failedPods = append(failedPods, pod.Name) + } + } + + // Ensure at least some pods were scanned + scannedCount := len(pods.Items) - len(skippedPods) + o.Expect(scannedCount).To(o.BeNumerically(">", 0), + "Could not retrieve logs from any pod - all %d pods skipped: %v", + len(pods.Items), skippedPods) + + // Assert no tokens were found + o.Expect(totalViolations).To(o.Equal(0), + "Found %d potential token exposures in pods: %v", + totalViolations, failedPods) + }) + + // Medium-77102: CIS File Permissions for CNI Config + g.It("[OTP][informing][77102] should have secure permissions on CNI configuration files", func() { + g.By("Checking multus config permissions via multus pods") + multusPods, err := clientset.CoreV1().Pods("openshift-multus").List(ctx, metav1.ListOptions{ + LabelSelector: "app=multus", + }) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(multusPods.Items)).To(o.BeNumerically(">", 0), "Expected at least one multus pod") + + // Check first multus pod for config file permissions + multusPod := multusPods.Items[0].Name + output, err := execInPod(ctx, clientset, config, "openshift-multus", multusPod, "kube-multus", + []string{"/bin/bash", "-c", "stat -c '%a %n' /host/etc/cni/net.d/*.conf"}) + o.Expect(err).NotTo(o.HaveOccurred(), "Failed to check multus config permissions") + + g.By("Verifying multus config has 600 permissions") + lines := strings.Split(strings.TrimSpace(output), "\n") + for _, line := range lines { + if line == "" { + continue + } + parts := strings.Fields(line) + o.Expect(len(parts)).To(o.BeNumerically(">=", 2), "Invalid stat output: %s", line) + perms := parts[0] + filename := parts[1] + o.Expect(perms).To(o.Equal("600"), + "CIS violation: %s has insecure permissions %s (expected 600)", filename, perms) + } + + g.By("Checking whereabouts config permissions") + // Get a worker node + nodes, err := clientset.CoreV1().Nodes().List(ctx, metav1.ListOptions{ + LabelSelector: "node-role.kubernetes.io/worker", + }) + o.Expect(err).NotTo(o.HaveOccurred()) + + if len(nodes.Items) == 0 { + // Fall back to any node if no workers labeled + nodes, err = clientset.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + } + o.Expect(len(nodes.Items)).To(o.BeNumerically(">", 0), "Expected at least one node") + + nodeName := nodes.Items[0].Name + + // Create debug pod on node + debugPodName := "cis-perms-check-77102" + debugPod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: debugPodName, + Namespace: "openshift-ovn-kubernetes", + }, + Spec: corev1.PodSpec{ + NodeName: nodeName, + HostNetwork: true, + HostPID: true, + Containers: []corev1.Container{ + { + Name: "debug", + Image: "registry.access.redhat.com/ubi8/ubi-minimal:latest", + Command: []string{"sleep", "300"}, + SecurityContext: &corev1.SecurityContext{ + Privileged: boolPtr(true), + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "host", + MountPath: "/host", + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "host", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/", + }, + }, + }, + }, + RestartPolicy: corev1.RestartPolicyNever, + }, + } + + _, err = clientset.CoreV1().Pods("openshift-ovn-kubernetes").Create(ctx, debugPod, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + defer func() { + _ = clientset.CoreV1().Pods("openshift-ovn-kubernetes").Delete(ctx, debugPodName, metav1.DeleteOptions{}) + }() + + // Wait for debug pod to be running + o.Eventually(func() corev1.PodPhase { + p, err := clientset.CoreV1().Pods("openshift-ovn-kubernetes").Get(ctx, debugPodName, metav1.GetOptions{}) + if err != nil { + return corev1.PodPending + } + return p.Status.Phase + }, 60, 5).Should(o.Equal(corev1.PodRunning), "Debug pod did not reach Running state") + + // Check whereabouts config file permissions + output, err = execInPod(ctx, clientset, config, "openshift-ovn-kubernetes", debugPodName, "debug", + []string{"/bin/bash", "-c", "stat -c '%a %n' /host/etc/kubernetes/cni/net.d/whereabouts.d/*.conf /host/etc/kubernetes/cni/net.d/whereabouts.d/*.kubeconfig 2>/dev/null || true"}) + o.Expect(err).NotTo(o.HaveOccurred(), "Failed to check whereabouts config permissions") + + g.By("Verifying whereabouts configs have 600 permissions") + if strings.TrimSpace(output) != "" { + lines = strings.Split(strings.TrimSpace(output), "\n") + for _, line := range lines { + if line == "" { + continue + } + parts := strings.Fields(line) + o.Expect(len(parts)).To(o.BeNumerically(">=", 2), "Invalid stat output: %s", line) + perms := parts[0] + filename := parts[1] + o.Expect(perms).To(o.Equal("600"), + "CIS violation: %s has insecure permissions %s (expected 600)", filename, perms) + } + } + }) +}) + +// Helper functions +func int64Ptr(i int64) *int64 { + return &i +} + +func boolPtr(b bool) *bool { + return &b +} + +// execInPod executes a command in a pod and returns the output +func execInPod(ctx context.Context, clientset *kubernetes.Clientset, config *rest.Config, + namespace, podName, containerName string, command []string) (string, error) { + + scheme := runtime.NewScheme() + if err := corev1.AddToScheme(scheme); err != nil { + return "", err + } + + req := clientset.CoreV1().RESTClient().Post(). + Resource("pods"). + Name(podName). + Namespace(namespace). + SubResource("exec"). + VersionedParams(&corev1.PodExecOptions{ + Container: containerName, + Command: command, + Stdout: true, + Stderr: true, + }, runtime.NewParameterCodec(scheme)) + + exec, err := remotecommand.NewSPDYExecutor(config, "POST", req.URL()) + if err != nil { + return "", err + } + + var stdout, stderr bytes.Buffer + err = exec.StreamWithContext(ctx, remotecommand.StreamOptions{ + Stdout: &stdout, + Stderr: &stderr, + }) + if err != nil { + return stdout.String() + "\n" + stderr.String(), err + } + + return stdout.String(), nil +} diff --git a/openshift/test/tests.go b/openshift/test/tests.go index 952b62e96e..a6c2ff7781 100644 --- a/openshift/test/tests.go +++ b/openshift/test/tests.go @@ -106,8 +106,22 @@ var InformingTests = []string{ "[Feature:EVPN][Feature:RouteAdvertisements][FeatureGate:EVPN][ovn-kubernetes-ote][sig-network] EVPN: VTEP API validations api-server should reject invalid VTEP updates Invalid VTEP update configurations [Suite:openshift/conformance/parallel]", "[Feature:EVPN][Feature:RouteAdvertisements][FeatureGate:EVPN][ovn-kubernetes-ote][sig-network] BGP: For BGP configured networks When the tested network is of type Layer 2 CUDN EVPN MAC-VRF and IP-VRF random VTEP When a pod runs on the tested network It can be reached by an external server on the same network When the network is IPv4 [Suite:openshift/conformance/parallel]", "[Feature:EVPN][Feature:RouteAdvertisements][FeatureGate:EVPN][ovn-kubernetes-ote][sig-network] BGP: For BGP configured networks When the tested network is of type Layer 3 CUDN EVPN IP-VRF random VTEP When a pod runs on the tested network It can be reached by an external server on the same network When the network is IPv6 [Suite:openshift/conformance/parallel]", + // OTP (OpenShift Tests Private) migration - informing tests + "[JIRA:Networking][OTP][sig-network] OTP Networking Tools [OTP][informing][case_id:67625] should trace pod-to-pod traffic successfully", + "[JIRA:Networking][OTP][sig-network] OTP Networking Tools [OTP][informing][case_id:67648] should trace pod-to-hostnetworkpod traffic successfully", + "[JIRA:Networking][OTP][sig-network] OTP Multus [OTP][informing][case_id:69947] should send Unsolicited Neighbor Advertisements when macvlan pod is created", } // BlockingTests lists tests that are considered stable and should block CI jobs // if they fail. -var BlockingTests = []string{} +var BlockingTests = []string{ + // OTP (OpenShift Tests Private) migration - blocking tests + "[JIRA:Networking][OTP][sig-network] OTP Security [OTP][blocking][case_id:49216] should not expose API tokens in ovnkube-node logs", + "[JIRA:Networking][OTP][sig-network] OTP Security [OTP][blocking][case_id:77102] should have secure permissions on CNI configuration files", + "[JIRA:Networking][OTP][sig-network] OTP Multus [OTP][blocking][case_id:57589] should handle large IPv6 exclude ranges without timeout", + "[JIRA:Networking][OTP][sig-network] OTP Multus [OTP][blocking][case_id:76652] should support Dummy CNI plugin with Multus", + "[JIRA:Networking][OTP][sig-network] OTP Networking Tools [OTP][blocking][case_id:55889] should execute ovn-db-run-command script successfully", + "[JIRA:Networking][OTP][sig-network] OTP Networking Tools [OTP][blocking][case_id:45146] should create healthy pod with single-stack gateway on dual-stack cluster", + "[JIRA:Networking][OTP][sig-network] OTP Networking Tools [OTP][blocking][case_id:69761] should show aggregated status from all zones in AdminPolicyBasedExternalRoute", + "[JIRA:Networking][OTP][sig-network] OTP Multus [OTP][blocking][case_id:66876] should assign dual-stack IPs with Whereabouts IPAM", +}