e2e: Refactor PF shutdown test case

Shutting down a PF to test if the NIC can be used for
pod-to-pod connectivity causes a race condition, as
the config-daemon would try to set it up again during
reconciliation. Also, it would trigger a node drain,
making the test case very flaky.

Instead, convert the test case to verify that a NIC with
 no carrier provides intra-node connectivity.

Signed-off-by: Andrea Panattoni <[email protected]>

Author: zeeke

test/conformance/tests/test_policy_configuration.go

@@ -456,15 +456,15 @@ var _ = Describe("[sriov] operator", Ordered, func() {
 			})
 			Context("PF shutdown", func() {
 				// 29398
-				It("Should be able to create pods successfully if PF is down.Pods are able to communicate with each other on the same node", func() {
+				It("Should be able to create pods successfully on a NIC with NO_CARRIER. Pods are able to communicate with each other on the same node", func() {
 					if cluster.VirtualCluster() {
 						// https://bugzilla.redhat.com/show_bug.cgi?id=2214976
 						Skip("Bug in IGB driver")
 					}
 
 					resourceName := testResourceName
 					var testNode string
-					var unusedSriovDevice *sriovv1.InterfaceExt
+					var noCarrierSriovDevice *sriovv1.InterfaceExt
 
 					if discovery.Enabled() {
 						Skip("PF Shutdown test not enabled in discovery mode")
@@ -473,17 +473,15 @@ var _ = Describe("[sriov] operator", Ordered, func() {
 					testNode = sriovInfos.Nodes[0]
 					sriovDeviceList, err := sriovInfos.FindSriovDevices(testNode)
 					Expect(err).ToNot(HaveOccurred())
-					unusedSriovDevices, err := findUnusedSriovDevices(testNode, sriovDeviceList)
+					noCarrierSriovDevices, err := findNoCarrierSriovDevices(testNode, sriovDeviceList)
 					if err != nil {
 						Skip(err.Error())
 					}
-					unusedSriovDevice = unusedSriovDevices[0]
+					noCarrierSriovDevice = noCarrierSriovDevices[0]
 
-					By("Using device " + unusedSriovDevice.Name + " on node " + testNode)
+					By("Using device " + noCarrierSriovDevice.Name + " on node " + testNode)
 
-					defer changeNodeInterfaceState(testNode, unusedSriovDevices[0].Name, true)
-					Expect(err).ToNot(HaveOccurred())
-					createSriovPolicy(unusedSriovDevice.Name, testNode, 2, resourceName)
+					createSriovPolicy(noCarrierSriovDevice.Name, testNode, 2, resourceName)
 
 					ipam := `{
 						"type":"host-local",
@@ -492,10 +490,10 @@ var _ = Describe("[sriov] operator", Ordered, func() {
 						"rangeEnd":"10.10.10.181"
 						}`
 
-					err = network.CreateSriovNetwork(clients, unusedSriovDevice, sriovNetworkName, namespaces.Test, operatorNamespace, resourceName, ipam)
+					err = network.CreateSriovNetwork(clients, noCarrierSriovDevice, sriovNetworkName, namespaces.Test, operatorNamespace, resourceName, ipam)
 					Expect(err).ToNot(HaveOccurred())
 					waitForNetAttachDef(sriovNetworkName, namespaces.Test)
-					changeNodeInterfaceState(testNode, unusedSriovDevice.Name, false)
+
 					pod := createTestPod(testNode, []string{sriovNetworkName})
 					ips, err := network.GetSriovNicIPs(pod, "net1")
 					Expect(err).ToNot(HaveOccurred())

test/conformance/tests/test_sriov_operator.go

@@ -1543,29 +1543,6 @@ func getDriver(ethtoolstdout string) string {
 	return ""
 }
 
-func changeNodeInterfaceState(testNode string, ifcName string, enable bool) {
-	state := "up"
-	if !enable {
-		state = "down"
-	}
-	podDefinition := pod.RedefineAsPrivileged(
-		pod.RedefineWithRestartPolicy(
-			pod.RedefineWithCommand(
-				pod.DefineWithHostNetwork(testNode),
-				[]string{"ip", "link", "set", "dev", ifcName, state}, []string{},
-			),
-			corev1.RestartPolicyNever,
-		),
-	)
-	createdPod, err := clients.Pods(namespaces.Test).Create(context.Background(), podDefinition, metav1.CreateOptions{})
-	Expect(err).ToNot(HaveOccurred())
-	Eventually(func() corev1.PodPhase {
-		runningPod, err := clients.Pods(namespaces.Test).Get(context.Background(), createdPod.Name, metav1.GetOptions{})
-		Expect(err).ToNot(HaveOccurred())
-		return runningPod.Status.Phase
-	}, 3*time.Minute, 1*time.Second).Should(Equal(corev1.PodSucceeded))
-}
-
 func discoverResourceForMainSriov(nodes *cluster.EnabledNodes) (*sriovv1.InterfaceExt, string, string, bool) {
 	for _, node := range nodes.Nodes {
 		nodeDevices, err := nodes.FindSriovDevices(node)
@@ -1690,6 +1667,33 @@ func isDefaultRouteInterface(intfName string, routes []string) bool {
 	return false
 }
 
+func findNoCarrierSriovDevices(testNode string, sriovDevices []*sriovv1.InterfaceExt) ([]*sriovv1.InterfaceExt, error) {
+	filteredDevices := []*sriovv1.InterfaceExt{}
+
+	for _, device := range sriovDevices {
+		stdout, stderr, err := runCommandOnConfigDaemon(testNode, "/bin/bash", "-c", fmt.Sprintf("ip link show %s", device.Name))
+		if err != nil {
+			fmt.Printf("Can't query link state for device [%s]: %s", device.Name, err.Error())
+			continue
+		}
+
+		if len(stdout) == 0 {
+			fmt.Printf("Can't query link state for device [%s]: stderr:[%s]", device.Name, stderr)
+			continue
+		}
+
+		// Check for NO-CARRIER state
+		if strings.Contains(stdout, "NO-CARRIER") {
+			filteredDevices = append(filteredDevices, device)
+		}
+	}
+	if len(filteredDevices) == 0 {
+		return nil, fmt.Errorf("no carrier sriov devices not found")
+	}
+
+	return filteredDevices, nil
+}
+
 // podVFIndexInHost retrieves the vf index on the host network namespace related to the given
 // interface that was passed to the pod, using the name in the pod's namespace.
 func podVFIndexInHost(hostNetPod *corev1.Pod, targetPod *corev1.Pod, interfaceName string) (int, error) {