functional tests for externally manage feature

Signed-off-by: Sebastian Sch <[email protected]>

test/conformance/tests/test_sriov_operator.go

@@ -21,6 +21,8 @@ import (
 	rbacv1 "k8s.io/api/rbac/v1"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/fields"
+	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/utils/pointer"
 	runtimeclient "sigs.k8s.io/controller-runtime/pkg/client"
@@ -907,28 +909,24 @@ var _ = Describe("[sriov] operator", func() {
 	})
 
 	Describe("Custom SriovNetworkNodePolicy", func() {
-		var vfioNode string
-		var vfioNic sriovv1.InterfaceExt
-		execute.BeforeAll(func() {
-			vfioNode, vfioNic = sriovInfos.FindOneVfioSriovDevice()
-		})
-
 		BeforeEach(func() {
 			err := namespaces.Clean(operatorNamespace, namespaces.Test, clients, discovery.Enabled())
 			Expect(err).ToNot(HaveOccurred())
 			WaitForSRIOVStable()
 		})
 
 		Describe("Configuration", func() {
-
 			Context("Create vfio-pci node policy", func() {
+				var vfioNode string
+				var vfioNic sriovv1.InterfaceExt
+
 				BeforeEach(func() {
 					if discovery.Enabled() {
 						Skip("Test unsuitable to be run in discovery mode")
 					}
-					if vfioNode == "" {
-						Skip("skip test as no vfio-pci capable PF was found")
-					}
+
+					vfioNode, vfioNic = sriovInfos.FindOneVfioSriovDevice()
+					Expect(vfioNode).ToNot(Equal(""))
 				})
 
 				It("Should be possible to create a vfio-pci resource", func() {
@@ -964,14 +962,16 @@ var _ = Describe("[sriov] operator", func() {
 			})
 
 			Context("PF Partitioning", func() {
+				var vfioNode string
+				var vfioNic sriovv1.InterfaceExt
+
 				// 27633
 				BeforeEach(func() {
 					if discovery.Enabled() {
 						Skip("Test unsuitable to be run in discovery mode")
 					}
-					if vfioNode == "" {
-						Skip("skip test as no vfio-pci capable PF was found")
-					}
+					vfioNode, vfioNic = sriovInfos.FindOneVfioSriovDevice()
+					Expect(vfioNode).ToNot(Equal(""))
 				})
 
 				It("Should be possible to partition the pf's vfs", func() {
@@ -1681,6 +1681,135 @@ var _ = Describe("[sriov] operator", func() {
 				Expect(err).ToNot(HaveOccurred())
 			})
 		})
+
+		Context("ExternallyCreated Validation", func() {
+			numVfs := 5
+			var node string
+			var nic sriovv1.InterfaceExt
+			externallyManage := func(policy *sriovv1.SriovNetworkNodePolicy) {
+				policy.Spec.ExternallyCreated = true
+			}
+
+			execute.BeforeAll(func() {
+				node, nic = sriovInfos.FindOneVfioSriovDevice()
+				Expect(node).ToNot(Equal(""))
+			})
+
+			It("Should not allow to create a policy if there are no vfs configured", func() {
+				resourceName := "test"
+				_, err := network.CreateSriovPolicy(clients, "test-policy-", operatorNamespace, nic.Name, node, numVfs, resourceName, "netdevice", externallyManage)
+				Expect(err).To(HaveOccurred())
+				Expect(err.Error()).To(ContainSubstring("is not equal to the number of virtual functions allocated for the PF externally"))
+			})
+
+			It("Should create a policy if the number of requested vfs is equal", func() {
+				resourceName := "testexternally"
+				By("allocating the 5 virtual functions to the selected device")
+				_, errOutput, err := runCommandOnConfigDaemon(node, "/bin/bash", "-c", fmt.Sprintf("echo 5 > /host/sys/class/net/%s/device/sriov_numvfs", nic.Name))
+				Expect(err).ToNot(HaveOccurred())
+				Expect(errOutput).To(Equal(""))
+
+				By("creating the policy that will use the 5 virtual functions we create manually on the system")
+				Eventually(func() error {
+					_, err := network.CreateSriovPolicy(clients, "test-policy-", operatorNamespace, nic.Name, node, numVfs, resourceName, "netdevice", externallyManage)
+					return err
+				}, 1*time.Minute, time.Second).ShouldNot(HaveOccurred())
+
+				Eventually(func() int64 {
+					testedNode, err := clients.CoreV1Interface.Nodes().Get(context.Background(), node, metav1.GetOptions{})
+					Expect(err).ToNot(HaveOccurred())
+					resNum := testedNode.Status.Allocatable[corev1.ResourceName("openshift.io/"+resourceName)]
+					allocatable, _ := resNum.AsInt64()
+					return allocatable
+				}, 2*time.Minute, time.Second).Should(Equal(int64(numVfs)))
+
+				By("cleaning the manual sriov created")
+				_, errOutput, err = runCommandOnConfigDaemon(node, "/bin/bash", "-c", fmt.Sprintf("echo 0 > /host/sys/class/net/%s/device/sriov_numvfs", nic.Name))
+				Expect(err).ToNot(HaveOccurred())
+				Expect(errOutput).To(Equal(""))
+			})
+
+			It("Should create a policy if the number of requested vfs is equal and not delete them when the policy is removed", func() {
+				resourceName := "testexternally"
+				var sriovPolicy *sriovv1.SriovNetworkNodePolicy
+				By("allocating the 5 virtual functions to the selected device")
+				_, errOutput, err := runCommandOnConfigDaemon(node, "/bin/bash", "-c", fmt.Sprintf("echo 5 > /host/sys/class/net/%s/device/sriov_numvfs", nic.Name))
+				Expect(err).ToNot(HaveOccurred())
+				Expect(errOutput).To(Equal(""))
+
+				By("creating the policy that will use the 5 virtual functions we create manually on the system")
+				Eventually(func() error {
+					sriovPolicy, err = network.CreateSriovPolicy(clients, "test-policy-", operatorNamespace, nic.Name, node, numVfs, resourceName, "netdevice", externallyManage)
+					return err
+				}, 2*time.Minute, time.Second).ShouldNot(HaveOccurred())
+
+				Eventually(func() int64 {
+					testedNode, err := clients.CoreV1Interface.Nodes().Get(context.Background(), node, metav1.GetOptions{})
+					Expect(err).ToNot(HaveOccurred())
+					resNum := testedNode.Status.Allocatable[corev1.ResourceName("openshift.io/"+resourceName)]
+					allocatable, _ := resNum.AsInt64()
+					return allocatable
+				}, 3*time.Minute, time.Second).Should(Equal(int64(numVfs)))
+
+				By("deleting the policy")
+				err = clients.Delete(context.Background(), sriovPolicy, &runtimeclient.DeleteOptions{})
+				Expect(err).ToNot(HaveOccurred())
+				WaitForSRIOVStable()
+
+				Eventually(func() int64 {
+					testedNode, err := clients.CoreV1Interface.Nodes().Get(context.Background(), node, metav1.GetOptions{})
+					Expect(err).ToNot(HaveOccurred())
+					resNum := testedNode.Status.Allocatable[corev1.ResourceName("openshift.io/"+resourceName)]
+					allocatable, _ := resNum.AsInt64()
+					return allocatable
+				}, 2*time.Minute, time.Second).Should(Equal(int64(0)))
+
+				By("checking the virtual functions are still on the host")
+				output, errOutput, err := runCommandOnConfigDaemon(node, "/bin/bash", "-c", fmt.Sprintf("cat /host/sys/class/net/%s/device/sriov_numvfs", nic.Name))
+				Expect(err).ToNot(HaveOccurred())
+				Expect(errOutput).To(Equal(""))
+				Expect(output).To(ContainSubstring("5"))
+
+				By("cleaning the manual sriov created")
+				_, errOutput, err = runCommandOnConfigDaemon(node, "/bin/bash", "-c", fmt.Sprintf("echo 0 > /host/sys/class/net/%s/device/sriov_numvfs", nic.Name))
+				Expect(err).ToNot(HaveOccurred())
+				Expect(errOutput).To(Equal(""))
+			})
+
+			It("should reset the virtual functions if externallyCreated is false", func() {
+				resourceName := "testexternally"
+				var sriovPolicy *sriovv1.SriovNetworkNodePolicy
+				By("creating the policy for 5 virtual functions")
+				sriovPolicy, err := network.CreateSriovPolicy(clients, "test-policy-", operatorNamespace, nic.Name, node, numVfs, resourceName, "netdevice")
+
+				Eventually(func() int64 {
+					testedNode, err := clients.CoreV1Interface.Nodes().Get(context.Background(), node, metav1.GetOptions{})
+					Expect(err).ToNot(HaveOccurred())
+					resNum := testedNode.Status.Allocatable[corev1.ResourceName("openshift.io/"+resourceName)]
+					allocatable, _ := resNum.AsInt64()
+					return allocatable
+				}, 3*time.Minute, time.Second).Should(Equal(int64(numVfs)))
+
+				By("deleting the policy")
+				err = clients.Delete(context.Background(), sriovPolicy, &runtimeclient.DeleteOptions{})
+				Expect(err).ToNot(HaveOccurred())
+				WaitForSRIOVStable()
+
+				Eventually(func() int64 {
+					testedNode, err := clients.CoreV1Interface.Nodes().Get(context.Background(), node, metav1.GetOptions{})
+					Expect(err).ToNot(HaveOccurred())
+					resNum := testedNode.Status.Allocatable[corev1.ResourceName("openshift.io/"+resourceName)]
+					allocatable, _ := resNum.AsInt64()
+					return allocatable
+				}, 3*time.Minute, time.Second).Should(Equal(int64(0)))
+
+				By("checking the virtual functions don't exist anymore on the system")
+				output, errOutput, err := runCommandOnConfigDaemon(node, "/bin/bash", "-c", fmt.Sprintf("cat /host/sys/class/net/%s/device/sriov_numvfs", nic.Name))
+				Expect(err).ToNot(HaveOccurred())
+				Expect(errOutput).To(Equal(""))
+				Expect(output).To(ContainSubstring("0"))
+			})
+		})
 	})
 })
 
@@ -2080,6 +2209,20 @@ func createVanillaNetworkPolicy(node string, sriovInfos *cluster.EnabledNodes, n
 		})))
 }
 
+func runCommandOnConfigDaemon(nodeName string, command ...string) (string, string, error) {
+	pods := &corev1.PodList{}
+	label, err := labels.Parse("app=sriov-network-config-daemon")
+	Expect(err).ToNot(HaveOccurred())
+	field, err := fields.ParseSelector(fmt.Sprintf("spec.nodeName=%s", nodeName))
+	Expect(err).ToNot(HaveOccurred())
+	err = clients.List(context.Background(), pods, &runtimeclient.ListOptions{Namespace: operatorNamespace, LabelSelector: label, FieldSelector: field})
+	Expect(err).ToNot(HaveOccurred())
+	Expect(len(pods.Items)).To(Equal(1))
+
+	output, errOutput, err := pod.ExecCommand(clients, &pods.Items[0], command...)
+	return output, errOutput, err
+}
+
 func defaultFilterPolicy(policy sriovv1.SriovNetworkNodePolicy) bool {
 	return policy.Spec.DeviceType == "netdevice"
 }