From 434118fa0bc2f99527f8e61a890e8711a567c5ca Mon Sep 17 00:00:00 2001 From: Andrea Panattoni Date: Tue, 27 Aug 2024 19:00:14 +0200 Subject: [PATCH] draft Signed-off-by: Andrea Panattoni --- .../e2esuite/nmstate/nmstate_sriov.go | 179 +++++++++++++++--- .../testsuites/e2esuite/test_suite_test.go | 11 +- 2 files changed, 157 insertions(+), 33 deletions(-) diff --git a/cnf-tests/testsuites/e2esuite/nmstate/nmstate_sriov.go b/cnf-tests/testsuites/e2esuite/nmstate/nmstate_sriov.go index 384bb7464d..f5222cfeb5 100644 --- a/cnf-tests/testsuites/e2esuite/nmstate/nmstate_sriov.go +++ b/cnf-tests/testsuites/e2esuite/nmstate/nmstate_sriov.go @@ -1,10 +1,27 @@ -package bond +package nmstate import ( + "context" + "errors" + "fmt" + "regexp" + "time" + + sriovv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" sriovtestclient "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/client" + "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/cluster" + sriovnetwork "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/network" + nmstateshared "github.com/nmstate/kubernetes-nmstate/api/shared" + nmstatev1 "github.com/nmstate/kubernetes-nmstate/api/v1" client "github.com/openshift-kni/cnf-features-deploy/cnf-tests/testsuites/pkg/client" "github.com/openshift-kni/cnf-features-deploy/cnf-tests/testsuites/pkg/namespaces" "github.com/openshift-kni/cnf-features-deploy/cnf-tests/testsuites/pkg/networks" + "github.com/openshift-kni/cnf-features-deploy/cnf-tests/testsuites/pkg/nodes" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtimeclient "sigs.k8s.io/controller-runtime/pkg/client" + + corev1 "k8s.io/api/core/v1" + //"k8s.io/apimachinery/pkg/api/errors" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -14,63 +31,169 @@ var sriovclient *sriovtestclient.ClientSet const testNamespace string = "nmstate-sriov-testing" +const nmstateDesiredStateTemplateStaticIP string = +`interfaces: +- name: %s + description: test for SR-IOV network operator integration + type: ethernet + state: up + ipv4: + dhcp: false + address: + - ip: 192.0.2.2 + prefix-length: 24 + enabled: true` + func init() { + sriovclient = sriovtestclient.New("") } -var _ = Describe("[sriov] NMState Operator Integration", func() { +var _ = Describe("[sriov] [nmstate] NMState Operator Integration", Ordered, func() { + + var sriovInfos *cluster.EnabledNodes BeforeAll(func() { - err := namespaces.Create(namespaces.BondTestNamespace, client.Client) + err := namespaces.Create(testNamespace, client.Client) Expect(err).ToNot(HaveOccurred()) By("CleanSriov...") networks.CleanSriov(sriovclient) By("Discover SRIOV devices") - + sriovInfos, err = cluster.DiscoverSriov(sriovclient, namespaces.SRIOVOperator) Expect(err).ToNot(HaveOccurred()) + Expect(len(sriovInfos.Nodes)).ToNot(BeZero(), "no node with SR-IOV NICs found") + //nmstate := nmstatev1.NMState{ObjectMeta: metav1.ObjectMeta{Name: "nmstate"}} + //err = client.Client.Create(context.Background(), &nmstate) + //Expect(err).ToNot(HaveOccurred()) + //DeferCleanup(client.Client.Delete, context.Background(), &nmstate) }) - Context("when a SriovNetworkNodePolicy and NodeNetworkConfigurationPolicySpec targets the same NIC", func() { + Context("when a SriovNetworkNodePolicy and NodeNetworkConfigurationPolicySpec target the same NIC", func() { BeforeAll(func() { namespaces.CleanPods(testNamespace, client.Client) }) - It("VFs should spawn correctly", func() { - + It("XXX VFs should spawn correctly", func() { + node, devices, err := findNonPrimarySriovDevicesAndNode(sriovInfos) + Expect(err).ToNot(HaveOccurred()) + Expect(len(devices)).ToNot(BeZero()) + By("Using node " + node) + testDevice := devices[0] + By("Using device " + testDevice.Name) + + By("Configuring IPAM on Physical Function via NMState") + nodeNetworkConfigPolicy := nmstatev1.NodeNetworkConfigurationPolicy{ + ObjectMeta: metav1.ObjectMeta{ Name: "test-sriov-device-"+testDevice.Name}, + Spec: nmstateshared.NodeNetworkConfigurationPolicySpec{ + NodeSelector: map[string]string{ + "kubernetes.io/hostname": node, + }, + DesiredState: nmstateshared.State{ + Raw: nmstateshared.RawState(fmt.Sprintf(nmstateDesiredStateTemplateStaticIP, testDevice.Name)), + }, + }, + } + err = client.Client.Create(context.Background(), &nodeNetworkConfigPolicy) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(client.Client.Delete, context.Background(), &nodeNetworkConfigPolicy) + + waitForNMStatePolicyToBeStable(&nodeNetworkConfigPolicy) + + By("Configuring VF on device via SriovNetworkNodePolicy") + sriovNetworkNodePolicy, err := sriovnetwork.CreateSriovPolicy(sriovclient, "test-nmstate-", namespaces.SRIOVOperator, testDevice.Name, node, 10, "testnmstateresource", "netdevice") + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(sriovclient.Delete, context.Background(), sriovNetworkNodePolicy) + + Eventually(func() int64 { + testedNode, err := sriovclient.Nodes().Get(context.Background(), node, metav1.GetOptions{}) + Expect(err).ToNot(HaveOccurred()) + resNum := testedNode.Status.Allocatable[corev1.ResourceName("openshift.io/testnmstateresource")] + capacity, _ := resNum.AsInt64() + return capacity + }, 10*time.Minute, time.Second).Should(Equal(int64(10))) }) }) }) -/* -// findUnusedDevice search through all the nodes and NICs to find an SRIOV capable device that -// is not used as primary device for the node. -func findUnusedSRIOVDevice() (string, sriovv1.InterfaceExt) { - ctx := context.Background() - - sriovCapableNodes, err := sriovcluster.DiscoverSriov(sriovclient, namespaces.SRIOVOperator) - for _, nodeName := range sriovCapableNodes.Nodes { - sriovDevices, err := sriovCapableNodes.FindSriovDevices(nodeName) - Expect(err).ToNot(HaveOccurred()) - - nodeNetworkState := nmstatev1beta1.NodeNetworkState{} - err = client.Client.Get(ctx, runtimeclient.ObjectKey{Name: nodeName, Namespace: namespaces.IntelOperator}, nodeNetworkState) - Expect(err).ToNot(HaveOccurred()) - getNodeStateInterface(nodeNetworkState) +func findNonPrimarySriovDevicesAndNode(n *cluster.EnabledNodes) (string, []*sriovv1.InterfaceExt, error) { + errs := []error{} + + retNode := "" + retDevices := []*sriovv1.InterfaceExt{} + + for _, node := range n.Nodes { + devices, err := n.FindSriovDevices(node) + if err != nil { + errs = append(errs, err) + continue + } + + mainDeviceName, err := getPrimaryNICForNode(node) + if err != nil { + errs = append(errs, err) + continue + } + + nonPrimaryDevices := []*sriovv1.InterfaceExt{} + for _, device := range devices { + if device.Name == mainDeviceName { + continue + } + nonPrimaryDevices = append(nonPrimaryDevices, device) + } + + + if len(nonPrimaryDevices) > len(retDevices) { + retNode = node + retDevices = nonPrimaryDevices + } } - for nodeName, sriovNetworkNodeState := range sriovCapableNodes.States { - nic, err := findUnusedDeviceOnNode(nodeName) + + if len(retDevices) == 0 { + return "", nil, fmt.Errorf("can't find any SR-IOV devices in cluster's nodes: %w", errors.Join(errs...)) } + + return retNode, retDevices, nil } -func getNodeStateInterface(state *nmstatev1beta1.NodeNetworkState) { - //for _, interface := range state.Status.CurrentState.Interface +func getPrimaryNICForNode(node string) (string, error) { + nodeObj, err := client.Client.Nodes().Get(context.Background(), node, metav1.GetOptions{}) + if err != nil { + return "", err + } -} + buf, err := nodes.ExecCommandOnMachineConfigDaemon(client.Client, nodeObj, + []string{"/bin/bash", "-c", "chroot /rootfs /usr/bin/ovs-vsctl list-ports br-ex | /usr/bin/grep -v patch"}) + if err != nil { + return "", err + } + exp, err := regexp.Compile("\r\n") + if err != nil { + return "", err + } + + return exp.ReplaceAllString(string(buf), ""), nil +} -*/ \ No newline at end of file +func waitForNMStatePolicyToBeStable(nmstatePolicy *nmstatev1.NodeNetworkConfigurationPolicy) { + key := runtimeclient.ObjectKey{Name: nmstatePolicy.Name} + Eventually(func(g Gomega) { + err := client.Client.Get(context.Background(), key, nmstatePolicy) + g.Expect(err).ToNot(HaveOccurred()) + availableCondition := nmstatePolicy.Status.Conditions. + Find(nmstateshared.NodeNetworkConfigurationPolicyConditionAvailable) + g.Expect(availableCondition).ToNot(BeNil()) + g.Expect(availableCondition.Status). + To(Equal(corev1.ConditionTrue)) + }). + WithOffset(1). + WithPolling(10*time.Second). + WithTimeout(1*time.Minute). + Should(Succeed(), "") +} diff --git a/cnf-tests/testsuites/e2esuite/test_suite_test.go b/cnf-tests/testsuites/e2esuite/test_suite_test.go index 7ba94fd2a9..f93a2ea51b 100644 --- a/cnf-tests/testsuites/e2esuite/test_suite_test.go +++ b/cnf-tests/testsuites/e2esuite/test_suite_test.go @@ -25,11 +25,12 @@ import ( _ "github.com/openshift-kni/cnf-features-deploy/cnf-tests/testsuites/e2esuite/dpdk" // this is needed otherwise the dpdk test won't be executed _ "github.com/openshift-kni/cnf-features-deploy/cnf-tests/testsuites/e2esuite/fec" // this is needed otherwise the fec test won't be executed _ "github.com/openshift-kni/cnf-features-deploy/cnf-tests/testsuites/e2esuite/multinetworkpolicy" // this is needed otherwise the multinetworkpolicy test won't be executed' - _ "github.com/openshift-kni/cnf-features-deploy/cnf-tests/testsuites/e2esuite/ovs_qos" // this is needed otherwise the ovs_qos test won't be executed - _ "github.com/openshift-kni/cnf-features-deploy/cnf-tests/testsuites/e2esuite/s2i" // this is needed otherwise the dpdk test won't be executed - _ "github.com/openshift-kni/cnf-features-deploy/cnf-tests/testsuites/e2esuite/sctp" // this is needed otherwise the sctp test won't be executed - _ "github.com/openshift-kni/cnf-features-deploy/cnf-tests/testsuites/e2esuite/security" // this is needed otherwise the security test won't be executed - _ "github.com/openshift-kni/cnf-features-deploy/cnf-tests/testsuites/e2esuite/vrf" // this is needed otherwise the vrf test won't be executed + _ "github.com/openshift-kni/cnf-features-deploy/cnf-tests/testsuites/e2esuite/nmstate" + _ "github.com/openshift-kni/cnf-features-deploy/cnf-tests/testsuites/e2esuite/ovs_qos" // this is needed otherwise the ovs_qos test won't be executed + _ "github.com/openshift-kni/cnf-features-deploy/cnf-tests/testsuites/e2esuite/s2i" // this is needed otherwise the dpdk test won't be executed + _ "github.com/openshift-kni/cnf-features-deploy/cnf-tests/testsuites/e2esuite/sctp" // this is needed otherwise the sctp test won't be executed + _ "github.com/openshift-kni/cnf-features-deploy/cnf-tests/testsuites/e2esuite/security" // this is needed otherwise the security test won't be executed + _ "github.com/openshift-kni/cnf-features-deploy/cnf-tests/testsuites/e2esuite/vrf" // this is needed otherwise the vrf test won't be executed ) // TODO: we should refactor tests to use client from controller-runtime package