draft

Signed-off-by: Andrea Panattoni <[email protected]>

cnf-tests/testsuites/e2esuite/nmstate/nmstate_sriov.go

@@ -1,10 +1,27 @@
-package bond
+package nmstate
 
 import (
+ "context"
+ "errors"
+ "fmt"
+ "regexp"
+ "time"
+
+ sriovv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1"
  sriovtestclient "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/client"
+ "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/cluster"
+ sriovnetwork "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/network"
+ nmstateshared "github.com/nmstate/kubernetes-nmstate/api/shared"
+ nmstatev1 "github.com/nmstate/kubernetes-nmstate/api/v1"
  client "github.com/openshift-kni/cnf-features-deploy/cnf-tests/testsuites/pkg/client"
  "github.com/openshift-kni/cnf-features-deploy/cnf-tests/testsuites/pkg/namespaces"
  "github.com/openshift-kni/cnf-features-deploy/cnf-tests/testsuites/pkg/networks"
+ "github.com/openshift-kni/cnf-features-deploy/cnf-tests/testsuites/pkg/nodes"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ runtimeclient "sigs.k8s.io/controller-runtime/pkg/client"
+
+ corev1 "k8s.io/api/core/v1"
+ //"k8s.io/apimachinery/pkg/api/errors"
 
  . "github.com/onsi/ginkgo/v2"
  . "github.com/onsi/gomega"
@@ -14,63 +31,169 @@ var sriovclient *sriovtestclient.ClientSet
 
 const testNamespace string = "nmstate-sriov-testing"
 
+const nmstateDesiredStateTemplateStaticIP string = 
+`interfaces:
+- name: %s
+ description: test for SR-IOV network operator integration
+ type: ethernet
+ state: up
+ ipv4:
+ dhcp: false
+ address:
+ - ip: 192.0.2.2
+ prefix-length: 24
+ enabled: true`
+
 func init() {
+ sriovclient = sriovtestclient.New("")
 }
 
-var _ = Describe("[sriov] NMState Operator Integration", func() {
+var _ = Describe("[sriov] [nmstate] NMState Operator Integration", Ordered, func() {
+
+ var sriovInfos *cluster.EnabledNodes
 
  BeforeAll(func() {
- err := namespaces.Create(namespaces.BondTestNamespace, client.Client)
+ err := namespaces.Create(testNamespace, client.Client)
  Expect(err).ToNot(HaveOccurred())
 
  By("CleanSriov...")
  networks.CleanSriov(sriovclient)
 
  By("Discover SRIOV devices")
-
+ sriovInfos, err = cluster.DiscoverSriov(sriovclient, namespaces.SRIOVOperator)
  Expect(err).ToNot(HaveOccurred())
+ Expect(len(sriovInfos.Nodes)).ToNot(BeZero(), "no node with SR-IOV NICs found")
 
+ //nmstate := nmstatev1.NMState{ObjectMeta: metav1.ObjectMeta{Name: "nmstate"}}
+ //err = client.Client.Create(context.Background(), &nmstate)
+ //Expect(err).ToNot(HaveOccurred())
+ //DeferCleanup(client.Client.Delete, context.Background(), &nmstate)
  })
 
- Context("when a SriovNetworkNodePolicy and NodeNetworkConfigurationPolicySpec targets the same NIC", func() {
+ Context("when a SriovNetworkNodePolicy and NodeNetworkConfigurationPolicySpec target the same NIC", func() {
 
  BeforeAll(func() {
  namespaces.CleanPods(testNamespace, client.Client)
  })
 
- It("VFs should spawn correctly", func() {
-
+ It("XXX VFs should spawn correctly", func() {
+ node, devices, err := findNonPrimarySriovDevicesAndNode(sriovInfos)
+ Expect(err).ToNot(HaveOccurred())
+ Expect(len(devices)).ToNot(BeZero())
 
+ By("Using node " + node)
 
+ testDevice := devices[0]
+ By("Using device " + testDevice.Name)
+
+ By("Configuring IPAM on Physical Function via NMState")
+ nodeNetworkConfigPolicy := nmstatev1.NodeNetworkConfigurationPolicy{
+ ObjectMeta: metav1.ObjectMeta{ Name: "test-sriov-device-"+testDevice.Name},
+ Spec: nmstateshared.NodeNetworkConfigurationPolicySpec{
+ NodeSelector: map[string]string{
+ "kubernetes.io/hostname": node,
+ },
+ DesiredState: nmstateshared.State{
+ Raw: nmstateshared.RawState(fmt.Sprintf(nmstateDesiredStateTemplateStaticIP, testDevice.Name)),
+ },
+ },
+ }
+ err = client.Client.Create(context.Background(), &nodeNetworkConfigPolicy)
+ Expect(err).ToNot(HaveOccurred())
+ DeferCleanup(client.Client.Delete, context.Background(), &nodeNetworkConfigPolicy)
+
+ waitForNMStatePolicyToBeStable(&nodeNetworkConfigPolicy)
+
+ By("Configuring VF on device via SriovNetworkNodePolicy")
+ sriovNetworkNodePolicy, err := sriovnetwork.CreateSriovPolicy(sriovclient, "test-nmstate-", namespaces.SRIOVOperator, testDevice.Name, node, 10, "testnmstateresource", "netdevice")
+ Expect(err).ToNot(HaveOccurred())
+ DeferCleanup(sriovclient.Delete, context.Background(), sriovNetworkNodePolicy)
+
+ Eventually(func() int64 {
+ testedNode, err := sriovclient.Nodes().Get(context.Background(), node, metav1.GetOptions{})
+ Expect(err).ToNot(HaveOccurred())
+ resNum := testedNode.Status.Allocatable[corev1.ResourceName("openshift.io/testnmstateresource")]
+ capacity, _ := resNum.AsInt64()
+ return capacity
+ }, 10*time.Minute, time.Second).Should(Equal(int64(10)))
  })
  })
 })
-/*
-// findUnusedDevice search through all the nodes and NICs to find an SRIOV capable device that
-// is not used as primary device for the node.
-func findUnusedSRIOVDevice() (string, sriovv1.InterfaceExt) {
- ctx := context.Background()
-
- sriovCapableNodes, err := sriovcluster.DiscoverSriov(sriovclient, namespaces.SRIOVOperator)
- for _, nodeName := range sriovCapableNodes.Nodes {
- sriovDevices, err := sriovCapableNodes.FindSriovDevices(nodeName)
- Expect(err).ToNot(HaveOccurred())
-
- nodeNetworkState := nmstatev1beta1.NodeNetworkState{}
- err = client.Client.Get(ctx, runtimeclient.ObjectKey{Name: nodeName, Namespace: namespaces.IntelOperator}, nodeNetworkState)
- Expect(err).ToNot(HaveOccurred())
 
- getNodeStateInterface(nodeNetworkState)
+func findNonPrimarySriovDevicesAndNode(n *cluster.EnabledNodes) (string, []*sriovv1.InterfaceExt, error) {
+ errs := []error{}
+
+ retNode := ""
+ retDevices := []*sriovv1.InterfaceExt{}
+
+ for _, node := range n.Nodes {
+ devices, err := n.FindSriovDevices(node)
+ if err != nil {
+ errs = append(errs, err)
+ continue
+ }
+
+ mainDeviceName, err := getPrimaryNICForNode(node)
+ if err != nil {
+ errs = append(errs, err)
+ continue
+ }
+
+ nonPrimaryDevices := []*sriovv1.InterfaceExt{}
+ for _, device := range devices {
+ if device.Name == mainDeviceName {
+ continue
+ }
+ nonPrimaryDevices = append(nonPrimaryDevices, device)
+ }
+
+
+ if len(nonPrimaryDevices) > len(retDevices) {
+ retNode = node
+ retDevices = nonPrimaryDevices
+ }
  }
- for nodeName, sriovNetworkNodeState := range sriovCapableNodes.States {
- nic, err := findUnusedDeviceOnNode(nodeName)
+
+ if len(retDevices) == 0 {
+ return "", nil, fmt.Errorf("can't find any SR-IOV devices in cluster's nodes: %w", errors.Join(errs...))
  }
+
+ return retNode, retDevices, nil
 }
 
-func getNodeStateInterface(state *nmstatev1beta1.NodeNetworkState) {
- //for _, interface := range state.Status.CurrentState.Interface
+func getPrimaryNICForNode(node string) (string, error) {
+ nodeObj, err := client.Client.Nodes().Get(context.Background(), node, metav1.GetOptions{})
+ if err != nil {
+ return "", err
+ }
 
-}
+ buf, err := nodes.ExecCommandOnMachineConfigDaemon(client.Client, nodeObj, 
+ []string{"/bin/bash", "-c", "chroot /rootfs /usr/bin/ovs-vsctl list-ports br-ex | /usr/bin/grep -v patch"})
+ if err != nil {
+ return "", err
+ }
 
+ exp, err := regexp.Compile("\r\n")
+ if err != nil {
+ return "", err
+ }
+
+ return exp.ReplaceAllString(string(buf), ""), nil
+}
 
-*/
+func waitForNMStatePolicyToBeStable(nmstatePolicy *nmstatev1.NodeNetworkConfigurationPolicy) {
+ key := runtimeclient.ObjectKey{Name: nmstatePolicy.Name}
+ Eventually(func(g Gomega) {
+ err := client.Client.Get(context.Background(), key, nmstatePolicy)
+ g.Expect(err).ToNot(HaveOccurred())
+ availableCondition := nmstatePolicy.Status.Conditions.
+ Find(nmstateshared.NodeNetworkConfigurationPolicyConditionAvailable)
+ g.Expect(availableCondition).ToNot(BeNil())
+ g.Expect(availableCondition.Status).
+ To(Equal(corev1.ConditionTrue))
+ }).
+ WithOffset(1).
+ WithPolling(10*time.Second).
+ WithTimeout(1*time.Minute).
+ Should(Succeed(), "")
+}

cnf-tests/testsuites/e2esuite/test_suite_test.go

@@ -25,11 +25,12 @@ import (
  _ "github.com/openshift-kni/cnf-features-deploy/cnf-tests/testsuites/e2esuite/dpdk" // this is needed otherwise the dpdk test won't be executed
  _ "github.com/openshift-kni/cnf-features-deploy/cnf-tests/testsuites/e2esuite/fec" // this is needed otherwise the fec test won't be executed
  _ "github.com/openshift-kni/cnf-features-deploy/cnf-tests/testsuites/e2esuite/multinetworkpolicy" // this is needed otherwise the multinetworkpolicy test won't be executed'
- _ "github.com/openshift-kni/cnf-features-deploy/cnf-tests/testsuites/e2esuite/ovs_qos" // this is needed otherwise the ovs_qos test won't be executed
- _ "github.com/openshift-kni/cnf-features-deploy/cnf-tests/testsuites/e2esuite/s2i" // this is needed otherwise the dpdk test won't be executed
- _ "github.com/openshift-kni/cnf-features-deploy/cnf-tests/testsuites/e2esuite/sctp" // this is needed otherwise the sctp test won't be executed
- _ "github.com/openshift-kni/cnf-features-deploy/cnf-tests/testsuites/e2esuite/security" // this is needed otherwise the security test won't be executed
- _ "github.com/openshift-kni/cnf-features-deploy/cnf-tests/testsuites/e2esuite/vrf" // this is needed otherwise the vrf test won't be executed
+ _ "github.com/openshift-kni/cnf-features-deploy/cnf-tests/testsuites/e2esuite/nmstate"
+ _ "github.com/openshift-kni/cnf-features-deploy/cnf-tests/testsuites/e2esuite/ovs_qos" // this is needed otherwise the ovs_qos test won't be executed
+ _ "github.com/openshift-kni/cnf-features-deploy/cnf-tests/testsuites/e2esuite/s2i" // this is needed otherwise the dpdk test won't be executed
+ _ "github.com/openshift-kni/cnf-features-deploy/cnf-tests/testsuites/e2esuite/sctp" // this is needed otherwise the sctp test won't be executed
+ _ "github.com/openshift-kni/cnf-features-deploy/cnf-tests/testsuites/e2esuite/security" // this is needed otherwise the security test won't be executed
+ _ "github.com/openshift-kni/cnf-features-deploy/cnf-tests/testsuites/e2esuite/vrf" // this is needed otherwise the vrf test won't be executed
 )
 
 // TODO: we should refactor tests to use client from controller-runtime package