ADD: Mellanox RDMA interface support

+ Add functions for creating rdma interface
+ Fix traffic generator

Signed-off-by: Peter Mikus <pmikus@cisco.com>
Change-Id: I81787b72ff5ee926ed652d350888c4f86da766f1

diff --git a/resources/api/vpp/supported_crcs.yaml b/resources/api/vpp/supported_crcs.yaml
index 0370500..dfcff35 100644 (file)
--- a/resources/api/vpp/supported_crcs.yaml
+++ b/resources/api/vpp/supported_crcs.yaml
@@ -195,6 +195,8 @@
     policer_classify_set_interface: '0xe09537b0' # dev
     policer_classify_set_interface_reply: '0xe8d4e804' # dev
     # 4x^ tc01-64B-ethip4-ip4base-ipolicemarkbase-dev
+    rdma_create: '0x541ffa8e' # perf
+    rdma_create_reply: '0xfda5941f' # perf
     # show_lisp_map_register_state / reply # honeycomb
     # show_lisp_map_request_mode / reply # honeycomb
     # show_lisp_pitr / reply # honeycomb

diff --git a/resources/libraries/python/Constants.py b/resources/libraries/python/Constants.py
index 5717666..71d5074 100644 (file)
--- a/resources/libraries/python/Constants.py
+++ b/resources/libraries/python/Constants.py
@@ -234,7 +234,6 @@ class Constants(object):
     # Mapping from NIC name to its bps limit.
     # TODO: Implement logic to lower limits to TG NIC or software. Or PCI.
     NIC_NAME_TO_LIMIT = {
-        # TODO: Explain why ~40Gbps NICs are using ~25Gbps limit.
         "Cisco-VIC-1227": 10000000000,
         "Cisco-VIC-1385": 24500000000,
         "Intel-X520-DA2": 10000000000,
@@ -242,6 +241,7 @@ class Constants(object):
         "Intel-X710": 10000000000,
         "Intel-XL710": 24500000000,
         "Intel-XXV710": 24500000000,
+        "Mellanox-CX556A": 100000000000,
         "virtual": 100000000,
     }
 
@@ -254,6 +254,7 @@ class Constants(object):
         "Intel-X710": "10ge2p1x710",
         "Intel-XL710": "40ge2p1xl710",
         "Intel-XXV710": "25ge2p1xxv710",
+        "Mellanox-CX556A": "100ge2p1cx556a",
     }
 
     # TODO CSIT-1481: Crypto HW should be read from topology file instead.

diff --git a/resources/libraries/python/InterfaceUtil.py b/resources/libraries/python/InterfaceUtil.py
index 1d1d669..d185137 100644 (file)
--- a/resources/libraries/python/InterfaceUtil.py
+++ b/resources/libraries/python/InterfaceUtil.py
@@ -124,6 +124,25 @@ class InterfaceUtil(object):
         return (int(pci[0], 16) | int(pci[1], 16) << 16 |
                 int(pci[2], 16) << 24 | int(pci[3], 16) << 29)
 
+    @staticmethod
+    def pci_to_eth(node, pci_str):
+        """Convert PCI address to Linux ethernet name.
+
+        :param pci_str: PCI address.
+        :type pci_str: str
+        :returns: Ethernet name.
+        :rtype: str
+        """
+        cmd = ('basename /sys/bus/pci/devices/{pci_str}/net/*'.
+               format(pci_str=pci_str))
+        try:
+            stdout, _ = exec_cmd_no_error(node, cmd)
+        except RuntimeError:
+            raise RuntimeError("Cannot convert {pci_str} to ethernet name!".
+                               format(pci_str=pci_str))
+
+        return stdout.strip()
+
     @staticmethod
     def get_interface_index(node, interface):
         """Get interface sw_if_index from topology file.
@@ -1170,7 +1189,7 @@ class InterfaceUtil(object):
         """Create AVF interface on VPP node.
 
         :param node: DUT node from topology.
-        :param vf_pci_addr: Virtual Function PCI address.
+        :param vf_pci_addr: PCI address binded to i40evf driver.
         :param num_rx_queues: Number of RX queues.
         :type node: dict
         :type vf_pci_addr: str
@@ -1191,8 +1210,40 @@ class InterfaceUtil(object):
         with PapiSocketExecutor(node) as papi_exec:
             sw_if_index = papi_exec.add(cmd, **args).get_sw_if_index(err_msg)
 
-        InterfaceUtil.add_eth_interface(node, sw_if_index=sw_if_index,
-                                        ifc_pfx='eth_avf')
+        InterfaceUtil.add_eth_interface(
+            node, sw_if_index=sw_if_index, ifc_pfx='eth_avf')
+        if_key = Topology.get_interface_by_sw_index(node, sw_if_index)
+
+        return if_key
+
+    @staticmethod
+    def vpp_create_rdma_interface(node, pci_addr, num_rx_queues=None):
+        """Create RDMA interface on VPP node.
+
+        :param node: DUT node from topology.
+        :param pci_addr: PCI address binded to rdma-core driver.
+        :param num_rx_queues: Number of RX queues.
+        :type node: dict
+        :type pci_addr: str
+        :type num_rx_queues: int
+        :returns: Interface key (name) in topology.
+        :rtype: str
+        :raises RuntimeError: If it is not possible to create RDMA interface on
+            the node.
+        """
+        cmd = 'rdma_create'
+        args = dict(name=InterfaceUtil.pci_to_eth(node, pci_addr),
+                    host_if=InterfaceUtil.pci_to_eth(node, pci_addr),
+                    rxq_num=int(num_rx_queues) if num_rx_queues else 0,
+                    rxq_size=0,
+                    txq_size=0)
+        err_msg = 'Failed to create RDMA interface on host {host}'.format(
+            host=node['host'])
+        with PapiSocketExecutor(node) as papi_exec:
+            sw_if_index = papi_exec.add(cmd, **args).get_sw_if_index(err_msg)
+
+        InterfaceUtil.add_eth_interface(
+            node, sw_if_index=sw_if_index, ifc_pfx='eth_rdma')
         if_key = Topology.get_interface_by_sw_index(node, sw_if_index)
 
         return if_key

diff --git a/resources/libraries/python/TrafficGenerator.py b/resources/libraries/python/TrafficGenerator.py
index 49c19b1..82a5bf8 100644 (file)
--- a/resources/libraries/python/TrafficGenerator.py
+++ b/resources/libraries/python/TrafficGenerator.py
@@ -328,12 +328,6 @@ class TrafficGenerator(AbstractMeasurer):
                 self._node, cmd, sudo=True,
                 message='Unbind PCI ports from driver failed!')
 
-            cmd = ("sh -c 'cd {dir}/scripts/ && ./trex-cfg "
-                   "--unbind-unused-ports'"
-                   .format(dir=Constants.TREX_INSTALL_DIR))
-            exec_cmd_no_error(
-                self._node, cmd, sudo=True, message='Config TRex failed!')
-
             # Start TRex.
             cmd = ("sh -c 'cd {dir}/scripts/ && "
                    "nohup ./t-rex-64 --hdrh{mode} -i -c 7 > "

diff --git a/resources/tools/testbed-setup/ansible/roles/tg/tasks/ubuntu_bionic.yaml b/resources/tools/testbed-setup/ansible/roles/tg/tasks/ubuntu_bionic.yaml
index 90703ed..d119f9c 100644 (file)
--- a/resources/tools/testbed-setup/ansible/roles/tg/tasks/ubuntu_bionic.yaml
+++ b/resources/tools/testbed-setup/ansible/roles/tg/tasks/ubuntu_bionic.yaml
@@ -4,7 +4,7 @@
 - name: Install CSIT dependencies
   apt:
     name:
-      - 'cpufrequtils'
+      - 'libmnl-dev'
       - 'libnuma-dev'
       - 'libpcap-dev'
       - 'libssl-dev'