CSIT-1205 Create AVF driver test

[csit.git] / resources / libraries / python / DUTSetup.py

diff --git a/resources/libraries/python/DUTSetup.py b/resources/libraries/python/DUTSetup.py
index 8f9e94d..84862d4 100644 (file)
--- a/resources/libraries/python/DUTSetup.py
+++ b/resources/libraries/python/DUTSetup.py
@@ -16,7 +16,7 @@
 from robot.api import logger
 
 from resources.libraries.python.topology import NodeType, Topology
 from robot.api import logger
 
 from resources.libraries.python.topology import NodeType, Topology

-from resources.libraries.python.ssh import SSH
+from resources.libraries.python.ssh import SSH, exec_cmd_no_error

 from resources.libraries.python.constants import Constants
 from resources.libraries.python.VatExecutor import VatExecutor
 from resources.libraries.python.VPPUtil import VPPUtil
 from resources.libraries.python.constants import Constants
 from resources.libraries.python.VatExecutor import VatExecutor
 from resources.libraries.python.VPPUtil import VPPUtil


@@ -252,7 +252,7 @@ class DUTSetup(object):
     def crypto_device_verify(node, force_init=False, numvfs=32):
         """Verify if Crypto QAT device virtual functions are initialized on all
         DUTs. If parameter force initialization is set to True, then try to
     def crypto_device_verify(node, force_init=False, numvfs=32):
         """Verify if Crypto QAT device virtual functions are initialized on all
         DUTs. If parameter force initialization is set to True, then try to

-        initialize or disable QAT.
+        initialize or remove VFs on QAT.

 
         :param node: DUT node.
         :param force_init: If True then try to initialize to specific value.
 
         :param node: DUT node.
         :param force_init: If True then try to initialize to specific value.


@@ -261,37 +261,19 @@ class DUTSetup(object):
         :type force_init: bool
         :type numvfs: int
         :returns: nothing
         :type force_init: bool
         :type numvfs: int
         :returns: nothing

-        :raises RuntimeError: If QAT is not initialized or failed to initialize.
+        :raises RuntimeError: If QAT VFs are not created and force init is set
+                              to False.

         """
         """

+        pci_addr = Topology.get_cryptodev(node)
+        sriov_numvfs = DUTSetup.get_sriov_numvfs(node, pci_addr)

 
 

-        ssh = SSH()
-        ssh.connect(node)
-
-        cryptodev = Topology.get_cryptodev(node)
-        cmd = 'cat /sys/bus/pci/devices/{0}/sriov_numvfs'.\
-            format(cryptodev.replace(':', r'\:'))
-
-        # Try to read number of VFs from PCI address of QAT device
-        for _ in range(3):
-            ret_code, stdout, _ = ssh.exec_command(cmd)
-            if not int(ret_code):
-                try:
-                    sriov_numvfs = int(stdout)
-                except ValueError:
-                    logger.trace('Reading sriov_numvfs info failed on {0}'.
-                                 format(node['host']))
-                else:
-                    if sriov_numvfs != numvfs:
-                        if force_init:
-                            # QAT is not initialized and we want to initialize
-                            # with numvfs
-                            DUTSetup.crypto_device_init(node, numvfs)
-                        else:
-                            raise RuntimeError('QAT device {0} is not '
-                                               'initialized to {1} on host {2}'
-                                               .format(cryptodev, numvfs,
-                                                       node['host']))
-                    break
+        if sriov_numvfs != numvfs:
+            if force_init:
+                # QAT is not initialized and we want to initialize with numvfs
+                DUTSetup.crypto_device_init(node, numvfs)
+            else:
+                raise RuntimeError('QAT device failed to create VFs on {host}'.
+                                   format(host=node['host']))

 
     @staticmethod
     def crypto_device_init(node, numvfs):
 
     @staticmethod
     def crypto_device_init(node, numvfs):


@@ -304,33 +286,98 @@ class DUTSetup(object):
         :returns: nothing
         :raises RuntimeError: If failed to stop VPP or QAT failed to initialize.
         """
         :returns: nothing
         :raises RuntimeError: If failed to stop VPP or QAT failed to initialize.
         """

-        cryptodev = Topology.get_cryptodev(node)
+        pci_addr = Topology.get_cryptodev(node)

 
 

-        # QAT device must be re-bound to kernel driver before initialization
-        driver = 'dh895xcc'
-        kernel_module = 'qat_dh895xcc'
-        current_driver = DUTSetup.get_pci_dev_driver(
-            node, cryptodev.replace(':', r'\:'))
-
-        DUTSetup.kernel_module_verify(node, kernel_module, force_load=True)
+        # QAT device must be re-bound to kernel driver before initialization.
+        DUTSetup.verify_kernel_module(node, 'qat_dh895xcc', force_load=True)

 
 

+        # Stop VPP to prevent deadlock.

         VPPUtil.stop_vpp_service(node)
         VPPUtil.stop_vpp_service(node)

+
+        current_driver = DUTSetup.get_pci_dev_driver(
+            node, pci_addr.replace(':', r'\:'))

         if current_driver is not None:
         if current_driver is not None:

-            DUTSetup.pci_driver_unbind(node, cryptodev)
-        DUTSetup.pci_driver_bind(node, cryptodev, driver)
+            DUTSetup.pci_driver_unbind(node, pci_addr)

 
 

-        ssh = SSH()
-        ssh.connect(node)
+        # Bind to kernel driver.
+        DUTSetup.pci_driver_bind(node, pci_addr, 'dh895xcc')

 
 

-        # Initialize QAT VFs
+        # Initialize QAT VFs.

         if numvfs > 0:
         if numvfs > 0:

-            cmd = 'echo "{0}" | tee /sys/bus/pci/devices/{1}/sriov_numvfs'.\
-                format(numvfs, cryptodev.replace(':', r'\:'), timeout=180)
-            ret_code, _, _ = ssh.exec_command_sudo("sh -c '{0}'".format(cmd))
+            DUTSetup.set_sriov_numvfs(node, pci_addr, numvfs)

 
 

-            if int(ret_code):
-                raise RuntimeError('Failed to initialize {0} VFs on QAT device '
-                                   ' on host {1}'.format(numvfs, node['host']))
+    @staticmethod
+    def get_virtfn_pci_addr(node, pf_pci_addr, vf_id):
+        """Get PCI address of Virtual Function.
+
+        :param node: DUT node.
+        :param pf_pci_addr: Physical Function PCI address.
+        :param vf_id: Virtual Function number.
+        :type node: dict
+        :type pf_pci_addr: str
+        :type vf_id: int
+        :returns: Virtual Function PCI address.
+        :rtype: int
+        :raises RuntimeError: If failed to get Virtual Function PCI address.
+        """
+        command = "sh -c "\
+            "'basename $(readlink /sys/bus/pci/devices/{pci}/virtfn{vf_id})'".\
+            format(pci=pf_pci_addr, vf_id=vf_id)
+        message = 'Failed to get virtual function PCI address.'
+
+        stdout, _ = exec_cmd_no_error(node, command, timeout=30, sudo=True,
+                                      message=message)
+
+        return stdout.strip()
+
+    @staticmethod
+    def get_sriov_numvfs(node, pf_pci_addr):
+        """Get number of SR-IOV VFs.
+
+        :param node: DUT node.
+        :param pf_pci_addr: Physical Function PCI device address.
+        :type node: dict
+        :type pf_pci_addr: str
+        :returns: Number of VFs.
+        :rtype: int
+        :raises RuntimeError: If PCI device is not SR-IOV capable.
+        """
+        command = 'cat /sys/bus/pci/devices/{pci}/sriov_numvfs'.\
+            format(pci=pf_pci_addr.replace(':', r'\:'))
+        message = 'PCI device {pci} is not a SR-IOV device.'.\
+            format(pci=pf_pci_addr)
+
+        for _ in range(3):
+            stdout, _ = exec_cmd_no_error(node, command, timeout=30, sudo=True,
+                                          message=message)
+            try:
+                sriov_numvfs = int(stdout)
+            except ValueError:
+                logger.trace('Reading sriov_numvfs info failed on {host}'.
+                             format(host=node['host']))
+            else:
+                return sriov_numvfs
+
+    @staticmethod
+    def set_sriov_numvfs(node, pf_pci_addr, numvfs=0):
+        """Init or reset SR-IOV virtual functions by setting its number on PCI
+        device on DUT. Setting to zero removes all VFs.
+
+        :param node: DUT node.
+        :param pf_pci_addr: Physical Function PCI device address.
+        :param numvfs: Number of VFs to initialize, 0 - removes the VFs.
+        :type node: dict
+        :type pf_pci_addr: str
+        :type numvfs: int
+        :raises RuntimeError: Failed to create VFs on PCI.
+        """
+        command = "sh -c "\
+            "'echo {num} | tee /sys/bus/pci/devices/{pci}/sriov_numvfs'".\
+            format(num=numvfs, pci=pf_pci_addr.replace(':', r'\:'))
+        message = 'Failed to create {num} VFs on {pci} device on {host}'.\
+            format(num=numvfs, pci=pf_pci_addr, host=node['host'])
+
+        exec_cmd_no_error(node, command, timeout=60, sudo=True, message=message)

 
     @staticmethod
     def pci_driver_unbind(node, pci_addr):
 
     @staticmethod
     def pci_driver_unbind(node, pci_addr):


@@ -340,20 +387,15 @@ class DUTSetup(object):
         :param pci_addr: PCI device address.
         :type node: dict
         :type pci_addr: str
         :param pci_addr: PCI device address.
         :type node: dict
         :type pci_addr: str

-        :returns: nothing

         :raises RuntimeError: If PCI device unbind failed.
         """
         :raises RuntimeError: If PCI device unbind failed.
         """

+        command = "sh -c "\
+            "'echo {pci} | tee /sys/bus/pci/devices/{pcie}/driver/unbind'".\
+            format(pci=pci_addr, pcie=pci_addr.replace(':', r'\:'))
+        message = 'Failed to unbind PCI device {pci} on {host}'.\
+            format(pci=pci_addr, host=node['host'])

 
 

-        ssh = SSH()
-        ssh.connect(node)
-
-        ret_code, _, _ = ssh.exec_command_sudo(
-            "sh -c 'echo {0} | tee /sys/bus/pci/devices/{1}/driver/unbind'"
-            .format(pci_addr, pci_addr.replace(':', r'\:')), timeout=180)
-
-        if int(ret_code):
-            raise RuntimeError('Failed to unbind PCI device {0} from driver on '
-                               'host {1}'.format(pci_addr, node['host']))
+        exec_cmd_no_error(node, command, timeout=60, sudo=True, message=message)

 
     @staticmethod
     def pci_driver_bind(node, pci_addr, driver):
 
     @staticmethod
     def pci_driver_bind(node, pci_addr, driver):


@@ -365,21 +407,92 @@ class DUTSetup(object):
         :type node: dict
         :type pci_addr: str
         :type driver: str
         :type node: dict
         :type pci_addr: str
         :type driver: str

-        :returns: nothing

         :raises RuntimeError: If PCI device bind failed.
         """
         :raises RuntimeError: If PCI device bind failed.
         """

+        message = 'Failed to bind PCI device {pci} to {driver} on host {host}'.\
+            format(pci=pci_addr, driver=driver, host=node['host'])

 
 

-        ssh = SSH()
-        ssh.connect(node)
+        command = "sh -c "\
+            "'echo {driver} | tee /sys/bus/pci/devices/{pci}/driver_override'".\
+            format(driver=driver, pci=pci_addr.replace(':', r'\:'))

 
 

-        ret_code, _, _ = ssh.exec_command_sudo(
-            "sh -c 'echo {0} | tee /sys/bus/pci/drivers/{1}/bind'".format(
-                pci_addr, driver), timeout=180)
+        exec_cmd_no_error(node, command, timeout=60, sudo=True, message=message)

 
 

-        if int(ret_code):
-            raise RuntimeError('Failed to bind PCI device {0} to {1} driver on '
-                               'host {2}'.format(pci_addr, driver,
-                                                 node['host']))
+        command = "sh -c "\
+            "'echo {pci} | tee /sys/bus/pci/drivers/{driver}/bind'".\
+            format(pci=pci_addr, driver=driver)
+
+        exec_cmd_no_error(node, command, timeout=60, sudo=True, message=message)
+
+        command = "sh -c "\
+            "'echo  | tee /sys/bus/pci/devices/{pci}/driver_override'".\
+            format(pci=pci_addr.replace(':', r'\:'))
+
+        exec_cmd_no_error(node, command, timeout=60, sudo=True, message=message)
+
+    @staticmethod
+    def pci_vf_driver_unbind(node, pf_pci_addr, vf_id):
+        """Unbind Virtual Function from driver on node.
+
+        :param node: DUT node.
+        :param pf_pci_addr: PCI device address.
+        :param vf_id: Virtual Function ID.
+        :type node: dict
+        :type pf_pci_addr: str
+        :type vf_id: int
+        :raises RuntimeError: If Virtual Function unbind failed.
+        """
+        vf_pci_addr = DUTSetup.get_virtfn_pci_addr(node, pf_pci_addr, vf_id)
+        vf_path = "/sys/bus/pci/devices/{pf_pci_addr}/virtfn{vf_id}".\
+            format(pf_pci_addr=pf_pci_addr.replace(':', r'\:'), vf_id=vf_id)
+
+        command = "sh -c "\
+            "'echo {vf_pci_addr} | tee {vf_path}/driver/unbind'".\
+            format(vf_pci_addr=vf_pci_addr, vf_path=vf_path)
+
+        message = 'Failed to unbind VF {vf_pci_addr} to on {host}'.\
+            format(vf_pci_addr=vf_pci_addr, host=node['host'])
+
+        exec_cmd_no_error(node, command, timeout=60, sudo=True, message=message)
+
+    @staticmethod
+    def pci_vf_driver_bind(node, pf_pci_addr, vf_id, driver):
+        """Bind Virtual Function to driver on node.
+
+        :param node: DUT node.
+        :param pf_pci_addr: PCI device address.
+        :param vf_id: Virtual Function ID.
+        :param driver: Driver to bind.
+        :type node: dict
+        :type pf_pci_addr: str
+        :type vf_id: int
+        :type driver: str
+        :raises RuntimeError: If PCI device bind failed.
+        """
+        vf_pci_addr = DUTSetup.get_virtfn_pci_addr(node, pf_pci_addr, vf_id)
+        vf_path = "/sys/bus/pci/devices/{pf_pci_addr}/virtfn{vf_id}".\
+            format(pf_pci_addr=pf_pci_addr.replace(':', r'\:'), vf_id=vf_id)
+
+        message = 'Failed to bind VF {vf_pci_addr} to {driver} on {host}'.\
+            format(vf_pci_addr=vf_pci_addr, driver=driver, host=node['host'])
+
+        command = "sh -c "\
+            "'echo {driver} | tee {vf_path}/driver_override'".\
+            format(driver=driver, vf_path=vf_path)
+
+        exec_cmd_no_error(node, command, timeout=60, sudo=True, message=message)
+
+        command = "sh -c "\
+            "'echo {vf_pci_addr} | tee /sys/bus/pci/drivers/{driver}/bind'".\
+            format(vf_pci_addr=vf_pci_addr, driver=driver)
+
+        exec_cmd_no_error(node, command, timeout=60, sudo=True, message=message)
+
+        command = "sh -c "\
+            "'echo  | tee {vf_path}/driver_override'".\
+            format(vf_path=vf_path)
+
+        exec_cmd_no_error(node, command, timeout=60, sudo=True, message=message)

 
     @staticmethod
     def get_pci_dev_driver(node, pci_addr):
 
     @staticmethod
     def get_pci_dev_driver(node, pci_addr):


@@ -442,7 +555,7 @@ class DUTSetup(object):
         return None
 
     @staticmethod
         return None
 
     @staticmethod

-    def kernel_module_verify(node, module, force_load=False):
+    def verify_kernel_module(node, module, force_load=False):

         """Verify if kernel module is loaded on node. If parameter force
         load is set to True, then try to load the modules.
 
         """Verify if kernel module is loaded on node. If parameter force
         load is set to True, then try to load the modules.
 


@@ -454,22 +567,22 @@ class DUTSetup(object):
         :type force_load: bool
         :raises RuntimeError: If module is not loaded or failed to load.
         """
         :type force_load: bool
         :raises RuntimeError: If module is not loaded or failed to load.
         """

-        ssh = SSH()
-        ssh.connect(node)
+        command = 'grep -w {module} /proc/modules'.format(module=module)
+        message = 'Kernel module {module} is not loaded on host {host}'.\
+            format(module=module, host=node['host'])

 
 

-        cmd = 'grep -w {0} /proc/modules'.format(module)
-        ret_code, _, _ = ssh.exec_command(cmd)
-
-        if int(ret_code):
+        try:
+            exec_cmd_no_error(node, command, timeout=30, sudo=False,
+                              message=message)
+        except RuntimeError:

             if force_load:
                 # Module is not loaded and we want to load it
             if force_load:
                 # Module is not loaded and we want to load it

-                DUTSetup.kernel_module_load(node, module)
+                DUTSetup.load_kernel_module(node, module)

             else:
             else:

-                raise RuntimeError('Kernel module {0} is not loaded on host '
-                                   '{1}'.format(module, node['host']))
+                raise

 
     @staticmethod
 
     @staticmethod

-    def kernel_module_verify_on_all_duts(nodes, module, force_load=False):
+    def verify_kernel_module_on_all_duts(nodes, module, force_load=False):

         """Verify if kernel module is loaded on all DUTs. If parameter force
         load is set to True, then try to load the modules.
 
         """Verify if kernel module is loaded on all DUTs. If parameter force
         load is set to True, then try to load the modules.
 


@@ -482,7 +595,7 @@ class DUTSetup(object):
         """
         for node in nodes.values():
             if node['type'] == NodeType.DUT:
         """
         for node in nodes.values():
             if node['type'] == NodeType.DUT:

-                DUTSetup.kernel_module_verify(node, module, force_load)
+                DUTSetup.verify_kernel_module(node, module, force_load)

 
     @staticmethod
     def verify_uio_driver_on_all_duts(nodes):
 
     @staticmethod
     def verify_uio_driver_on_all_duts(nodes):


@@ -495,10 +608,10 @@ class DUTSetup(object):
         for node in nodes.values():
             if node['type'] == NodeType.DUT:
                 uio_driver = Topology.get_uio_driver(node)
         for node in nodes.values():
             if node['type'] == NodeType.DUT:
                 uio_driver = Topology.get_uio_driver(node)

-                DUTSetup.kernel_module_verify(node, uio_driver, force_load=True)
+                DUTSetup.verify_kernel_module(node, uio_driver, force_load=True)

 
     @staticmethod
 
     @staticmethod

-    def kernel_module_load(node, module):
+    def load_kernel_module(node, module):

         """Load kernel module on node.
 
         :param node: DUT node.
         """Load kernel module on node.
 
         :param node: DUT node.


@@ -508,15 +621,11 @@ class DUTSetup(object):
         :returns: nothing
         :raises RuntimeError: If loading failed.
         """
         :returns: nothing
         :raises RuntimeError: If loading failed.
         """

+        command = 'modprobe {module}'.format(module=module)
+        message = 'Failed to load {module} on host {host}'.\
+            format(module=module, host=node['host'])

 
 

-        ssh = SSH()
-        ssh.connect(node)
-
-        ret_code, _, _ = ssh.exec_command_sudo("modprobe {0}".format(module))
-
-        if int(ret_code):
-            raise RuntimeError('Failed to load {0} kernel module on host {1}'.
-                               format(module, node['host']))
+        exec_cmd_no_error(node, command, timeout=30, sudo=True, message=message)

 
     @staticmethod
     def vpp_enable_traces_on_all_duts(nodes):
 
     @staticmethod
     def vpp_enable_traces_on_all_duts(nodes):


@@ -614,6 +723,21 @@ class DUTSetup(object):
 
                 ssh.disconnect(node)
 
 
                 ssh.disconnect(node)
 

+    @staticmethod
+    def verify_vpp_on_dut(node):
+        """Verify that VPP is installed on DUT node.
+
+        :param node: DUT node.
+        :type node: dict
+        :raises RuntimeError: If failed to restart VPP, get VPP version
+            or get VPP interfaces.
+        """
+
+        logger.debug("Verify VPP on node {0}".format(node['host']))
+
+        DUTSetup.vpp_show_version_verbose(node)
+        DUTSetup.vpp_show_interfaces(node)
+

     @staticmethod
     def verify_vpp_on_all_duts(nodes):
         """Verify that VPP is installed on all DUT nodes.
     @staticmethod
     def verify_vpp_on_all_duts(nodes):
         """Verify that VPP is installed on all DUT nodes.


@@ -630,20 +754,6 @@ class DUTSetup(object):
             if node['type'] == NodeType.DUT:
                 DUTSetup.verify_vpp_on_dut(node)
 
             if node['type'] == NodeType.DUT:
                 DUTSetup.verify_vpp_on_dut(node)
 

-    @staticmethod
-    def verify_vpp_on_dut(node):
-        """Verify that VPP is installed on DUT node.
-
-        :param node: DUT node.
-        :type node: dict
-        :raises RuntimeError: If failed to restart VPP, get VPP version
-            or get VPP interfaces.
-        """
-
-        logger.debug("Verify VPP on node {0}".format(node['host']))
-
-        DUTSetup.vpp_show_version_verbose(node)
-        DUTSetup.vpp_show_interfaces(node)

 
     @staticmethod
     def get_huge_page_size(node):
 
     @staticmethod
     def get_huge_page_size(node):