FIX: Remove old restart sequence - Honeycomb

[csit.git] / resources / libraries / python / DUTSetup.py

diff --git a/resources/libraries/python/DUTSetup.py b/resources/libraries/python/DUTSetup.py
index 7a28e09..95b2301 100644 (file)
--- a/resources/libraries/python/DUTSetup.py
+++ b/resources/libraries/python/DUTSetup.py
@@ -15,7 +15,7 @@
 
 from robot.api import logger
 
 
 from robot.api import logger
 

-from resources.libraries.python.constants import Constants
+from resources.libraries.python.Constants import Constants

 from resources.libraries.python.ssh import SSH, exec_cmd_no_error
 from resources.libraries.python.topology import NodeType, Topology
 
 from resources.libraries.python.ssh import SSH, exec_cmd_no_error
 from resources.libraries.python.topology import NodeType, Topology
 


@@ -60,6 +60,40 @@ class DUTSetup(object):
             if node['type'] == NodeType.DUT:
                 DUTSetup.get_service_logs(node, service)
 
             if node['type'] == NodeType.DUT:
                 DUTSetup.get_service_logs(node, service)
 

+    @staticmethod
+    def restart_service(node, service):
+        """Restart the named service on node.
+
+        :param node: Node in the topology.
+        :param service: Service unit name.
+        :type node: dict
+        :type service: str
+        """
+        if DUTSetup.running_in_container(node):
+            command = 'supervisorctl restart {name}'.format(name=service)
+        else:
+            command = 'service {name} restart'.format(name=service)
+        message = 'Node {host} failed to restart service {name}'.\
+            format(host=node['host'], name=service)
+
+        exec_cmd_no_error(
+            node, command, timeout=180, sudo=True, message=message)
+
+        DUTSetup.get_service_logs(node, service)
+
+    @staticmethod
+    def restart_service_on_all_duts(nodes, service):
+        """Restart the named service on all DUTs.
+
+        :param node: Nodes in the topology.
+        :param service: Service unit name.
+        :type node: dict
+        :type service: str
+        """
+        for node in nodes.values():
+            if node['type'] == NodeType.DUT:
+                DUTSetup.restart_service(node, service)
+

     @staticmethod
     def start_service(node, service):
         """Start up the named service on node.
     @staticmethod
     def start_service(node, service):
         """Start up the named service on node.


@@ -69,6 +103,7 @@ class DUTSetup(object):
         :type node: dict
         :type service: str
         """
         :type node: dict
         :type service: str
         """

+        # TODO: change command to start once all parent function updated.

         if DUTSetup.running_in_container(node):
             command = 'supervisorctl restart {name}'.format(name=service)
         else:
         if DUTSetup.running_in_container(node):
             command = 'supervisorctl restart {name}'.format(name=service)
         else:


@@ -76,7 +111,8 @@ class DUTSetup(object):
         message = 'Node {host} failed to start service {name}'.\
             format(host=node['host'], name=service)
 
         message = 'Node {host} failed to start service {name}'.\
             format(host=node['host'], name=service)
 

-        exec_cmd_no_error(node, command, timeout=30, sudo=True, message=message)
+        exec_cmd_no_error(
+            node, command, timeout=180, sudo=True, message=message)

 
         DUTSetup.get_service_logs(node, service)
 
 
         DUTSetup.get_service_logs(node, service)
 


@@ -109,7 +145,8 @@ class DUTSetup(object):
         message = 'Node {host} failed to stop service {name}'.\
             format(host=node['host'], name=service)
 
         message = 'Node {host} failed to stop service {name}'.\
             format(host=node['host'], name=service)
 

-        exec_cmd_no_error(node, command, timeout=30, sudo=True, message=message)
+        exec_cmd_no_error(
+            node, command, timeout=180, sudo=True, message=message)

 
         DUTSetup.get_service_logs(node, service)
 
 
         DUTSetup.get_service_logs(node, service)
 


@@ -126,34 +163,6 @@ class DUTSetup(object):
             if node['type'] == NodeType.DUT:
                 DUTSetup.stop_service(node, service)
 
             if node['type'] == NodeType.DUT:
                 DUTSetup.stop_service(node, service)
 

-    @staticmethod
-    def setup_dut(node):
-        """Run script over SSH to setup the DUT node.
-
-        :param node: DUT node to set up.
-        :type node: dict
-
-        :raises Exception: If the DUT setup fails.
-        """
-        command = 'bash {0}/{1}/dut_setup.sh'.\
-            format(Constants.REMOTE_FW_DIR, Constants.RESOURCES_LIB_SH)
-        message = 'DUT test setup script failed at node {name}'.\
-            format(name=node['host'])
-
-        exec_cmd_no_error(node, command, timeout=120, sudo=True,
-                          message=message)
-
-    @staticmethod
-    def setup_all_duts(nodes):
-        """Run script over SSH to setup all DUT nodes.
-
-        :param nodes: Topology nodes.
-        :type nodes: dict
-        """
-        for node in nodes.values():
-            if node['type'] == NodeType.DUT:
-                DUTSetup.setup_dut(node)
-

     @staticmethod
     def get_vpp_pid(node):
         """Get PID of running VPP process.
     @staticmethod
     def get_vpp_pid(node):
         """Get PID of running VPP process.


@@ -176,19 +185,17 @@ class DUTSetup(object):
                                    'on node: {0}\n {1}'.
                                    format(node['host'], stdout + stderr))
 
                                    'on node: {0}\n {1}'.
                                    format(node['host'], stdout + stderr))
 

-            if len(stdout.splitlines()) == 1:
+            pid_list = stdout.split()
+            if len(pid_list) == 1:

                 return int(stdout)
                 return int(stdout)

-            elif not stdout.splitlines():
+            elif not pid_list:

                 logger.debug("No VPP PID found on node {0}".
                              format(node['host']))
                 continue
             else:
                 logger.debug("More then one VPP PID found on node {0}".
                              format(node['host']))
                 logger.debug("No VPP PID found on node {0}".
                              format(node['host']))
                 continue
             else:
                 logger.debug("More then one VPP PID found on node {0}".
                              format(node['host']))

-                ret_list = list()
-                for line in stdout.splitlines():
-                    ret_list.append(int(line))
-                return ret_list
+                return [int(pid) for pid in pid_list]

 
         return None
 
 
         return None
 


@@ -208,17 +215,19 @@ class DUTSetup(object):
         return pids
 
     @staticmethod
         return pids
 
     @staticmethod

-    def crypto_device_verify(node, force_init=False, numvfs=32):
+    def crypto_device_verify(node, crypto_type, numvfs, force_init=False):

         """Verify if Crypto QAT device virtual functions are initialized on all
         DUTs. If parameter force initialization is set to True, then try to
         initialize or remove VFs on QAT.
 
         :param node: DUT node.
         """Verify if Crypto QAT device virtual functions are initialized on all
         DUTs. If parameter force initialization is set to True, then try to
         initialize or remove VFs on QAT.
 
         :param node: DUT node.

-        :param force_init: If True then try to initialize to specific value.
+        :crypto_type: Crypto device type - HW_DH895xcc or HW_C3xxx.

         :param numvfs: Number of VFs to initialize, 0 - disable the VFs.
         :param numvfs: Number of VFs to initialize, 0 - disable the VFs.

+        :param force_init: If True then try to initialize to specific value.

         :type node: dict
         :type node: dict

-        :type force_init: bool
+        :type crypto_type: string

         :type numvfs: int
         :type numvfs: int

+        :type force_init: bool

         :returns: nothing
         :raises RuntimeError: If QAT VFs are not created and force init is set
                               to False.
         :returns: nothing
         :raises RuntimeError: If QAT VFs are not created and force init is set
                               to False.


@@ -229,26 +238,38 @@ class DUTSetup(object):
         if sriov_numvfs != numvfs:
             if force_init:
                 # QAT is not initialized and we want to initialize with numvfs
         if sriov_numvfs != numvfs:
             if force_init:
                 # QAT is not initialized and we want to initialize with numvfs

-                DUTSetup.crypto_device_init(node, numvfs)
+                DUTSetup.crypto_device_init(node, crypto_type, numvfs)

             else:
                 raise RuntimeError('QAT device failed to create VFs on {host}'.
                                    format(host=node['host']))
 
     @staticmethod
             else:
                 raise RuntimeError('QAT device failed to create VFs on {host}'.
                                    format(host=node['host']))
 
     @staticmethod

-    def crypto_device_init(node, numvfs):
+    def crypto_device_init(node, crypto_type, numvfs):

         """Init Crypto QAT device virtual functions on DUT.
 
         :param node: DUT node.
         """Init Crypto QAT device virtual functions on DUT.
 
         :param node: DUT node.

+        :crypto_type: Crypto device type - HW_DH895xcc or HW_C3xxx.

         :param numvfs: Number of VFs to initialize, 0 - disable the VFs.
         :type node: dict
         :param numvfs: Number of VFs to initialize, 0 - disable the VFs.
         :type node: dict

+        :type crypto_type: string

         :type numvfs: int
         :returns: nothing
         :raises RuntimeError: If failed to stop VPP or QAT failed to initialize.
         """
         :type numvfs: int
         :returns: nothing
         :raises RuntimeError: If failed to stop VPP or QAT failed to initialize.
         """

+        if crypto_type == "HW_DH895xcc":
+            kernel_mod = "qat_dh895xcc"
+            kernel_drv = "dh895xcc"
+        elif crypto_type == "HW_C3xxx":
+            kernel_mod = "qat_c3xxx"
+            kernel_drv = "c3xxx"
+        else:
+            raise RuntimeError('Unsupported crypto device type on {host}'.
+                               format(host=node['host']))
+

         pci_addr = Topology.get_cryptodev(node)
 
         # QAT device must be re-bound to kernel driver before initialization.
         pci_addr = Topology.get_cryptodev(node)
 
         # QAT device must be re-bound to kernel driver before initialization.

-        DUTSetup.verify_kernel_module(node, 'qat_dh895xcc', force_load=True)
+        DUTSetup.verify_kernel_module(node, kernel_mod, force_load=True)

 
         # Stop VPP to prevent deadlock.
         DUTSetup.stop_service(node, Constants.VPP_UNIT)
 
         # Stop VPP to prevent deadlock.
         DUTSetup.stop_service(node, Constants.VPP_UNIT)


@@ -259,7 +280,7 @@ class DUTSetup(object):
             DUTSetup.pci_driver_unbind(node, pci_addr)
 
         # Bind to kernel driver.
             DUTSetup.pci_driver_unbind(node, pci_addr)
 
         # Bind to kernel driver.

-        DUTSetup.pci_driver_bind(node, pci_addr, 'dh895xcc')
+        DUTSetup.pci_driver_bind(node, pci_addr, kernel_drv)

 
         # Initialize QAT VFs.
         if numvfs > 0:
 
         # Initialize QAT VFs.
         if numvfs > 0:


@@ -597,7 +618,8 @@ class DUTSetup(object):
 
     @staticmethod
     def install_vpp_on_all_duts(nodes, vpp_pkg_dir):
 
     @staticmethod
     def install_vpp_on_all_duts(nodes, vpp_pkg_dir):

-        """Install VPP on all DUT nodes.
+        """Install VPP on all DUT nodes. Start the VPP service in case of
+        systemd is not available or does not support autostart.

 
         :param nodes: Nodes in the topology.
         :param vpp_pkg_dir: Path to directory where VPP packages are stored.
 
         :param nodes: Nodes in the topology.
         :param vpp_pkg_dir: Path to directory where VPP packages are stored.


@@ -606,8 +628,12 @@ class DUTSetup(object):
         :raises RuntimeError: If failed to remove or install VPP.
         """
         for node in nodes.values():
         :raises RuntimeError: If failed to remove or install VPP.
         """
         for node in nodes.values():

-            message='Failed to install VPP on {host}!'.format(host=node['host'])
+            message = 'Failed to install VPP on host {host}!'.\
+                format(host=node['host'])

             if node['type'] == NodeType.DUT:
             if node['type'] == NodeType.DUT:

+                command = 'ln -s /dev/null /etc/sysctl.d/80-vpp.conf || true'
+                exec_cmd_no_error(node, command, sudo=True)
+

                 command = '. /etc/lsb-release; echo "${DISTRIB_ID}"'
                 stdout, _ = exec_cmd_no_error(node, command)
 
                 command = '. /etc/lsb-release; echo "${DISTRIB_ID}"'
                 stdout, _ = exec_cmd_no_error(node, command)
 


@@ -615,16 +641,19 @@ class DUTSetup(object):
                     exec_cmd_no_error(node, 'apt-get purge -y "*vpp*" || true',
                                       timeout=120, sudo=True)
                     exec_cmd_no_error(node, 'dpkg -i --force-all {dir}*.deb'.
                     exec_cmd_no_error(node, 'apt-get purge -y "*vpp*" || true',
                                       timeout=120, sudo=True)
                     exec_cmd_no_error(node, 'dpkg -i --force-all {dir}*.deb'.

-                        format(dir=vpp_pkg_dir), timeout=120, sudo=True,
-                        message=message)
+                                      format(dir=vpp_pkg_dir), timeout=120,
+                                      sudo=True, message=message)

                     exec_cmd_no_error(node, 'dpkg -l | grep vpp', sudo=True)
                     exec_cmd_no_error(node, 'dpkg -l | grep vpp', sudo=True)

+                    if DUTSetup.running_in_container(node):
+                        DUTSetup.restart_service(node, Constants.VPP_UNIT)

                 else:
                     exec_cmd_no_error(node, 'yum -y remove "*vpp*" || true',
                                       timeout=120, sudo=True)
                 else:
                     exec_cmd_no_error(node, 'yum -y remove "*vpp*" || true',
                                       timeout=120, sudo=True)

-                    exec_cmd_no_error(node, 'rpm -ivh {dir}*.rpm'.\
-                        format(dir=vpp_pkg_dir), timeout=120, sudo=True,
-                        message=message)
+                    exec_cmd_no_error(node, 'rpm -ivh {dir}*.rpm'.
+                                      format(dir=vpp_pkg_dir), timeout=120,
+                                      sudo=True, message=message)

                     exec_cmd_no_error(node, 'rpm -qai *vpp*', sudo=True)
                     exec_cmd_no_error(node, 'rpm -qai *vpp*', sudo=True)

+                    DUTSetup.restart_service(node, Constants.VPP_UNIT)

 
     @staticmethod
     def running_in_container(node):
 
     @staticmethod
     def running_in_container(node):


@@ -645,6 +674,26 @@ class DUTSetup(object):
             return False
         return True
 
             return False
         return True
 

+    @staticmethod
+    def get_docker_mergeddir(node, uuid):
+        """Get Docker overlay for MergedDir diff.
+
+        :param node: DUT node.
+        :param uuid: Docker UUID.
+        :type node: dict
+        :type uuid: str
+        :returns: Docker container MergedDir.
+        :rtype: str
+        :raises RuntimeError: If getting output failed.
+        """
+        command = "docker inspect --format='"\
+            "{{{{.GraphDriver.Data.MergedDir}}}}' {uuid}".format(uuid=uuid)
+        message = 'Failed to get directory of {uuid} on host {host}'.\
+            format(uuid=uuid, host=node['host'])
+
+        stdout, _ = exec_cmd_no_error(node, command, sudo=True, message=message)
+        return stdout.strip()
+

     @staticmethod
     def get_huge_page_size(node):
         """Get default size of huge pages in system.
     @staticmethod
     def get_huge_page_size(node):
         """Get default size of huge pages in system.