Compatibility fixes with Ubuntu 18.04
[csit.git] / resources / libraries / python / DUTSetup.py
index ca37d9e..4ead762 100644 (file)
@@ -13,8 +13,6 @@
 
 """DUT setup library."""
 
-import os
-
 from robot.api import logger
 
 from resources.libraries.python.topology import NodeType, Topology
@@ -26,6 +24,63 @@ from resources.libraries.python.VPPUtil import VPPUtil
 
 class DUTSetup(object):
     """Contains methods for setting up DUTs."""
+
+    @staticmethod
+    def get_service_logs(node, service):
+        """Get specific service unit logs by journalctl from node.
+
+        :param node: Node in the topology.
+        :param service: Service unit name.
+        :type node: dict
+        :type service: str
+        """
+        ssh = SSH()
+        ssh.connect(node)
+        ret_code, _, _ = \
+            ssh.exec_command_sudo('journalctl --no-pager --unit={name} '
+                                  '--since="$(echo `systemctl show -p '
+                                  'ActiveEnterTimestamp {name}` | '
+                                  'awk \'{{print $2 $3}}\')"'.
+                                  format(name=service))
+        if int(ret_code) != 0:
+            raise RuntimeError('DUT {host} failed to get logs from unit {name}'.
+                               format(host=node['host'], name=service))
+
+    @staticmethod
+    def get_service_logs_on_all_duts(nodes, service):
+        """Get specific service unit logs by journalctl from all DUTs.
+
+        :param nodes: Nodes in the topology.
+        :param service: Service unit name.
+        :type nodes: dict
+        :type service: str
+        """
+        for node in nodes.values():
+            if node['type'] == NodeType.DUT:
+                DUTSetup.get_service_logs(node, service)
+
+    @staticmethod
+    def start_service(node, service):
+        """Start up the named service on node.
+
+        :param node: Node in the topology.
+        :param service: Service unit name.
+        :type node: dict
+        :type service: str
+        """
+        ssh = SSH()
+        ssh.connect(node)
+        # We are doing restart. With this we do not care if service
+        # was running or not.
+        ret_code, _, _ = \
+            ssh.exec_command_sudo('service {name} restart'.
+                                  format(name=service), timeout=120)
+        if int(ret_code) != 0:
+            raise RuntimeError('DUT {host} failed to start service {name}'.
+                               format(host=node['host'], name=service))
+
+        DUTSetup.get_service_logs(node, service)
+
     @staticmethod
     def start_vpp_service_on_all_duts(nodes):
         """Start up the VPP service on all nodes.
@@ -33,15 +88,9 @@ class DUTSetup(object):
         :param nodes: Nodes in the topology.
         :type nodes: dict
         """
-        ssh = SSH()
         for node in nodes.values():
             if node['type'] == NodeType.DUT:
-                ssh.connect(node)
-                (ret_code, stdout, stderr) = \
-                    ssh.exec_command_sudo('service vpp restart', timeout=120)
-                if int(ret_code) != 0:
-                    raise Exception('DUT {0} failed to start VPP service'.
-                                    format(node['host']))
+                DUTSetup.start_service(node, Constants.VPP_UNIT)
 
     @staticmethod
     def vpp_show_version_verbose(node):
@@ -56,8 +105,8 @@ class DUTSetup(object):
         try:
             vat.script_should_have_passed()
         except AssertionError:
-            raise RuntimeError('Failed to get VPP version on host: {}'.
-                               format(node['host']))
+            raise RuntimeError('Failed to get VPP version on host: {name}'.
+                               format(name=node['host']))
 
     @staticmethod
     def show_vpp_version_on_all_duts(nodes):
@@ -83,8 +132,8 @@ class DUTSetup(object):
         try:
             vat.script_should_have_passed()
         except AssertionError:
-            raise RuntimeError('Failed to get VPP interfaces on host: {}'.
-                               format(node['host']))
+            raise RuntimeError('Failed to get VPP interfaces on host: {name}'.
+                               format(name=node['host']))
 
     @staticmethod
     def vpp_api_trace_save(node):
@@ -125,15 +174,13 @@ class DUTSetup(object):
         ssh = SSH()
         ssh.connect(node)
 
-        (ret_code, stdout, stderr) = \
+        ret_code, _, _ = \
             ssh.exec_command('sudo -Sn bash {0}/{1}/dut_setup.sh'.
                              format(Constants.REMOTE_FW_DIR,
                                     Constants.RESOURCES_LIB_SH), timeout=120)
         if int(ret_code) != 0:
-            logger.debug('DUT {0} setup script failed: "{1}"'.
-                         format(node['host'], stdout + stderr))
-            raise Exception('DUT test setup script failed at node {}'.
-                            format(node['host']))
+            raise RuntimeError('DUT test setup script failed at node {name}'.
+                               format(name=node['host']))
 
     @staticmethod
     def get_vpp_pid(node):
@@ -143,7 +190,7 @@ class DUTSetup(object):
         :type node: dict
         :returns: PID
         :rtype: int
-        :raises RuntimeError if it is not possible to get the PID.
+        :raises RuntimeError: If it is not possible to get the PID.
         """
 
         ssh = SSH()
@@ -338,24 +385,31 @@ class DUTSetup(object):
     def get_pci_dev_driver(node, pci_addr):
         """Get current PCI device driver on node.
 
+        .. note::
+            # lspci -vmmks 0000:00:05.0
+            Slot:   00:05.0
+            Class:  Ethernet controller
+            Vendor: Red Hat, Inc
+            Device: Virtio network device
+            SVendor:        Red Hat, Inc
+            SDevice:        Device 0001
+            PhySlot:        5
+            Driver: virtio-pci
+
         :param node: DUT node.
         :param pci_addr: PCI device address.
         :type node: dict
         :type pci_addr: str
         :returns: Driver or None
         :raises RuntimeError: If PCI rescan or lspci command execution failed.
+        :raises RuntimeError: If it is not possible to get the interface driver
+            information from the node.
         """
         ssh = SSH()
         ssh.connect(node)
 
         for i in range(3):
-            logger.trace('Try {0}: Get interface driver'.format(i))
-            cmd = 'sh -c "echo 1 > /sys/bus/pci/rescan"'
-            ret_code, _, _ = ssh.exec_command_sudo(cmd)
-            if int(ret_code) != 0:
-                raise RuntimeError("'{0}' failed on '{1}'"
-                                   .format(cmd, node['host']))
-
+            logger.trace('Try number {0}: Get PCI device driver'.format(i))
             cmd = 'lspci -vmmks {0}'.format(pci_addr)
             ret_code, stdout, _ = ssh.exec_command(cmd)
             if int(ret_code) != 0:
@@ -374,24 +428,31 @@ class DUTSetup(object):
                         return None
                 if name == 'Driver:':
                     return value
-        else:
-            return None
+
+            if i < 2:
+                logger.trace('Driver for PCI device {} not found, executing '
+                             'pci rescan and retrying'.format(pci_addr))
+                cmd = 'sh -c "echo 1 > /sys/bus/pci/rescan"'
+                ret_code, _, _ = ssh.exec_command_sudo(cmd)
+                if int(ret_code) != 0:
+                    raise RuntimeError("'{0}' failed on '{1}'"
+                                       .format(cmd, node['host']))
+
+        return None
 
     @staticmethod
     def kernel_module_verify(node, module, force_load=False):
-        """Verify if kernel module is loaded on all DUTs. If parameter force
+        """Verify if kernel module is loaded on node. If parameter force
         load is set to True, then try to load the modules.
 
-        :param node: DUT node.
+        :param node: Node.
         :param module: Module to verify.
         :param force_load: If True then try to load module.
         :type node: dict
         :type module: str
         :type force_load: bool
-        :returns: nothing
         :raises RuntimeError: If module is not loaded or failed to load.
         """
-
         ssh = SSH()
         ssh.connect(node)
 
@@ -406,6 +467,22 @@ class DUTSetup(object):
                 raise RuntimeError('Kernel module {0} is not loaded on host '
                                    '{1}'.format(module, node['host']))
 
+    @staticmethod
+    def kernel_module_verify_on_all_duts(nodes, module, force_load=False):
+        """Verify if kernel module is loaded on all DUTs. If parameter force
+        load is set to True, then try to load the modules.
+
+        :param node: DUT nodes.
+        :param module: Module to verify.
+        :param force_load: If True then try to load module.
+        :type node: dict
+        :type module: str
+        :type force_load: bool
+        """
+        for node in nodes.values():
+            if node['type'] == NodeType.DUT:
+                DUTSetup.kernel_module_verify(node, module, force_load)
+
     @staticmethod
     def kernel_module_load(node, module):
         """Load kernel module on node.
@@ -449,6 +526,7 @@ class DUTSetup(object):
         vat = VatExecutor()
         vat.execute_script("enable_dpdk_traces.vat", node, json_out=False)
         vat.execute_script("enable_vhost_user_traces.vat", node, json_out=False)
+        vat.execute_script("enable_memif_traces.vat", node, json_out=False)
 
     @staticmethod
     def install_vpp_on_all_duts(nodes, vpp_pkg_dir, vpp_rpm_pkgs, vpp_deb_pkgs):
@@ -462,7 +540,7 @@ class DUTSetup(object):
         :type vpp_pkg_dir: str
         :type vpp_rpm_pkgs: list
         :type vpp_deb_pkgs: list
-        :raises: RuntimeError if failed to remove or install VPP
+        :raises RuntimeError: If failed to remove or install VPP.
         """
 
         logger.debug("Installing VPP")
@@ -474,24 +552,26 @@ class DUTSetup(object):
                 ssh = SSH()
                 ssh.connect(node)
 
-                if os.path.isfile("/etc/redhat-release"):
+                cmd = "[[ -f /etc/redhat-release ]]"
+                return_code, _, _ = ssh.exec_command(cmd)
+                if int(return_code) == 0:
                     # workaroud - uninstall existing vpp installation until
                     # start-testcase script is updated on all virl servers
-                    rpm_pkgs_remove = " ".join(vpp_rpm_pkgs)
-                    r_rcode, _, r_err = ssh.exec_command_sudo(
-                        "rpm -e {0}".format(rpm_pkgs_remove), timeout=90)
+                    rpm_pkgs_remove = "vpp*"
+                    cmd_u = 'yum -y remove "{0}"'.format(rpm_pkgs_remove)
+                    r_rcode, _, r_err = ssh.exec_command_sudo(cmd_u, timeout=90)
                     if int(r_rcode) != 0:
                         raise RuntimeError('Failed to remove previous VPP'
                                            'installation on host {0}:\n{1}'
-                                           .format(node['host']), r_err)
+                                           .format(node['host'], r_err))
 
                     rpm_pkgs = "*.rpm ".join(str(vpp_pkg_dir + pkg)
                                              for pkg in vpp_rpm_pkgs) + "*.rpm"
-                    ret_code, _, err = ssh.exec_command_sudo(
-                        "rpm -ivh {0}".format(rpm_pkgs), timeout=90)
+                    cmd_i = "rpm -ivh {0}".format(rpm_pkgs)
+                    ret_code, _, err = ssh.exec_command_sudo(cmd_i, timeout=90)
                     if int(ret_code) != 0:
                         raise RuntimeError('Failed to install VPP on host {0}:'
-                                           '\n{1}'.format(node['host']), err)
+                                           '\n{1}'.format(node['host'], err))
                     else:
                         ssh.exec_command_sudo("rpm -qai vpp*")
                         logger.info("VPP installed on node {0}".
@@ -499,20 +579,20 @@ class DUTSetup(object):
                 else:
                     # workaroud - uninstall existing vpp installation until
                     # start-testcase script is updated on all virl servers
-                    deb_pkgs_remove = " ".join(vpp_deb_pkgs)
-                    r_rcode, _, r_err = ssh.exec_command_sudo(
-                        "dpkg --purge {0}".format(deb_pkgs_remove), timeout=90)
+                    deb_pkgs_remove = "vpp*"
+                    cmd_u = 'apt-get purge -y "{0}"'.format(deb_pkgs_remove)
+                    r_rcode, _, r_err = ssh.exec_command_sudo(cmd_u, timeout=90)
                     if int(r_rcode) != 0:
                         raise RuntimeError('Failed to remove previous VPP'
                                            'installation on host {0}:\n{1}'
-                                           .format(node['host']), r_err)
+                                           .format(node['host'], r_err))
                     deb_pkgs = "*.deb ".join(str(vpp_pkg_dir + pkg)
                                              for pkg in vpp_deb_pkgs) + "*.deb"
-                    ret_code, _, err = ssh.exec_command_sudo(
-                        "dpkg -i --force-all {0}".format(deb_pkgs), timeout=90)
+                    cmd_i = "dpkg -i --force-all {0}".format(deb_pkgs)
+                    ret_code, _, err = ssh.exec_command_sudo(cmd_i, timeout=90)
                     if int(ret_code) != 0:
                         raise RuntimeError('Failed to install VPP on host {0}:'
-                                           '\n{1}'.format(node['host']), err)
+                                           '\n{1}'.format(node['host'], err))
                     else:
                         ssh.exec_command_sudo("dpkg -l | grep vpp")
                         logger.info("VPP installed on node {0}".
@@ -542,8 +622,8 @@ class DUTSetup(object):
 
         :param node: DUT node.
         :type node: dict
-        :raises: RuntimeError if failed to restart VPP, get VPP version or
-        get VPP interfaces
+        :raises RuntimeError: If failed to restart VPP, get VPP version
+            or get VPP interfaces.
         """
 
         logger.debug("Verify VPP on node {0}".format(node['host']))