CSIT-1493 VPP restart handling code

Change-Id: Ibe52125089f39e0ff17ec607a3ed00c61d52ab8c
Signed-off-by: Peter Mikus <pmikus@cisco.com>

diff --git a/resources/libraries/python/DUTSetup.py b/resources/libraries/python/DUTSetup.py
index 67edefb..e2fba06 100644 (file)
--- a/resources/libraries/python/DUTSetup.py
+++ b/resources/libraries/python/DUTSetup.py
@@ -60,6 +60,39 @@ class DUTSetup(object):
             if node['type'] == NodeType.DUT:
                 DUTSetup.get_service_logs(node, service)
 
             if node['type'] == NodeType.DUT:
                 DUTSetup.get_service_logs(node, service)
 

+    @staticmethod
+    def restart_service(node, service):
+        """Restarts the named service on node.
+
+        :param node: Node in the topology.
+        :param service: Service unit name.
+        :type node: dict
+        :type service: str
+        """
+        if DUTSetup.running_in_container(node):
+            command = 'supervisorctl restart {name}'.format(name=service)
+        else:
+            command = 'service {name} restart'.format(name=service)
+        message = 'Node {host} failed to restart service {name}'.\
+            format(host=node['host'], name=service)
+
+        exec_cmd_no_error(node, command, timeout=30, sudo=True, message=message)
+
+        DUTSetup.get_service_logs(node, service)
+
+    @staticmethod
+    def restart_service_on_all_duts(nodes, service):
+        """Retarts the named service on all DUTs.
+
+        :param node: Nodes in the topology.
+        :param service: Service unit name.
+        :type node: dict
+        :type service: str
+        """
+        for node in nodes.values():
+            if node['type'] == NodeType.DUT:
+                DUTSetup.restart_service(node, service)
+

     @staticmethod
     def start_service(node, service):
         """Start up the named service on node.
     @staticmethod
     def start_service(node, service):
         """Start up the named service on node.


@@ -69,6 +102,7 @@ class DUTSetup(object):
         :type node: dict
         :type service: str
         """
         :type node: dict
         :type service: str
         """

+        # TODO: change command to start once all parent function updated.

         if DUTSetup.running_in_container(node):
             command = 'supervisorctl restart {name}'.format(name=service)
         else:
         if DUTSetup.running_in_container(node):
             command = 'supervisorctl restart {name}'.format(name=service)
         else:


@@ -609,7 +643,8 @@ class DUTSetup(object):
 
     @staticmethod
     def install_vpp_on_all_duts(nodes, vpp_pkg_dir):
 
     @staticmethod
     def install_vpp_on_all_duts(nodes, vpp_pkg_dir):

-        """Install VPP on all DUT nodes.
+        """Install VPP on all DUT nodes. Start the VPP service in case of
+        systemd is not available or does not support autostart.

 
         :param nodes: Nodes in the topology.
         :param vpp_pkg_dir: Path to directory where VPP packages are stored.
 
         :param nodes: Nodes in the topology.
         :param vpp_pkg_dir: Path to directory where VPP packages are stored.


@@ -634,6 +669,8 @@ class DUTSetup(object):
                                       format(dir=vpp_pkg_dir), timeout=120,
                                       sudo=True, message=message)
                     exec_cmd_no_error(node, 'dpkg -l | grep vpp', sudo=True)
                                       format(dir=vpp_pkg_dir), timeout=120,
                                       sudo=True, message=message)
                     exec_cmd_no_error(node, 'dpkg -l | grep vpp', sudo=True)

+                    if DUTSetup.running_in_container(node):
+                        DUTSetup.restart_service(node, Constants.VPP_UNIT)

                 else:
                     exec_cmd_no_error(node, 'yum -y remove "*vpp*" || true',
                                       timeout=120, sudo=True)
                 else:
                     exec_cmd_no_error(node, 'yum -y remove "*vpp*" || true',
                                       timeout=120, sudo=True)


@@ -641,6 +678,7 @@ class DUTSetup(object):
                                       format(dir=vpp_pkg_dir), timeout=120,
                                       sudo=True, message=message)
                     exec_cmd_no_error(node, 'rpm -qai *vpp*', sudo=True)
                                       format(dir=vpp_pkg_dir), timeout=120,
                                       sudo=True, message=message)
                     exec_cmd_no_error(node, 'rpm -qai *vpp*', sudo=True)

+                    DUTSetup.restart_service(node, Constants.VPP_UNIT)

 
     @staticmethod
     def running_in_container(node):
 
     @staticmethod
     def running_in_container(node):

diff --git a/resources/libraries/python/KubernetesUtils.py b/resources/libraries/python/KubernetesUtils.py
index e932492..60e1286 100644 (file)
--- a/resources/libraries/python/KubernetesUtils.py
+++ b/resources/libraries/python/KubernetesUtils.py
@@ -483,7 +483,6 @@ class KubernetesUtils(object):
         vpp_config.set_node(kwargs['node'])
         vpp_config.add_unix_cli_listen(value='0.0.0.0:5002')
         vpp_config.add_unix_nodaemon()
         vpp_config.set_node(kwargs['node'])
         vpp_config.add_unix_cli_listen(value='0.0.0.0:5002')
         vpp_config.add_unix_nodaemon()

-        vpp_config.add_dpdk_socketmem('1024,1024')

         vpp_config.add_heapsize('4G')
         vpp_config.add_ip_heap_size('4G')
         vpp_config.add_ip6_heap_size('4G')
         vpp_config.add_heapsize('4G')
         vpp_config.add_ip_heap_size('4G')
         vpp_config.add_ip6_heap_size('4G')


@@ -500,7 +499,7 @@ class KubernetesUtils(object):
         if cpuset_cpus:
             corelist_workers = ','.join(str(cpu) for cpu in cpuset_cpus)
             vpp_config.add_cpu_corelist_workers(corelist_workers)
         if cpuset_cpus:
             corelist_workers = ','.join(str(cpu) for cpu in cpuset_cpus)
             vpp_config.add_cpu_corelist_workers(corelist_workers)

-        vpp_config.apply_config(filename=kwargs['filename'], restart_vpp=False)
+        vpp_config.write_config(filename=kwargs['filename'])

 
     @staticmethod
     def create_kubernetes_vnf_startup_config(**kwargs):
 
     @staticmethod
     def create_kubernetes_vnf_startup_config(**kwargs):


@@ -536,4 +535,4 @@ class KubernetesUtils(object):
             corelist_workers = ','.join(str(cpu) for cpu in cpuset_cpus)
             vpp_config.add_cpu_corelist_workers(corelist_workers)
         vpp_config.add_plugin('disable', 'dpdk_plugin.so')
             corelist_workers = ','.join(str(cpu) for cpu in cpuset_cpus)
             vpp_config.add_cpu_corelist_workers(corelist_workers)
         vpp_config.add_plugin('disable', 'dpdk_plugin.so')

-        vpp_config.apply_config(filename=kwargs['filename'], restart_vpp=False)
+        vpp_config.write_config(filename=kwargs['filename'])

diff --git a/resources/libraries/python/QemuUtils.py b/resources/libraries/python/QemuUtils.py
index 5404b51..735feab 100644 (file)
--- a/resources/libraries/python/QemuUtils.py
+++ b/resources/libraries/python/QemuUtils.py
@@ -212,7 +212,7 @@ class QemuUtils(object):
             vpp_config.add_dpdk_no_tx_checksum_offload()
         vpp_config.add_plugin('disable', 'default')
         vpp_config.add_plugin('enable', 'dpdk_plugin.so')
             vpp_config.add_dpdk_no_tx_checksum_offload()
         vpp_config.add_plugin('disable', 'default')
         vpp_config.add_plugin('enable', 'dpdk_plugin.so')

-        vpp_config.apply_config(startup, restart_vpp=False)
+        vpp_config.write_config(startup)

 
         # Create VPP running configuration.
         template = '{res}/{tpl}.exec'.format(res=Constants.RESOURCES_TPL_VM,
 
         # Create VPP running configuration.
         template = '{res}/{tpl}.exec'.format(res=Constants.RESOURCES_TPL_VM,

diff --git a/resources/libraries/python/VPPUtil.py b/resources/libraries/python/VPPUtil.py
index 726ba2b..5b77284 100644 (file)
--- a/resources/libraries/python/VPPUtil.py
+++ b/resources/libraries/python/VPPUtil.py
@@ -13,14 +13,12 @@
 
 """VPP util library."""
 
 
 """VPP util library."""
 

-import time
-

 from robot.api import logger
 
 from resources.libraries.python.Constants import Constants
 from resources.libraries.python.DUTSetup import DUTSetup
 from resources.libraries.python.PapiExecutor import PapiExecutor
 from robot.api import logger
 
 from resources.libraries.python.Constants import Constants
 from resources.libraries.python.DUTSetup import DUTSetup
 from resources.libraries.python.PapiExecutor import PapiExecutor

-from resources.libraries.python.ssh import exec_cmd, exec_cmd_no_error
+from resources.libraries.python.ssh import exec_cmd_no_error

 from resources.libraries.python.topology import NodeType
 from resources.libraries.python.VatExecutor import VatExecutor
 
 from resources.libraries.python.topology import NodeType
 from resources.libraries.python.VatExecutor import VatExecutor
 


@@ -58,55 +56,39 @@ class VPPUtil(object):
             exec_cmd_no_error(node, command, timeout=30, sudo=True)
 
     @staticmethod
             exec_cmd_no_error(node, command, timeout=30, sudo=True)
 
     @staticmethod

-    def start_vpp_service(node, retries=120):
-        """Start VPP service on the specified node.
+    def restart_vpp_service(node):
+        """Restart VPP service on the specified topology node.

 
 

-        :param node: VPP node.
-        :param retries: Number of times (default 60) to re-try waiting.
+        :param node: Topology node.

         :type node: dict
         :type node: dict

-        :type retries: int
-        :raises RuntimeError: If VPP service fails to start.

         """
         """

-        DUTSetup.start_service(node, Constants.VPP_UNIT)
-        # Sleep 1 second, up to <retry> times,
-        # and verify if VPP is running.
-        for _ in range(retries):
-            time.sleep(1)
-            command = 'vppctl show pci'
-            ret, stdout, _ = exec_cmd(node, command, timeout=30, sudo=True)
-            if not ret and 'Connection refused' not in stdout:
-                break
-        else:
-            raise RuntimeError('VPP failed to start on host {name}'.
-                               format(name=node['host']))
-        DUTSetup.get_service_logs(node, Constants.VPP_UNIT)
+        DUTSetup.restart_service(node, Constants.VPP_UNIT)

 
     @staticmethod
 
     @staticmethod

-    def start_vpp_service_on_all_duts(nodes):
-        """Start up the VPP service on all nodes.
+    def restart_vpp_service_on_all_duts(nodes):
+        """Restart VPP service on all DUT nodes.

 
 

-        :param nodes: Nodes in the topology.
+        :param nodes: Topology nodes.

         :type nodes: dict
         """
         for node in nodes.values():
             if node['type'] == NodeType.DUT:
         :type nodes: dict
         """
         for node in nodes.values():
             if node['type'] == NodeType.DUT:

-                VPPUtil.start_vpp_service(node)
+                VPPUtil.restart_vpp_service(node)

 
     @staticmethod
     def stop_vpp_service(node):
 
     @staticmethod
     def stop_vpp_service(node):

-        """Stop VPP service on the specified node.
+        """Stop VPP service on the specified topology node.

 
 

-        :param node: VPP node.
+        :param node: Topology node.

         :type node: dict
         :type node: dict

-        :raises RuntimeError: If VPP service fails to stop.

         """
         DUTSetup.stop_service(node, Constants.VPP_UNIT)
 
     @staticmethod
     def stop_vpp_service_on_all_duts(nodes):
         """
         DUTSetup.stop_service(node, Constants.VPP_UNIT)
 
     @staticmethod
     def stop_vpp_service_on_all_duts(nodes):

-        """Stop VPP service on all nodes.
+        """Stop VPP service on all DUT nodes.

 
 

-        :param nodes: Nodes in the topology.
+        :param nodes: Topology nodes.

         :type nodes: dict
         """
         for node in nodes.values():
         :type nodes: dict
         """
         for node in nodes.values():


@@ -114,31 +96,60 @@ class VPPUtil(object):
                 VPPUtil.stop_vpp_service(node)
 
     @staticmethod
                 VPPUtil.stop_vpp_service(node)
 
     @staticmethod

-    def verify_vpp_on_dut(node):
-        """Verify that VPP is installed on DUT node.
+    def verify_vpp_installed(node):
+        """Verify that VPP is installed on the specified topology node.
+
+        :param node: Topology node.
+        :type node: dict
+        """
+        cmd = 'command -v vpp'
+        exec_cmd_no_error(
+            node, cmd, message='VPP is not installed!')
+
+    @staticmethod
+    def verify_vpp_started(node):
+        """Verify that VPP is started on the specified topology node.

 
 

-        :param node: DUT node.
+        :param node: Topology node.

         :type node: dict
         :type node: dict

-        :raises RuntimeError: If failed to restart VPP, get VPP version
-            or get VPP interfaces.

         """
         """

-        VPPUtil.vpp_show_version_verbose(node)
-        VPPUtil.vpp_show_interfaces(node)
+        cmd = ('vppctl show pci 2>&1 | '
+               'fgrep -v "Connection refused" | '
+               'fgrep -v "No such file or directory"')
+        exec_cmd_no_error(
+            node, cmd, sudo=True, message='VPP failed to start!', retries=120)
+
+    @staticmethod
+    def verify_vpp(node):
+        """Verify that VPP is installed and started on the specified topology
+        node.
+
+        :param node: Topology node.
+        :type node: dict
+        :raises RuntimeError: If VPP service fails to start.
+        """
+        VPPUtil.verify_vpp_installed(node)
+        try:
+            # Verify responsivness of vppctl.
+            VPPUtil.verify_vpp_started(node)
+            # Verify responsivness of PAPI.
+            VPPUtil.show_log(node)
+        finally:
+            DUTSetup.get_service_logs(node, Constants.VPP_UNIT)

 
     @staticmethod
     def verify_vpp_on_all_duts(nodes):
 
     @staticmethod
     def verify_vpp_on_all_duts(nodes):

-        """Verify that VPP is installed on all DUT nodes.
+        """Verify that VPP is installed and started on all DUT nodes.

 
         :param nodes: Nodes in the topology.
         :type nodes: dict
         """
         for node in nodes.values():
             if node['type'] == NodeType.DUT:
 
         :param nodes: Nodes in the topology.
         :type nodes: dict
         """
         for node in nodes.values():
             if node['type'] == NodeType.DUT:

-                VPPUtil.start_vpp_service(node)
-                VPPUtil.verify_vpp_on_dut(node)
+                VPPUtil.verify_vpp(node)

 
     @staticmethod
 
     @staticmethod

-    def vpp_show_version(node, verbose=False):
+    def vpp_show_version(node, verbose=True):

         """Run "show_version" PAPI command.
 
         :param node: Node to run command on.
         """Run "show_version" PAPI command.
 
         :param node: Node to run command on.


@@ -149,7 +160,6 @@ class VPPUtil(object):
         :returns: VPP version.
         :rtype: str
         """
         :returns: VPP version.
         :rtype: str
         """

-

         with PapiExecutor(node) as papi_exec:
             data = papi_exec.add('show_version').execute_should_pass().\
                 verify_reply()
         with PapiExecutor(node) as papi_exec:
             data = papi_exec.add('show_version').execute_should_pass().\
                 verify_reply()


@@ -163,26 +173,16 @@ class VPPUtil(object):
         logger.info(version)
         return data['version'].rstrip('\0x00')
 
         logger.info(version)
         return data['version'].rstrip('\0x00')
 

-    @staticmethod
-    def vpp_show_version_verbose(node):
-        """Run "show_version" API command and return verbose string of version
-        data.
-
-        :param node: Node to run command on.
-        :type node: dict
-        """
-        VPPUtil.vpp_show_version(node, verbose=True)
-

     @staticmethod
     def show_vpp_version_on_all_duts(nodes):
     @staticmethod
     def show_vpp_version_on_all_duts(nodes):

-        """Show VPP version on all DUTs.
+        """Show VPP version verbose on all DUTs.

 
 

-        :param nodes: VPP nodes.
+        :param nodes: Nodes in the topology.

         :type nodes: dict
         """
         for node in nodes.values():
             if node['type'] == NodeType.DUT:
         :type nodes: dict
         """
         for node in nodes.values():
             if node['type'] == NodeType.DUT:

-                VPPUtil.vpp_show_version_verbose(node)
+                VPPUtil.vpp_show_version(node)

 
     @staticmethod
     def vpp_show_interfaces(node):
 
     @staticmethod
     def vpp_show_interfaces(node):


@@ -211,26 +211,6 @@ class VPPUtil(object):
         vat.execute_script("show_crypto_device_mapping.vat", node,
                            json_out=False)
 
         vat.execute_script("show_crypto_device_mapping.vat", node,
                            json_out=False)
 

-    @staticmethod
-    def vpp_api_trace_dump(node):
-        """Run "api trace custom-dump" CLI command.
-
-        :param node: Node to run command on.
-        :type node: dict
-        """
-        vat = VatExecutor()
-        vat.execute_script("api_trace_dump.vat", node, json_out=False)
-
-    @staticmethod
-    def vpp_api_trace_save(node):
-        """Run "api trace save" CLI command.
-
-        :param node: Node to run command on.
-        :type node: dict
-        """
-        vat = VatExecutor()
-        vat.execute_script("api_trace_save.vat", node, json_out=False)
-

     @staticmethod
     def vpp_enable_traces_on_dut(node):
         """Enable vpp packet traces on the DUT node.
     @staticmethod
     def vpp_enable_traces_on_dut(node):
         """Enable vpp packet traces on the DUT node.


@@ -297,6 +277,19 @@ class VPPUtil(object):
             if node['type'] == NodeType.DUT:
                 VPPUtil.show_event_logger_on_dut(node)
 
             if node['type'] == NodeType.DUT:
                 VPPUtil.show_event_logger_on_dut(node)
 

+    @staticmethod
+    def show_log(node):
+        """Show log on the specified topology node.
+
+        :param node: Topology node.
+        :type node: dict
+        :returns: VPP log data.
+        :rtype: list
+        """
+        with PapiExecutor(node) as papi_exec:
+            return papi_exec.add('cli_inband', cmd='show log').get_replies().\
+                verify_reply()["reply"]
+

     @staticmethod
     def vpp_show_threads(node):
         """Show VPP threads on node.
     @staticmethod
     def vpp_show_threads(node):
         """Show VPP threads on node.

diff --git a/resources/libraries/python/VppConfigGenerator.py b/resources/libraries/python/VppConfigGenerator.py
index 350e1c0..883197f 100644 (file)
--- a/resources/libraries/python/VppConfigGenerator.py
+++ b/resources/libraries/python/VppConfigGenerator.py
@@ -14,13 +14,11 @@
 """VPP Configuration File Generator library."""
 
 import re
 """VPP Configuration File Generator library."""
 
 import re

-import time

 
 

-from resources.libraries.python.ssh import SSH
-from resources.libraries.python.Constants import Constants
-from resources.libraries.python.DUTSetup import DUTSetup
+from resources.libraries.python.ssh import exec_cmd_no_error

 from resources.libraries.python.topology import NodeType
 from resources.libraries.python.topology import Topology
 from resources.libraries.python.topology import NodeType
 from resources.libraries.python.topology import Topology

+from resources.libraries.python.VPPUtil import VPPUtil

 
 __all__ = ['VppConfigGenerator']
 
 
 __all__ = ['VppConfigGenerator']
 


@@ -559,74 +557,51 @@ class VppConfigGenerator(object):
         path = ['session', 'local-endpoints-table-memory']
         self.add_config_item(self._nodeconfig, value, path)
 
         path = ['session', 'local-endpoints-table-memory']
         self.add_config_item(self._nodeconfig, value, path)
 

-    def apply_config(self, filename=None, retries=60, restart_vpp=True):
-        """Generate and apply VPP configuration for node.
+    def write_config(self, filename=None):
+        """Generate and write VPP startup configuration to file.

 
         Use data from calls to this class to form a startup.conf file and
 
         Use data from calls to this class to form a startup.conf file and

-        replace /etc/vpp/startup.conf with it on node.
+        replace /etc/vpp/startup.conf with it on topology node.

 
         :param filename: Startup configuration file name.
 
         :param filename: Startup configuration file name.

-        :param retries: Number of times (default 60) to re-try waiting.
-        :param restart_vpp: Whether to restart VPP.

         :type filename: str
         :type filename: str

-        :type retries: int
-        :type restart_vpp: bool.
-        :raises RuntimeError: If writing config file failed or restart of VPP
-            failed or backup of VPP startup.conf failed.

         """
         self.dump_config(self._nodeconfig)
 
         """
         self.dump_config(self._nodeconfig)
 

-        ssh = SSH()
-        ssh.connect(self._node)
-

         if filename is None:
             filename = self._vpp_startup_conf
 
         if self._vpp_startup_conf_backup is not None:
         if filename is None:
             filename = self._vpp_startup_conf
 
         if self._vpp_startup_conf_backup is not None:

-            ret, _, _ = \
-                ssh.exec_command('sudo cp {src} {dest}'.
-                                 format(src=self._vpp_startup_conf,
-                                        dest=self._vpp_startup_conf_backup))
-            if ret != 0:
-                raise RuntimeError('Backup of config file failed on node '
-                                   '{name}'.format(name=self._hostname))
-
-        ret, _, _ = \
-            ssh.exec_command('echo "{config}" | sudo tee {filename}'.
-                             format(config=self._vpp_config,
-                                    filename=filename))
-
-        if ret != 0:
-            raise RuntimeError('Writing config file failed to node {name}'.
-                               format(name=self._hostname))
-
-        if restart_vpp:
-            DUTSetup.start_service(self._node, Constants.VPP_UNIT)
-
-            # Sleep <waittime> seconds, up to <retry> times,
-            # and verify if VPP is running.
-            for _ in range(retries):
-                time.sleep(1)
-                ret, stdout, _ = \
-                    ssh.exec_command_sudo('vppctl show pci')
-                if ret == 0 and 'Connection refused' not in stdout:
-                    break
-            else:
-                raise RuntimeError('VPP failed to restart on node {name}'.
-                                   format(name=self._hostname))
+            cmd = ('cp {src} {dest}'.format(
+                src=self._vpp_startup_conf, dest=self._vpp_startup_conf_backup))
+            exec_cmd_no_error(
+                self._node, cmd, sudo=True, message='Copy config file failed!')

 
 

-    def restore_config(self):
-        """Restore VPP startup.conf from backup.
+        cmd = ('echo "{config}" | sudo tee {filename}'.format(
+            config=self._vpp_config, filename=filename))
+        exec_cmd_no_error(
+            self._node, cmd, message='Writing config file failed!')
+
+    def apply_config(self, filename=None, verify_vpp=True):
+        """Generate and write VPP startup configuration to file and restart VPP.
+
+        Use data from calls to this class to form a startup.conf file and
+        replace /etc/vpp/startup.conf with it on topology node.

 
 

-        :raises RuntimeError: When restoration of startup.conf file failed.
+        :param filename: Startup configuration file name.
+        :param verify_vpp: Verify VPP is running after restart.
+        :type filename: str
+        :type verify_vpp: bool

         """
         """

+        self.write_config(filename=filename)

 
 

-        ssh = SSH()
-        ssh.connect(self._node)
+        VPPUtil.restart_vpp_service(self._node)
+        if verify_vpp:
+            VPPUtil.verify_vpp(self._node)

 
 

-        ret, _, _ = ssh.exec_command('sudo cp {src} {dest}'.
-                                     format(src=self._vpp_startup_conf_backup,
-                                            dest=self._vpp_startup_conf))
-        if ret != 0:
-            raise RuntimeError('Restoration of config file failed on node '
-                               '{name}'.format(name=self._hostname))
+    def restore_config(self):
+        """Restore VPP startup.conf from backup."""
+        cmd = ('cp {src} {dest}'.format(
+            src=self._vpp_startup_conf_backup, dest=self._vpp_startup_conf))
+        exec_cmd_no_error(
+            self._node, cmd, sudo=True, message='Copy config file failed!')

diff --git a/resources/libraries/python/ssh.py b/resources/libraries/python/ssh.py
index 966d1b0..9c7adc4 100644 (file)
--- a/resources/libraries/python/ssh.py
+++ b/resources/libraries/python/ssh.py
@@ -452,10 +452,14 @@ def exec_cmd(node, cmd, timeout=600, sudo=False, disconnect=False):
 
 
 def exec_cmd_no_error(
 
 
 def exec_cmd_no_error(

-        node, cmd, timeout=600, sudo=False, message=None, disconnect=False):
+        node, cmd, timeout=600, sudo=False, message=None, disconnect=False,
+        retries=0):

     """Convenience function to ssh/exec/return out & err.
 
     Verifies that return code is zero.
     """Convenience function to ssh/exec/return out & err.
 
     Verifies that return code is zero.

+    Supports retries, timeout is related to each try separately then. There is
+    sleep(1) before each retry.
+    Disconnect (if enabled) is applied after each try.

 
     :param node: DUT node.
     :param cmd: Command to be executed.
 
     :param node: DUT node.
     :param cmd: Command to be executed.


@@ -463,21 +467,27 @@ def exec_cmd_no_error(
     :param sudo: Sudo privilege execution flag. Default: False.
     :param message: Error message in case of failure. Default: None.
     :param disconnect: Close the opened SSH connection if True.
     :param sudo: Sudo privilege execution flag. Default: False.
     :param message: Error message in case of failure. Default: None.
     :param disconnect: Close the opened SSH connection if True.

+    :param retries: How many times to retry on failure.

     :type node: dict
     :type cmd: str or OptionString
     :type timeout: int
     :type sudo: bool
     :type message: str
     :type disconnect: bool
     :type node: dict
     :type cmd: str or OptionString
     :type timeout: int
     :type sudo: bool
     :type message: str
     :type disconnect: bool

+    :type retries: int

     :returns: Stdout, Stderr.
     :rtype: tuple(str, str)
     :raises RuntimeError: If bash return code is not 0.
     """
     :returns: Stdout, Stderr.
     :rtype: tuple(str, str)
     :raises RuntimeError: If bash return code is not 0.
     """

-    ret_code, stdout, stderr = exec_cmd(
-        node, cmd, timeout=timeout, sudo=sudo, disconnect=disconnect)
-    msg = ('Command execution failed: "{cmd}"\n{stderr}'.
-           format(cmd=cmd, stderr=stderr) if message is None else message)
-    if ret_code != 0:
+    for _ in range(retries + 1):
+        ret_code, stdout, stderr = exec_cmd(
+            node, cmd, timeout=timeout, sudo=sudo, disconnect=disconnect)
+        if ret_code == 0:
+            break
+        sleep(1)
+    else:
+        msg = ('Command execution failed: "{cmd}"\n{stderr}'.
+               format(cmd=cmd, stderr=stderr) if message is None else message)

         raise RuntimeError(msg)
 
     return stdout, stderr
         raise RuntimeError(msg)
 
     return stdout, stderr

diff --git a/resources/libraries/robot/crypto/ipsec.robot b/resources/libraries/robot/crypto/ipsec.robot
index 91a4847..b7e9ff1 100644 (file)
--- a/resources/libraries/robot/crypto/ipsec.robot
+++ b/resources/libraries/robot/crypto/ipsec.robot
@@ -312,7 +312,7 @@
 | | | ... | ${sw_dev_count}
 | | | Run keyword | ${dut}.Add DPDK SW Cryptodev | aesni_gcm | ${socket_id}
 | | | ... | ${sw_dev_count}
 | | | ... | ${sw_dev_count}
 | | | Run keyword | ${dut}.Add DPDK SW Cryptodev | aesni_gcm | ${socket_id}
 | | | ... | ${sw_dev_count}

-| | Apply startup configuration on all VPP DUTs | restart_vpp=${FALSE}
+| | Write startup configuration on all VPP DUTs

 | | Set up functional test
 | | Run Keyword | Configure topology for ${ip_version} IPsec testing
 
 | | Set up functional test
 | | Run Keyword | Configure topology for ${ip_version} IPsec testing
 

diff --git a/resources/libraries/robot/performance/performance_setup.robot b/resources/libraries/robot/performance/performance_setup.robot
index 6e7cc70..7c72783 100644 (file)
--- a/resources/libraries/robot/performance/performance_setup.robot
+++ b/resources/libraries/robot/performance/performance_setup.robot
@@ -654,7 +654,8 @@
 | | Configure VPP in all '${container_group}' containers
 | | Stop VPP service on all DUTs | ${nodes}
 | | Start VPP in all '${container_group}' containers
 | | Configure VPP in all '${container_group}' containers
 | | Stop VPP service on all DUTs | ${nodes}
 | | Start VPP in all '${container_group}' containers

-| | Start VPP service on all DUTs | ${nodes}
+| | Restart VPP service on all DUTs | ${nodes}
+| | Verify VPP on all DUTs | ${nodes}

 | | Append To List | ${container_groups} | ${container_group}
 
 # Tests teardowns
 | | Append To List | ${container_groups} | ${container_group}
 
 # Tests teardowns

diff --git a/resources/libraries/robot/shared/default.robot b/resources/libraries/robot/shared/default.robot
index 6e26abc..09ca17d 100644 (file)
--- a/resources/libraries/robot/shared/default.robot
+++ b/resources/libraries/robot/shared/default.robot
@@ -54,14 +54,6 @@
 | | :FOR | ${dut} | IN | @{duts}
 | | | Vpp Show Errors | ${nodes['${dut}']}
 
 | | :FOR | ${dut} | IN | @{duts}
 | | | Vpp Show Errors | ${nodes['${dut}']}
 

-| Show VPP trace dump on all DUTs
-| | [Documentation] | Save API trace and dump output on all DUTs.
-| | ...
-| | ${duts}= | Get Matches | ${nodes} | DUT*
-| | :FOR | ${dut} | IN | @{duts}
-| | | Vpp api trace save | ${nodes['${dut}']}
-| | | Vpp api trace dump | ${nodes['${dut}']}
-

 | Show Bridge Domain Data On All DUTs
 | | [Documentation] | Show Bridge Domain data on all DUTs.
 | | ...
 | Show Bridge Domain Data On All DUTs
 | | [Documentation] | Show Bridge Domain data on all DUTs.
 | | ...


@@ -419,21 +411,20 @@
 | | Run keyword | DUT2.Add DPDK SW Cryptodev | ${sw_pmd_type} | ${socket_id}
 | | ... | ${thr_count_int}
 
 | | Run keyword | DUT2.Add DPDK SW Cryptodev | ${sw_pmd_type} | ${socket_id}
 | | ... | ${thr_count_int}
 

-| Apply startup configuration on all VPP DUTs
-| | [Documentation] | Write startup configuration and restart VPP on all DUTs.
-| | ...
-| | ... | *Arguments:*
-| | ... | - restart_vpp - Whether to restart VPP (Optional). Type: boolean
-| | ...
-| | ... | *Example:*
+| Write startup configuration on all VPP DUTs
+| | [Documentation] | Write VPP startup configuration on all DUTs.

 | | ...
 | | ...

-| | ... | \| Apply startup configuration on all VPP DUTs \| ${False} \|
-| | ...
-| | [Arguments] | ${restart_vpp}=${True}
+| | ${duts}= | Get Matches | ${nodes} | DUT*
+| | :FOR | ${dut} | IN | @{duts}
+| | | Run keyword | ${dut}.Write Config
+
+| Apply startup configuration on all VPP DUTs
+| | [Documentation] | Write VPP startup configuration and restart VPP on all
+| | ... | DUTs.

 | | ...
 | | ${duts}= | Get Matches | ${nodes} | DUT*
 | | :FOR | ${dut} | IN | @{duts}
 | | ...
 | | ${duts}= | Get Matches | ${nodes} | DUT*
 | | :FOR | ${dut} | IN | @{duts}

-| | | Run keyword | ${dut}.Apply Config | restart_vpp=${restart_vpp}
+| | | Run keyword | ${dut}.Apply Config

 | | Enable Coredump Limit VPP on All DUTs | ${nodes}
 | | Update All Interface Data On All Nodes | ${nodes} | skip_tg=${True}
 
 | | Enable Coredump Limit VPP on All DUTs | ${nodes}
 | | Update All Interface Data On All Nodes | ${nodes} | skip_tg=${True}
 


@@ -617,16 +608,3 @@
 | | ...
 | | [Arguments] | ${node}
 | | Stop VPP Service | ${node}
 | | ...
 | | [Arguments] | ${node}
 | | Stop VPP Service | ${node}

-
-| Start VPP Service on DUT
-| | [Documentation] | Start the VPP service on the specified node.
-| | ...
-| | ... | *Arguments:*
-| | ... | - node - information about a DUT node. Type: dictionary
-| | ...
-| | ... | *Example:*
-| | ...
-| | ... | \| Start VPP Service on DUT \| ${nodes['DUT1']} \|
-| | ...
-| | [Arguments] | ${node}
-| | Start VPP Service | ${node}

diff --git a/tests/vpp/device/__init__.robot b/tests/vpp/device/__init__.robot
index 8b07e4f..56ce814 100644 (file)
--- a/tests/vpp/device/__init__.robot
+++ b/tests/vpp/device/__init__.robot
@@ -22,7 +22,6 @@
 | ... | AND | Setup Framework | ${nodes}
 | ... | AND | Setup Corekeeper on All Nodes | ${nodes}
 | ... | AND | Install Vpp On All Duts | ${nodes} | ${packages_dir}
 | ... | AND | Setup Framework | ${nodes}
 | ... | AND | Setup Corekeeper on All Nodes | ${nodes}
 | ... | AND | Install Vpp On All Duts | ${nodes} | ${packages_dir}

-| ... | AND | Sleep | 10s

 | ... | AND | Verify Vpp On All Duts | ${nodes}
 | ... | AND | Get CPU Layout from all nodes | ${nodes}
 | ... | AND | Update All Interface Data On All Nodes | ${nodes}
 | ... | AND | Verify Vpp On All Duts | ${nodes}
 | ... | AND | Get CPU Layout from all nodes | ${nodes}
 | ... | AND | Update All Interface Data On All Nodes | ${nodes}

diff --git a/tests/vpp/func/__init__.robot b/tests/vpp/func/__init__.robot
index f63fb67..563d1ea 100644 (file)
--- a/tests/vpp/func/__init__.robot
+++ b/tests/vpp/func/__init__.robot
@@ -19,7 +19,6 @@
 | Suite Setup | Run Keywords | Setup Functional Global Variables
 | ... | AND | Setup Framework | ${nodes}
 | ... | AND | Install Vpp On All Duts | ${nodes} | ${VPP_PKG_DIR}
 | Suite Setup | Run Keywords | Setup Functional Global Variables
 | ... | AND | Setup Framework | ${nodes}
 | ... | AND | Install Vpp On All Duts | ${nodes} | ${VPP_PKG_DIR}

-| ... | AND | Sleep | 10s

 | ... | AND | Verify Vpp On All Duts | ${nodes}
 | ... | AND | Setup All DUTs | ${nodes}
 | ... | AND | Update All Interface Data On All Nodes | ${nodes}
 | ... | AND | Verify Vpp On All Duts | ${nodes}
 | ... | AND | Setup All DUTs | ${nodes}
 | ... | AND | Update All Interface Data On All Nodes | ${nodes}

diff --git a/tests/vpp/perf/__init__.robot b/tests/vpp/perf/__init__.robot
index 33c2901..37ff3a8 100644 (file)
--- a/tests/vpp/perf/__init__.robot
+++ b/tests/vpp/perf/__init__.robot
@@ -22,7 +22,6 @@
 | ... | AND | Setup Framework | ${nodes}
 | ... | AND | Setup Corekeeper on All Nodes | ${nodes}
 | ... | AND | Install Vpp on All Duts | ${nodes} | ${packages_dir}
 | ... | AND | Setup Framework | ${nodes}
 | ... | AND | Setup Corekeeper on All Nodes | ${nodes}
 | ... | AND | Install Vpp on All Duts | ${nodes} | ${packages_dir}

-| ... | AND | Sleep | 10s

 | ... | AND | Verify Vpp on All Duts | ${nodes}
 | ... | AND | Verify UIO Driver on all DUTs | ${nodes}
 | ... | AND | Setup All DUTs | ${nodes}
 | ... | AND | Verify Vpp on All Duts | ${nodes}
 | ... | AND | Verify UIO Driver on all DUTs | ${nodes}
 | ... | AND | Setup All DUTs | ${nodes}