CSIT-1291 Improve service handling with container detection

Change-Id: Iaeac10676f46e0267041afee927bc6e3201146b2
Signed-off-by: Peter Mikus <pmikus@cisco.com>

diff --git a/resources/libraries/bash/dut_setup.sh b/resources/libraries/bash/dut_setup.sh
index 28b495c..aee6386 100644 (file)
--- a/resources/libraries/bash/dut_setup.sh
+++ b/resources/libraries/bash/dut_setup.sh
@@ -33,7 +33,7 @@ cmd 'ps aux | grep vpp'
 
 cmd 'cat /etc/vpp/startup.conf'
 
 
 cmd 'cat /etc/vpp/startup.conf'
 

-cmd 'sudo -S service vpp restart'
+cmd 'if fgrep docker /proc/1/cgroup; then supervisorctl restart vpp; else sudo -S service vpp restart; fi'

 
 echo "[Command_desc] SLEEP for three seconds, so that VPP is up for sure"
 cmd 'sleep 3'
 
 echo "[Command_desc] SLEEP for three seconds, so that VPP is up for sure"
 cmd 'sleep 3'

diff --git a/resources/libraries/python/DUTSetup.py b/resources/libraries/python/DUTSetup.py
index 84862d4..7885ad0 100644 (file)
--- a/resources/libraries/python/DUTSetup.py
+++ b/resources/libraries/python/DUTSetup.py
@@ -15,11 +15,9 @@
 
 from robot.api import logger
 
 
 from robot.api import logger
 

-from resources.libraries.python.topology import NodeType, Topology
-from resources.libraries.python.ssh import SSH, exec_cmd_no_error

 from resources.libraries.python.constants import Constants
 from resources.libraries.python.constants import Constants

-from resources.libraries.python.VatExecutor import VatExecutor
-from resources.libraries.python.VPPUtil import VPPUtil
+from resources.libraries.python.ssh import SSH, exec_cmd_no_error
+from resources.libraries.python.topology import NodeType, Topology

 
 
 class DUTSetup(object):
 
 
 class DUTSetup(object):


@@ -27,28 +25,30 @@ class DUTSetup(object):
 
     @staticmethod
     def get_service_logs(node, service):
 
     @staticmethod
     def get_service_logs(node, service):

-        """Get specific service unit logs by journalctl from node.
+        """Get specific service unit logs from node.

 
         :param node: Node in the topology.
         :param service: Service unit name.
         :type node: dict
         :type service: str
         """
 
         :param node: Node in the topology.
         :param service: Service unit name.
         :type node: dict
         :type service: str
         """

-        ssh = SSH()
-        ssh.connect(node)
-        ret_code, _, _ = \
-            ssh.exec_command_sudo('journalctl --no-pager --unit={name} '
-                                  '--since="$(echo `systemctl show -p '
-                                  'ActiveEnterTimestamp {name}` | '
-                                  'awk \'{{print $2 $3}}\')"'.
-                                  format(name=service))
-        if int(ret_code):
-            raise RuntimeError('DUT {host} failed to get logs from unit {name}'.
-                               format(host=node['host'], name=service))
+        if DUTSetup.running_in_container(node):
+            command = 'echo $(< /var/log/supervisord.log)'
+        else:
+            command = ('journalctl --no-pager --unit={name} '
+                       '--since="$(echo `systemctl show -p '
+                       'ActiveEnterTimestamp {name}` | '
+                       'awk \'{{print $2 $3}}\')"'.
+                       format(name=service))
+        message = 'Node {host} failed to get logs from unit {name}'.\
+            format(host=node['host'], name=service)
+
+        exec_cmd_no_error(node, command, timeout=30, sudo=True,
+                          message=message)

 
     @staticmethod
     def get_service_logs_on_all_duts(nodes, service):
 
     @staticmethod
     def get_service_logs_on_all_duts(nodes, service):

-        """Get specific service unit logs by journalctl from all DUTs.
+        """Get specific service unit logs from all DUTs.

 
         :param nodes: Nodes in the topology.
         :param service: Service unit name.
 
         :param nodes: Nodes in the topology.
         :param service: Service unit name.


@@ -68,99 +68,62 @@ class DUTSetup(object):
         :type node: dict
         :type service: str
         """
         :type node: dict
         :type service: str
         """

-        ssh = SSH()
-        ssh.connect(node)
-        # We are doing restart. With this we do not care if service
-        # was running or not.
-        ret_code, _, _ = \
-            ssh.exec_command_sudo('service {name} restart'.
-                                  format(name=service), timeout=120)
-        if int(ret_code):
-            raise RuntimeError('DUT {host} failed to start service {name}'.
-                               format(host=node['host'], name=service))
-
-        DUTSetup.get_service_logs(node, service)
+        if DUTSetup.running_in_container(node):
+            command = 'supervisorctl restart {name}'.format(name=service)
+        else:
+            command = 'service {name} restart'.format(name=service)
+        message = 'Node {host} failed to start service {name}'.\
+            format(host=node['host'], name=service)

 
 

-    @staticmethod
-    def start_vpp_service_on_all_duts(nodes):
-        """Start up the VPP service on all nodes.
+        exec_cmd_no_error(node, command, timeout=30, sudo=True, message=message)

 
 

-        :param nodes: Nodes in the topology.
-        :type nodes: dict
-        """
-        for node in nodes.values():
-            if node['type'] == NodeType.DUT:
-                DUTSetup.start_service(node, Constants.VPP_UNIT)
+        DUTSetup.get_service_logs(node, service)

 
     @staticmethod
 
     @staticmethod

-    def vpp_show_version_verbose(node):
-        """Run "show version verbose" CLI command.
+    def start_service_on_all_duts(nodes, service):
+        """Start up the named service on all DUTs.

 
 

-        :param node: Node to run command on.
+        :param node: Nodes in the topology.
+        :param service: Service unit name.

         :type node: dict
         :type node: dict

-        """
-        vat = VatExecutor()
-        vat.execute_script("show_version_verbose.vat", node, json_out=False)
-
-        try:
-            vat.script_should_have_passed()
-        except AssertionError:
-            raise RuntimeError('Failed to get VPP version on host: {name}'.
-                               format(name=node['host']))
-
-    @staticmethod
-    def show_vpp_version_on_all_duts(nodes):
-        """Show VPP version verbose on all DUTs.
-
-        :param nodes: VPP nodes
-        :type nodes: dict
+        :type service: str

         """
         for node in nodes.values():
             if node['type'] == NodeType.DUT:
         """
         for node in nodes.values():
             if node['type'] == NodeType.DUT:

-                DUTSetup.vpp_show_version_verbose(node)
+                DUTSetup.start_service(node, service)

 
     @staticmethod
 
     @staticmethod

-    def vpp_show_interfaces(node):
-        """Run "show interface" CLI command.
+    def stop_service(node, service):
+        """Stop the named service on node.

 
 

-        :param node: Node to run command on.
+        :param node: Node in the topology.
+        :param service: Service unit name.

         :type node: dict
         :type node: dict

+        :type service: str

         """
         """

-        vat = VatExecutor()
-        vat.execute_script("show_interface.vat", node, json_out=False)
-
-        try:
-            vat.script_should_have_passed()
-        except AssertionError:
-            raise RuntimeError('Failed to get VPP interfaces on host: {name}'.
-                               format(name=node['host']))
+        if DUTSetup.running_in_container(node):
+            command = 'supervisorctl stop {name}'.format(name=service)
+        else:
+            command = 'service {name} stop'.format(name=service)
+        message = 'Node {host} failed to stop service {name}'.\
+            format(host=node['host'], name=service)

 
 

-    @staticmethod
-    def vpp_api_trace_save(node):
-        """Run "api trace save" CLI command.
+        exec_cmd_no_error(node, command, timeout=30, sudo=True, message=message)

 
 

-        :param node: Node to run command on.
-        :type node: dict
-        """
-        vat = VatExecutor()
-        vat.execute_script("api_trace_save.vat", node, json_out=False)
+        DUTSetup.get_service_logs(node, service)

 
     @staticmethod
 
     @staticmethod

-    def vpp_api_trace_dump(node):
-        """Run "api trace custom-dump" CLI command.
+    def stop_service_on_all_duts(nodes, service):
+        """Stop the named service on all DUTs.

 
 

-        :param node: Node to run command on.
+        :param node: Nodes in the topology.
+        :param service: Service unit name.

         :type node: dict
         :type node: dict

+        :type service: str

         """
         """

-        vat = VatExecutor()
-        vat.execute_script("api_trace_dump.vat", node, json_out=False)
-
-    @staticmethod
-    def setup_all_duts(nodes):
-        """Prepare all DUTs in given topology for test execution."""

         for node in nodes.values():
             if node['type'] == NodeType.DUT:
         for node in nodes.values():
             if node['type'] == NodeType.DUT:

-                DUTSetup.setup_dut(node)
+                DUTSetup.stop_service(node, service)

 
     @staticmethod
     def setup_dut(node):
 
     @staticmethod
     def setup_dut(node):


@@ -171,16 +134,24 @@ class DUTSetup(object):
 
         :raises Exception: If the DUT setup fails.
         """
 
         :raises Exception: If the DUT setup fails.
         """

-        ssh = SSH()
-        ssh.connect(node)
+        command = 'bash {0}/{1}/dut_setup.sh'.\
+            format(Constants.REMOTE_FW_DIR, Constants.RESOURCES_LIB_SH)
+        message = 'DUT test setup script failed at node {name}'.\
+            format(name=node['host'])

 
 

-        ret_code, _, _ = \
-            ssh.exec_command('sudo -Sn bash {0}/{1}/dut_setup.sh'.
-                             format(Constants.REMOTE_FW_DIR,
-                                    Constants.RESOURCES_LIB_SH), timeout=120)
-        if int(ret_code):
-            raise RuntimeError('DUT test setup script failed at node {name}'.
-                               format(name=node['host']))
+        exec_cmd_no_error(node, command, timeout=120, sudo=True,
+                          message=message)
+
+    @staticmethod
+    def setup_all_duts(nodes):
+        """Run script over SSH to setup all DUT nodes.
+
+        :param nodes: Topology nodes.
+        :type nodes: dict
+        """
+        for node in nodes.values():
+            if node['type'] == NodeType.DUT:
+                DUTSetup.setup_dut(node)

 
     @staticmethod
     def get_vpp_pid(node):
 
     @staticmethod
     def get_vpp_pid(node):


@@ -192,7 +163,6 @@ class DUTSetup(object):
         :rtype: int
         :raises RuntimeError: If it is not possible to get the PID.
         """
         :rtype: int
         :raises RuntimeError: If it is not possible to get the PID.
         """

-

         ssh = SSH()
         ssh.connect(node)
 
         ssh = SSH()
         ssh.connect(node)
 


@@ -230,24 +200,12 @@ class DUTSetup(object):
         :returns: PIDs
         :rtype: dict
         """
         :returns: PIDs
         :rtype: dict
         """

-

         pids = dict()
         for node in nodes.values():
             if node['type'] == NodeType.DUT:
                 pids[node['host']] = DUTSetup.get_vpp_pid(node)
         return pids
 
         pids = dict()
         for node in nodes.values():
             if node['type'] == NodeType.DUT:
                 pids[node['host']] = DUTSetup.get_vpp_pid(node)
         return pids
 

-    @staticmethod
-    def vpp_show_crypto_device_mapping(node):
-        """Run "show crypto device mapping" CLI command.
-
-        :param node: Node to run command on.
-        :type node: dict
-        """
-        vat = VatExecutor()
-        vat.execute_script("show_crypto_device_mapping.vat", node,
-                           json_out=False)
-

     @staticmethod
     def crypto_device_verify(node, force_init=False, numvfs=32):
         """Verify if Crypto QAT device virtual functions are initialized on all
     @staticmethod
     def crypto_device_verify(node, force_init=False, numvfs=32):
         """Verify if Crypto QAT device virtual functions are initialized on all


@@ -292,7 +250,7 @@ class DUTSetup(object):
         DUTSetup.verify_kernel_module(node, 'qat_dh895xcc', force_load=True)
 
         # Stop VPP to prevent deadlock.
         DUTSetup.verify_kernel_module(node, 'qat_dh895xcc', force_load=True)
 
         # Stop VPP to prevent deadlock.

-        VPPUtil.stop_vpp_service(node)
+        DUTSetup.stop_service(node, Constants.VPP_UNIT)

 
         current_driver = DUTSetup.get_pci_dev_driver(
             node, pci_addr.replace(':', r'\:'))
 
         current_driver = DUTSetup.get_pci_dev_driver(
             node, pci_addr.replace(':', r'\:'))


@@ -627,30 +585,6 @@ class DUTSetup(object):
 
         exec_cmd_no_error(node, command, timeout=30, sudo=True, message=message)
 
 
         exec_cmd_no_error(node, command, timeout=30, sudo=True, message=message)
 

-    @staticmethod
-    def vpp_enable_traces_on_all_duts(nodes):
-        """Enable vpp packet traces on all DUTs in the given topology.
-
-        :param nodes: Nodes in the topology.
-        :type nodes: dict
-        """
-        for node in nodes.values():
-            if node['type'] == NodeType.DUT:
-                DUTSetup.vpp_enable_traces_on_dut(node)
-
-    @staticmethod
-    def vpp_enable_traces_on_dut(node):
-        """Enable vpp packet traces on the DUT node.
-
-        :param node: DUT node to set up.
-        :type node: dict
-        """
-
-        vat = VatExecutor()
-        vat.execute_script("enable_dpdk_traces.vat", node, json_out=False)
-        vat.execute_script("enable_vhost_user_traces.vat", node, json_out=False)
-        vat.execute_script("enable_memif_traces.vat", node, json_out=False)
-

     @staticmethod
     def install_vpp_on_all_duts(nodes, vpp_pkg_dir, vpp_rpm_pkgs, vpp_deb_pkgs):
         """Install VPP on all DUT nodes.
     @staticmethod
     def install_vpp_on_all_duts(nodes, vpp_pkg_dir, vpp_rpm_pkgs, vpp_deb_pkgs):
         """Install VPP on all DUT nodes.


@@ -665,9 +599,6 @@ class DUTSetup(object):
         :type vpp_deb_pkgs: list
         :raises RuntimeError: If failed to remove or install VPP.
         """
         :type vpp_deb_pkgs: list
         :raises RuntimeError: If failed to remove or install VPP.
         """

-
-        logger.debug("Installing VPP")
-

         for node in nodes.values():
             if node['type'] == NodeType.DUT:
                 logger.debug("Installing VPP on node {0}".format(node['host']))
         for node in nodes.values():
             if node['type'] == NodeType.DUT:
                 logger.debug("Installing VPP on node {0}".format(node['host']))


@@ -724,36 +655,23 @@ class DUTSetup(object):
                 ssh.disconnect(node)
 
     @staticmethod
                 ssh.disconnect(node)
 
     @staticmethod

-    def verify_vpp_on_dut(node):
-        """Verify that VPP is installed on DUT node.
+    def running_in_container(node):
+        """This method tests if topology node is running inside container.

 
 

-        :param node: DUT node.
+        :param node: Topology node.

         :type node: dict
         :type node: dict

-        :raises RuntimeError: If failed to restart VPP, get VPP version
-            or get VPP interfaces.
+        :returns: True if running in docker container, false if not or failed
+        to detect.
+        :rtype: bool

         """
         """

-
-        logger.debug("Verify VPP on node {0}".format(node['host']))
-
-        DUTSetup.vpp_show_version_verbose(node)
-        DUTSetup.vpp_show_interfaces(node)
-
-    @staticmethod
-    def verify_vpp_on_all_duts(nodes):
-        """Verify that VPP is installed on all DUT nodes.
-
-        :param nodes: Nodes in the topology.
-        :type nodes: dict
-        """
-
-        logger.debug("Verify VPP on all DUTs")
-
-        DUTSetup.start_vpp_service_on_all_duts(nodes)
-
-        for node in nodes.values():
-            if node['type'] == NodeType.DUT:
-                DUTSetup.verify_vpp_on_dut(node)
-
+        command = "fgrep docker /proc/1/cgroup"
+        message = 'Failed to get cgroup settings.'
+        try:
+            exec_cmd_no_error(node, command, timeout=30, sudo=False,
+                              message=message)
+        except RuntimeError:
+            return False
+        return True

 
     @staticmethod
     def get_huge_page_size(node):
 
     @staticmethod
     def get_huge_page_size(node):

diff --git a/resources/libraries/python/VPPUtil.py b/resources/libraries/python/VPPUtil.py
index 3a0148a..de3caad 100644 (file)
--- a/resources/libraries/python/VPPUtil.py
+++ b/resources/libraries/python/VPPUtil.py
@@ -11,8 +11,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 # See the License for the specific language governing permissions and
 # limitations under the License.
 

-"""VPP util library"""
-from resources.libraries.python.ssh import SSH
+"""VPP util library."""
+from resources.libraries.python.constants import Constants
+from resources.libraries.python.DUTSetup import DUTSetup
+from resources.libraries.python.ssh import exec_cmd_no_error
+from resources.libraries.python.topology import NodeType
+from resources.libraries.python.VatExecutor import VatExecutor

 
 
 class VPPUtil(object):
 
 
 class VPPUtil(object):


@@ -42,10 +46,30 @@ class VPPUtil(object):
         if additional_cmds:
             for cmd in additional_cmds:
                 def_setting_tb_displayed['Custom Setting: {}'.format(cmd)] = cmd
         if additional_cmds:
             for cmd in additional_cmds:
                 def_setting_tb_displayed['Custom Setting: {}'.format(cmd)] = cmd

-        ssh = SSH()
-        ssh.connect(node)
-        for _, value in def_setting_tb_displayed.items():
-            ssh.exec_command_sudo('vppctl sh {}'.format(value))
+
+        for _, cmd in def_setting_tb_displayed.items():
+            command = 'vppctl sh {cmd}'.format(cmd=cmd)
+            exec_cmd_no_error(node, command, timeout=30, sudo=True)
+
+    @staticmethod
+    def start_vpp_service(node):
+        """Start VPP service on the specified node.
+
+        :param node: VPP node.
+        :type node: dict
+        :raises RuntimeError: If VPP service fails to start.
+        """
+        DUTSetup.start_service(node, Constants.VPP_UNIT)
+        DUTSetup.get_service_logs(node, Constants.VPP_UNIT)
+
+    @staticmethod
+    def start_vpp_service_on_all_duts(nodes):
+        """Start up the VPP service on all nodes.
+
+        :param nodes: Nodes in the topology.
+        :type nodes: dict
+        """
+        DUTSetup.start_service_on_all_duts(nodes, Constants.VPP_UNIT)

 
     @staticmethod
     def stop_vpp_service(node):
 
     @staticmethod
     def stop_vpp_service(node):


@@ -53,28 +77,128 @@ class VPPUtil(object):
 
         :param node: VPP node.
         :type node: dict
 
         :param node: VPP node.
         :type node: dict

-        :raises RuntimeError: If VPP fails to stop.
+        :raises RuntimeError: If VPP service fails to stop.
+        """
+        DUTSetup.stop_service(node, Constants.VPP_UNIT)
+
+    @staticmethod
+    def verify_vpp_on_dut(node):
+        """Verify that VPP is installed on DUT node.
+
+        :param node: DUT node.
+        :type node: dict
+        :raises RuntimeError: If failed to restart VPP, get VPP version
+            or get VPP interfaces.

         """
         """

+        VPPUtil.vpp_show_version_verbose(node)
+        VPPUtil.vpp_show_interfaces(node)

 
 

-        ssh = SSH()
-        ssh.connect(node)
-        cmd = "service vpp stop"
-        ret_code, _, _ = ssh.exec_command_sudo(cmd, timeout=80)
-        if int(ret_code) != 0:
-            raise RuntimeError("VPP service did not shut down gracefully.")
+    @staticmethod
+    def verify_vpp_on_all_duts(nodes):
+        """Verify that VPP is installed on all DUT nodes.
+
+        :param nodes: Nodes in the topology.
+        :type nodes: dict
+        """
+        for node in nodes.values():
+            if node['type'] == NodeType.DUT:
+                DUTSetup.start_service(node, Constants.VPP_UNIT)
+                VPPUtil.vpp_show_version_verbose(node)
+                VPPUtil.vpp_show_interfaces(node)

 
     @staticmethod
 
     @staticmethod

-    def start_vpp_service(node):
-        """start VPP service on the specified node.
+    def vpp_show_version_verbose(node):
+        """Run "show version verbose" CLI command.

 
 

-        :param node: VPP node.
+        :param node: Node to run command on.
+        :type node: dict
+        """
+        vat = VatExecutor()
+        vat.execute_script("show_version_verbose.vat", node, json_out=False)
+
+        try:
+            vat.script_should_have_passed()
+        except AssertionError:
+            raise RuntimeError('Failed to get VPP version on host: {name}'.
+                               format(name=node['host']))
+
+    @staticmethod
+    def show_vpp_version_on_all_duts(nodes):
+        """Show VPP version verbose on all DUTs.
+
+        :param nodes: VPP nodes.
+        :type nodes: dict
+        """
+        for node in nodes.values():
+            if node['type'] == NodeType.DUT:
+                VPPUtil.vpp_show_version_verbose(node)
+
+    @staticmethod
+    def vpp_show_interfaces(node):
+        """Run "show interface" CLI command.
+
+        :param node: Node to run command on.
+        :type node: dict
+        """
+        vat = VatExecutor()
+        vat.execute_script("show_interface.vat", node, json_out=False)
+
+        try:
+            vat.script_should_have_passed()
+        except AssertionError:
+            raise RuntimeError('Failed to get VPP interfaces on host: {name}'.
+                               format(name=node['host']))
+
+    @staticmethod
+    def vpp_show_crypto_device_mapping(node):
+        """Run "show crypto device mapping" CLI command.
+
+        :param node: Node to run command on.
+        :type node: dict
+        """
+        vat = VatExecutor()
+        vat.execute_script("show_crypto_device_mapping.vat", node,
+                           json_out=False)
+
+    @staticmethod
+    def vpp_api_trace_dump(node):
+        """Run "api trace custom-dump" CLI command.
+
+        :param node: Node to run command on.
+        :type node: dict
+        """
+        vat = VatExecutor()
+        vat.execute_script("api_trace_dump.vat", node, json_out=False)
+
+    @staticmethod
+    def vpp_api_trace_save(node):
+        """Run "api trace save" CLI command.
+
+        :param node: Node to run command on.
+        :type node: dict
+        """
+        vat = VatExecutor()
+        vat.execute_script("api_trace_save.vat", node, json_out=False)
+
+    @staticmethod
+    def vpp_enable_traces_on_dut(node):
+        """Enable vpp packet traces on the DUT node.
+
+        :param node: DUT node to set up.

         :type node: dict
         :type node: dict

-        :raises RuntimeError: If VPP fails to start.

         """
         """

+        vat = VatExecutor()
+        vat.execute_script("enable_dpdk_traces.vat", node, json_out=False)
+        vat.execute_script("enable_vhost_user_traces.vat", node, json_out=False)
+        vat.execute_script("enable_memif_traces.vat", node, json_out=False)

 
 

-        ssh = SSH()
-        ssh.connect(node)
-        cmd = "service vpp start"
-        ret_code, _, _ = ssh.exec_command_sudo(cmd)
-        if int(ret_code) != 0:
-            raise RuntimeError("VPP service did not start.")
+    @staticmethod
+    def vpp_enable_traces_on_all_duts(nodes):
+        """Enable vpp packet traces on all DUTs in the given topology.
+
+        :param nodes: Nodes in the topology.
+        :type nodes: dict
+        """
+        for node in nodes.values():
+            if node['type'] == NodeType.DUT:
+                VPPUtil.vpp_enable_traces_on_dut(node)