perf: add TCP Nginx+LDPRELOAD suites

[csit.git] / resources / libraries / python / HoststackUtil.py

diff --git a/resources/libraries/python/HoststackUtil.py b/resources/libraries/python/HoststackUtil.py
index 9e6e200..e797c3c 100644 (file)
--- a/resources/libraries/python/HoststackUtil.py
+++ b/resources/libraries/python/HoststackUtil.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2020 Cisco and/or its affiliates.
+# Copyright (c) 2021 Cisco and/or its affiliates.

 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:


@@ -12,9 +12,11 @@
 # limitations under the License.
 
 """Host Stack util library."""
 # limitations under the License.
 
 """Host Stack util library."""

+import json

 from time import sleep
 from robot.api import logger
 
 from time import sleep
 from robot.api import logger
 

+from resources.libraries.python.Constants import Constants

 from resources.libraries.python.ssh import exec_cmd, exec_cmd_no_error
 from resources.libraries.python.PapiExecutor import PapiSocketExecutor
 from resources.libraries.python.DUTSetup import DUTSetup
 from resources.libraries.python.ssh import exec_cmd, exec_cmd_no_error
 from resources.libraries.python.PapiExecutor import PapiSocketExecutor
 from resources.libraries.python.DUTSetup import DUTSetup


@@ -43,6 +45,8 @@ class HoststackUtil():
             f"socket-name {vpp_echo_attributes[u'vpp_api_socket']} " \
             f"{vpp_echo_attributes[u'json_output']} " \
             f"uri {proto}://{addr}/{port} " \
             f"socket-name {vpp_echo_attributes[u'vpp_api_socket']} " \
             f"{vpp_echo_attributes[u'json_output']} " \
             f"uri {proto}://{addr}/{port} " \

+            f"nthreads {vpp_echo_attributes[u'nthreads']} " \
+            f"mq-size {vpp_echo_attributes[u'mq_size']} " \

             f"nclients {vpp_echo_attributes[u'nclients']} " \
             f"quic-streams {vpp_echo_attributes[u'quic_streams']} " \
             f"time {vpp_echo_attributes[u'time']} " \
             f"nclients {vpp_echo_attributes[u'nclients']} " \
             f"quic-streams {vpp_echo_attributes[u'quic_streams']} " \
             f"time {vpp_echo_attributes[u'time']} " \


@@ -55,6 +59,49 @@ class HoststackUtil():
             vpp_echo_cmd[u"args"] += u" tx-results-diff"
         return vpp_echo_cmd
 
             vpp_echo_cmd[u"args"] += u" tx-results-diff"
         return vpp_echo_cmd
 

+    @staticmethod
+    def get_iperf3_command(iperf3_attributes):
+        """Construct the iperf3 command using the specified attributes.
+
+        :param iperf3_attributes: iperf3 test program attributes.
+        :type iperf3_attributes: dict
+        :returns: Command line components of the iperf3 command
+            'env_vars' - environment variables
+            'name' - program name
+            'args' - command arguments.
+        :rtype: dict
+        """
+        # TODO: Use a python class instead of dictionary for the return type
+        iperf3_cmd = {}
+        iperf3_cmd[u"env_vars"] = f"VCL_CONFIG={Constants.REMOTE_FW_DIR}/" \
+            f"{Constants.RESOURCES_TPL_VCL}/" \
+            f"{iperf3_attributes[u'vcl_config']}"
+        if iperf3_attributes[u"ld_preload"]:
+            iperf3_cmd[u"env_vars"] += \
+                f" LD_PRELOAD={Constants.VCL_LDPRELOAD_LIBRARY}"
+        if iperf3_attributes[u'transparent_tls']:
+            iperf3_cmd[u"env_vars"] += u" LDP_ENV_TLS_TRANS=1"
+
+        json_results = u" --json" if iperf3_attributes[u'json'] else u""
+        ip_address = f" {iperf3_attributes[u'ip_address']}" if u"ip_address" \
+                     in iperf3_attributes else u""
+        iperf3_cmd[u"name"] = u"iperf3"
+        iperf3_cmd[u"args"] = f"--{iperf3_attributes[u'role']}{ip_address} " \
+                              f"--interval 0{json_results} " \
+                              f"--version{iperf3_attributes[u'ip_version']}"
+
+        if iperf3_attributes[u"role"] == u"server":
+            iperf3_cmd[u"args"] += u" --one-off"
+        else:
+            iperf3_cmd[u"args"] += u" --get-server-output"
+            if u"parallel" in iperf3_attributes:
+                iperf3_cmd[u"args"] += \
+                    f" --parallel {iperf3_attributes[u'parallel']}"
+            if u"time" in iperf3_attributes:
+                iperf3_cmd[u"args"] += \
+                    f" --time {iperf3_attributes[u'time']}"
+        return iperf3_cmd
+

     @staticmethod
     def set_hoststack_quic_fifo_size(node, fifo_size):
         """Set the QUIC protocol fifo size.
     @staticmethod
     def set_hoststack_quic_fifo_size(node, fifo_size):
         """Set the QUIC protocol fifo size.


@@ -77,7 +124,7 @@ class HoststackUtil():
         :type node: dict
         :type quic_crypto_engine: str
         """
         :type node: dict
         :type quic_crypto_engine: str
         """

-        vpp_crypto_engines = {u"openssl", u"ia32", u"ipsecmb"}
+        vpp_crypto_engines = {u"openssl", u"native", u"ipsecmb"}

         if quic_crypto_engine == u"nocrypto":
             logger.trace(u"No QUIC crypto engine.")
             return
         if quic_crypto_engine == u"nocrypto":
             logger.trace(u"No QUIC crypto engine.")
             return


@@ -118,20 +165,55 @@ class HoststackUtil():
         return stdout_log, stderr_log
 
     @staticmethod
         return stdout_log, stderr_log
 
     @staticmethod

-    def start_hoststack_test_program(node, namespace, program):
+    def get_nginx_command(nginx_attributes, nginx_version, nginx_ins_dir):
+        """Construct the NGINX command using the specified attributes.
+
+        :param nginx_attributes: NGINX test program attributes.
+        :param nginx_version: NGINX version.
+        :param nginx_ins_dir: NGINX install dir.
+        :type nginx_attributes: dict
+        :type nginx_version: str
+        :type nginx_ins_dir: str
+        :returns: Command line components of the NGINX command
+            'env_vars' - environment variables
+            'name' - program name
+            'args' - command arguments.
+            'path' - program path.
+        :rtype: dict
+        """
+        nginx_cmd = dict()
+        nginx_cmd[u"env_vars"] = f"VCL_CONFIG={Constants.REMOTE_FW_DIR}/" \
+                                 f"{Constants.RESOURCES_TPL_VCL}/" \
+                                 f"{nginx_attributes[u'vcl_config']}"
+        if nginx_attributes[u"ld_preload"]:
+            nginx_cmd[u"env_vars"] += \
+                f" LD_PRELOAD={Constants.VCL_LDPRELOAD_LIBRARY}"
+        if nginx_attributes[u'transparent_tls']:
+            nginx_cmd[u"env_vars"] += u" LDP_ENV_TLS_TRANS=1"
+
+        nginx_cmd[u"name"] = u"nginx"
+        nginx_cmd[u"path"] = f"{nginx_ins_dir}nginx-{nginx_version}/sbin/"
+        nginx_cmd[u"args"] = f"-c {nginx_ins_dir}/" \
+                             f"nginx-{nginx_version}/conf/nginx.conf"
+        return nginx_cmd
+
+    @staticmethod
+    def start_hoststack_test_program(node, namespace, core_list, program):

         """Start the specified HostStack test program.
 
         :param node: DUT node.
         :param namespace: Net Namespace to run program in.
         """Start the specified HostStack test program.
 
         :param node: DUT node.
         :param namespace: Net Namespace to run program in.

+        :param core_list: List of cpu's to pass to taskset to pin the test
+            program to a different set of cores on the same numa node as VPP.

         :param program: Test program.
         :type node: dict
         :type namespace: str
         :param program: Test program.
         :type node: dict
         :type namespace: str

+        :type core_list: str

         :type program: dict
         :returns: Process ID
         :rtype: int
         :raises RuntimeError: If node subtype is not a DUT or startup failed.
         """
         :type program: dict
         :returns: Process ID
         :rtype: int
         :raises RuntimeError: If node subtype is not a DUT or startup failed.
         """

-        # TODO: Pin test program to core(s) on same numa node as VPP.

         if node[u"type"] != u"DUT":
             raise RuntimeError(u"Node type is not a DUT!")
 
         if node[u"type"] != u"DUT":
             raise RuntimeError(u"Node type is not a DUT!")
 


@@ -145,9 +227,13 @@ class HoststackUtil():
 
         env_vars = f"{program[u'env_vars']} " if u"env_vars" in program else u""
         args = program[u"args"]
 
         env_vars = f"{program[u'env_vars']} " if u"env_vars" in program else u""
         args = program[u"args"]

-        cmd = f"nohup {shell_cmd} \'{env_vars}{program_name} {args} " \
-            f">/tmp/{program_name}_stdout.log " \
-            f"2>/tmp/{program_name}_stderr.log &\'"
+        program_path = program.get(u"path", u"")
+        # NGINX used `worker_cpu_affinity` in configuration file
+        taskset_cmd = u"" if program_name == u"nginx" else \
+                                             f"taskset --cpu-list {core_list}"
+        cmd = f"nohup {shell_cmd} \'{env_vars}{taskset_cmd} " \
+              f"{program_path}{program_name} {args} >/tmp/{program_name}_" \
+              f"stdout.log 2>/tmp/{program_name}_stderr.log &\'"

         try:
             exec_cmd_no_error(node, cmd, sudo=True)
             return DUTSetup.get_pid(node, program_name)[0]
         try:
             exec_cmd_no_error(node, cmd, sudo=True)
             return DUTSetup.get_pid(node, program_name)[0]


@@ -201,10 +287,16 @@ class HoststackUtil():
         sleep(1)
 
     @staticmethod
         sleep(1)
 
     @staticmethod

-    def analyze_hoststack_test_program_output(node, role, nsim_attr,
-                                              program):
+    def analyze_hoststack_test_program_output(
+            node, role, nsim_attr, program):

         """Gather HostStack test program output and check for errors.
 
         """Gather HostStack test program output and check for errors.
 

+        The [defer_fail] return bool is used instead of failing immediately
+        to allow the analysis of both the client and server instances of
+        the test program for debugging a test failure.  When [defer_fail]
+        is true, then the string returned is debug output instead of
+        JSON formatted test program results.
+

         :param node: DUT node.
         :param role: Role (client|server) of test program.
         :param nsim_attr: Network Simulation Attributes.
         :param node: DUT node.
         :param role: Role (client|server) of test program.
         :param nsim_attr: Network Simulation Attributes.


@@ -214,7 +306,8 @@ class HoststackUtil():
         :type role: str
         :type nsim_attr: dict
         :type program: dict
         :type role: str
         :type nsim_attr: dict
         :type program: dict

-        :returns: tuple of no results bool and test program results.
+        :returns: tuple of [defer_fail] bool and either JSON formatted hoststack
+            test program output or failure debug output.

         :rtype: bool, str
         :raises RuntimeError: If node subtype is not a DUT.
         """
         :rtype: bool, str
         :raises RuntimeError: If node subtype is not a DUT.
         """


@@ -229,14 +322,13 @@ class HoststackUtil():
             program_stdout, program_stderr = \
                HoststackUtil.get_hoststack_test_program_logs(node, program)
 
             program_stdout, program_stderr = \
                HoststackUtil.get_hoststack_test_program_logs(node, program)
 

-        no_results = False

         env_vars = f"{program[u'env_vars']} " if u"env_vars" in program else u""
         program_cmd = f"{env_vars}{program_name} {program[u'args']}"
         test_results = f"Test Results of '{program_cmd}':\n"
 
         env_vars = f"{program[u'env_vars']} " if u"env_vars" in program else u""
         program_cmd = f"{env_vars}{program_name} {program[u'args']}"
         test_results = f"Test Results of '{program_cmd}':\n"
 

-        if nsim_attr[u"output_feature_enable"] or \
-            nsim_attr[u"cross_connect_feature_enable"]:
-            if nsim_attr[u"output_feature_enable"]:
+        if nsim_attr[u"output_nsim_enable"] or \
+            nsim_attr[u"xc_nsim_enable"]:
+            if nsim_attr[u"output_nsim_enable"]:

                 feature_name = u"output"
             else:
                 feature_name = u"cross-connect"
                 feature_name = u"output"
             else:
                 feature_name = u"cross-connect"


@@ -244,44 +336,69 @@ class HoststackUtil():
                 f"NSIM({feature_name}): delay " \
                 f"{nsim_attr[u'delay_in_usec']} usecs, " \
                 f"avg-pkt-size {nsim_attr[u'average_packet_size']}, " \
                 f"NSIM({feature_name}): delay " \
                 f"{nsim_attr[u'delay_in_usec']} usecs, " \
                 f"avg-pkt-size {nsim_attr[u'average_packet_size']}, " \

-                f"bandwidth {nsim_attr[u'bandwidth_in_bits_per_second']} " \
+                f"bandwidth {nsim_attr[u'bw_in_bits_per_second']} " \

                 f"bits/sec, pkt-drop-rate {nsim_attr[u'packets_per_drop']} " \
                 f"pkts/drop\n"
 
                 f"bits/sec, pkt-drop-rate {nsim_attr[u'packets_per_drop']} " \
                 f"pkts/drop\n"
 

+        # TODO: Incorporate show error stats into results analysis
+        test_results += \
+            f"\n{role} VPP 'show errors' on host {node[u'host']}:\n" \
+            f"{PapiSocketExecutor.run_cli_cmd(node, u'show error')}\n"
+

         if u"error" in program_stderr.lower():
             test_results += f"ERROR DETECTED:\n{program_stderr}"
         if u"error" in program_stderr.lower():
             test_results += f"ERROR DETECTED:\n{program_stderr}"

-            raise RuntimeError(test_results)
-        if program_stdout:
-            bad_test_results = False
-            if program == u"vpp_echo" and u"JSON stats" not in program_stdout:
-                test_results += u"Invalid test data output!\n"
-                bad_test_results = True
-            test_results += program_stdout
-            if bad_test_results:
-                raise RuntimeError(test_results)
-        else:
-            no_results = True
+            return (True, test_results)
+        if not program_stdout:

             test_results += f"\nNo {program} test data retrieved!\n"
             test_results += f"\nNo {program} test data retrieved!\n"

-            cmd = u"ls -l /tmp/*.log"
-            ls_stdout, _ = exec_cmd_no_error(node, cmd, sudo=True)
+            ls_stdout, _ = exec_cmd_no_error(node, u"ls -l /tmp/*.log",
+                                             sudo=True)

             test_results += f"{ls_stdout}\n"
             test_results += f"{ls_stdout}\n"

+            return (True, test_results)
+        if program[u"name"] == u"vpp_echo":
+            if u"JSON stats" in program_stdout and \
+                    u'"has_failed": "0"' in program_stdout:
+                json_start = program_stdout.find(u"{")
+                #TODO: Fix parsing once vpp_echo produces valid
+                # JSON output. Truncate for now.
+                json_end = program_stdout.find(u',\n  "closing"')
+                json_results = f"{program_stdout[json_start:json_end]}\n}}"
+                program_json = json.loads(json_results)
+            else:
+                test_results += u"Invalid test data output!\n" + program_stdout
+                return (True, test_results)
+        elif program[u"name"] == u"iperf3":
+            test_results += program_stdout
+            iperf3_json = json.loads(program_stdout)
+            program_json = iperf3_json[u"intervals"][0][u"sum"]
+        else:
+            test_results += u"Unknown HostStack Test Program!\n" + \
+                            program_stdout
+            return (True, program_stdout)
+        return (False, json.dumps(program_json))

 
 

-        # TODO: Incorporate show error stats into results analysis
-        host = node[u"host"]
-        test_results += \
-            f"\n{role} VPP 'show errors' on host {host}:\n" \
-            f"{PapiSocketExecutor.run_cli_cmd(node, u'show error')}\n"
+    @staticmethod
+    def hoststack_test_program_defer_fail(server_defer_fail, client_defer_fail):
+        """Return True if either HostStack test program fail was deferred.

 
 

-        return no_results, test_results
+        :param server_defer_fail: server no results value.
+        :param client_defer_fail: client no results value.
+        :type server_defer_fail: bool
+        :type client_defer_fail: bool
+        :rtype: bool
+        """
+        return server_defer_fail and client_defer_fail

 
     @staticmethod
 
     @staticmethod

-    def no_hoststack_test_program_results(server_no_results, client_no_results):
-        """Return True if no HostStack test program output was gathered.
+    def log_vpp_hoststack_data(node):
+        """Retrieve and log VPP HostStack data.

 
 

-        :param server_no_results: server no results value.
-        :param client_no_results: client no results value.
-        :type server_no_results: bool
-        :type client_no_results: bool
-        :rtype: bool
+        :param node: DUT node.
+        :type node: dict
+        :raises RuntimeError: If node subtype is not a DUT or startup failed.

         """
         """

-        return server_no_results and client_no_results
+
+        if node[u"type"] != u"DUT":
+            raise RuntimeError(u"Node type is not a DUT!")
+
+        PapiSocketExecutor.run_cli_cmd(node, u"show error")
+        PapiSocketExecutor.run_cli_cmd(node, u"show interface")