perf: Clean up Hoststack tests

- Update test names with clients/streams
- Convert test results to JSON output
  * iperf3 results include bits_per_second
  * vpp_echo results include both client
    and server output which includes time in seconds
    and rx_data/tx_data in bytes which can be used
    to calculate the average bits per second.
    Tx and Rx data will always be the same:
    BPS = (client tx_data * 8) / ((client time + server time) / 2)
- Fix WRK test results data formatting errors

Change-Id: Ie2aeb665e3cc0739b16f97ba2628eebe6e041d22
Signed-off-by: Dave Wallace <dwallacelf@gmail.com>

diff --git a/resources/libraries/python/HoststackUtil.py b/resources/libraries/python/HoststackUtil.py
index dde5cf6..d184b5a 100644 (file)
--- a/resources/libraries/python/HoststackUtil.py
+++ b/resources/libraries/python/HoststackUtil.py
@@ -12,6 +12,7 @@
 # limitations under the License.
 
 """Host Stack util library."""
 # limitations under the License.
 
 """Host Stack util library."""

+import json

 from time import sleep
 from robot.api import logger
 
 from time import sleep
 from robot.api import logger
 


@@ -249,10 +250,16 @@ class HoststackUtil():
         sleep(1)
 
     @staticmethod
         sleep(1)
 
     @staticmethod

-    def analyze_hoststack_test_program_output(node, role, nsim_attr,
-                                              program):
+    def analyze_hoststack_test_program_output(
+            node, role, nsim_attr, program):

         """Gather HostStack test program output and check for errors.
 
         """Gather HostStack test program output and check for errors.
 

+        The [defer_fail] return bool is used instead of failing immediately
+        to allow the analysis of both the client and server instances of
+        the test program for debugging a test failure.  When [defer_fail]
+        is true, then the string returned is debug output instead of
+        JSON formatted test program results.
+

         :param node: DUT node.
         :param role: Role (client|server) of test program.
         :param nsim_attr: Network Simulation Attributes.
         :param node: DUT node.
         :param role: Role (client|server) of test program.
         :param nsim_attr: Network Simulation Attributes.


@@ -262,7 +269,8 @@ class HoststackUtil():
         :type role: str
         :type nsim_attr: dict
         :type program: dict
         :type role: str
         :type nsim_attr: dict
         :type program: dict

-        :returns: tuple of no results bool and test program results.
+        :returns: tuple of [defer_fail] bool and either JSON formatted hoststack
+            test program output or failure debug output.

         :rtype: bool, str
         :raises RuntimeError: If node subtype is not a DUT.
         """
         :rtype: bool, str
         :raises RuntimeError: If node subtype is not a DUT.
         """


@@ -277,7 +285,6 @@ class HoststackUtil():
             program_stdout, program_stderr = \
                HoststackUtil.get_hoststack_test_program_logs(node, program)
 
             program_stdout, program_stderr = \
                HoststackUtil.get_hoststack_test_program_logs(node, program)
 

-        no_results = False

         env_vars = f"{program[u'env_vars']} " if u"env_vars" in program else u""
         program_cmd = f"{env_vars}{program_name} {program[u'args']}"
         test_results = f"Test Results of '{program_cmd}':\n"
         env_vars = f"{program[u'env_vars']} " if u"env_vars" in program else u""
         program_cmd = f"{env_vars}{program_name} {program[u'args']}"
         test_results = f"Test Results of '{program_cmd}':\n"


@@ -296,50 +303,50 @@ class HoststackUtil():
                 f"bits/sec, pkt-drop-rate {nsim_attr[u'packets_per_drop']} " \
                 f"pkts/drop\n"
 
                 f"bits/sec, pkt-drop-rate {nsim_attr[u'packets_per_drop']} " \
                 f"pkts/drop\n"
 

-        if u"error" in program_stderr.lower():
-            test_results += f"ERROR DETECTED:\n{program_stderr}"
-            raise RuntimeError(test_results)
-        if program_stdout:
-            bad_test_results = False
-            if program[u"name"] == u"vpp_echo":
-                if u"JSON stats" in program_stdout:
-                    test_results += program_stdout
-                    # TODO: Decode vpp_echo output when JSON format is correct.
-                    # json_start = program_stdout.find(u"{")
-                    # vpp_echo_results = json.loads(program_stdout[json_start:])
-                    if u'"has_failed": "0"' not in program_stdout:
-                        bad_test_results = True
-                else:
-                    test_results += u"Invalid test data output!\n" + \
-                                    program_stdout
-                    bad_test_results = True
-            else:
-                test_results += program_stdout
-            if bad_test_results:
-                raise RuntimeError(test_results)
-        else:
-            no_results = True
-            test_results += f"\nNo {program} test data retrieved!\n"
-            cmd = u"ls -l /tmp/*.log"
-            ls_stdout, _ = exec_cmd_no_error(node, cmd, sudo=True)
-            test_results += f"{ls_stdout}\n"
-

         # TODO: Incorporate show error stats into results analysis
         # TODO: Incorporate show error stats into results analysis

-        host = node[u"host"]

         test_results += \
         test_results += \

-            f"\n{role} VPP 'show errors' on host {host}:\n" \
+            f"\n{role} VPP 'show errors' on host {node[u'host']}:\n" \

             f"{PapiSocketExecutor.run_cli_cmd(node, u'show error')}\n"
 
             f"{PapiSocketExecutor.run_cli_cmd(node, u'show error')}\n"
 

-        return no_results, test_results
+        if u"error" in program_stderr.lower():
+            test_results += f"ERROR DETECTED:\n{program_stderr}"
+            return (True, test_results)
+        if not program_stdout:
+            test_results += f"\nNo {program} test data retrieved!\n"
+            ls_stdout, _ = exec_cmd_no_error(node, u"ls -l /tmp/*.log",
+                                             sudo=True)
+            test_results += f"{ls_stdout}\n"
+            return (True, test_results)
+        if program[u"name"] == u"vpp_echo":
+            if u"JSON stats" in program_stdout and \
+                    u'"has_failed": "0"' in program_stdout:
+                json_start = program_stdout.find(u"{")
+                #TODO: Fix parsing once vpp_echo produces valid
+                # JSON output. Truncate for now.
+                json_end = program_stdout.find(u',\n  "closing"')
+                json_results = f"{program_stdout[json_start:json_end]}\n}}"
+                program_json = json.loads(json_results)
+            else:
+                test_results += u"Invalid test data output!\n" + program_stdout
+                return (True, test_results)
+        elif program[u"name"] == u"iperf3":
+            test_results += program_stdout
+            iperf3_json = json.loads(program_stdout)
+            program_json = iperf3_json[u"intervals"][0][u"sum"]
+        else:
+            test_results += u"Unknown HostStack Test Program!\n" + \
+                            program_stdout
+            return (True, program_stdout)
+        return (False, json.dumps(program_json))

 
     @staticmethod
 
     @staticmethod

-    def no_hoststack_test_program_results(server_no_results, client_no_results):
-        """Return True if no HostStack test program output was gathered.
+    def hoststack_test_program_defer_fail(server_defer_fail, client_defer_fail):
+        """Return True if either HostStack test program fail was deferred.

 
 

-        :param server_no_results: server no results value.
-        :param client_no_results: client no results value.
-        :type server_no_results: bool
-        :type client_no_results: bool
+        :param server_defer_fail: server no results value.
+        :param client_defer_fail: client no results value.
+        :type server_defer_fail: bool
+        :type client_defer_fail: bool

         :rtype: bool
         """
         :rtype: bool
         """

-        return server_no_results and client_no_results
+        return server_defer_fail and client_defer_fail

diff --git a/resources/libraries/python/autogen/Regenerator.py b/resources/libraries/python/autogen/Regenerator.py
index 5cd8bcc..d40ae7c 100644 (file)
--- a/resources/libraries/python/autogen/Regenerator.py
+++ b/resources/libraries/python/autogen/Regenerator.py
@@ -440,50 +440,24 @@ class Regenerator:
 
         min_frame_size = PROTOCOL_TO_MIN_FRAME_SIZE[protocol]
         default_kwargs_list = [
 
         min_frame_size = PROTOCOL_TO_MIN_FRAME_SIZE[protocol]
         default_kwargs_list = [

-            {u"frame_size": min_frame_size, u"phy_cores": 1, u"clients": 1,
-             u"streams": 1, u"bytes_str": u"1G"},
-            {u"frame_size": min_frame_size, u"phy_cores": 2, u"clients": 1,
-             u"streams": 1, u"bytes_str": u"1G"},
-            {u"frame_size": min_frame_size, u"phy_cores": 4, u"clients": 1,
-             u"streams": 1, u"bytes_str": u"1G"},
-            {u"frame_size": 1518, u"phy_cores": 1, u"clients": 1,
-             u"streams": 1, u"bytes_str": u"1G"},
-            {u"frame_size": 1518, u"phy_cores": 2, u"clients": 1,
-             u"streams": 1, u"bytes_str": u"1G"},
-            {u"frame_size": 1518, u"phy_cores": 4, u"clients": 1,
-             u"streams": 1, u"bytes_str": u"1G"},
-            {u"frame_size": 9000, u"phy_cores": 1, u"clients": 1,
-             u"streams": 1, u"bytes_str": u"1G"},
-            {u"frame_size": 9000, u"phy_cores": 2, u"clients": 1,
-             u"streams": 1, u"bytes_str": u"1G"},
-            {u"frame_size": 9000, u"phy_cores": 4, u"clients": 1,
-             u"streams": 1, u"bytes_str": u"1G"},
-            {u"frame_size": u"IMIX_v4_1", u"phy_cores": 1, u"clients": 1,
-             u"streams": 1, u"bytes_str": u"1G"},
-            {u"frame_size": u"IMIX_v4_1", u"phy_cores": 2, u"clients": 1,
-             u"streams": 1, u"bytes_str": u"1G"},
-            {u"frame_size": u"IMIX_v4_1", u"phy_cores": 4, u"clients": 1,
-             u"streams": 1, u"bytes_str": u"1G"}
+            {u"frame_size": min_frame_size, u"phy_cores": 1},
+            {u"frame_size": min_frame_size, u"phy_cores": 2},
+            {u"frame_size": min_frame_size, u"phy_cores": 4},
+            {u"frame_size": 1518, u"phy_cores": 1},
+            {u"frame_size": 1518, u"phy_cores": 2},
+            {u"frame_size": 1518, u"phy_cores": 4},
+            {u"frame_size": 9000, u"phy_cores": 1},
+            {u"frame_size": 9000, u"phy_cores": 2},
+            {u"frame_size": 9000, u"phy_cores": 4},
+            {u"frame_size": u"IMIX_v4_1", u"phy_cores": 1},
+            {u"frame_size": u"IMIX_v4_1", u"phy_cores": 2},
+            {u"frame_size": u"IMIX_v4_1", u"phy_cores": 4}

         ]
         ]

-        hoststack_wrk_kwargs_list = [
-            {u"frame_size": 0, u"phy_cores": i, u"clients": 1,
-             u"streams": 1, u"bytes_str": u"1G"} for i in (1, 2, 4)
+        hs_wrk_kwargs_list = [
+            {u"frame_size": 0, u"phy_cores": i} for i in (1, 2, 4)

         ]
         ]

-        hoststack_iperf3_kwargs_list = [
-            {u"frame_size": 0, u"phy_cores": 1, u"clients": 1,
-             u"streams": 1, u"bytes_str": u"1G"},
-            {u"frame_size": 0, u"phy_cores": 1, u"clients": 1,
-             u"streams": 10, u"bytes_str": u"1G"},
-        ]
-        hoststack_quic_kwargs_list = [
-            {u"phy_cores": 1, u"frame_size": 0, u"clients": 1,
-             u"streams": 1, u"bytes_str": u"10G"},
-            {u"phy_cores": 1, u"frame_size": 0, u"clients": 1,
-             u"streams": 10, u"bytes_str": u"1G"},
-            {u"phy_cores": 1, u"frame_size": 0, u"clients": 10,
-             u"streams": 1, u"bytes_str": u"1G"},
-            {u"phy_cores": 1, u"frame_size": 0, u"clients": 10,
-             u"streams": 10, u"bytes_str": u"100M"},
+        hs_bps_kwargs_list = [
+            {u"frame_size": 0, u"phy_cores": 1},

         ]
 
         for in_filename in glob(pattern):
         ]
 
         for in_filename in glob(pattern):


@@ -510,14 +484,9 @@ class Regenerator:
             elif in_filename.endswith(u"-reconf.robot"):
                 write_reconf_files(in_filename, in_prolog, default_kwargs_list)
             elif in_filename[-10:] in (u"-cps.robot", u"-rps.robot"):
             elif in_filename.endswith(u"-reconf.robot"):
                 write_reconf_files(in_filename, in_prolog, default_kwargs_list)
             elif in_filename[-10:] in (u"-cps.robot", u"-rps.robot"):

-                write_tcp_files(in_filename, in_prolog,
-                                hoststack_wrk_kwargs_list)
+                write_tcp_files(in_filename, in_prolog, hs_wrk_kwargs_list)

             elif in_filename[-10:] in (u"-bps.robot"):
             elif in_filename[-10:] in (u"-bps.robot"):

-                if u"ldpreload-iperf3" in in_filename:
-                    hoststack_kwargs_list = hoststack_iperf3_kwargs_list
-                else:
-                    hoststack_kwargs_list = hoststack_quic_kwargs_list
-                write_tcp_files(in_filename, in_prolog, hoststack_kwargs_list)
+                write_tcp_files(in_filename, in_prolog, hs_bps_kwargs_list)

             else:
                 raise RuntimeError(
                     f"Error in {in_filename}: non-primary suite type found."
             else:
                 raise RuntimeError(
                     f"Error in {in_filename}: non-primary suite type found."

diff --git a/resources/libraries/python/autogen/Testcase.py b/resources/libraries/python/autogen/Testcase.py
index c540d8c..d419b29 100644 (file)
--- a/resources/libraries/python/autogen/Testcase.py
+++ b/resources/libraries/python/autogen/Testcase.py
@@ -34,8 +34,7 @@ class Testcase:
         """
         self.template = Template(template_string)
 
         """
         self.template = Template(template_string)
 

-    def generate(self, num, frame_size, phy_cores, clients, streams,
-                 bytes_str):
+    def generate(self, num, frame_size, phy_cores):

         """Return string of test case code with placeholders filled.
 
         Fail if there are placeholders left unfilled.
         """Return string of test case code with placeholders filled.
 
         Fail if there are placeholders left unfilled.


@@ -44,16 +43,9 @@ class Testcase:
         :param num: Test case number. Example value: 4.
         :param frame_size: Imix string or numeric frame size. Example: 74.
         :param phy_cores: Number of physical cores to use. Example: 2.
         :param num: Test case number. Example value: 4.
         :param frame_size: Imix string or numeric frame size. Example: 74.
         :param phy_cores: Number of physical cores to use. Example: 2.

-        :param clients: Number of clients used by test program. Example: 4.
-        :param streams: Number of streams used by test program. Example: 10.
-        :param bytes_str: Size in bytes of stream sent by test program.
-            Example: 1G

         :type num: int
         :type frame_size: str or int
         :type phy_cores: int or str
         :type num: int
         :type frame_size: str or int
         :type phy_cores: int or str

-        :type clients: int
-        :type streams: int
-        :type bytes_str: str

         :returns: Filled template, usable as test case code.
         :rtype: str
         """
         :returns: Filled template, usable as test case code.
         :rtype: str
         """


@@ -75,11 +67,6 @@ class Testcase:
                 u"cores_num": f"${{{cores_num:d}}}",
                 u"cores_str": phy_cores,
                 u"tc_num": f"tc{num:02d}",
                 u"cores_num": f"${{{cores_num:d}}}",
                 u"cores_str": phy_cores,
                 u"tc_num": f"tc{num:02d}",

-                u"clients_num": f"${{{clients:d}}}",
-                u"clients_str": str(clients),
-                u"streams_num": f"${{{streams:d}}}",
-                u"streams_str": str(streams),
-                u"bytes_str": bytes_str,

             }
         )
         return self.template.substitute(subst_dict)
             }
         )
         return self.template.substitute(subst_dict)


@@ -122,17 +109,9 @@ class Testcase:
 | | [Tags] | ${{cores_str}}C
 | | phy_cores=${{cores_num}}
 '''
 | | [Tags] | ${{cores_str}}C
 | | phy_cores=${{cores_num}}
 '''

-        elif u"iperf3" in suite_id:
-            template_string = f'''
-| ${{tc_num}}-9000B-${{cores_str}}c-{suite_id}
-| | [Tags] | ${{cores_str}}C | ${{clients_str}}CLIENT | ${{streams_str}}STREAM
-| | phy_cores=${{cores_num}} | clients=${{clients_num}}'''
-            template_string += f" | streams=${{streams_num}}\n"

         else:
         else:

-            template_string = f'''
-| ${{tc_num}}-9000B-${{cores_str}}c-{suite_id}
-| | [Tags] | ${{cores_str}}C | ${{clients_str}}CLIENT | ${{streams_str}}STREAM
-| | phy_cores=${{cores_num}} | clients=${{clients_num}}'''
-            template_string += f" | streams=${{streams_num}}" \
-                               f" | bytes=${{bytes_str}}\n"
+            template_string = \
+                f"\n| ${{tc_num}}-9000B-${{cores_str}}c-{suite_id[:-4]}" \
+                f"-{suite_id[-3:]}\n" \
+                f"| | [Tags] | ${{cores_str}}C\n| | phy_cores=${{cores_num}}\n"

         return cls(template_string)
         return cls(template_string)

diff --git a/resources/libraries/robot/hoststack/hoststack.robot b/resources/libraries/robot/hoststack/hoststack.robot
index 16a390a..c6bc7de 100644 (file)
--- a/resources/libraries/robot/hoststack/hoststack.robot
+++ b/resources/libraries/robot/hoststack/hoststack.robot
@@ -91,7 +91,7 @@
 | ... | vcl_config=vcl_iperf3.conf
 | ... | ld_preload=${True}
 | ... | transparent_tls=${False}
 | ... | vcl_config=vcl_iperf3.conf
 | ... | ld_preload=${True}
 | ... | transparent_tls=${False}

-| ... | json=${False}
+| ... | json=${True}

 | ... | ip_version=${4}
 | &{iperf3_client_attr}=
 | ... | role=client
 | ... | ip_version=${4}
 | &{iperf3_client_attr}=
 | ... | role=client


@@ -101,7 +101,7 @@
 | ... | vcl_config=vcl_iperf3.conf
 | ... | ld_preload=${True}
 | ... | transparent_tls=${False}
 | ... | vcl_config=vcl_iperf3.conf
 | ... | ld_preload=${True}
 | ... | transparent_tls=${False}

-| ... | json=${False}
+| ... | json=${True}

 | ... | ip_version=${4}
 | ... | ip_address=${EMPTY}
 | ... | parallel=${1}
 | ... | ip_version=${4}
 | ... | ip_address=${EMPTY}
 | ... | parallel=${1}


@@ -464,9 +464,9 @@
 | | [Documentation]
 | | ... | Configure IP address on the port, set it up and start the specified
 | | ... | HostStack test programs on the DUTs. Gather test program
 | | [Documentation]
 | | ... | Configure IP address on the port, set it up and start the specified
 | | ... | HostStack test programs on the DUTs. Gather test program

-| | ... | output and append test results in message.
-| | ... | Return boolean indicating when no results were available from
-| | ... | both the server and client test programs.
+| | ... | output and append JSON formatted test data in message.
+| | ... | Return boolean indicating there was a defered failure of either the
+| | ... | server and/or client test programs.

 | |
 | | Set To Dictionary | ${vpp_echo_server_attr} | uri_ip4_addr
 | | ... | ${dut2_if1_ip4_addr}
 | |
 | | Set To Dictionary | ${vpp_echo_server_attr} | uri_ip4_addr
 | | ... | ${dut2_if1_ip4_addr}


@@ -492,25 +492,25 @@
 | | ... | ${vpp_echo_client_attr}[namespace] | ${core_list}
 | | ... | ${vpp_echo_client_attr}[cfg_vpp_feature] | ${vpp_echo_client}
 | | When Hoststack Test Program Finished | ${dut1} | ${client_pid}
 | | ... | ${vpp_echo_client_attr}[namespace] | ${core_list}
 | | ... | ${vpp_echo_client_attr}[cfg_vpp_feature] | ${vpp_echo_client}
 | | When Hoststack Test Program Finished | ${dut1} | ${client_pid}

-| | ${client_no_results} | ${client_output}=
+| | ${client_defer_fail} | ${client_output}=

 | | ... | Analyze hoststack test program output | ${dut1} | Client
 | | ... | ${vpp_nsim_attr} | ${vpp_echo_client}
 | | Then Set test message | ${client_output}
 | | And Hoststack Test Program Finished | ${dut2} | ${server_pid}
 | | ... | Analyze hoststack test program output | ${dut1} | Client
 | | ... | ${vpp_nsim_attr} | ${vpp_echo_client}
 | | Then Set test message | ${client_output}
 | | And Hoststack Test Program Finished | ${dut2} | ${server_pid}

-| | ${server_no_results} | ${server_output}=
+| | ${server_defer_fail} | ${server_output}=

 | | ... | Analyze hoststack test program output | ${dut2} | Server
 | | ... | ${vpp_nsim_attr} | ${vpp_echo_server}
 | | Set test message | ${server_output} | append=True
 | | ... | Analyze hoststack test program output | ${dut2} | Server
 | | ... | ${vpp_nsim_attr} | ${vpp_echo_server}
 | | Set test message | ${server_output} | append=True

-| | Run Keyword And Return | No Hoststack Test Program Results
-| | ... | ${server_no_results} | ${client_no_results}
+| | Run Keyword And Return | Hoststack Test Program Defer Fail
+| | ... | ${server_defer_fail} | ${client_defer_fail}

 
 | Get Test Results From Hoststack Iperf3 Test
 | | [Documentation]
 | | ... | Configure IP address on the port, set it up and start the specified
 | | ... | HostStack test programs on the DUTs. Gather test program
 
 | Get Test Results From Hoststack Iperf3 Test
 | | [Documentation]
 | | ... | Configure IP address on the port, set it up and start the specified
 | | ... | HostStack test programs on the DUTs. Gather test program

-| | ... | output and append test results in message.
-| | ... | Return boolean indicating when no results were available from
-| | ... | both the server and client test programs.
+| | ... | output and append JSON formatted test data in message.
+| | ... | Return boolean indicating there was a defered failure of either the
+| | ... | server and/or client test programs.

 | |
 | | Set To Dictionary | ${iperf3_client_attr} | ip_address
 | | ... | ${dut2_if1_ip4_addr}
 | |
 | | Set To Dictionary | ${iperf3_client_attr} | ip_address
 | | ... | ${dut2_if1_ip4_addr}


@@ -534,8 +534,8 @@
 | | ... | ${iperf3_client_attr}[namespace] | ${core_list}
 | | ... | ${iperf3_client_attr}[cfg_vpp_feature] | ${iperf3_client}
 | | When Hoststack Test Program Finished | ${dut1} | ${client_pid}
 | | ... | ${iperf3_client_attr}[namespace] | ${core_list}
 | | ... | ${iperf3_client_attr}[cfg_vpp_feature] | ${iperf3_client}
 | | When Hoststack Test Program Finished | ${dut1} | ${client_pid}

-| | ${client_no_results} | ${client_output}=
+| | ${client_defer_fail} | ${client_output}=

 | | ... | Analyze hoststack test program output | ${dut1} | Client
 | | ... | ${vpp_nsim_attr} | ${iperf3_client}
 | | Then Set test message | ${client_output}
 | | ... | Analyze hoststack test program output | ${dut1} | Client
 | | ... | ${vpp_nsim_attr} | ${iperf3_client}
 | | Then Set test message | ${client_output}

-| | Return From Keyword | ${client_no_results}
+| | Return From Keyword | ${client_defer_fail}

diff --git a/resources/tools/wrk/wrk.py b/resources/tools/wrk/wrk.py
index 381e9b9..85d18d4 100644 (file)
--- a/resources/tools/wrk/wrk.py
+++ b/resources/tools/wrk/wrk.py
@@ -186,12 +186,12 @@ def run_wrk(tg_node, profile_name, tg_numa, test_type, warm_up=False):
     if test_type == u"cps":
         log_msg += u"Connections/sec: Avg / Stdev / Max  / +/- Stdev\n"
         for item in stats[u"rps-stats-lst"]:
     if test_type == u"cps":
         log_msg += u"Connections/sec: Avg / Stdev / Max  / +/- Stdev\n"
         for item in stats[u"rps-stats-lst"]:

-            log_msg += f"{0} / {1} / {2} / {3}\n".format(*item)
+            log_msg += u" / ".join(map(str, item)) + u"\n"

         log_msg += f"Total cps: {stats[u'rps-sum']}cps\n"
     elif test_type == u"rps":
         log_msg += u"Requests/sec: Avg / Stdev / Max  / +/- Stdev\n"
         for item in stats[u"rps-stats-lst"]:
         log_msg += f"Total cps: {stats[u'rps-sum']}cps\n"
     elif test_type == u"rps":
         log_msg += u"Requests/sec: Avg / Stdev / Max  / +/- Stdev\n"
         for item in stats[u"rps-stats-lst"]:

-            log_msg += f"{0} / {1} / {2} / {3}\n".format(*item)
+            log_msg += u" / ".join(map(str, item)) + u"\n"

         log_msg += f"Total rps: {stats[u'rps-sum']}rps\n"
     elif test_type == u"bw":
         log_msg += f"Transfer/sec: {stats[u'bw-sum']}Bps"
         log_msg += f"Total rps: {stats[u'rps-sum']}rps\n"
     elif test_type == u"bw":
         log_msg += f"Transfer/sec: {stats[u'bw-sum']}Bps"

diff --git a/tests/vpp/perf/hoststack/10ge2p1x710-eth-ip4tcpbase-ldpreload-iperf3-bps.robot b/tests/vpp/perf/hoststack/10ge2p1x710-eth-ip4tcpbase-ldpreload-iperf3-bps.robot
new file mode 100644 (file)
index 0000000..f2c7165
--- /dev/null
+++ b/tests/vpp/perf/hoststack/10ge2p1x710-eth-ip4tcpbase-ldpreload-iperf3-bps.robot
@@ -0,0 +1,60 @@
+# Copyright (c) 2020 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+*** Settings ***
+| Library  | resources.libraries.python.HoststackUtil
+| Resource | resources/libraries/robot/shared/default.robot
+| Resource | resources/libraries/robot/hoststack/hoststack.robot
+|
+| Force Tags | 3_NODE_SINGLE_LINK_TOPO | PERFTEST | HW_ENV
+| ... | TCP | NIC_Intel-X710 | DRV_VFIO_PCI | HOSTSTACK
+| ... | LDPRELOAD | IPERF3 | 1CLIENT | 1STREAM | 9000B
+| ... | eth-ip4tcpbase-ldpreload-iperf3
+|
+| Suite Setup | Setup suite single link no tg
+| Suite Teardown | Tear down suite | hoststack
+| Test Setup | Setup test
+| Test Teardown | Tear down test
+|
+| Test Template | Local template
+|
+| Documentation | *Iperf3 client -> Iperf3 server throughput.
+|
+| ... | *[Top] Network Topologies:* DUT-DUT 2-node topology
+| ... | with single link between nodes.
+| ... | *[Enc] Packet Encapsulations:* Eth-IPv4-TCP
+| ... | *[Cfg] DUT configuration:*
+| ... | *[Ref] Applicable standard specifications:*
+
+*** Variables ***
+| @{plugins_to_enable}= | dpdk_plugin.so
+| ${nic_name}= | Intel-X710
+| ${nic_driver}= | vfio-pci
+| ${overhead}= | ${0}
+| ${frame_size}= | ${9000}
+| ${crypto_type}= | ${None}
+
+*** Keywords ***
+| Local template
+| | [Arguments] | ${phy_cores}
+| |
+| | Set Test Variable | ${dpdk_no_tx_checksum_offload} | ${False}
+| | Set VPP Hoststack Attributes | phy_cores=${phy_cores}
+| | ${defer_fail}= | Get Test Results From Hoststack Iperf3 Test
+| | Run Keyword If | ${defer_fail}==True | FAIL
+| | ... | Defered Failure From Hoststack Iperf3 Test Program
+
+*** Test Cases ***
+| tc01-9000B-1c-eth-ip4tcpbase-ldpreload-iperf3-bps
+| | [Tags] | 1C
+| | phy_cores=${1}

diff --git a/tests/vpp/perf/hoststack/10ge2p1x710-eth-ip4tcpbase-nsim-ldpreload-iperf3-bps.robot b/tests/vpp/perf/hoststack/10ge2p1x710-eth-ip4tcpbase-nsim-ldpreload-iperf3-bps.robot
new file mode 100644 (file)
index 0000000..2eaed06
--- /dev/null
+++ b/tests/vpp/perf/hoststack/10ge2p1x710-eth-ip4tcpbase-nsim-ldpreload-iperf3-bps.robot
@@ -0,0 +1,62 @@
+# Copyright (c) 2020 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+*** Settings ***
+| Library  | resources.libraries.python.HoststackUtil
+| Resource | resources/libraries/robot/shared/default.robot
+| Resource | resources/libraries/robot/hoststack/hoststack.robot
+|
+| Force Tags | 3_NODE_SINGLE_LINK_TOPO | PERFTEST | HW_ENV
+| ... | TCP | NIC_Intel-X710 | DRV_VFIO_PCI | HOSTSTACK
+| ... | NSIM | LDPRELOAD | IPERF3 | 1CLIENT | 1STREAM | 9000B
+| ... | eth-ip4tcpbase-nsim-ldpreload-iperf3
+|
+| Suite Setup | Setup suite single link no tg
+| Suite Teardown | Tear down suite
+| Test Setup | Setup test
+| Test Teardown | Tear down test
+|
+| Test Template | Local template
+|
+| Documentation | *Iperf3 client -> Iperf3 server throughput.
+|
+| ... | *[Top] Network Topologies:* DUT-DUT 2-node topology
+| ... | with single link between nodes.
+| ... | *[Enc] Packet Encapsulations:* Eth-IPv4-TCP
+| ... | *[Cfg] DUT configuration:*
+| ... | *[Ref] Applicable standard specifications:*
+
+*** Variables ***
+| @{plugins_to_enable}= | dpdk_plugin.so | nsim_plugin.so
+| ${nic_name}= | Intel-X710
+| ${nic_driver}= | vfio-pci
+| ${overhead}= | ${0}
+| ${frame_size}= | ${9000}
+| ${crypto_type}= | ${None}
+| ${pkts_per_drop}= | ${100}
+
+*** Keywords ***
+| Local template
+| | [Arguments] | ${phy_cores}
+| |
+| | Set VPP Hoststack Attributes | phy_cores=${phy_cores}
+| | Set VPP NSIM Attributes | output_feature_enable=${True} |
+| | ... | packets_per_drop=${pkts_per_drop}
+| | ${defer_fail}= | Get Test Results From Hoststack Iperf3 Test
+| | Run Keyword If | ${defer_fail}==True | FAIL
+| | ... | Defered Failure From Hoststack Iperf3 Test Program
+
+*** Test Cases ***
+| tc01-9000B-1c-eth-ip4tcpbase-nsim-ldpreload-iperf3-bps
+| | [Tags] | 1C
+| | phy_cores=${1}

diff --git a/tests/vpp/perf/hoststack/10ge2p1x710-eth-ip4tcp-ldpreload-iperf3-bps.robot b/tests/vpp/perf/hoststack/10ge2p1x710-eth-ip4tcpscale1cl10s-ldpreload-iperf3-bps.robot
similarity index 77%
rename from tests/vpp/perf/hoststack/10ge2p1x710-eth-ip4tcp-ldpreload-iperf3-bps.robot
rename to tests/vpp/perf/hoststack/10ge2p1x710-eth-ip4tcpscale1cl10s-ldpreload-iperf3-bps.robot
index a907c61..7d314bb 100644 (file)
--- a/tests/vpp/perf/hoststack/10ge2p1x710-eth-ip4tcp-ldpreload-iperf3-bps.robot
+++ b/tests/vpp/perf/hoststack/10ge2p1x710-eth-ip4tcpscale1cl10s-ldpreload-iperf3-bps.robot
@@ -18,7 +18,8 @@
 |
 | Force Tags | 3_NODE_SINGLE_LINK_TOPO | PERFTEST | HW_ENV
 | ... | TCP | NIC_Intel-X710 | DRV_VFIO_PCI | HOSTSTACK
 |
 | Force Tags | 3_NODE_SINGLE_LINK_TOPO | PERFTEST | HW_ENV
 | ... | TCP | NIC_Intel-X710 | DRV_VFIO_PCI | HOSTSTACK

-| ... | LDPRELOAD | IPERF3 | eth-ip4tcp-ldpreload-iperf3
+| ... | LDPRELOAD | IPERF3 | 1CLIENT | 10STREAM | 9000B
+| ... | eth-ip4tcpscale1cl10s-ldpreload-iperf3

 |
 | Suite Setup | Setup suite single link no tg
 | Suite Teardown | Tear down suite | hoststack
 |
 | Suite Setup | Setup suite single link no tg
 | Suite Teardown | Tear down suite | hoststack


@@ -42,23 +43,20 @@
 | ${overhead}= | ${0}
 | ${frame_size}= | ${9000}
 | ${crypto_type}= | ${None}
 | ${overhead}= | ${0}
 | ${frame_size}= | ${9000}
 | ${crypto_type}= | ${None}

+| ${streams} | ${10}

 
 *** Keywords ***
 | Local template
 
 *** Keywords ***
 | Local template

-| | [Arguments] | ${phy_cores} | ${clients} | ${streams}
+| | [Arguments] | ${phy_cores}

 | |
 | | Set Test Variable | ${dpdk_no_tx_checksum_offload} | ${False}
 | | Set VPP Hoststack Attributes | phy_cores=${phy_cores}
 | | Set Iperf3 Client Attributes | parallel=${streams}
 | |
 | | Set Test Variable | ${dpdk_no_tx_checksum_offload} | ${False}
 | | Set VPP Hoststack Attributes | phy_cores=${phy_cores}
 | | Set Iperf3 Client Attributes | parallel=${streams}

-| | ${no_results}= | Get Test Results From Hoststack Iperf3 Test
-| | Run Keyword If | ${no_results}==True | FAIL
-| | ... | No Test Results From Iperf3 client
+| | ${defer_fail}= | Get Test Results From Hoststack Iperf3 Test
+| | Run Keyword If | ${defer_fail}==True | FAIL
+| | ... | Defered Failure From Hoststack Iperf3 Test Program

 
 *** Test Cases ***
 
 *** Test Cases ***

-| tc01-9000B-1c-eth-ip4tcp-ldpreload-iperf3-bps
-| | [Tags] | 1C | 1CLIENT | 1STREAM
-| | phy_cores=${1} | clients=${1} | streams=${1}
-
-| tc02-9000B-1c-eth-ip4tcp-ldpreload-iperf3-bps
-| | [Tags] | 1C | 1CLIENT | 10STREAM
-| | phy_cores=${1} | clients=${1} | streams=${10}
+| tc01-9000B-1c-eth-ip4tcpscale1cl10s-ldpreload-iperf3-bps
+| | [Tags] | 1C
+| | phy_cores=${1}

diff --git a/tests/vpp/perf/hoststack/10ge2p1x710-eth-ip4tcp-nsim-ldpreload-iperf3-bps.robot b/tests/vpp/perf/hoststack/10ge2p1x710-eth-ip4tcpscale1cl10s-nsim-ldpreload-iperf3-bps.robot
similarity index 77%
rename from tests/vpp/perf/hoststack/10ge2p1x710-eth-ip4tcp-nsim-ldpreload-iperf3-bps.robot
rename to tests/vpp/perf/hoststack/10ge2p1x710-eth-ip4tcpscale1cl10s-nsim-ldpreload-iperf3-bps.robot
index 0a98ee3..036414f 100644 (file)
--- a/tests/vpp/perf/hoststack/10ge2p1x710-eth-ip4tcp-nsim-ldpreload-iperf3-bps.robot
+++ b/tests/vpp/perf/hoststack/10ge2p1x710-eth-ip4tcpscale1cl10s-nsim-ldpreload-iperf3-bps.robot
@@ -18,7 +18,8 @@
 |
 | Force Tags | 3_NODE_SINGLE_LINK_TOPO | PERFTEST | HW_ENV
 | ... | TCP | NIC_Intel-X710 | DRV_VFIO_PCI | HOSTSTACK
 |
 | Force Tags | 3_NODE_SINGLE_LINK_TOPO | PERFTEST | HW_ENV
 | ... | TCP | NIC_Intel-X710 | DRV_VFIO_PCI | HOSTSTACK

-| ... | NSIM | LDPRELOAD | IPERF3 | eth-ip4tcp-nsim-ldpreload-iperf3
+| ... | NSIM | LDPRELOAD | IPERF3 | 1CLIENT | 10STREAM | 9000B
+| ... | eth-ip4tcpscale1cl10s-nsim-ldpreload-iperf3

 |
 | Suite Setup | Setup suite single link no tg
 | Suite Teardown | Tear down suite
 |
 | Suite Setup | Setup suite single link no tg
 | Suite Teardown | Tear down suite


@@ -43,24 +44,21 @@
 | ${frame_size}= | ${9000}
 | ${crypto_type}= | ${None}
 | ${pkts_per_drop}= | ${100}
 | ${frame_size}= | ${9000}
 | ${crypto_type}= | ${None}
 | ${pkts_per_drop}= | ${100}

+| ${streams}= | ${10}

 
 *** Keywords ***
 | Local template
 
 *** Keywords ***
 | Local template

-| | [Arguments] | ${phy_cores} | ${clients} |  ${streams}
+| | [Arguments] | ${phy_cores}

 | |
 | | Set VPP Hoststack Attributes | phy_cores=${phy_cores}
 | | Set Iperf3 Client Attributes | parallel=${streams}
 | | Set VPP NSIM Attributes | output_feature_enable=${True} |
 | | ... | packets_per_drop=${pkts_per_drop}
 | |
 | | Set VPP Hoststack Attributes | phy_cores=${phy_cores}
 | | Set Iperf3 Client Attributes | parallel=${streams}
 | | Set VPP NSIM Attributes | output_feature_enable=${True} |
 | | ... | packets_per_drop=${pkts_per_drop}

-| | ${no_results}= | Get Test Results From Hoststack Iperf3 Test
-| | Run Keyword If | ${no_results}==True | FAIL
-| | ... | No Test Results From Iperf3 client
+| | ${defer_fail}= | Get Test Results From Hoststack Iperf3 Test
+| | Run Keyword If | ${defer_fail}==True | FAIL
+| | ... | Defered Failure From Hoststack Iperf3 Test Program

 
 *** Test Cases ***
 
 *** Test Cases ***

-| tc01-9000B-1c-eth-ip4tcp-nsim-ldpreload-iperf3-bps
-| | [Tags] | 1C | 1CLIENT | 1STREAM
-| | phy_cores=${1} | clients=${1} | streams=${1}
-
-| tc02-9000B-1c-eth-ip4tcp-nsim-ldpreload-iperf3-bps
-| | [Tags] | 1C | 1CLIENT | 10STREAM
-| | phy_cores=${1} | clients=${1} | streams=${10}
+| tc01-9000B-1c-eth-ip4tcpscale1cl10s-nsim-ldpreload-iperf3-bps
+| | [Tags] | 1C
+| | phy_cores=${1}

diff --git a/tests/vpp/perf/hoststack/10ge2p1x710-eth-ip4udpquicbase-vppecho-bps.robot b/tests/vpp/perf/hoststack/10ge2p1x710-eth-ip4udpquicbase-vppecho-bps.robot
new file mode 100644 (file)
index 0000000..564844e
--- /dev/null
+++ b/tests/vpp/perf/hoststack/10ge2p1x710-eth-ip4udpquicbase-vppecho-bps.robot
@@ -0,0 +1,61 @@
+# Copyright (c) 2020 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+*** Settings ***
+| Library  | resources.libraries.python.HoststackUtil
+| Resource | resources/libraries/robot/shared/default.robot
+| Resource | resources/libraries/robot/hoststack/hoststack.robot
+|
+| Force Tags | 3_NODE_SINGLE_LINK_TOPO | PERFTEST | HW_ENV
+| ... | NIC_Intel-X710 | DRV_VFIO_PCI | UDP | QUIC | VPPECHO
+| ... | 1CLIENT | 1STREAM | HOSTSTACK | 9000B | eth-ip4udpquicbase-vppecho
+|
+| Suite Setup | Setup suite single link no tg
+| Suite Teardown | Tear down suite
+| Test Setup | Setup test
+| Test Teardown | Tear down test
+|
+| Test Template | Local template
+|
+| Documentation | *QUIC Unidirectional Echo Client -> Echo Server throughput.
+|
+| ... | *[Top] Network Topologies:* DUT-DUT 2-node topology
+| ... | with single link between nodes.
+| ... | *[Enc] Packet Encapsulations:* Eth-IPv4-UDP-QUIC
+| ... | *[Cfg] DUT configuration:*
+| ... | *[Ref] Applicable standard specifications:*
+
+*** Variables ***
+| @{plugins_to_enable}= | dpdk_plugin.so | quic_plugin.so
+| ${nic_name}= | Intel-X710
+| ${nic_driver}= | vfio-pci
+| ${overhead}= | ${0}
+| ${frame_size}= | ${9000}
+| ${crypto_type}= | ${None}
+| ${bytes}= | 10G
+
+*** Keywords ***
+| Local template
+| | [Arguments] | ${phy_cores}
+| |
+| | Set VPP Hoststack Attributes | phy_cores=${phy_cores}
+| | Set VPP Echo Server Attributes | cfg_vpp_feature=quic | rx_bytes=${bytes}
+| | Set VPP Echo Client Attributes | cfg_vpp_feature=quic | tx_bytes=${bytes}
+| | ${defer_fail}= | Get Test Results From Hoststack VPP Echo Test
+| | Run Keyword If | ${defer_fail}==True | FAIL
+| | ... | Defered Failure From Hoststack VPP Echo Test Program
+
+*** Test Cases ***
+| tc01-9000B-1c-eth-ip4udpquicbase-vppecho-bps
+| | [Tags] | 1C
+| | phy_cores=${1}

diff --git a/tests/vpp/perf/hoststack/10ge2p1x710-eth-ip4udpquic-vppecho-bps.robot b/tests/vpp/perf/hoststack/10ge2p1x710-eth-ip4udpquicscale10cl10s-vppecho-bps.robot
similarity index 70%
rename from tests/vpp/perf/hoststack/10ge2p1x710-eth-ip4udpquic-vppecho-bps.robot
rename to tests/vpp/perf/hoststack/10ge2p1x710-eth-ip4udpquicscale10cl10s-vppecho-bps.robot
index a93afa9..f4e5c14 100644 (file)
--- a/tests/vpp/perf/hoststack/10ge2p1x710-eth-ip4udpquic-vppecho-bps.robot
+++ b/tests/vpp/perf/hoststack/10ge2p1x710-eth-ip4udpquicscale10cl10s-vppecho-bps.robot
@@ -18,7 +18,8 @@
 |
 | Force Tags | 3_NODE_SINGLE_LINK_TOPO | PERFTEST | HW_ENV
 | ... | NIC_Intel-X710 | DRV_VFIO_PCI | UDP | QUIC | VPPECHO
 |
 | Force Tags | 3_NODE_SINGLE_LINK_TOPO | PERFTEST | HW_ENV
 | ... | NIC_Intel-X710 | DRV_VFIO_PCI | UDP | QUIC | VPPECHO

-| ... | HOSTSTACK | eth-ip4udpquic-vppecho
+| ... | HOSTSTACK | 10CLIENT | 10STREAM | 9000B
+| ... | eth-ip4udpquicscale10cl10s-vppecho

 |
 | Suite Setup | Setup suite single link no tg
 | Suite Teardown | Tear down suite
 |
 | Suite Setup | Setup suite single link no tg
 | Suite Teardown | Tear down suite


@@ -42,33 +43,24 @@
 | ${overhead}= | ${0}
 | ${frame_size}= | ${9000}
 | ${crypto_type}= | ${None}
 | ${overhead}= | ${0}
 | ${frame_size}= | ${9000}
 | ${crypto_type}= | ${None}

+| ${clients}= | ${10}
+| ${streams}= | ${10}
+| ${bytes}= | 100M

 
 *** Keywords ***
 | Local template
 
 *** Keywords ***
 | Local template

-| | [Arguments] | ${phy_cores} | ${clients} |  ${streams} | ${bytes}
+| | [Arguments] | ${phy_cores}

 | |
 | | Set VPP Hoststack Attributes | phy_cores=${phy_cores}
 | | Set VPP Echo Server Attributes | cfg_vpp_feature=quic | nclients=${clients}
 | | ... | quic_streams=${streams} | rx_bytes=${bytes}
 | | Set VPP Echo Client Attributes | cfg_vpp_feature=quic | nclients=${clients}
 | | ... | quic_streams=${streams} | tx_bytes=${bytes}
 | |
 | | Set VPP Hoststack Attributes | phy_cores=${phy_cores}
 | | Set VPP Echo Server Attributes | cfg_vpp_feature=quic | nclients=${clients}
 | | ... | quic_streams=${streams} | rx_bytes=${bytes}
 | | Set VPP Echo Client Attributes | cfg_vpp_feature=quic | nclients=${clients}
 | | ... | quic_streams=${streams} | tx_bytes=${bytes}

-| | ${no_results}= | Get Test Results From Hoststack VPP Echo Test
-| | Run Keyword If | ${no_results}==True | FAIL
-| | ... | No Test Results From External Hoststack Apps
+| | ${defer_fail}= | Get Test Results From Hoststack VPP Echo Test
+| | Run Keyword If | ${defer_fail}==True | FAIL
+| | ... | Defered Failure From Hoststack VPP Echo Test Program

 
 *** Test Cases ***
 
 *** Test Cases ***

-| tc01-9000B-1c-eth-ip4udpquic-vppecho-bps
-| | [Tags] | 1C | 1CLIENT | 1STREAM
-| | phy_cores=${1} | clients=${1} | streams=${1} | bytes=10G
-
-| tc02-9000B-1c-eth-ip4udpquic-vppecho-bps
-| | [Tags] | 1C | 1CLIENT | 10STREAM
-| | phy_cores=${1} | clients=${1} | streams=${10} | bytes=1G
-
-| tc03-9000B-1c-eth-ip4udpquic-vppecho-bps
-| | [Tags] | 1C | 10CLIENT | 1STREAM
-| | phy_cores=${1} | clients=${10} | streams=${1} | bytes=1G
-
-| tc04-9000B-1c-eth-ip4udpquic-vppecho-bps
-| | [Tags] | 1C | 10CLIENT | 10STREAM
-| | phy_cores=${1} | clients=${10} | streams=${10} | bytes=100M
+| tc01-9000B-1c-eth-ip4udpquicscale10cl10s-vppecho-bps
+| | [Tags] | 1C
+| | phy_cores=${1}

diff --git a/tests/vpp/perf/hoststack/10ge2p1x710-eth-ip4udpquicscale10cl1s-vppecho-bps.robot b/tests/vpp/perf/hoststack/10ge2p1x710-eth-ip4udpquicscale10cl1s-vppecho-bps.robot
new file mode 100644 (file)
index 0000000..e160f05
--- /dev/null
+++ b/tests/vpp/perf/hoststack/10ge2p1x710-eth-ip4udpquicscale10cl1s-vppecho-bps.robot
@@ -0,0 +1,65 @@
+# Copyright (c) 2020 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+*** Settings ***
+| Library  | resources.libraries.python.HoststackUtil
+| Resource | resources/libraries/robot/shared/default.robot
+| Resource | resources/libraries/robot/hoststack/hoststack.robot
+|
+| Force Tags | 3_NODE_SINGLE_LINK_TOPO | PERFTEST | HW_ENV
+| ... | NIC_Intel-X710 | DRV_VFIO_PCI | UDP | QUIC | VPPECHO
+| ... | HOSTSTACK | 10CLIENT | 1STREAM | 9000B
+| ... | eth-ip4udpquicscale10cl1s-vppecho
+|
+| Suite Setup | Setup suite single link no tg
+| Suite Teardown | Tear down suite
+| Test Setup | Setup test
+| Test Teardown | Tear down test
+|
+| Test Template | Local template
+|
+| Documentation | *QUIC Unidirectional Echo Client -> Echo Server throughput.
+|
+| ... | *[Top] Network Topologies:* DUT-DUT 2-node topology
+| ... | with single link between nodes.
+| ... | *[Enc] Packet Encapsulations:* Eth-IPv4-UDP-QUIC
+| ... | *[Cfg] DUT configuration:*
+| ... | *[Ref] Applicable standard specifications:*
+
+*** Variables ***
+| @{plugins_to_enable}= | dpdk_plugin.so | quic_plugin.so
+| ${nic_name}= | Intel-X710
+| ${nic_driver}= | vfio-pci
+| ${overhead}= | ${0}
+| ${frame_size}= | ${9000}
+| ${crypto_type}= | ${None}
+| ${clients}= | ${10}
+| ${bytes}= | 1G
+
+*** Keywords ***
+| Local template
+| | [Arguments] | ${phy_cores}
+| |
+| | Set VPP Hoststack Attributes | phy_cores=${phy_cores}
+| | Set VPP Echo Server Attributes | cfg_vpp_feature=quic | nclients=${clients}
+| | ... | rx_bytes=${bytes}
+| | Set VPP Echo Client Attributes | cfg_vpp_feature=quic | nclients=${clients}
+| | ... | tx_bytes=${bytes}
+| | ${defer_fail}= | Get Test Results From Hoststack VPP Echo Test
+| | Run Keyword If | ${defer_fail}==True | FAIL
+| | ... | Defered Failure From Hoststack VPP Echo Test Program
+
+*** Test Cases ***
+| tc01-9000B-1c-eth-ip4udpquicscale10cl1s-vppecho-bps
+| | [Tags] | 1C
+| | phy_cores=${1}

diff --git a/tests/vpp/perf/hoststack/10ge2p1x710-eth-ip4udpquicscale1cl10s-vppecho-bps.robot b/tests/vpp/perf/hoststack/10ge2p1x710-eth-ip4udpquicscale1cl10s-vppecho-bps.robot
new file mode 100644 (file)
index 0000000..d03e247
--- /dev/null
+++ b/tests/vpp/perf/hoststack/10ge2p1x710-eth-ip4udpquicscale1cl10s-vppecho-bps.robot
@@ -0,0 +1,65 @@
+# Copyright (c) 2020 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+*** Settings ***
+| Library  | resources.libraries.python.HoststackUtil
+| Resource | resources/libraries/robot/shared/default.robot
+| Resource | resources/libraries/robot/hoststack/hoststack.robot
+|
+| Force Tags | 3_NODE_SINGLE_LINK_TOPO | PERFTEST | HW_ENV
+| ... | NIC_Intel-X710 | DRV_VFIO_PCI | UDP | QUIC | VPPECHO
+| ... | HOSTSTACK | 1CLIENT | 10STREAM | 9000B
+| ... | eth-ip4udpquicscale1cl10s-vppecho
+|
+| Suite Setup | Setup suite single link no tg
+| Suite Teardown | Tear down suite
+| Test Setup | Setup test
+| Test Teardown | Tear down test
+|
+| Test Template | Local template
+|
+| Documentation | *QUIC Unidirectional Echo Client -> Echo Server throughput.
+|
+| ... | *[Top] Network Topologies:* DUT-DUT 2-node topology
+| ... | with single link between nodes.
+| ... | *[Enc] Packet Encapsulations:* Eth-IPv4-UDP-QUIC
+| ... | *[Cfg] DUT configuration:*
+| ... | *[Ref] Applicable standard specifications:*
+
+*** Variables ***
+| @{plugins_to_enable}= | dpdk_plugin.so | quic_plugin.so
+| ${nic_name}= | Intel-X710
+| ${nic_driver}= | vfio-pci
+| ${overhead}= | ${0}
+| ${frame_size}= | ${9000}
+| ${crypto_type}= | ${None}
+| ${streams}= | ${10}
+| ${bytes}= | 1G
+
+*** Keywords ***
+| Local template
+| | [Arguments] | ${phy_cores}
+| |
+| | Set VPP Hoststack Attributes | phy_cores=${phy_cores}
+| | Set VPP Echo Server Attributes | cfg_vpp_feature=quic
+| | ... | quic_streams=${streams} | rx_bytes=${bytes}
+| | Set VPP Echo Client Attributes | cfg_vpp_feature=quic
+| | ... | quic_streams=${streams} | tx_bytes=${bytes}
+| | ${defer_fail}= | Get Test Results From Hoststack VPP Echo Test
+| | Run Keyword If | ${defer_fail}==True | FAIL
+| | ... | Defered Failure From Hoststack VPP Echo Test Program
+
+*** Test Cases ***
+| tc01-9000B-1c-eth-ip4udpquicscale1cl10s-vppecho-bps
+| | [Tags] | 1C
+| | phy_cores=${1}

diff --git a/tests/vpp/perf/hoststack/2n1l-10ge2p1x710-eth-ip4tcphttp-wrk8u8c50con-cps.robot b/tests/vpp/perf/hoststack/2n1l-10ge2p1x710-eth-ip4tcphttp-wrk8u8c50con-cps.robot
index d7a7e6c..6b4f3c3 100644 (file)
--- a/tests/vpp/perf/hoststack/2n1l-10ge2p1x710-eth-ip4tcphttp-wrk8u8c50con-cps.robot
+++ b/tests/vpp/perf/hoststack/2n1l-10ge2p1x710-eth-ip4tcphttp-wrk8u8c50con-cps.robot
@@ -19,7 +19,7 @@
 | Resource | resources/libraries/robot/hoststack/tcp_setup.robot
 |
 | Force Tags | 2_NODE_SINGLE_LINK_TOPO | PERFTEST | HW_ENV
 | Resource | resources/libraries/robot/hoststack/tcp_setup.robot
 |
 | Force Tags | 2_NODE_SINGLE_LINK_TOPO | PERFTEST | HW_ENV

-| ... | HTTP | TCP | TCP_CPS | NIC_Intel-X710 | DRV_VFIO_PCI
+| ... | HTTP | TCP | TCP_CPS | NIC_Intel-X710 | DRV_VFIO_PCI | IMIX

 | ... | eth-ip4tcphttp-wrk8u8c50con
 |
 | Suite Setup | Setup suite single link | wrk
 | ... | eth-ip4tcphttp-wrk8u8c50con
 |
 | Suite Setup | Setup suite single link | wrk

diff --git a/tests/vpp/perf/hoststack/2n1l-10ge2p1x710-eth-ip4tcphttp-wrk8u8c50con-rps.robot b/tests/vpp/perf/hoststack/2n1l-10ge2p1x710-eth-ip4tcphttp-wrk8u8c50con-rps.robot
index 27469bb..b9b2226 100644 (file)
--- a/tests/vpp/perf/hoststack/2n1l-10ge2p1x710-eth-ip4tcphttp-wrk8u8c50con-rps.robot
+++ b/tests/vpp/perf/hoststack/2n1l-10ge2p1x710-eth-ip4tcphttp-wrk8u8c50con-rps.robot
@@ -19,7 +19,7 @@
 | Resource | resources/libraries/robot/hoststack/tcp_setup.robot
 |
 | Force Tags | 2_NODE_SINGLE_LINK_TOPO | PERFTEST | HW_ENV
 | Resource | resources/libraries/robot/hoststack/tcp_setup.robot
 |
 | Force Tags | 2_NODE_SINGLE_LINK_TOPO | PERFTEST | HW_ENV

-| ... | HTTP | TCP | TCP_RPS | NIC_Intel-X710 | DRV_VFIO_PCI
+| ... | HTTP | TCP | TCP_RPS | NIC_Intel-X710 | DRV_VFIO_PCI | IMIX

 | ... | eth-ip4tcphttp-wrk8u8c50con
 |
 | Suite Setup | Setup suite single link | wrk
 | ... | eth-ip4tcphttp-wrk8u8c50con
 |
 | Suite Setup | Setup suite single link | wrk