From a95c54b7821596402e0aa7136cd7d1de71a5b187 Mon Sep 17 00:00:00 2001 From: Tibor Frank Date: Mon, 4 Dec 2017 16:41:57 +0100 Subject: [PATCH] CSIT-866: wrk onboarding in CSIT - CSIT-867: Low Level Description - CSIT-868: wrk traffic profile - parsing - CSIT-869: wrk implementation into CSIT Change-Id: I65e1037f5ae05b3a5b2020e4a6c54462766ae1b4 Signed-off-by: Tibor Frank --- bootstrap-verify-perf.sh | 1 - docs/tag_documentation.rst | 16 ++ resources/libraries/python/IPUtil.py | 52 +++- resources/libraries/python/TrafficGenerator.py | 18 ++ resources/libraries/python/VppConfigGenerator.py | 92 ++++++- resources/libraries/python/constants.py | 2 +- resources/libraries/python/tcp.py | 36 +++ resources/libraries/python/topology.py | 4 +- .../robot/performance/performance_setup.robot | 54 +++- resources/libraries/robot/tcp/tcp_setup.robot | 43 +++ resources/libraries/robot/wrk/wrk_utils.robot | 78 ++++++ resources/templates/vat/start_http_server.vat | 1 + resources/tools/__init__.py | 16 ++ resources/tools/wrk/__init__.py | 16 ++ resources/tools/wrk/doc/wrk_lld.rst | 293 +++++++++++++++++++++ resources/tools/wrk/wrk.py | 291 ++++++++++++++++++++ resources/tools/wrk/wrk_errors.py | 55 ++++ resources/tools/wrk/wrk_traffic_profile_parser.py | 286 ++++++++++++++++++++ resources/tools/wrk/wrk_utils.sh | 290 ++++++++++++++++++++ .../wrk/wrk-sf-2n-ethip4tcphttp-1u1c1con-cps.yaml | 47 ++++ .../wrk/wrk-sf-2n-ethip4tcphttp-1u1c50con-bw.yaml | 47 ++++ .../wrk/wrk-sf-2n-ethip4tcphttp-1u1c50con-rps.yaml | 47 ++++ .../tcp/10ge2p1x520-ethip4tcphttp-httpserver.robot | 127 +++++++++ 23 files changed, 1905 insertions(+), 7 deletions(-) create mode 100644 resources/libraries/python/tcp.py create mode 100644 resources/libraries/robot/tcp/tcp_setup.robot create mode 100644 resources/libraries/robot/wrk/wrk_utils.robot create mode 100644 resources/templates/vat/start_http_server.vat create mode 100644 resources/tools/__init__.py create mode 100644 resources/tools/wrk/__init__.py create mode 100644 resources/tools/wrk/doc/wrk_lld.rst create mode 100644 resources/tools/wrk/wrk.py create mode 100644 resources/tools/wrk/wrk_errors.py create mode 100644 resources/tools/wrk/wrk_traffic_profile_parser.py create mode 100755 resources/tools/wrk/wrk_utils.sh create mode 100644 resources/traffic_profiles/wrk/wrk-sf-2n-ethip4tcphttp-1u1c1con-cps.yaml create mode 100644 resources/traffic_profiles/wrk/wrk-sf-2n-ethip4tcphttp-1u1c50con-bw.yaml create mode 100644 resources/traffic_profiles/wrk/wrk-sf-2n-ethip4tcphttp-1u1c50con-rps.yaml create mode 100644 tests/vpp/perf/tcp/10ge2p1x520-ethip4tcphttp-httpserver.robot diff --git a/bootstrap-verify-perf.sh b/bootstrap-verify-perf.sh index 4b86c6ea37..e2d37e2d8c 100755 --- a/bootstrap-verify-perf.sh +++ b/bootstrap-verify-perf.sh @@ -244,7 +244,6 @@ case "$TEST_TAG" in --include pdrdiscANDnic_intel-xl710AND1t1cANDipsechwANDbase \ --include pdrdiscANDnic_intel-xl710AND2t2cANDipsechwANDbase \ tests/ - RETURN_STATUS=$(echo $?) ;; VPP-VERIFY-PERF-IP4 ) pybot ${PYBOT_ARGS} \ diff --git a/docs/tag_documentation.rst b/docs/tag_documentation.rst index c6064c61a3..165feeec0a 100644 --- a/docs/tag_documentation.rst +++ b/docs/tag_documentation.rst @@ -235,6 +235,22 @@ Test type tags Functional test cases for TLDK. +.. topic:: TCP + + Tests which use TCP. + +.. topic:: TCP_CPS + + Performance tests which measure connections per second using http requests. + +.. topic:: TCP_RPS + + Performance tests which measure requests per second using http requests. + +.. topic:: HTTP + + Tests which use HTTP. + Forwarding mode tags -------------------- diff --git a/resources/libraries/python/IPUtil.py b/resources/libraries/python/IPUtil.py index d2f2adcf28..e215b301b9 100644 --- a/resources/libraries/python/IPUtil.py +++ b/resources/libraries/python/IPUtil.py @@ -1,4 +1,4 @@ -# Copyright (c) 2016 Cisco and/or its affiliates. +# Copyright (c) 2018 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -13,6 +13,8 @@ """Common IP utilities library.""" +import re + from ipaddress import IPv4Network, ip_address from resources.libraries.python.ssh import SSH @@ -137,6 +139,54 @@ class IPUtil(object): cmd = 'sysctl -w net.{0}.ip_forward=1'.format(ip_ver) exec_cmd_no_error(node, cmd, sudo=True) + @staticmethod + def get_linux_interface_name(node, pci_addr): + """Get the interface name. + + :param node: Node where to execute command. + :param pci_addr: PCI address + :type node: dict + :type pci_addr: str + :returns: Interface name + :rtype: str + :raises RuntimeError: If cannot get the information about interfaces. + """ + + regex_intf_info = r"pci@" \ + r"([0-9a-f]{4}:[0-9a-f]{2}:[0-9a-f]{2}.[0-9a-f])\s*" \ + r"([a-zA-Z0-9]*)\s*network" + + cmd = "lshw -class network -businfo" + ret_code, stdout, stderr = exec_cmd(node, cmd, timeout=30, sudo=True) + if ret_code != 0: + raise RuntimeError('Could not get information about interfaces, ' + 'reason:{0}'.format(stderr)) + + for line in stdout.splitlines()[2:]: + try: + if re.search(regex_intf_info, line).group(1) == pci_addr: + return re.search(regex_intf_info, line).group(2) + except AttributeError: + continue + return None + + @staticmethod + def set_linux_interface_up(node, interface): + """Set the specified interface up. + + :param node: Node where to execute command. + :param interface: Interface in namespace. + :type node: dict + :type interface: str + :raises RuntimeError: If the interface could not be set up. + """ + + cmd = "ip link set {0} up".format(interface) + ret_code, _, stderr = exec_cmd(node, cmd, timeout=30, sudo=True) + if ret_code != 0: + raise RuntimeError('Could not set the interface up, reason:{0}'. + format(stderr)) + @staticmethod def set_linux_interface_ip(node, interface, ip_addr, prefix, namespace=None): diff --git a/resources/libraries/python/TrafficGenerator.py b/resources/libraries/python/TrafficGenerator.py index 698b67ead2..f363fe3f55 100644 --- a/resources/libraries/python/TrafficGenerator.py +++ b/resources/libraries/python/TrafficGenerator.py @@ -307,6 +307,24 @@ class TrafficGenerator(object): # critical error occurred raise RuntimeError('t-rex-64 startup failed') + @staticmethod + def is_trex_running(node): + """Check if TRex is running using pidof. + + :param node: Traffic generator node. + :type node: dict + :returns: True if TRex is running otherwise False. + :rtype: bool + :raises: RuntimeError if node type is not a TG. + """ + if node['type'] != NodeType.TG: + raise RuntimeError('Node type is not a TG') + + ssh = SSH() + ssh.connect(node) + ret, _, _ = ssh.exec_command_sudo("pidof t-rex") + return bool(int(ret) == 0) + @staticmethod def teardown_traffic_generator(node): """TG teardown. diff --git a/resources/libraries/python/VppConfigGenerator.py b/resources/libraries/python/VppConfigGenerator.py index c1bde49742..d4fde0a6c8 100644 --- a/resources/libraries/python/VppConfigGenerator.py +++ b/resources/libraries/python/VppConfigGenerator.py @@ -1,4 +1,4 @@ -# Copyright (c) 2016 Cisco and/or its affiliates. +# Copyright (c) 2018 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -382,6 +382,96 @@ class VppConfigGenerator(object): path = ['nat'] self.add_config_item(self._nodeconfig, value, path) + def add_tcp_preallocated_connections(self, value): + """Add TCP pre-allocated connections. + + :param value: The number of pre-allocated connections. + :type value: int + """ + path = ['tcp', 'preallocated-connections'] + self.add_config_item(self._nodeconfig, value, path) + + def add_tcp_preallocated_half_open_connections(self, value): + """Add TCP pre-allocated half open connections. + + :param value: The number of pre-allocated half open connections. + :type value: int + """ + path = ['tcp', 'preallocated-half-open-connections'] + self.add_config_item(self._nodeconfig, value, path) + + def add_session_event_queue_length(self, value): + """Add session event queue length. + + :param value: Session event queue length. + :type value: int + """ + path = ['session', 'event-queue-length'] + self.add_config_item(self._nodeconfig, value, path) + + def add_session_preallocated_sessions(self, value): + """Add the number of pre-allocated sessions. + + :param value: Number of pre-allocated sessions. + :type value: int + """ + path = ['session', 'preallocated-sessions'] + self.add_config_item(self._nodeconfig, value, path) + + def add_session_v4_session_table_buckets(self, value): + """Add number of v4 session table buckets to the config. + + :param value: Number of v4 session table buckets. + :type value: int + """ + path = ['session', 'v4-session-table-buckets'] + self.add_config_item(self._nodeconfig, value, path) + + def add_session_v4_session_table_memory(self, value): + """Add the size of v4 session table memory. + + :param value: Size of v4 session table memory. + :type value: str + """ + path = ['session', 'v4-session-table-memory'] + self.add_config_item(self._nodeconfig, value, path) + + def add_session_v4_halfopen_table_buckets(self, value): + """Add the number of v4 halfopen table buckets. + + :param value: Number of v4 halfopen table buckets. + :type value: int + """ + path = ['session', 'v4-halfopen-table-buckets'] + self.add_config_item(self._nodeconfig, value, path) + + def add_session_v4_halfopen_table_memory(self, value): + """Add the size of v4 halfopen table memory. + + :param value: Size of v4 halfopen table memory. + :type value: str + """ + path = ['session', 'v4-halfopen-table-memory'] + self.add_config_item(self._nodeconfig, value, path) + + def add_session_local_endpoints_table_buckets(self, value): + """Add the number of local endpoints table buckets. + + :param value: Number of local endpoints table buckets. + :type value: int + """ + path = ['session', 'local-endpoints-table-buckets'] + self.add_config_item(self._nodeconfig, value, path) + + def add_session_local_endpoints_table_memory(self, value): + """Add the size of local endpoints table memory. + + :param value: Size of local endpoints table memory. + :type value: str + """ + path = ['session', 'local-endpoints-table-memory'] + self.add_config_item(self._nodeconfig, value, path) + def apply_config(self, filename=None, waittime=5, retries=12, restart_vpp=True): """Generate and apply VPP configuration for node. diff --git a/resources/libraries/python/constants.py b/resources/libraries/python/constants.py index 30f7531947..c3f1551c55 100644 --- a/resources/libraries/python/constants.py +++ b/resources/libraries/python/constants.py @@ -1,4 +1,4 @@ -# Copyright (c) 2016 Cisco and/or its affiliates. +# Copyright (c) 2018 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: diff --git a/resources/libraries/python/tcp.py b/resources/libraries/python/tcp.py new file mode 100644 index 0000000000..5ae1ebf929 --- /dev/null +++ b/resources/libraries/python/tcp.py @@ -0,0 +1,36 @@ +# Copyright (c) 2018 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""TCP util library. +""" + +from resources.libraries.python.VatExecutor import VatTerminal + + +class TCPUtils(object): + """Implementation of the TCP utilities. + """ + + def __init__(self): + pass + + @staticmethod + def start_http_server(node): + """Start HTTP server on the given node. + + :param node: Node to start HTTP server on. + :type node: dict + """ + + with VatTerminal(node) as vat: + vat.vat_terminal_exec_cmd_from_template("start_http_server.vat") diff --git a/resources/libraries/python/topology.py b/resources/libraries/python/topology.py index 94652dbd23..38e08d23d3 100644 --- a/resources/libraries/python/topology.py +++ b/resources/libraries/python/topology.py @@ -1,4 +1,4 @@ -# Copyright (c) 2016 Cisco and/or its affiliates. +# Copyright (c) 2018 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -21,7 +21,7 @@ from robot.api import logger from robot.libraries.BuiltIn import BuiltIn, RobotNotRunningError from robot.api.deco import keyword -__all__ = ["DICT__nodes", 'Topology'] +__all__ = ["DICT__nodes", 'Topology', 'NodeType'] def load_topo_from_yaml(): diff --git a/resources/libraries/robot/performance/performance_setup.robot b/resources/libraries/robot/performance/performance_setup.robot index 0453fad275..0e909aa455 100644 --- a/resources/libraries/robot/performance/performance_setup.robot +++ b/resources/libraries/robot/performance/performance_setup.robot @@ -1,4 +1,4 @@ -# Copyright (c) 2017 Cisco and/or its affiliates. +# Copyright (c) 2018 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -13,6 +13,7 @@ *** Settings *** | Library | resources.libraries.python.DUTSetup +| Library | resources.tools.wrk.wrk | Resource | resources/libraries/robot/performance/performance_configuration.robot | Resource | resources/libraries/robot/performance/performance_utils.robot | Documentation | Performance suite keywords - Suite and test setups and @@ -385,6 +386,45 @@ | | Configure VPP in all 'VNF' containers | | Install VPP in all 'VNF' containers +| Set up 3-node performance topology with wrk and DUT's NIC model +| | [Documentation] +| | ... | Suite preparation phase that setup default startup configuration of +| | ... | VPP on all DUTs. Updates interfaces on all nodes and setup global +| | ... | variables used in test cases based on interface model provided as an +| | ... | argument. Installs the traffic generator. +| | ... +| | ... | *Arguments:* +| | ... | - iface_model - Interface model. Type: string +| | ... +| | ... | *Example:* +| | ... +| | ... | \| Set up 3-node performance topology with wrk and DUT's NIC model\ +| | ... | \| Intel-X520-DA2 \| +| | ... +| | [Arguments] | ${iface_model} +| | ... +| | Set variables in 3-node circular topology with DUT interface model +| | ... | ${iface_model} +| | Iface update numa node | ${tg} +# Make sure TRex is stopped +| | ${running}= | Is TRex running | ${tg} +| | Run keyword if | ${running}==${True} | Teardown traffic generator | ${tg} +| | ${curr_driver}= | Get PCI dev driver | ${tg} +| | ... | ${tg['interfaces']['${tg_if1}']['pci_address']} +| | Run keyword if | '${curr_driver}'!='${None}' +| | ... | PCI Driver Unbind | ${tg} | +| | ... | ${tg['interfaces']['${tg_if1}']['pci_address']} +# Bind tg_if1 to driver specified in the topology +| | ${driver}= | Get Variable Value | ${tg['interfaces']['${tg_if1}']['driver']} +| | PCI Driver Bind | ${tg} +| | ... | ${tg['interfaces']['${tg_if1}']['pci_address']} | ${driver} +# Set IP on tg_if1 +| | ${intf_name}= | Get Linux interface name | ${tg} +| | ... | ${tg['interfaces']['${tg_if1}']['pci_address']} +| | Set Linux interface IP | ${tg} | ${intf_name} | 192.168.10.1 | 24 +| | Set Linux interface up | ${tg} | ${intf_name} +| | Install wrk | ${tg} + # Suite teardowns | Tear down 3-node performance topology @@ -480,6 +520,18 @@ | | Show VAT History On All DUTs | ${nodes} | | Show statistics on all DUTs | ${nodes} +| Tear down performance test with wrk +| | [Documentation] | Common test teardown for ndrdisc and pdrdisc performance \ +| | ... | tests. +| | ... +| | ... | *Example:* +| | ... +| | ... | \| Tear down performance test with wrk \| +| | ... +| | Remove All Added Ports On All DUTs From Topology | ${nodes} +| | Show VAT History On All DUTs | ${nodes} +| | Show statistics on all DUTs | ${nodes} + | Tear down performance test with vhost and VM with dpdk-testpmd | | [Documentation] | Common test teardown for performance tests which use | | ... | vhost(s) and VM(s) with dpdk-testpmd. diff --git a/resources/libraries/robot/tcp/tcp_setup.robot b/resources/libraries/robot/tcp/tcp_setup.robot new file mode 100644 index 0000000000..09f6afd592 --- /dev/null +++ b/resources/libraries/robot/tcp/tcp_setup.robot @@ -0,0 +1,43 @@ +# Copyright (c) 2018 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +*** Settings *** +| Library | resources.libraries.python.IPv4Util.IPv4Util +| Library | resources.libraries.python.InterfaceUtil +| Library | resources.libraries.python.tcp.TCPUtils +| Resource | resources/libraries/robot/ip/ip4.robot +| ... +| Documentation | L2 keywords to set up VPP to test tcp. + +*** Keywords *** +| Set up HTTP server on the VPP node +| | [Documentation] +| | ... | Configure IP address on the port, set it up and start HTTP server on +| | ... | the VPP. +| | ... +| | ... | *Arguments:* +| | ... | - ${dut1_if1_ip4} - IP address to be set on the dut1_if1 interface. +| | ... | Type: string +| | ... | - ${ip4_len} - Length of the netmask. Type: integer +| | ... +| | ... | *Example:* +| | ... +| | ... | \| Set up HTTP server on the VPP node \| 192.168.10.2 \| 24 \| +| | ... +| | [Arguments] | ${dut1_if1_ip4} | ${ip4_len} +| | ... +| | Set Interface State | ${dut1} | ${dut1_if1} | up +| | Set Interface Address | ${dut1} | ${dut1_if1} | ${dut1_if1_ip4} | ${ip4_len} +| | Vpp Node Interfaces Ready Wait | ${dut1} +| | Start HTTP server | ${dut1} +| | Sleep | 30 diff --git a/resources/libraries/robot/wrk/wrk_utils.robot b/resources/libraries/robot/wrk/wrk_utils.robot new file mode 100644 index 0000000000..fd18a5d686 --- /dev/null +++ b/resources/libraries/robot/wrk/wrk_utils.robot @@ -0,0 +1,78 @@ +# Copyright (c) 2018 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +*** Settings *** +| Library | resources.tools.wrk.wrk +| Library | resources.libraries.python.IPUtil +| Library | resources.libraries.python.DUTSetup +| Library | resources.libraries.python.TrafficGenerator +| Library | resources.libraries.python.topology.Topology +| Resource | resources/libraries/robot/performance/performance_setup.robot +| ... +| Documentation | L2 keywords to set up wrk and to measure performance +| ... | parameters using wrk. + +*** Keywords *** +| Measure throughput +| | [Documentation] +| | ... | Measure throughput using wrk. +| | ... +| | ... | *Arguments:* +| | ... | - ${profile} - The name of the wrk traffic profile defining the +| | ... | traffic. Type: string +| | ... +| | ... | *Example:* +| | ... +| | ... | \| Measure throughput \| wrk-bw-1url-1core-50con \| +| | ... +| | [Arguments] | ${profile} +| | ... +| | ${tg_numa}= | Get interfaces numa node | ${tg} | ${tg_if1} | ${tg_if2} +| | ${output}= | Run wrk | ${tg} | ${profile} | ${tg_numa} | bw +| | Set test message | ${output} + +| Measure requests per second +| | [Documentation] +| | ... | Measure number of requests per second using wrk. +| | ... +| | ... | *Arguments:* +| | ... | - ${profile} - The name of the wrk traffic profile defining the +| | ... | traffic. Type: string +| | ... +| | ... | *Example:* +| | ... +| | ... | \| Measure requests per second \| wrk-bw-1url-1core-50con \| +| | ... +| | [Arguments] | ${profile} +| | ... +| | ${tg_numa}= | Get interfaces numa node | ${tg} | ${tg_if1} | ${tg_if2} +| | ${output}= | Run wrk | ${tg} | ${profile} | ${tg_numa} | rps +| | Set test message | ${output} + +| Measure connections per second +| | [Documentation] +| | ... | Measure number of connections per second using wrk. +| | ... +| | ... | *Arguments:* +| | ... | - ${profile} - The name of the wrk traffic profile defining the +| | ... | traffic. Type: string +| | ... +| | ... | *Example:* +| | ... +| | ... | \| Measure connections per second \| wrk-bw-1url-1core-50con \| +| | ... +| | [Arguments] | ${profile} +| | ... +| | ${tg_numa}= | Get interfaces numa node | ${tg} | ${tg_if1} | ${tg_if2} +| | ${output}= | Run wrk | ${tg} | ${profile} | ${tg_numa} | cps +| | Set test message | ${output} diff --git a/resources/templates/vat/start_http_server.vat b/resources/templates/vat/start_http_server.vat new file mode 100644 index 0000000000..1d00285a91 --- /dev/null +++ b/resources/templates/vat/start_http_server.vat @@ -0,0 +1 @@ +exec test http server static \ No newline at end of file diff --git a/resources/tools/__init__.py b/resources/tools/__init__.py new file mode 100644 index 0000000000..e9d6d476eb --- /dev/null +++ b/resources/tools/__init__.py @@ -0,0 +1,16 @@ +# Copyright (c) 2018 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +__init__ file for directory tools +""" diff --git a/resources/tools/wrk/__init__.py b/resources/tools/wrk/__init__.py new file mode 100644 index 0000000000..977169c00f --- /dev/null +++ b/resources/tools/wrk/__init__.py @@ -0,0 +1,16 @@ +# Copyright (c) 2018 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +__init__ file for directory tools/wrk +""" diff --git a/resources/tools/wrk/doc/wrk_lld.rst b/resources/tools/wrk/doc/wrk_lld.rst new file mode 100644 index 0000000000..1437fd8948 --- /dev/null +++ b/resources/tools/wrk/doc/wrk_lld.rst @@ -0,0 +1,293 @@ +Onboarding of wrk as a http traffic generator in CSIT +----------------------------------------------------- + +wrk is a modern HTTP benchmarking tool capable of generating significant +load when run on a single multi-core CPU. + +An optional LuaJIT script can perform HTTP request generation, response +processing, and custom reporting. + + +wrk installation on TG node +''''''''''''''''''''''''''' + +**Procedure** + + #. Check if wrk is installed on the TG node. + #. If not, install it. + +**wrk installation** + +:: + + # Install pre-requisites: + sudo apt-get install build-essential libssl-dev git -y + + # Get the specified version: + wget ${WRK_DWNLD_PATH}/${WRK_TAR} + tar xzf ${WRK_TAR} + cd wrk-${WRK_VERSION} + + # Build the wrk: + cd wrk + make + + # Move the executable to somewhere in the PATH, e.q: + sudo cp wrk /usr/local/bin + + +wrk traffic profile +''''''''''''''''''' + +**The traffic profile can include these items:** + + - List of URLs - mandatory, + - The first CPU used to run wrk - mandatory, + - Number of CPUs used for wrk - mandatory, + - Test duration - mandatory, + - Number of threads - mandatory, + - Number of connections - mandatory, + - LuaJIT script - optional, defaults to no script, + - HTTP header - optional, defaults to no header, + - Latency - optional, defaults to False, + - Timeout - optional, defaults to wrk default. + +**List of URLs** + +List of URLs for requests. Each URL is requested in a separate instance of wrk. +Type: list + +*Example:* + +:: + + urls: + - "http://192.168.1.1/1kB.bin" + - "http://192.168.1.2/1kB.bin" + - "http://192.168.1.3/1kB.bin" + +**The first CPU used to run wrk** +The first CPU used to run wrk. The other CPUs follow this one. +Type: integer + +*Example:* + +:: + + first-cpu: 1 + +**Number of CPUs used for wrk** + +The number of CPUs used for wrk. The number of CPUs must be a multiplication +of the number of URLs. +Type: integer + +*Example:* + +:: + + cpus: 6 + +.. note:: + + The combinations of URLs and a number of CPUs create following use cases: + + - One URL and one CPU - One instance of wrk sends one request (URL) via + one NIC + - One URL and n CPUs - n instances of wrk send the same request (URL) + via one or more NICs + - n URLs and n CPUs - n instances of wrk send n requests (URL) via one + or more NICs + - n URLs and m CPUs, m = a * n - m instances of wrk send n requests + (URL) via one or more NICs + +**Test duration** + +Duration of the test in seconds. +Type: integer + +*Example:* + +:: + + duration: 30 + +**Number of threads** + +Total number of threads to use by wrk to send traffic. +Type: integer + +*Example:* + +:: + + nr-of-threads: 1 + +**Number of connections** + +Total number of HTTP connections to keep open with each thread handling +N = connections / threads. +Type: integer + +*Example:* + +:: + + nr-of-connections: 50 + +**LuaJIT script** + +Path to LuaJIT script. +Type: string + +For more information see: https://github.com/wg/wrk/blob/master/SCRIPTING + +*Example:* + +:: + + script: "scripts/report.lua" + +**HTTP header** + +HTTP header to add to request. +Type: string (taken as it is) or dictionary + +*Example:* + +:: + + # Dictionary: + header: + Connection: "close" + +or + +:: + + # String: + header: "Connection: close" + +**Latency** + +Print detailed latency statistics. +Type: boolean + +*Example:* + +:: + + latency: False + +**Timeout** + +Record a timeout if a response is not received within this amount of time. +Type: integer + +:: + + timeout: 5 + +**Examples of a wrk traffic profile** + +*Get the number of connections per second:* + +- Use 3 CPUs to send 3 different requests via 3 NICs. +- The test takes 30 seconds. +- wrk sends traffic in one thread per CPU. +- There will be open max 50 connection at the same time. +- The header is set to 'Connection: "close"' so wrk opens separate connection + for each request. Then the number of requests equals to the number of + connections. +- Timeout for responses from the server is set to 5 seconds. + +:: + + urls: + - "http://192.168.1.1/0B.bin" + - "http://192.168.1.2/0B.bin" + - "http://192.168.1.3/0B.bin" + cpus: 3 + duration: 30 + nr-of-threads: 1 + nr-of-connections: 50 + header: + Connection: "close" + timeout: 5 + +*Get the number of requests per second:* + +- Use 3 CPUs to send 3 different requests via 3 NICs. +- The test takes 30 seconds. +- wrk sends traffic in one thread per CPU. +- There will be max 50 concurrent open connections. + +:: + + urls: + - "http://192.168.1.1/1kB.bin" + - "http://192.168.1.2/1kB.bin" + - "http://192.168.1.3/1kB.bin" + cpus: 3 + duration: 30 + nr-of-threads: 1 + nr-of-connections: 50 + +*Get the bandwidth:* + +- Use 3 CPUs to send 3 different requests via 3 NICs. +- The test takes 30 seconds. +- wrk sends traffic in one thread per CPU. +- There will be open max 50 connection at the same time. +- Timeout for responses from the server is set to 5 seconds. + +:: + + urls: + - "http://192.168.1.1/1MB.bin" + - "http://192.168.1.2/1MB.bin" + - "http://192.168.1.3/1MB.bin" + cpus: 3 + duration: 30 + nr-of-threads: 1 + nr-of-connections: 50 + timeout: 5 + + +Running wrk +''''''''''' + +**Suite setup phase** + +CSIT framework checks if wrk is installed on the TG node. If not, or if the +installation is forced, it installs it on the TG node. + +*Procedure:* + + #. Make sure TRex is stopped. + #. Bind used TG interfaces to corresponding drivers (defined in the topology + file). + #. If the wrk installation is forced: + + - Destroy existing wrk + + #. If the wrk installation is not forced: + + - Check if wrk is installed. + - If installed, exit. + + #. Clone wrk from git (https://github.com/wg/wrk.git) + #. Build wrk. + #. Copy the executable to /usr/local/bin so it is in the PATH. + +**Test phase** + +*Procedure:* + +#. Read the wrk traffic profile. +#. Verify the profile. +#. Use the information from the profile to set the wrk parameters. +#. Run wrk. +#. Read the output. +#. Evaluate and log the output. + diff --git a/resources/tools/wrk/wrk.py b/resources/tools/wrk/wrk.py new file mode 100644 index 0000000000..33cfd08174 --- /dev/null +++ b/resources/tools/wrk/wrk.py @@ -0,0 +1,291 @@ +# Copyright (c) 2018 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""wrk implementation into CSIT framework. +""" + +import re + +from robot.api import logger + +from resources.libraries.python.ssh import SSH +from resources.libraries.python.topology import NodeType +from resources.libraries.python.CpuUtils import CpuUtils +from resources.libraries.python.constants import Constants + +from resources.tools.wrk.wrk_traffic_profile_parser import WrkTrafficProfile +from resources.tools.wrk.wrk_errors import WrkError + + +REGEX_LATENCY_STATS = \ + r"Latency\s*" \ + r"(\d*\.*\d*\S*)\s*" \ + r"(\d*\.*\d*\S*)\s*" \ + r"(\d*\.*\d*\S*)\s*" \ + r"(\d*\.*\d*\%)" +REGEX_RPS_STATS = \ + r"Req/Sec\s*" \ + r"(\d*\.*\d*\S*)\s*" \ + r"(\d*\.*\d*\S*)\s*" \ + r"(\d*\.*\d*\S*)\s*" \ + r"(\d*\.*\d*\%)" +REGEX_RPS = r"Requests/sec:\s*" \ + r"(\d*\.*\S*)" +REGEX_BW = r"Transfer/sec:\s*" \ + r"(\d*\.*\S*)" +REGEX_LATENCY_DIST = \ + r"Latency Distribution\n" \ + r"\s*50\%\s*(\d*\.*\d*\D*)\n" \ + r"\s*75\%\s*(\d*\.*\d*\D*)\n" \ + r"\s*90\%\s*(\d*\.*\d*\D*)\n" \ + r"\s*99\%\s*(\d*\.*\d*\D*)\n" + +# Split number and multiplicand, e.g. 14.25k --> 14.25 and k +REGEX_NUM = r"(\d*\.*\d*)(\D*)" + + +def install_wrk(tg_node): + """Install wrk on the TG node. + + :param tg_node: Traffic generator node. + :type tg_node: dict + :raises: RuntimeError if the given node is not a TG node or if the + installation fails. + """ + + if tg_node['type'] != NodeType.TG: + raise RuntimeError('Node type is not a TG.') + + ssh = SSH() + ssh.connect(tg_node) + + ret, _, _ = ssh.exec_command( + "sudo -E " + "sh -c '{0}/resources/tools/wrk/wrk_utils.sh install false'". + format(Constants.REMOTE_FW_DIR), timeout=1800) + if int(ret) != 0: + raise RuntimeError('Installation of wrk on TG node failed.') + + +def destroy_wrk(tg_node): + """Destroy wrk on the TG node. + + :param tg_node: Traffic generator node. + :type tg_node: dict + :raises: RuntimeError if the given node is not a TG node or the removal of + wrk failed. + """ + + if tg_node['type'] != NodeType.TG: + raise RuntimeError('Node type is not a TG.') + + ssh = SSH() + ssh.connect(tg_node) + + ret, _, _ = ssh.exec_command( + "sudo -E " + "sh -c '{0}/resources/tools/wrk/wrk_utils.sh destroy'". + format(Constants.REMOTE_FW_DIR), timeout=1800) + if int(ret) != 0: + raise RuntimeError('Removal of wrk from the TG node failed.') + + +def run_wrk(tg_node, profile_name, tg_numa, test_type): + """Send the traffic as defined in the profile. + + :param tg_node: Traffic generator node. + :param profile_name: The name of wrk traffic profile. + :param tg_numa: Numa node on which wrk will run. + :param test_type: The type of the tests: cps, rps, bw + :type profile_name: str + :type tg_node: dict + :type tg_numa: int + :type test_type: str + :returns: Message with measured data. + :rtype: str + :raises: RuntimeError if node type is not a TG. + """ + + if tg_node['type'] != NodeType.TG: + raise RuntimeError('Node type is not a TG.') + + # Parse and validate the profile + profile_path = ("resources/traffic_profiles/wrk/{0}.yaml". + format(profile_name)) + profile = WrkTrafficProfile(profile_path).traffic_profile + + cores = CpuUtils.cpu_list_per_node(tg_node, tg_numa) + first_cpu = cores[profile["first-cpu"]] + + if len(profile["urls"]) == 1 and profile["cpus"] == 1: + params = [ + "traffic_1_url_1_core", + str(first_cpu), + str(profile["nr-of-threads"]), + str(profile["nr-of-connections"]), + "{0}s".format(profile["duration"]), + "'{0}'".format(profile["header"]), + str(profile["timeout"]), + str(profile["script"]), + str(profile["latency"]), + "'{0}'".format(" ".join(profile["urls"])) + ] + elif len(profile["urls"]) == profile["cpus"]: + params = [ + "traffic_n_urls_n_cores", + str(first_cpu), + str(profile["nr-of-threads"]), + str(profile["nr-of-connections"]), + "{0}s".format(profile["duration"]), + "'{0}'".format(profile["header"]), + str(profile["timeout"]), + str(profile["script"]), + str(profile["latency"]), + "'{0}'".format(" ".join(profile["urls"])) + ] + else: + params = [ + "traffic_n_urls_m_cores", + str(first_cpu), + str(profile["cpus"] / len(profile["urls"])), + str(profile["nr-of-threads"]), + str(profile["nr-of-connections"]), + "{0}s".format(profile["duration"]), + "'{0}'".format(profile["header"]), + str(profile["timeout"]), + str(profile["script"]), + str(profile["latency"]), + "'{0}'".format(" ".join(profile["urls"])) + ] + args = " ".join(params) + + ssh = SSH() + ssh.connect(tg_node) + + ret, stdout, _ = ssh.exec_command( + "{0}/resources/tools/wrk/wrk_utils.sh {1}". + format(Constants.REMOTE_FW_DIR, args), timeout=1800) + if int(ret) != 0: + raise RuntimeError('wrk runtime error.') + + stats = _parse_wrk_output(stdout) + + log_msg = "\nMeasured values:\n" + if test_type == "cps": + log_msg += "Connections/sec: Avg / Stdev / Max / +/- Stdev\n" + for item in stats["rps-stats-lst"]: + log_msg += "{0} / {1} / {2} / {3}\n".format(*item) + log_msg += "Total cps: {0}cps\n".format(stats["rps-sum"]) + elif test_type == "rps": + log_msg += "Requests/sec: Avg / Stdev / Max / +/- Stdev\n" + for item in stats["rps-stats-lst"]: + log_msg += "{0} / {1} / {2} / {3}\n".format(*item) + log_msg += "Total rps: {0}cps\n".format(stats["rps-sum"]) + elif test_type == "bw": + log_msg += "Transfer/sec: {0}Bps".format(stats["bw-sum"]) + + logger.info(log_msg) + + return log_msg + + +def _parse_wrk_output(msg): + """Parse the wrk stdout with the results. + + :param msg: stdout of wrk. + :type msg: str + :returns: Parsed results. + :rtype: dict + :raises: WrkError if the message does not include the results. + """ + + if "Thread Stats" not in msg: + raise WrkError("The output of wrk does not include the results.") + + msg_lst = msg.splitlines(False) + + stats = { + "latency-dist-lst": list(), + "latency-stats-lst": list(), + "rps-stats-lst": list(), + "rps-lst": list(), + "bw-lst": list(), + "rps-sum": 0, + "bw-sum": None + } + + for line in msg_lst: + if "Latency Distribution" in line: + # Latency distribution - 50%, 75%, 90%, 99% + pass + elif "Latency" in line: + # Latency statistics - Avg, Stdev, Max, +/- Stdev + pass + elif "Req/Sec" in line: + # rps statistics - Avg, Stdev, Max, +/- Stdev + stats["rps-stats-lst"].append(( + _evaluate_number(re.search(REGEX_RPS_STATS, line).group(1)), + _evaluate_number(re.search(REGEX_RPS_STATS, line).group(2)), + _evaluate_number(re.search(REGEX_RPS_STATS, line).group(3)), + _evaluate_number(re.search(REGEX_RPS_STATS, line).group(4)))) + elif "Requests/sec:" in line: + # rps (cps) + stats["rps-lst"].append( + _evaluate_number(re.search(REGEX_RPS, line).group(1))) + elif "Transfer/sec:" in line: + # BW + stats["bw-lst"].append( + _evaluate_number(re.search(REGEX_BW, line).group(1))) + + for item in stats["rps-stats-lst"]: + stats["rps-sum"] += item[0] + stats["bw-sum"] = sum(stats["bw-lst"]) + + return stats + + +def _evaluate_number(num): + """Evaluate the numeric value of the number with multiplicands, e.g.: + 12.25k --> 12250 + + :param num: Number to evaluate. + :type num: str + :returns: Evaluated number. + :rtype: float + :raises: WrkError if it is not possible to evaluate the given number. + """ + + val = re.search(REGEX_NUM, num) + try: + val_num = float(val.group(1)) + except ValueError: + raise WrkError("The output of wrk does not include the results " + "or the format of results has changed.") + val_mul = val.group(2).lower() + if val_mul: + if "k" in val_mul: + val_num *= 1000 + elif "m" in val_mul: + val_num *= 1000000 + elif "g" in val_mul: + val_num *= 1000000000 + elif "b" in val_mul: + pass + elif "%" in val_mul: + pass + elif "" in val_mul: + pass + else: + raise WrkError("The multiplicand {0} is not defined.". + format(val_mul)) + return val_num diff --git a/resources/tools/wrk/wrk_errors.py b/resources/tools/wrk/wrk_errors.py new file mode 100644 index 0000000000..3173dd4223 --- /dev/null +++ b/resources/tools/wrk/wrk_errors.py @@ -0,0 +1,55 @@ +# Copyright (c) 2018 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Implementation of exceptions used in the wrk traffic generator. +""" + + +from robot.api import logger + + +class WrkError(Exception): + """Exception(s) raised by the wrk traffic generator. + + When raising this exception, put this information to the message in this + order: + - short description of the encountered problem (parameter msg), + - relevant messages if there are any collected, e.g., from caught + exception (optional parameter details), + - relevant data if there are any collected (optional parameter details). + """ + + def __init__(self, msg, details=''): + """Sets the exception message and the level. + + :param msg: Short description of the encountered problem. + :param details: Relevant messages if there are any collected, e.g.: + from caught exception (optional parameter details), or relevant data if + there are any collected (optional parameter details). + :type msg: str + :type details: str + """ + + super(WrkError, self).__init__() + self._msg = msg + self._details = details + + logger.error(self._msg) + if self._details: + logger.error(self._details) + + def __repr__(self): + return repr(self._msg) + + def __str__(self): + return str(self._msg) diff --git a/resources/tools/wrk/wrk_traffic_profile_parser.py b/resources/tools/wrk/wrk_traffic_profile_parser.py new file mode 100644 index 0000000000..e1f8365345 --- /dev/null +++ b/resources/tools/wrk/wrk_traffic_profile_parser.py @@ -0,0 +1,286 @@ +# Copyright (c) 2018 Cisco and / or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy +# of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions +# and limitations under the License. + +"""wrk traffic profile parser. + +See LLD for the structure of a wrk traffic profile. +""" + + +from os.path import isfile +from pprint import pformat + +from yaml import load, YAMLError +from robot.api import logger + +from resources.tools.wrk.wrk_errors import WrkError + + +class WrkTrafficProfile(object): + """The wrk traffic profile. + """ + + MANDATORY_PARAMS = ("urls", + "first-cpu", + "cpus", + "duration", + "nr-of-threads", + "nr-of-connections") + + def __init__(self, profile_name): + """Read the traffic profile from the yaml file. + + :param profile_name: Path to the yaml file with the profile. + :type profile_name: str + :raises: WrkError if it is not possible to parse the profile. + """ + + self._profile_name = None + self._traffic_profile = None + + self.profile_name = profile_name + + try: + with open(self.profile_name, 'r') as profile_file: + self.traffic_profile = load(profile_file) + except IOError as err: + raise WrkError(msg="An error occurred while opening the file '{0}'." + .format(self.profile_name), + details=str(err)) + except YAMLError as err: + raise WrkError(msg="An error occurred while parsing the traffic " + "profile '{0}'.".format(self.profile_name), + details=str(err)) + + self._validate_traffic_profile() + + if self.traffic_profile: + logger.debug("\nThe wrk traffic profile '{0}' is valid.\n". + format(self.profile_name)) + logger.debug("wrk traffic profile '{0}':".format(self.profile_name)) + logger.debug(pformat(self.traffic_profile)) + else: + logger.debug("\nThe wrk traffic profile '{0}' is invalid.\n". + format(self.profile_name)) + raise WrkError("\nThe wrk traffic profile '{0}' is invalid.\n". + format(self.profile_name)) + + def __repr__(self): + return pformat(self.traffic_profile) + + def __str__(self): + return pformat(self.traffic_profile) + + def _validate_traffic_profile(self): + """Validate the traffic profile. + + The specification, the structure and the rules are described in + doc/wrk_lld.rst + """ + + logger.debug("\nValidating the wrk traffic profile '{0}'...\n". + format(self.profile_name)) + + # Level 1: Check if the profile is a dictionary: + if not isinstance(self.traffic_profile, dict): + logger.error("The wrk traffic profile must be a dictionary.") + self.traffic_profile = None + return + + # Level 2: Check if all mandatory parameters are present: + is_valid = True + for param in self.MANDATORY_PARAMS: + if self.traffic_profile.get(param, None) is None: + logger.error("The parameter '{0}' in mandatory.".format(param)) + is_valid = False + if not is_valid: + self.traffic_profile = None + return + + # Level 3: Mandatory params: Check if urls is a list: + is_valid = True + if not isinstance(self.traffic_profile["urls"], list): + logger.error("The parameter 'urls' must be a list.") + is_valid = False + + # Level 3: Mandatory params: Check if cpus is a valid integer: + try: + cpus = int(self.traffic_profile["cpus"]) + if cpus < 1: + raise ValueError + self.traffic_profile["cpus"] = cpus + except ValueError: + logger.error("The parameter 'cpus' must be an integer greater than " + "1.") + is_valid = False + + # Level 3: Mandatory params: Check if first-cpu is a valid integer: + try: + first_cpu = int(self.traffic_profile["first-cpu"]) + if first_cpu < 0: + raise ValueError + self.traffic_profile["first-cpu"] = first_cpu + except ValueError: + logger.error("The parameter 'first-cpu' must be an integer greater " + "than 1.") + is_valid = False + + # Level 3: Mandatory params: Check if duration is a valid integer: + try: + duration = int(self.traffic_profile["duration"]) + if duration < 1: + raise ValueError + self.traffic_profile["duration"] = duration + except ValueError: + logger.error("The parameter 'duration' must be an integer " + "greater than 1.") + is_valid = False + + # Level 3: Mandatory params: Check if nr-of-threads is a valid integer: + try: + nr_of_threads = int(self.traffic_profile["nr-of-threads"]) + if nr_of_threads < 1: + raise ValueError + self.traffic_profile["nr-of-threads"] = nr_of_threads + except ValueError: + logger.error("The parameter 'nr-of-threads' must be an integer " + "greater than 1.") + is_valid = False + + # Level 3: Mandatory params: Check if nr-of-connections is a valid + # integer: + try: + nr_of_connections = int(self.traffic_profile["nr-of-connections"]) + if nr_of_connections < 1: + raise ValueError + self.traffic_profile["nr-of-connections"] = nr_of_connections + except ValueError: + logger.error("The parameter 'nr-of-connections' must be an integer " + "greater than 1.") + is_valid = False + + # Level 4: Optional params: Check if script is present: + script = self.traffic_profile.get("script", None) + if script is not None: + if not isinstance(script, str): + logger.error("The path to LuaJIT script in invalid") + is_valid = False + else: + if not isfile(script): + logger.error("The file '{0}' in not present.". + format(script)) + is_valid = False + else: + self.traffic_profile["script"] = None + logger.debug("The optional parameter 'LuaJIT script' is not " + "defined. No problem.") + + # Level 4: Optional params: Check if header is present: + header = self.traffic_profile.get("header", None) + if header: + if not (isinstance(header, dict) or isinstance(header, str)): + logger.error("The parameter 'header' is not valid.") + is_valid = False + else: + if isinstance(header, dict): + header_lst = list() + for key, val in header.items(): + header_lst.append("{0}: {1}".format(key, val)) + if header_lst: + self.traffic_profile["header"] = ", ".join(header_lst) + else: + logger.error("The parameter 'header' is defined but " + "empty.") + is_valid = False + else: + self.traffic_profile["header"] = None + logger.debug("The optional parameter 'header' is not defined. " + "No problem.") + + # Level 4: Optional params: Check if latency is present: + latency = self.traffic_profile.get("latency", None) + if latency is not None: + try: + latency = bool(latency) + self.traffic_profile["latency"] = latency + except ValueError: + logger.error("The parameter 'latency' must be boolean.") + is_valid = False + else: + self.traffic_profile["latency"] = False + logger.debug("The optional parameter 'latency' is not defined. " + "No problem.") + + # Level 4: Optional params: Check if timeout is present: + timeout = self.traffic_profile.get("timeout", None) + if timeout: + try: + timeout = int(timeout) + if timeout < 1: + raise ValueError + self.traffic_profile["timeout"] = timeout + except ValueError: + logger.error("The parameter 'timeout' must be integer greater " + "than 1.") + is_valid = False + else: + self.traffic_profile["timeout"] = None + logger.debug("The optional parameter 'timeout' is not defined. " + "No problem.") + + if not is_valid: + self.traffic_profile = None + return + + # Level 5: Check dependencies between parameters: + # Level 5: Check urls and cpus: + if self.traffic_profile["cpus"] % len(self.traffic_profile["urls"]): + logger.error("The number of CPUs must be a multiplication of the " + "number of URLs.") + self.traffic_profile = None + + @property + def profile_name(self): + """Getter - Profile name. + + :returns: The traffic profile file path + :rtype: str + """ + return self._profile_name + + @profile_name.setter + def profile_name(self, profile_name): + """ + + :param profile_name: + :type profile_name: str + """ + self._profile_name = profile_name + + @property + def traffic_profile(self): + """Getter: Traffic profile. + + :returns: The traffic profile. + :rtype: dict + """ + return self._traffic_profile + + @traffic_profile.setter + def traffic_profile(self, profile): + """Setter - Traffic profile. + + :param profile: The new traffic profile. + :type profile: dict + """ + self._traffic_profile = profile diff --git a/resources/tools/wrk/wrk_utils.sh b/resources/tools/wrk/wrk_utils.sh new file mode 100755 index 0000000000..2b9f6cf296 --- /dev/null +++ b/resources/tools/wrk/wrk_utils.sh @@ -0,0 +1,290 @@ +#!/bin/bash +# Copyright (c) 2018 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x + +WRK_VERSION="4.0.2" +WRK_TAR=${WRK_VERSION}".tar.gz" +WRK_DWNLD_PATH="https://github.com/wg/wrk/archive" +WRK_TARGET="/opt" +WRK_INSTALL_DIR=${WRK_TARGET}/wrk-${WRK_VERSION} + +function wrk_utils.install { + # Install wrk + + # Directory for wrk: + dir=${1} + # Force the installation: + force=${2:-false} + + # Check if wrk is installed: + if [ "${force}" = true ]; then + wrk_utils.destroy + else + which wrk + if [ $? -eq 0 ]; then + test -d ${dir}/${WRK_INSTALL_DIR} && echo "WRK already installed: ${dir}/${WRK_INSTALL_DIR}" && exit 0 + fi + fi + + # Install pre-requisites: + apt-get update + apt-get install build-essential libssl-dev -y + + # Remove previous installation: + wrk_utils.destroy + + # Change the directory: + cd ${WRK_TARGET} + + # Get the specified version: + wget ${WRK_DWNLD_PATH}/${WRK_TAR} + tar xzf ${WRK_TAR} + rm ${WRK_TAR} + cd ${WRK_INSTALL_DIR} + + # Build the wrk: + make + + # Move the executable to somewhere in the PATH: + cp wrk /usr/local/bin +} + +function wrk_utils.destroy { + # Remove wrk + + sudo rm /usr/local/bin/wrk || true + sudo rm -rf ${WRK_INSTALL_DIR} || true +} + +function wrk_utils.traffic_1_url_1_core { + # Send traffic + # - to n URL (NIC) + # - using n instances of wrk, each on separate core. + + # The CPU used for wrk + cpu=${1} + # Total number of threads to use by one instance of wrk to send traffic. + threads=${2} + # Total number of HTTP connections to keep open with each thread handling + # N = connections / threads. + connections=${3} + # Duration of the test. + duration=${4} + # HTTP header to add to request. + header=${5} + # Record a timeout if a response is not received within this amount of time. + timeout=${6} + # Path to LuaJIT script. + script=${7} + # Print detailed latency statistics. + latency=${8} + # URL to send the traffic to. + url=${9} + + if [ "${timeout}" != "None" ]; then + timeout="--timeout ${timeout}" + else + timeout="" + fi + + if [ "${latency}" = "True" ]; then + latency="--latency" + else + latency="" + fi + + if [ "${script}" != "None" ]; then + script="--script '${script}'" + else + script="" + fi + + if [ "${header}" != "None" ]; then + header="${header}" + else + header="" + fi + + taskset --cpu-list ${cpu} \ + wrk --threads ${threads} \ + --connections ${connections} \ + --duration ${duration} \ + --header "${header}" \ + ${timeout} \ + ${script} \ + ${latency} \ + ${url} +} + +function wrk_utils.traffic_n_urls_n_cores { + # Send traffic + # - to n URL (NIC) + # - using n instances of wrk, each on separate core. + + # The first CPU used for wrk + first_cpu=${1} + # Total number of threads to use by one instance of wrk to send traffic. + threads=${2} + # Total number of HTTP connections to keep open with each thread handling + # N = connections / threads. + connections=${3} + # Duration of the test. + duration=${4} + # HTTP header to add to request. + header=${5} + # Record a timeout if a response is not received within this amount of time. + timeout=${6} + # Path to LuaJIT script. + script=${7} + # Print detailed latency statistics. + latency=${8} + # URL to send the traffic to. + urls=${9} + + if [ "${timeout}" != "None" ]; then + timeout="--timeout ${timeout}" + else + timeout="" + fi + + if [ "${latency}" = "True" ]; then + latency="--latency" + else + latency="" + fi + + if [ "${script}" != "None" ]; then + script="--script '${script}'" + else + script="" + fi + + if [ "${header}" != "None" ]; then + header="${header}" + else + header="" + fi + + urls=$(echo ${urls} | tr ";" "\n") + cpu=${first_cpu} + for url in ${urls}; do + taskset --cpu-list ${cpu} \ + wrk --threads ${threads} \ + --connections ${connections} \ + --duration ${duration} \ + --header "${header}" \ + ${timeout} \ + ${script} \ + ${latency} \ + ${url} & + cpu=$((cpu+1)) + done + + sleep ${duration} + sleep 2 +} + +function wrk_utils.traffic_n_urls_m_cores { + # Send traffic + # - to n URL (NIC) + # - using m instances of wrk, each on separate core. + + # The first CPU used for wrk + first_cpu=${1} + # The last CPU used for wrk + cpus_per_url=${2} + # Total number of threads to use by one instance of wrk to send traffic. + threads=${3} + # Total number of HTTP connections to keep open with each thread handling + # N = connections / threads. + connections=${4} + # Duration of the test. + duration=${5} + # HTTP header to add to request. + header=${6} + # Record a timeout if a response is not received within this amount of time. + timeout=${7} + # Path to LuaJIT script. + script=${8} + # Print detailed latency statistics. + latency=${9} + # URL to send the traffic to. + urls=${10} + + if [ "${timeout}" != "None" ]; then + timeout="--timeout ${timeout}" + else + timeout="" + fi + + if [ "${latency}" = "True" ]; then + latency="--latency" + else + latency="" + fi + + if [ "${script}" != "None" ]; then + script="--script '${script}'" + else + script="" + fi + + if [ "${header}" != "None" ]; then + header="${header}" + else + header="" + fi + + urls=$(echo ${urls} | tr ";" "\n") + + cpu=${first_cpu} + for i in `seq 1 ${cpus_per_url}`; do + for url in ${urls}; do + taskset --cpu-list ${cpu} \ + wrk --threads ${threads} \ + --connections ${connections} \ + --duration ${duration} \ + --header "${header}" \ + ${timeout} \ + ${script} \ + ${latency} \ + ${url} & + cpu=$((cpu+1)) + done + done + + sleep ${duration} + sleep 2 +} + +args=("$@") +case ${1} in + install) + force=${2} + wrk_utils.install ${force} + ;; + destroy) + wrk_utils.destroy + ;; + traffic_1_url_1_core) + wrk_utils.traffic_1_url_1_core "${args[@]:1}" + ;; + traffic_n_urls_n_cores) + wrk_utils.traffic_n_urls_n_cores "${args[@]:1}" + ;; + traffic_n_urls_m_cores) + wrk_utils.traffic_n_urls_m_cores "${args[@]:1}" + ;; +esac diff --git a/resources/traffic_profiles/wrk/wrk-sf-2n-ethip4tcphttp-1u1c1con-cps.yaml b/resources/traffic_profiles/wrk/wrk-sf-2n-ethip4tcphttp-1u1c1con-cps.yaml new file mode 100644 index 0000000000..cf120e7e1d --- /dev/null +++ b/resources/traffic_profiles/wrk/wrk-sf-2n-ethip4tcphttp-1u1c1con-cps.yaml @@ -0,0 +1,47 @@ +# This is an example wrk traffic profile. + +# List of urls for requests. Each url is requested in a separate instance of +# wrk. +# Type: list +urls: + # There must be an empty file (0B) requested but vpp does not support it. + - "http://192.168.10.2" + +# Index of the first CPU on the numa used to run wrk. +# Type: integer +first-cpu: 1 + +# The number of cpus used for wrk. The number of cpus must be a +# multiplication of the number of urls. +# Type: integer +cpus: 1 + +# Duration of the test in seconds. +# Type: integer +duration: 30 + +# Total number of threads to use. +# Type: integer +nr-of-threads: 1 + +# Total number of HTTP connections to keep open with each thread handling +# N = connections/threads. +# Type: integer +nr-of-connections: 1 + +# Path to LuaJIT script. +# Type: string +# script: "" + +# HTTP header to add to request, e.g. "Connection: close". +# Type: string (taken as it is) or dictionary +header: + Connection: "close" + +# Print detailed latency statistics. +# Type: boolean +latency: False + +# Record a timeout if a response is not received within this amount of time. +# Type: integer +timeout: 5 diff --git a/resources/traffic_profiles/wrk/wrk-sf-2n-ethip4tcphttp-1u1c50con-bw.yaml b/resources/traffic_profiles/wrk/wrk-sf-2n-ethip4tcphttp-1u1c50con-bw.yaml new file mode 100644 index 0000000000..93ce51d6d0 --- /dev/null +++ b/resources/traffic_profiles/wrk/wrk-sf-2n-ethip4tcphttp-1u1c50con-bw.yaml @@ -0,0 +1,47 @@ +# This is an example wrk traffic profile. + +# List of urls for requests. Each url is requested in a separate instance of +# wrk. +# Type: list +urls: + # There must be a big file (10MB) requested but vpp does not support it. + - "http://192.168.10.2" + +# Index of the first CPU on the numa used to run wrk. +# Type: integer +first-cpu: 1 + +# The number of cpus used for wrk. The number of cpus must be a +# multiplication of the number of urls. +# Type: integer +cpus: 1 + +# Duration of the test in seconds. +# Type: integer +duration: 30 + +# Total number of threads to use. +# Type: integer +nr-of-threads: 1 + +# Total number of HTTP connections to keep open with each thread handling +# N = connections/threads. +# Type: integer +nr-of-connections: 50 + +# Path to LuaJIT script. +# Type: string +# script: "" + +# HTTP header to add to request, e.g. "Connection: close". +# Type: string (taken as it is) or dictionary +# header: +# Connection: "close" + +# Print detailed latency statistics. +# Type: boolean +latency: False + +# Record a timeout if a response is not received within this amount of time. +# Type: integer +timeout: 5 diff --git a/resources/traffic_profiles/wrk/wrk-sf-2n-ethip4tcphttp-1u1c50con-rps.yaml b/resources/traffic_profiles/wrk/wrk-sf-2n-ethip4tcphttp-1u1c50con-rps.yaml new file mode 100644 index 0000000000..939703638b --- /dev/null +++ b/resources/traffic_profiles/wrk/wrk-sf-2n-ethip4tcphttp-1u1c50con-rps.yaml @@ -0,0 +1,47 @@ +# This is an example wrk traffic profile. + +# List of urls for requests. Each url is requested in a separate instance of +# wrk. +# Type: list +urls: + # There must be a file (1kB - 1MB) requested but vpp does not support it. + - "http://192.168.10.2" + +# Index of the first CPU on the numa used to run wrk. +# Type: integer +first-cpu: 1 + +# The number of cpus used for wrk. The number of cpus must be a +# multiplication of the number of urls. +# Type: integer +cpus: 1 + +# Duration of the test in seconds. +# Type: integer +duration: 30 + +# Total number of threads to use. +# Type: integer +nr-of-threads: 1 + +# Total number of HTTP connections to keep open with each thread handling +# N = connections/threads. +# Type: integer +nr-of-connections: 50 + +# Path to LuaJIT script. +# Type: string +# script: "" + +# HTTP header to add to request, e.g. "Connection: close". +# Type: string (taken as it is) or dictionary +# header: +# Connection: "close" + +# Print detailed latency statistics. +# Type: boolean +latency: False + +# Record a timeout if a response is not received within this amount of time. +# Type: integer +timeout: 5 diff --git a/tests/vpp/perf/tcp/10ge2p1x520-ethip4tcphttp-httpserver.robot b/tests/vpp/perf/tcp/10ge2p1x520-ethip4tcphttp-httpserver.robot new file mode 100644 index 0000000000..867c3177d5 --- /dev/null +++ b/tests/vpp/perf/tcp/10ge2p1x520-ethip4tcphttp-httpserver.robot @@ -0,0 +1,127 @@ +# Copyright (c) 2018 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +*** Settings *** + +| Library | resources.tools.wrk.wrk +| Resource | resources/libraries/robot/wrk/wrk_utils.robot +| Resource | resources/libraries/robot/performance/performance_setup.robot +| Resource | resources/libraries/robot/tcp/tcp_setup.robot +| ... +| Force Tags | 3_NODE_SINGLE_LINK_TOPO | PERFTEST | HW_ENV | HTTP | TCP +| ... +| Suite Setup | Set up 3-node performance topology with wrk and DUT's NIC model +| ... | Intel-XL710 +| ... +| Test Setup | Set up performance test +| Test Teardown | Tear down performance test with wrk +| ... +| Documentation | *HTTP requests per seconds, connections per seconds and +| ... | throughput measurement.* +| ... +| ... | *[Top] Network Topologies:* TG-DUT-TG 2-node topology +| ... | with single link between nodes. +| ... | *[Enc] Packet Encapsulations:* Eth-IPv4 for IPv4 routing. +| ... | *[Cfg] DUT configuration:* +| ... | *[Ver] TG verification:* +| ... | *[Ref] Applicable standard specifications:* + +*** Keywords *** +| Measure throughput or rps or cps +| | [Arguments] | ${traffic_profile} | ${wt} | ${rxq} | ${test_type} +| | ... +| | Add '${wt}' worker threads and '${rxq}' rxqueues in 3-node single-link circular topology +| | Add PCI devices to DUTs in 3-node single link topology +| | ${duts}= | Get Matches | ${nodes} | DUT* +| | :FOR | ${dut} | IN | @{duts} +| | | Import Library | resources.libraries.python.VppConfigGenerator +| | | ... | WITH NAME | ${dut} +| | | Run keyword | ${dut}.Add TCP preallocated connections | 1000000 +| | | Run keyword | ${dut}.Add TCP preallocated half open connections | 1000000 +| | | Run keyword | ${dut}.Add session event queue length | 1000000 +| | | Run keyword | ${dut}.Add session preallocated sessions | 1000000 +| | | Run keyword | ${dut}.Add session v4 session table buckets | 500000 +| | | Run keyword | ${dut}.Add session v4 session table memory | 1g +| | | Run keyword | ${dut}.Add session v4 halfopen table buckets | 2500000 +| | | Run keyword | ${dut}.Add session v4 halfopen table memory | 3g +| | | Run keyword | ${dut}.Add session local endpoints table buckets | 2500000 +| | | Run keyword | ${dut}.Add session local endpoints table memory | 3g +| | Apply startup configuration on all VPP DUTs +| | Set up HTTP server on the VPP node | 192.168.10.2 | 24 +| | Run Keyword If | '${test_type}' == 'bw' +| | ... | Measure throughput | ${traffic_profile} +| | ... | ELSE IF | '${test_type}' == 'rps' +| | ... | Measure requests per second | ${traffic_profile} +| | ... | ELSE IF | '${test_type}' == 'cps' +| | ... | Measure connections per second | ${traffic_profile} + +*** Test Cases *** +| tc01-1t1c-ethip4tcphttp-httpserver-cps +| | [Documentation] +| | ... | Measure number of connections per second using wrk. +| | ... +| | [Tags] | 1T1C | TCP_CPS +| | ... +| | [Template] | Measure throughput or rps or cps +| | traffic_profile=wrk-sf-2n-ethip4tcphttp-1u1c1con-cps | wt=1 | rxq=1 +| | ... | test_type=cps + +| tc02-2t2c-ethip4tcphttp-httpserver-cps +| | [Documentation] +| | ... | Measure number of connections per second using wrk. +| | ... +| | [Tags] | 2T2C | TCP_CPS +| | ... +| | [Template] | Measure throughput or rps or cps +| | traffic_profile=wrk-sf-2n-ethip4tcphttp-1u1c1con-cps | wt=2 | rxq=1 +| | ... | test_type=cps + +| tc03-4t4c-ethip4tcphttp-httpserver-cps +| | [Documentation] +| | ... | Measure number of connections per second using wrk. +| | ... +| | [Tags] | 4T4C | TCP_CPS +| | ... +| | [Template] | Measure throughput or rps or cps +| | traffic_profile=wrk-sf-2n-ethip4tcphttp-1u1c1con-cps | wt=4 | rxq=2 +| | ... | test_type=cps + +| tc04-1t1c-ethip4tcphttp-httpserver-rps +| | [Documentation] +| | ... | Measure and report number of requests per second using wrk. +| | ... +| | [Tags] | 1T1C | TCP_RPS +| | ... +| | [Template] | Measure throughput or rps or cps +| | traffic_profile=wrk-sf-2n-ethip4tcphttp-1u1c50con-rps | wt=1 | rxq=1 +| | ... | test_type=rps + +| tc05-2t2c-ethip4tcphttp-httpserver-rps +| | [Documentation] +| | ... | Measure and report number of requests per second using wrk. +| | ... +| | [Tags] | 2T2C | TCP_RPS +| | ... +| | [Template] | Measure throughput or rps or cps +| | traffic_profile=wrk-sf-2n-ethip4tcphttp-1u1c50con-rps | wt=2 | rxq=1 +| | ... | test_type=rps + +| tc06-4t4c-ethip4tcphttp-httpserver-rps +| | [Documentation] +| | ... | Measure and report number of requests per second using wrk. +| | ... +| | [Tags] | 4T4C | TCP_RPS +| | ... +| | [Template] | Measure throughput or rps or cps +| | traffic_profile=wrk-sf-2n-ethip4tcphttp-1u1c50con-rps | wt=4 | rxq=2 +| | ... | test_type=rps -- 2.16.6