1 # Copyright (c) 2021 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
14 """Host Stack util library."""
16 from time import sleep
17 from robot.api import logger
19 from resources.libraries.python.Constants import Constants
20 from resources.libraries.python.ssh import exec_cmd, exec_cmd_no_error
21 from resources.libraries.python.PapiExecutor import PapiSocketExecutor
22 from resources.libraries.python.DUTSetup import DUTSetup
24 class HoststackUtil():
25 """Utilities for Host Stack tests."""
28 def get_vpp_echo_command(vpp_echo_attributes):
29 """Construct the vpp_echo command using the specified attributes.
31 :param vpp_echo_attributes: vpp_echo test program attributes.
32 :type vpp_echo_attributes: dict
33 :returns: Command line components of the vpp_echo command
35 'args' - command arguments.
38 proto = vpp_echo_attributes[u"uri_protocol"]
39 addr = vpp_echo_attributes[u"uri_ip4_addr"]
40 port = vpp_echo_attributes[u"uri_port"]
42 vpp_echo_cmd[u"name"] = u"vpp_echo"
43 vpp_echo_cmd[u"args"] = f"{vpp_echo_attributes[u'role']} " \
44 f"socket-name {vpp_echo_attributes[u'vpp_api_socket']} " \
45 f"{vpp_echo_attributes[u'json_output']} " \
46 f"uri {proto}://{addr}/{port} " \
47 f"nthreads {vpp_echo_attributes[u'nthreads']} " \
48 f"mq-size {vpp_echo_attributes[u'mq_size']} " \
49 f"nclients {vpp_echo_attributes[u'nclients']} " \
50 f"quic-streams {vpp_echo_attributes[u'quic_streams']} " \
51 f"time {vpp_echo_attributes[u'time']} " \
52 f"fifo-size {vpp_echo_attributes[u'fifo_size']} " \
53 f"TX={vpp_echo_attributes[u'tx_bytes']} " \
54 f"RX={vpp_echo_attributes[u'rx_bytes']}"
55 if vpp_echo_attributes[u"rx_results_diff"]:
56 vpp_echo_cmd[u"args"] += u" rx-results-diff"
57 if vpp_echo_attributes[u"tx_results_diff"]:
58 vpp_echo_cmd[u"args"] += u" tx-results-diff"
62 def get_iperf3_command(iperf3_attributes):
63 """Construct the iperf3 command using the specified attributes.
65 :param iperf3_attributes: iperf3 test program attributes.
66 :type iperf3_attributes: dict
67 :returns: Command line components of the iperf3 command
68 'env_vars' - environment variables
70 'args' - command arguments.
74 iperf3_cmd[u"env_vars"] = f"VCL_CONFIG={Constants.REMOTE_FW_DIR}/" \
75 f"{Constants.RESOURCES_TPL_VCL}/" \
76 f"{iperf3_attributes[u'vcl_config']}"
77 if iperf3_attributes[u"ld_preload"]:
78 iperf3_cmd[u"env_vars"] += \
79 f" LD_PRELOAD={Constants.VCL_LDPRELOAD_LIBRARY}"
80 if iperf3_attributes[u'transparent_tls']:
81 iperf3_cmd[u"env_vars"] += u" LDP_ENV_TLS_TRANS=1"
83 json_results = u" --json" if iperf3_attributes[u'json'] else u""
84 ip_address = f" {iperf3_attributes[u'ip_address']}" if u"ip_address" \
85 in iperf3_attributes else u""
86 iperf3_cmd[u"name"] = u"iperf3"
87 # TODO: Use OptionString library.
88 iperf3_cmd[u"args"] = f"--{iperf3_attributes[u'role']}{ip_address} " \
89 f"--interval 0{json_results} " \
90 f"--version{iperf3_attributes[u'ip_version']}"
92 if iperf3_attributes[u"role"] == u"server":
93 iperf3_cmd[u"args"] += u" --one-off"
95 iperf3_cmd[u"args"] += u" --get-server-output"
96 if u"parallel" in iperf3_attributes:
97 iperf3_cmd[u"args"] += \
98 f" --parallel {iperf3_attributes[u'parallel']}"
99 if u"time" in iperf3_attributes:
100 iperf3_cmd[u"args"] += \
101 f" --time {iperf3_attributes[u'time']}"
102 if iperf3_attributes[u"udp"]:
103 iperf3_cmd[u"args"] += u" --udp"
104 iperf3_cmd[u"args"] += \
105 f" --bandwidth {iperf3_attributes[u'bandwidth']}"
106 if iperf3_attributes[u"length"] > 0:
107 iperf3_cmd[u"args"] += \
108 f" --length {iperf3_attributes[u'length']}"
112 def set_hoststack_quic_fifo_size(node, fifo_size):
113 """Set the QUIC protocol fifo size.
115 :param node: Node to set the QUIC fifo size on.
116 :param fifo_size: fifo size, passed to the quic set fifo-size command.
120 cmd = f"quic set fifo-size {fifo_size}"
121 PapiSocketExecutor.run_cli_cmd(node, cmd)
124 def set_hoststack_quic_crypto_engine(node, quic_crypto_engine,
125 fail_on_error=False):
126 """Set the Hoststack QUIC crypto engine on node
128 :param node: Node to enable/disable HostStack.
129 :param quic_crypto_engine: type of crypto engine
131 :type quic_crypto_engine: str
133 vpp_crypto_engines = {u"openssl", u"native", u"ipsecmb"}
134 if quic_crypto_engine == u"nocrypto":
135 logger.trace(u"No QUIC crypto engine.")
138 if quic_crypto_engine in vpp_crypto_engines:
139 cmds = [u"quic set crypto api vpp",
140 f"set crypto handler aes-128-gcm {quic_crypto_engine}",
141 f"set crypto handler aes-256-gcm {quic_crypto_engine}"]
142 elif quic_crypto_engine == u"picotls":
143 cmds = [u"quic set crypto api picotls"]
145 raise ValueError(f"Unknown QUIC crypto_engine {quic_crypto_engine}")
149 PapiSocketExecutor.run_cli_cmd(node, cmd)
150 except AssertionError:
155 def get_hoststack_test_program_logs(node, program):
156 """Get HostStack test program stdout log.
158 :param node: DUT node.
159 :param program: test program.
163 program_name = program[u"name"]
164 cmd = f"sh -c \'cat /tmp/{program_name}_stdout.log\'"
165 stdout_log, _ = exec_cmd_no_error(node, cmd, sudo=True, \
166 message=f"Get {program_name} stdout log failed!")
168 cmd = f"sh -c \'cat /tmp/{program_name}_stderr.log\'"
169 stderr_log, _ = exec_cmd_no_error(node, cmd, sudo=True, \
170 message=f"Get {program_name} stderr log failed!")
171 return stdout_log, stderr_log
174 def get_nginx_command(nginx_attributes, nginx_version, nginx_ins_dir):
175 """Construct the NGINX command using the specified attributes.
177 :param nginx_attributes: NGINX test program attributes.
178 :param nginx_version: NGINX version.
179 :param nginx_ins_dir: NGINX install dir.
180 :type nginx_attributes: dict
181 :type nginx_version: str
182 :type nginx_ins_dir: str
183 :returns: Command line components of the NGINX command
184 'env_vars' - environment variables
185 'name' - program name
186 'args' - command arguments.
187 'path' - program path.
191 nginx_cmd[u"env_vars"] = f"VCL_CONFIG={Constants.REMOTE_FW_DIR}/" \
192 f"{Constants.RESOURCES_TPL_VCL}/" \
193 f"{nginx_attributes[u'vcl_config']}"
194 if nginx_attributes[u"ld_preload"]:
195 nginx_cmd[u"env_vars"] += \
196 f" LD_PRELOAD={Constants.VCL_LDPRELOAD_LIBRARY}"
197 if nginx_attributes[u'transparent_tls']:
198 nginx_cmd[u"env_vars"] += u" LDP_ENV_TLS_TRANS=1"
200 nginx_cmd[u"name"] = u"nginx"
201 nginx_cmd[u"path"] = f"{nginx_ins_dir}nginx-{nginx_version}/sbin/"
202 nginx_cmd[u"args"] = f"-c {nginx_ins_dir}/" \
203 f"nginx-{nginx_version}/conf/nginx.conf"
207 def start_hoststack_test_program(node, namespace, core_list, program):
208 """Start the specified HostStack test program.
210 :param node: DUT node.
211 :param namespace: Net Namespace to run program in.
212 :param core_list: List of cpu's to pass to taskset to pin the test
213 program to a different set of cores on the same numa node as VPP.
214 :param program: Test program.
221 :raises RuntimeError: If node subtype is not a DUT or startup failed.
223 if node[u"type"] != u"DUT":
224 raise RuntimeError(u"Node type is not a DUT!")
226 program_name = program[u"name"]
227 DUTSetup.kill_program(node, program_name, namespace)
229 if namespace == u"default":
232 shell_cmd = f"ip netns exec {namespace} sh -c"
234 env_vars = f"{program[u'env_vars']} " if u"env_vars" in program else u""
235 args = program[u"args"]
236 program_path = program.get(u"path", u"")
237 # NGINX used `worker_cpu_affinity` in configuration file
238 taskset_cmd = u"" if program_name == u"nginx" else \
239 f"taskset --cpu-list {core_list}"
240 cmd = f"nohup {shell_cmd} \'{env_vars}{taskset_cmd} " \
241 f"{program_path}{program_name} {args} >/tmp/{program_name}_" \
242 f"stdout.log 2>/tmp/{program_name}_stderr.log &\'"
244 exec_cmd_no_error(node, cmd, sudo=True)
245 return DUTSetup.get_pid(node, program_name)[0]
247 stdout_log, stderr_log = \
248 HoststackUtil.get_hoststack_test_program_logs(node,
250 raise RuntimeError(f"Start {program_name} failed!\nSTDERR:\n" \
251 f"{stderr_log}\nSTDOUT:\n{stdout_log}")
255 def stop_hoststack_test_program(node, program, pid):
256 """Stop the specified Hoststack test program.
258 :param node: DUT node.
259 :param program: Test program.
260 :param pid: Process ID of test program.
265 program_name = program[u"name"]
266 if program_name == u"nginx":
267 cmd = u"nginx -s quit"
268 errmsg = u"Quit nginx failed!"
270 cmd = f'if [ -n "$(ps {pid} | grep {program_name})" ] ; ' \
271 f'then kill -s SIGTERM {pid}; fi'
272 errmsg = f"Kill {program_name} ({pid}) failed!"
274 exec_cmd_no_error(node, cmd, message=errmsg, sudo=True)
277 def hoststack_test_program_finished(node, program_pid):
278 """Wait for the specified HostStack test program process to complete.
280 :param node: DUT node.
281 :param program_pid: test program pid.
283 :type program_pid: str
284 :raises RuntimeError: If node subtype is not a DUT.
286 if node[u"type"] != u"DUT":
287 raise RuntimeError(u"Node type is not a DUT!")
289 cmd = f"sh -c 'strace -qqe trace=none -p {program_pid}'"
290 exec_cmd(node, cmd, sudo=True)
291 # Wait a bit for stdout/stderr to be flushed to log files
292 # TODO: see if sub-second sleep works e.g. sleep(0.1)
296 def analyze_hoststack_test_program_output(
297 node, role, nsim_attr, program):
298 """Gather HostStack test program output and check for errors.
300 The [defer_fail] return bool is used instead of failing immediately
301 to allow the analysis of both the client and server instances of
302 the test program for debugging a test failure. When [defer_fail]
303 is true, then the string returned is debug output instead of
304 JSON formatted test program results.
306 :param node: DUT node.
307 :param role: Role (client|server) of test program.
308 :param nsim_attr: Network Simulation Attributes.
309 :param program: Test program.
310 :param program_args: List of test program args.
313 :type nsim_attr: dict
315 :returns: tuple of [defer_fail] bool and either JSON formatted hoststack
316 test program output or failure debug output.
318 :raises RuntimeError: If node subtype is not a DUT.
320 if node[u"type"] != u"DUT":
321 raise RuntimeError(u"Node type is not a DUT!")
323 program_name = program[u"name"]
324 program_stdout, program_stderr = \
325 HoststackUtil.get_hoststack_test_program_logs(node, program)
326 if len(program_stdout) == 0 and len(program_stderr) == 0:
327 logger.trace(f"Retrying {program_name} log retrieval")
328 program_stdout, program_stderr = \
329 HoststackUtil.get_hoststack_test_program_logs(node, program)
331 env_vars = f"{program[u'env_vars']} " if u"env_vars" in program else u""
332 program_cmd = f"{env_vars}{program_name} {program[u'args']}"
333 test_results = f"Test Results of '{program_cmd}':\n"
335 if nsim_attr[u"output_nsim_enable"] or \
336 nsim_attr[u"xc_nsim_enable"]:
337 if nsim_attr[u"output_nsim_enable"]:
338 feature_name = u"output"
340 feature_name = u"cross-connect"
342 f"NSIM({feature_name}): delay " \
343 f"{nsim_attr[u'delay_in_usec']} usecs, " \
344 f"avg-pkt-size {nsim_attr[u'average_packet_size']}, " \
345 f"bandwidth {nsim_attr[u'bw_in_bits_per_second']} " \
346 f"bits/sec, pkt-drop-rate {nsim_attr[u'packets_per_drop']} " \
349 # TODO: Incorporate show error stats into results analysis
351 f"\n{role} VPP 'show errors' on host {node[u'host']}:\n" \
352 f"{PapiSocketExecutor.run_cli_cmd(node, u'show error')}\n"
354 if u"error" in program_stderr.lower():
355 test_results += f"ERROR DETECTED:\n{program_stderr}"
356 return (True, test_results)
357 if not program_stdout:
358 test_results += f"\nNo {program} test data retrieved!\n"
359 ls_stdout, _ = exec_cmd_no_error(node, u"ls -l /tmp/*.log",
361 test_results += f"{ls_stdout}\n"
362 return (True, test_results)
363 if program[u"name"] == u"vpp_echo":
364 if u"JSON stats" in program_stdout and \
365 u'"has_failed": "0"' in program_stdout:
366 json_start = program_stdout.find(u"{")
367 #TODO: Fix parsing once vpp_echo produces valid
368 # JSON output. Truncate for now.
369 json_end = program_stdout.find(u',\n "closing"')
370 json_results = f"{program_stdout[json_start:json_end]}\n}}"
371 program_json = json.loads(json_results)
373 test_results += u"Invalid test data output!\n" + program_stdout
374 return (True, test_results)
375 elif program[u"name"] == u"iperf3":
376 test_results += program_stdout
377 iperf3_json = json.loads(program_stdout)
378 program_json = iperf3_json[u"intervals"][0][u"sum"]
380 test_results += u"Unknown HostStack Test Program!\n" + \
382 return (True, program_stdout)
383 return (False, json.dumps(program_json))
386 def hoststack_test_program_defer_fail(server_defer_fail, client_defer_fail):
387 """Return True if either HostStack test program fail was deferred.
389 :param server_defer_fail: server no results value.
390 :param client_defer_fail: client no results value.
391 :type server_defer_fail: bool
392 :type client_defer_fail: bool
395 return server_defer_fail and client_defer_fail
398 def log_vpp_hoststack_data(node):
399 """Retrieve and log VPP HostStack data.
401 :param node: DUT node.
403 :raises RuntimeError: If node subtype is not a DUT or startup failed.
406 if node[u"type"] != u"DUT":
407 raise RuntimeError(u"Node type is not a DUT!")
409 PapiSocketExecutor.run_cli_cmd(node, u"show error")
410 PapiSocketExecutor.run_cli_cmd(node, u"show interface")