1 # Copyright (c) 2023 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
14 """Host Stack util library."""
16 from time import sleep
17 from robot.api import logger
19 from resources.libraries.python.Constants import Constants
20 from resources.libraries.python.DUTSetup import DUTSetup
21 from resources.libraries.python.model.ExportResult import (
22 export_hoststack_results
24 from resources.libraries.python.PapiExecutor import PapiSocketExecutor
25 from resources.libraries.python.ssh import exec_cmd, exec_cmd_no_error
27 class HoststackUtil():
28 """Utilities for Host Stack tests."""
31 def get_vpp_echo_command(vpp_echo_attributes):
32 """Construct the vpp_echo command using the specified attributes.
34 :param vpp_echo_attributes: vpp_echo test program attributes.
35 :type vpp_echo_attributes: dict
36 :returns: Command line components of the vpp_echo command
38 'args' - command arguments.
41 proto = vpp_echo_attributes[u"uri_protocol"]
42 addr = vpp_echo_attributes[u"uri_ip4_addr"]
43 port = vpp_echo_attributes[u"uri_port"]
45 vpp_echo_cmd[u"name"] = u"vpp_echo"
46 vpp_echo_cmd[u"args"] = f"{vpp_echo_attributes[u'role']} " \
47 f"socket-name {vpp_echo_attributes[u'app_api_socket']} " \
48 f"{vpp_echo_attributes[u'json_output']} " \
49 f"uri {proto}://{addr}/{port} " \
50 f"nthreads {vpp_echo_attributes[u'nthreads']} " \
51 f"mq-size {vpp_echo_attributes[u'mq_size']} " \
52 f"nclients {vpp_echo_attributes[u'nclients']} " \
53 f"quic-streams {vpp_echo_attributes[u'quic_streams']} " \
54 f"time {vpp_echo_attributes[u'time']} " \
55 f"fifo-size {vpp_echo_attributes[u'fifo_size']} " \
56 f"TX={vpp_echo_attributes[u'tx_bytes']} " \
57 f"RX={vpp_echo_attributes[u'rx_bytes']}"
58 if vpp_echo_attributes[u"rx_results_diff"]:
59 vpp_echo_cmd[u"args"] += u" rx-results-diff"
60 if vpp_echo_attributes[u"tx_results_diff"]:
61 vpp_echo_cmd[u"args"] += u" tx-results-diff"
62 if vpp_echo_attributes[u"use_app_socket_api"]:
63 vpp_echo_cmd[u"args"] += u" use-app-socket-api"
67 def get_iperf3_command(iperf3_attributes):
68 """Construct the iperf3 command using the specified attributes.
70 :param iperf3_attributes: iperf3 test program attributes.
71 :type iperf3_attributes: dict
72 :returns: Command line components of the iperf3 command
73 'env_vars' - environment variables
75 'args' - command arguments.
79 iperf3_cmd[u"env_vars"] = f"VCL_CONFIG={Constants.REMOTE_FW_DIR}/" \
80 f"{Constants.RESOURCES_TPL_VCL}/" \
81 f"{iperf3_attributes[u'vcl_config']}"
82 if iperf3_attributes[u"ld_preload"]:
83 iperf3_cmd[u"env_vars"] += \
84 f" LD_PRELOAD={Constants.VCL_LDPRELOAD_LIBRARY}"
85 if iperf3_attributes[u'transparent_tls']:
86 iperf3_cmd[u"env_vars"] += u" LDP_ENV_TLS_TRANS=1"
88 json_results = u" --json" if iperf3_attributes[u'json'] else u""
89 ip_address = f" {iperf3_attributes[u'ip_address']}" if u"ip_address" \
90 in iperf3_attributes else u""
91 iperf3_cmd[u"name"] = u"iperf3"
92 iperf3_cmd[u"args"] = f"--{iperf3_attributes[u'role']}{ip_address} " \
93 f"--interval 0{json_results} " \
94 f"--version{iperf3_attributes[u'ip_version']}"
96 if iperf3_attributes[u"role"] == u"server":
97 iperf3_cmd[u"args"] += u" --one-off"
99 iperf3_cmd[u"args"] += u" --get-server-output"
100 if u"parallel" in iperf3_attributes:
101 iperf3_cmd[u"args"] += \
102 f" --parallel {iperf3_attributes[u'parallel']}"
103 if u"time" in iperf3_attributes:
104 iperf3_cmd[u"args"] += \
105 f" --time {iperf3_attributes[u'time']}"
106 if iperf3_attributes[u"udp"]:
107 iperf3_cmd[u"args"] += u" --udp"
108 iperf3_cmd[u"args"] += \
109 f" --bandwidth {iperf3_attributes[u'bandwidth']}"
110 if iperf3_attributes[u"length"] > 0:
111 iperf3_cmd[u"args"] += \
112 f" --length {iperf3_attributes[u'length']}"
116 def set_hoststack_quic_fifo_size(node, fifo_size):
117 """Set the QUIC protocol fifo size.
119 :param node: Node to set the QUIC fifo size on.
120 :param fifo_size: fifo size, passed to the quic set fifo-size command.
124 cmd = f"quic set fifo-size {fifo_size}"
125 PapiSocketExecutor.run_cli_cmd(node, cmd)
128 def set_hoststack_quic_crypto_engine(node, quic_crypto_engine,
129 fail_on_error=False):
130 """Set the Hoststack QUIC crypto engine on node
132 :param node: Node to enable/disable HostStack.
133 :param quic_crypto_engine: type of crypto engine
135 :type quic_crypto_engine: str
137 vpp_crypto_engines = {u"openssl", u"native", u"ipsecmb"}
138 if quic_crypto_engine == u"nocrypto":
139 logger.trace(u"No QUIC crypto engine.")
142 if quic_crypto_engine in vpp_crypto_engines:
143 cmds = [u"quic set crypto api vpp",
144 f"set crypto handler aes-128-gcm {quic_crypto_engine}",
145 f"set crypto handler aes-256-gcm {quic_crypto_engine}"]
146 elif quic_crypto_engine == u"picotls":
147 cmds = [u"quic set crypto api picotls"]
149 raise ValueError(f"Unknown QUIC crypto_engine {quic_crypto_engine}")
153 PapiSocketExecutor.run_cli_cmd(node, cmd)
154 except AssertionError:
159 def _get_hoststack_test_program_logs(node, program_name):
160 """Get HostStack test program stdout log.
162 :param node: DUT node.
163 :param program_name: test program.
165 :type program_name: str
167 cmd = f"sh -c \'cat /tmp/{program_name}_stdout.log\'"
168 stdout_log, _ = exec_cmd_no_error(node, cmd, sudo=True, \
169 message=f"Get {program_name} stdout log failed!")
171 cmd = f"sh -c \'cat /tmp/{program_name}_stderr.log\'"
172 stderr_log, _ = exec_cmd_no_error(node, cmd, sudo=True, \
173 message=f"Get {program_name} stderr log failed!")
175 return stdout_log, stderr_log
178 def get_hoststack_test_program_logs(node, program):
179 """Get HostStack test program stdout log.
181 :param node: DUT node.
182 :param program: test program.
186 program_name = program[u"name"]
187 program_stdout_log, program_stderr_log = \
188 HoststackUtil._get_hoststack_test_program_logs(node,
190 if len(program_stdout_log) == 0 and len(program_stderr_log) == 0:
191 logger.trace(f"Retrying {program_name} log retrieval")
192 program_stdout_log, program_stderr_log = \
193 HoststackUtil._get_hoststack_test_program_logs(node,
195 return program_stdout_log, program_stderr_log
198 def get_nginx_command(nginx_attributes, nginx_version, nginx_ins_dir):
199 """Construct the NGINX command using the specified attributes.
201 :param nginx_attributes: NGINX test program attributes.
202 :param nginx_version: NGINX version.
203 :param nginx_ins_dir: NGINX install dir.
204 :type nginx_attributes: dict
205 :type nginx_version: str
206 :type nginx_ins_dir: str
207 :returns: Command line components of the NGINX command
208 'env_vars' - environment variables
209 'name' - program name
210 'args' - command arguments.
211 'path' - program path.
215 nginx_cmd[u"env_vars"] = f"VCL_CONFIG={Constants.REMOTE_FW_DIR}/" \
216 f"{Constants.RESOURCES_TPL_VCL}/" \
217 f"{nginx_attributes[u'vcl_config']}"
218 if nginx_attributes[u"ld_preload"]:
219 nginx_cmd[u"env_vars"] += \
220 f" LD_PRELOAD={Constants.VCL_LDPRELOAD_LIBRARY}"
221 if nginx_attributes[u'transparent_tls']:
222 nginx_cmd[u"env_vars"] += u" LDP_ENV_TLS_TRANS=1"
224 nginx_cmd[u"name"] = u"nginx"
225 nginx_cmd[u"path"] = f"{nginx_ins_dir}nginx-{nginx_version}/sbin/"
226 nginx_cmd[u"args"] = f"-c {nginx_ins_dir}/" \
227 f"nginx-{nginx_version}/conf/nginx.conf"
231 def start_hoststack_test_program(node, namespace, core_list, program):
232 """Start the specified HostStack test program.
234 :param node: DUT node.
235 :param namespace: Net Namespace to run program in.
236 :param core_list: List of cpu's to pass to taskset to pin the test
237 program to a different set of cores on the same numa node as VPP.
238 :param program: Test program.
245 :raises RuntimeError: If node subtype is not a DUT or startup failed.
247 if node[u"type"] != u"DUT":
248 raise RuntimeError(u"Node type is not a DUT!")
250 program_name = program[u"name"]
251 DUTSetup.kill_program(node, program_name, namespace)
253 if namespace == u"default":
256 shell_cmd = f"ip netns exec {namespace} sh -c"
258 env_vars = f"{program[u'env_vars']} " if u"env_vars" in program else u""
259 args = program[u"args"]
260 program_path = program.get(u"path", u"")
261 # NGINX used `worker_cpu_affinity` in configuration file
262 taskset_cmd = u"" if program_name == u"nginx" else \
263 f"taskset --cpu-list {core_list}"
264 cmd = f"nohup {shell_cmd} \'{env_vars}{taskset_cmd} " \
265 f"{program_path}{program_name} {args} >/tmp/{program_name}_" \
266 f"stdout.log 2>/tmp/{program_name}_stderr.log &\'"
268 exec_cmd_no_error(node, cmd, sudo=True)
269 return DUTSetup.get_pid(node, program_name)[0]
271 stdout_log, stderr_log = \
272 HoststackUtil.get_hoststack_test_program_logs(node,
274 raise RuntimeError(f"Start {program_name} failed!\nSTDERR:\n" \
275 f"{stderr_log}\nSTDOUT:\n{stdout_log}")
279 def stop_hoststack_test_program(node, program, pid):
280 """Stop the specified Hoststack test program.
282 :param node: DUT node.
283 :param program: Test program.
284 :param pid: Process ID of test program.
289 program_name = program[u"name"]
290 if program_name == u"nginx":
291 cmd = u"nginx -s quit"
292 errmsg = u"Quit nginx failed!"
294 cmd = f'if [ -n "$(ps {pid} | grep {program_name})" ] ; ' \
295 f'then kill -s SIGTERM {pid}; fi'
296 errmsg = f"Kill {program_name} ({pid}) failed!"
298 exec_cmd_no_error(node, cmd, message=errmsg, sudo=True)
301 def hoststack_test_program_finished(node, program_pid, program,
302 other_node, other_program):
303 """Wait for the specified HostStack test program process to complete.
305 :param node: DUT node.
306 :param program_pid: test program pid.
307 :param program: test program
308 :param other_node: DUT node of other hoststack program
309 :param other_program: other test program
311 :type program_pid: str
313 :type other_node: dict
314 :type other_program: dict
315 :raises RuntimeError: If node subtype is not a DUT.
317 if node[u"type"] != u"DUT":
318 raise RuntimeError(u"Node type is not a DUT!")
319 if other_node[u"type"] != u"DUT":
320 raise RuntimeError(u"Other node type is not a DUT!")
322 cmd = f"sh -c 'strace -qqe trace=none -p {program_pid}'"
324 exec_cmd(node, cmd, sudo=True)
327 if u"client" in program[u"args"]:
331 program_stdout, program_stderr = \
332 HoststackUtil.get_hoststack_test_program_logs(node, program)
333 if len(program_stdout) > 0:
334 logger.debug(f"{program[u'name']} {role} stdout log:\n"
337 logger.debug(f"Empty {program[u'name']} {role} stdout log :(")
338 if len(program_stderr) > 0:
339 logger.debug(f"{program[u'name']} stderr log:\n"
342 logger.debug(f"Empty {program[u'name']} stderr log :(")
343 if u"client" in other_program[u"args"]:
347 program_stdout, program_stderr = \
348 HoststackUtil.get_hoststack_test_program_logs(other_node,
350 if len(program_stdout) > 0:
351 logger.debug(f"{other_program[u'name']} {role} stdout log:\n"
354 logger.debug(f"Empty {other_program[u'name']} "
355 f"{role} stdout log :(")
356 if len(program_stderr) > 0:
357 logger.debug(f"{other_program[u'name']} {role} stderr log:\n"
360 logger.debug(f"Empty {other_program[u'name']} "
361 f"{role} stderr log :(")
363 # Wait a bit for stdout/stderr to be flushed to log files
367 def analyze_hoststack_test_program_output(
368 node, role, nsim_attr, program):
369 """Gather HostStack test program output and check for errors.
371 The [defer_fail] return bool is used instead of failing immediately
372 to allow the analysis of both the client and server instances of
373 the test program for debugging a test failure. When [defer_fail]
374 is true, then the string returned is debug output instead of
375 JSON formatted test program results.
377 :param node: DUT node.
378 :param role: Role (client|server) of test program.
379 :param nsim_attr: Network Simulation Attributes.
380 :param program: Test program.
381 :param program_args: List of test program args.
384 :type nsim_attr: dict
386 :returns: tuple of [defer_fail] bool and either JSON formatted hoststack
387 test program output or failure debug output.
389 :raises RuntimeError: If node subtype is not a DUT.
391 if node[u"type"] != u"DUT":
392 raise RuntimeError(u"Node type is not a DUT!")
394 program_name = program[u"name"]
395 program_stdout, program_stderr = \
396 HoststackUtil.get_hoststack_test_program_logs(node, program)
398 env_vars = f"{program[u'env_vars']} " if u"env_vars" in program else u""
399 program_cmd = f"{env_vars}{program_name} {program[u'args']}"
400 test_results = f"Test Results of '{program_cmd}':\n"
402 if nsim_attr[u"output_nsim_enable"] or \
403 nsim_attr[u"xc_nsim_enable"]:
404 if nsim_attr[u"output_nsim_enable"]:
405 feature_name = u"output"
407 feature_name = u"cross-connect"
409 f"NSIM({feature_name}): delay " \
410 f"{nsim_attr[u'delay_in_usec']} usecs, " \
411 f"avg-pkt-size {nsim_attr[u'average_packet_size']}, " \
412 f"bandwidth {nsim_attr[u'bw_in_bits_per_second']} " \
413 f"bits/sec, pkt-drop-rate {nsim_attr[u'packets_per_drop']} " \
417 f"\n{role} VPP 'show errors' on host {node[u'host']}:\n" \
418 f"{PapiSocketExecutor.run_cli_cmd(node, u'show error')}\n"
420 if u"error" in program_stderr.lower():
421 test_results += f"ERROR DETECTED:\n{program_stderr}"
422 return (True, test_results)
423 if not program_stdout:
424 test_results += f"\nNo {program} test data retrieved!\n"
425 ls_stdout, _ = exec_cmd_no_error(node, u"ls -l /tmp/*.log",
427 test_results += f"{ls_stdout}\n"
428 return (True, test_results)
429 if program[u"name"] == u"vpp_echo":
430 if u"JSON stats" in program_stdout and \
431 u'"has_failed": "0"' in program_stdout:
432 json_start = program_stdout.find(u"{")
433 json_end = program_stdout.find(u',\n "closing"')
434 json_results = f"{program_stdout[json_start:json_end]}\n}}"
435 program_json = json.loads(json_results)
436 export_hoststack_results(
437 bandwidth=program_json["rx_bits_per_second"],
438 duration=float(program_json["time"])
441 test_results += u"Invalid test data output!\n" + program_stdout
442 return (True, test_results)
443 elif program[u"name"] == u"iperf3":
444 test_results += program_stdout
445 program_json = json.loads(program_stdout)[u"intervals"][0][u"sum"]
447 retransmits = program_json["retransmits"]
450 export_hoststack_results(
451 bandwidth=program_json["bits_per_second"],
452 duration=program_json["seconds"],
453 retransmits=retransmits
456 test_results += u"Unknown HostStack Test Program!\n" + \
458 return (True, program_stdout)
459 return (False, json.dumps(program_json))
462 def hoststack_test_program_defer_fail(server_defer_fail, client_defer_fail):
463 """Return True if either HostStack test program fail was deferred.
465 :param server_defer_fail: server no results value.
466 :param client_defer_fail: client no results value.
467 :type server_defer_fail: bool
468 :type client_defer_fail: bool
471 return server_defer_fail and client_defer_fail
474 def log_vpp_hoststack_data(node):
475 """Retrieve and log VPP HostStack data.
477 :param node: DUT node.
479 :raises RuntimeError: If node subtype is not a DUT or startup failed.
482 if node[u"type"] != u"DUT":
483 raise RuntimeError(u"Node type is not a DUT!")
485 PapiSocketExecutor.run_cli_cmd(node, u"show error")
486 PapiSocketExecutor.run_cli_cmd(node, u"show interface")