JumpAvg: Fix string format
[csit.git] / resources / libraries / python / HoststackUtil.py
1 # Copyright (c) 2021 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
5 #
6 #     http://www.apache.org/licenses/LICENSE-2.0
7 #
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
13
14 """Host Stack util library."""
15 import json
16 from time import sleep
17 from robot.api import logger
18
19 from resources.libraries.python.Constants import Constants
20 from resources.libraries.python.ssh import exec_cmd, exec_cmd_no_error
21 from resources.libraries.python.PapiExecutor import PapiSocketExecutor
22 from resources.libraries.python.DUTSetup import DUTSetup
23
24 class HoststackUtil():
25     """Utilities for Host Stack tests."""
26
27     @staticmethod
28     def get_vpp_echo_command(vpp_echo_attributes):
29         """Construct the vpp_echo command using the specified attributes.
30
31         :param vpp_echo_attributes: vpp_echo test program attributes.
32         :type vpp_echo_attributes: dict
33         :returns: Command line components of the vpp_echo command
34             'name' - program name
35             'args' - command arguments.
36         :rtype: dict
37         """
38         proto = vpp_echo_attributes[u"uri_protocol"]
39         addr = vpp_echo_attributes[u"uri_ip4_addr"]
40         port = vpp_echo_attributes[u"uri_port"]
41         vpp_echo_cmd = {}
42         vpp_echo_cmd[u"name"] = u"vpp_echo"
43         vpp_echo_cmd[u"args"] = f"{vpp_echo_attributes[u'role']} " \
44             f"socket-name {vpp_echo_attributes[u'vpp_api_socket']} " \
45             f"{vpp_echo_attributes[u'json_output']} " \
46             f"uri {proto}://{addr}/{port} " \
47             f"nthreads {vpp_echo_attributes[u'nthreads']} " \
48             f"mq-size {vpp_echo_attributes[u'mq_size']} " \
49             f"nclients {vpp_echo_attributes[u'nclients']} " \
50             f"quic-streams {vpp_echo_attributes[u'quic_streams']} " \
51             f"time {vpp_echo_attributes[u'time']} " \
52             f"fifo-size {vpp_echo_attributes[u'fifo_size']} " \
53             f"TX={vpp_echo_attributes[u'tx_bytes']} " \
54             f"RX={vpp_echo_attributes[u'rx_bytes']}"
55         if vpp_echo_attributes[u"rx_results_diff"]:
56             vpp_echo_cmd[u"args"] += u" rx-results-diff"
57         if vpp_echo_attributes[u"tx_results_diff"]:
58             vpp_echo_cmd[u"args"] += u" tx-results-diff"
59         return vpp_echo_cmd
60
61     @staticmethod
62     def get_iperf3_command(iperf3_attributes):
63         """Construct the iperf3 command using the specified attributes.
64
65         :param iperf3_attributes: iperf3 test program attributes.
66         :type iperf3_attributes: dict
67         :returns: Command line components of the iperf3 command
68             'env_vars' - environment variables
69             'name' - program name
70             'args' - command arguments.
71         :rtype: dict
72         """
73         iperf3_cmd = {}
74         iperf3_cmd[u"env_vars"] = f"VCL_CONFIG={Constants.REMOTE_FW_DIR}/" \
75             f"{Constants.RESOURCES_TPL_VCL}/" \
76             f"{iperf3_attributes[u'vcl_config']}"
77         if iperf3_attributes[u"ld_preload"]:
78             iperf3_cmd[u"env_vars"] += \
79                 f" LD_PRELOAD={Constants.VCL_LDPRELOAD_LIBRARY}"
80         if iperf3_attributes[u'transparent_tls']:
81             iperf3_cmd[u"env_vars"] += u" LDP_ENV_TLS_TRANS=1"
82
83         json_results = u" --json" if iperf3_attributes[u'json'] else u""
84         ip_address = f" {iperf3_attributes[u'ip_address']}" if u"ip_address" \
85                      in iperf3_attributes else u""
86         iperf3_cmd[u"name"] = u"iperf3"
87         # TODO: Use OptionString library.
88         iperf3_cmd[u"args"] = f"--{iperf3_attributes[u'role']}{ip_address} " \
89                               f"--interval 0{json_results} " \
90                               f"--version{iperf3_attributes[u'ip_version']}"
91
92         if iperf3_attributes[u"role"] == u"server":
93             iperf3_cmd[u"args"] += u" --one-off"
94         else:
95             iperf3_cmd[u"args"] += u" --get-server-output"
96             if u"parallel" in iperf3_attributes:
97                 iperf3_cmd[u"args"] += \
98                     f" --parallel {iperf3_attributes[u'parallel']}"
99             if u"time" in iperf3_attributes:
100                 iperf3_cmd[u"args"] += \
101                     f" --time {iperf3_attributes[u'time']}"
102             if iperf3_attributes[u"udp"]:
103                 iperf3_cmd[u"args"] += u" --udp"
104                 iperf3_cmd[u"args"] += \
105                     f" --bandwidth {iperf3_attributes[u'bandwidth']}"
106             if iperf3_attributes[u"length"] > 0:
107                 iperf3_cmd[u"args"] += \
108                     f" --length {iperf3_attributes[u'length']}"
109         return iperf3_cmd
110
111     @staticmethod
112     def set_hoststack_quic_fifo_size(node, fifo_size):
113         """Set the QUIC protocol fifo size.
114
115         :param node: Node to set the QUIC fifo size on.
116         :param fifo_size: fifo size, passed to the quic set fifo-size command.
117         :type node: dict
118         :type fifo_size: str
119         """
120         cmd = f"quic set fifo-size {fifo_size}"
121         PapiSocketExecutor.run_cli_cmd(node, cmd)
122
123     @staticmethod
124     def set_hoststack_quic_crypto_engine(node, quic_crypto_engine,
125                                          fail_on_error=False):
126         """Set the Hoststack QUIC crypto engine on node
127
128         :param node: Node to enable/disable HostStack.
129         :param quic_crypto_engine: type of crypto engine
130         :type node: dict
131         :type quic_crypto_engine: str
132         """
133         vpp_crypto_engines = {u"openssl", u"native", u"ipsecmb"}
134         if quic_crypto_engine == u"nocrypto":
135             logger.trace(u"No QUIC crypto engine.")
136             return
137
138         if quic_crypto_engine in vpp_crypto_engines:
139             cmds = [u"quic set crypto api vpp",
140                     f"set crypto handler aes-128-gcm {quic_crypto_engine}",
141                     f"set crypto handler aes-256-gcm {quic_crypto_engine}"]
142         elif quic_crypto_engine == u"picotls":
143             cmds = [u"quic set crypto api picotls"]
144         else:
145             raise ValueError(f"Unknown QUIC crypto_engine {quic_crypto_engine}")
146
147         for cmd in cmds:
148             try:
149                 PapiSocketExecutor.run_cli_cmd(node, cmd)
150             except AssertionError:
151                 if fail_on_error:
152                     raise
153
154     @staticmethod
155     def get_hoststack_test_program_logs(node, program):
156         """Get HostStack test program stdout log.
157
158         :param node: DUT node.
159         :param program: test program.
160         :type node: dict
161         :type program: dict
162         """
163         program_name = program[u"name"]
164         cmd = f"sh -c \'cat /tmp/{program_name}_stdout.log\'"
165         stdout_log, _ = exec_cmd_no_error(node, cmd, sudo=True, \
166             message=f"Get {program_name} stdout log failed!")
167
168         cmd = f"sh -c \'cat /tmp/{program_name}_stderr.log\'"
169         stderr_log, _ = exec_cmd_no_error(node, cmd, sudo=True, \
170             message=f"Get {program_name} stderr log failed!")
171         return stdout_log, stderr_log
172
173     @staticmethod
174     def get_nginx_command(nginx_attributes, nginx_version, nginx_ins_dir):
175         """Construct the NGINX command using the specified attributes.
176
177         :param nginx_attributes: NGINX test program attributes.
178         :param nginx_version: NGINX version.
179         :param nginx_ins_dir: NGINX install dir.
180         :type nginx_attributes: dict
181         :type nginx_version: str
182         :type nginx_ins_dir: str
183         :returns: Command line components of the NGINX command
184             'env_vars' - environment variables
185             'name' - program name
186             'args' - command arguments.
187             'path' - program path.
188         :rtype: dict
189         """
190         nginx_cmd = dict()
191         nginx_cmd[u"env_vars"] = f"VCL_CONFIG={Constants.REMOTE_FW_DIR}/" \
192                                  f"{Constants.RESOURCES_TPL_VCL}/" \
193                                  f"{nginx_attributes[u'vcl_config']}"
194         if nginx_attributes[u"ld_preload"]:
195             nginx_cmd[u"env_vars"] += \
196                 f" LD_PRELOAD={Constants.VCL_LDPRELOAD_LIBRARY}"
197         if nginx_attributes[u'transparent_tls']:
198             nginx_cmd[u"env_vars"] += u" LDP_ENV_TLS_TRANS=1"
199
200         nginx_cmd[u"name"] = u"nginx"
201         nginx_cmd[u"path"] = f"{nginx_ins_dir}nginx-{nginx_version}/sbin/"
202         nginx_cmd[u"args"] = f"-c {nginx_ins_dir}/" \
203                              f"nginx-{nginx_version}/conf/nginx.conf"
204         return nginx_cmd
205
206     @staticmethod
207     def start_hoststack_test_program(node, namespace, core_list, program):
208         """Start the specified HostStack test program.
209
210         :param node: DUT node.
211         :param namespace: Net Namespace to run program in.
212         :param core_list: List of cpu's to pass to taskset to pin the test
213             program to a different set of cores on the same numa node as VPP.
214         :param program: Test program.
215         :type node: dict
216         :type namespace: str
217         :type core_list: str
218         :type program: dict
219         :returns: Process ID
220         :rtype: int
221         :raises RuntimeError: If node subtype is not a DUT or startup failed.
222         """
223         if node[u"type"] != u"DUT":
224             raise RuntimeError(u"Node type is not a DUT!")
225
226         program_name = program[u"name"]
227         DUTSetup.kill_program(node, program_name, namespace)
228
229         if namespace == u"default":
230             shell_cmd = u"sh -c"
231         else:
232             shell_cmd = f"ip netns exec {namespace} sh -c"
233
234         env_vars = f"{program[u'env_vars']} " if u"env_vars" in program else u""
235         args = program[u"args"]
236         program_path = program.get(u"path", u"")
237         # NGINX used `worker_cpu_affinity` in configuration file
238         taskset_cmd = u"" if program_name == u"nginx" else \
239                                              f"taskset --cpu-list {core_list}"
240         cmd = f"nohup {shell_cmd} \'{env_vars}{taskset_cmd} " \
241               f"{program_path}{program_name} {args} >/tmp/{program_name}_" \
242               f"stdout.log 2>/tmp/{program_name}_stderr.log &\'"
243         try:
244             exec_cmd_no_error(node, cmd, sudo=True)
245             return DUTSetup.get_pid(node, program_name)[0]
246         except RuntimeError:
247             stdout_log, stderr_log = \
248                 HoststackUtil.get_hoststack_test_program_logs(node,
249                                                               program)
250             raise RuntimeError(f"Start {program_name} failed!\nSTDERR:\n" \
251                                f"{stderr_log}\nSTDOUT:\n{stdout_log}")
252         return None
253
254     @staticmethod
255     def stop_hoststack_test_program(node, program, pid):
256         """Stop the specified Hoststack test program.
257
258         :param node: DUT node.
259         :param program: Test program.
260         :param pid: Process ID of test program.
261         :type node: dict
262         :type program: dict
263         :type pid: int
264         """
265         program_name = program[u"name"]
266         if program_name == u"nginx":
267             cmd = u"nginx -s quit"
268             errmsg = u"Quit nginx failed!"
269         else:
270             cmd = f'if [ -n "$(ps {pid} | grep {program_name})" ] ; ' \
271                 f'then kill -s SIGTERM {pid}; fi'
272             errmsg = f"Kill {program_name} ({pid}) failed!"
273
274         exec_cmd_no_error(node, cmd, message=errmsg, sudo=True)
275
276     @staticmethod
277     def hoststack_test_program_finished(node, program_pid):
278         """Wait for the specified HostStack test program process to complete.
279
280         :param node: DUT node.
281         :param program_pid: test program pid.
282         :type node: dict
283         :type program_pid: str
284         :raises RuntimeError: If node subtype is not a DUT.
285         """
286         if node[u"type"] != u"DUT":
287             raise RuntimeError(u"Node type is not a DUT!")
288
289         cmd = f"sh -c 'strace -qqe trace=none -p {program_pid}'"
290         exec_cmd(node, cmd, sudo=True)
291         # Wait a bit for stdout/stderr to be flushed to log files
292         # TODO: see if sub-second sleep works e.g. sleep(0.1)
293         sleep(1)
294
295     @staticmethod
296     def analyze_hoststack_test_program_output(
297             node, role, nsim_attr, program):
298         """Gather HostStack test program output and check for errors.
299
300         The [defer_fail] return bool is used instead of failing immediately
301         to allow the analysis of both the client and server instances of
302         the test program for debugging a test failure.  When [defer_fail]
303         is true, then the string returned is debug output instead of
304         JSON formatted test program results.
305
306         :param node: DUT node.
307         :param role: Role (client|server) of test program.
308         :param nsim_attr: Network Simulation Attributes.
309         :param program: Test program.
310         :param program_args: List of test program args.
311         :type node: dict
312         :type role: str
313         :type nsim_attr: dict
314         :type program: dict
315         :returns: tuple of [defer_fail] bool and either JSON formatted hoststack
316             test program output or failure debug output.
317         :rtype: bool, str
318         :raises RuntimeError: If node subtype is not a DUT.
319         """
320         if node[u"type"] != u"DUT":
321             raise RuntimeError(u"Node type is not a DUT!")
322
323         program_name = program[u"name"]
324         program_stdout, program_stderr = \
325             HoststackUtil.get_hoststack_test_program_logs(node, program)
326         if len(program_stdout) == 0 and len(program_stderr) == 0:
327             logger.trace(f"Retrying {program_name} log retrieval")
328             program_stdout, program_stderr = \
329                HoststackUtil.get_hoststack_test_program_logs(node, program)
330
331         env_vars = f"{program[u'env_vars']} " if u"env_vars" in program else u""
332         program_cmd = f"{env_vars}{program_name} {program[u'args']}"
333         test_results = f"Test Results of '{program_cmd}':\n"
334
335         if nsim_attr[u"output_nsim_enable"] or \
336             nsim_attr[u"xc_nsim_enable"]:
337             if nsim_attr[u"output_nsim_enable"]:
338                 feature_name = u"output"
339             else:
340                 feature_name = u"cross-connect"
341             test_results += \
342                 f"NSIM({feature_name}): delay " \
343                 f"{nsim_attr[u'delay_in_usec']} usecs, " \
344                 f"avg-pkt-size {nsim_attr[u'average_packet_size']}, " \
345                 f"bandwidth {nsim_attr[u'bw_in_bits_per_second']} " \
346                 f"bits/sec, pkt-drop-rate {nsim_attr[u'packets_per_drop']} " \
347                 f"pkts/drop\n"
348
349         # TODO: Incorporate show error stats into results analysis
350         test_results += \
351             f"\n{role} VPP 'show errors' on host {node[u'host']}:\n" \
352             f"{PapiSocketExecutor.run_cli_cmd(node, u'show error')}\n"
353
354         if u"error" in program_stderr.lower():
355             test_results += f"ERROR DETECTED:\n{program_stderr}"
356             return (True, test_results)
357         if not program_stdout:
358             test_results += f"\nNo {program} test data retrieved!\n"
359             ls_stdout, _ = exec_cmd_no_error(node, u"ls -l /tmp/*.log",
360                                              sudo=True)
361             test_results += f"{ls_stdout}\n"
362             return (True, test_results)
363         if program[u"name"] == u"vpp_echo":
364             if u"JSON stats" in program_stdout and \
365                     u'"has_failed": "0"' in program_stdout:
366                 json_start = program_stdout.find(u"{")
367                 #TODO: Fix parsing once vpp_echo produces valid
368                 # JSON output. Truncate for now.
369                 json_end = program_stdout.find(u',\n  "closing"')
370                 json_results = f"{program_stdout[json_start:json_end]}\n}}"
371                 program_json = json.loads(json_results)
372             else:
373                 test_results += u"Invalid test data output!\n" + program_stdout
374                 return (True, test_results)
375         elif program[u"name"] == u"iperf3":
376             test_results += program_stdout
377             iperf3_json = json.loads(program_stdout)
378             program_json = iperf3_json[u"intervals"][0][u"sum"]
379         else:
380             test_results += u"Unknown HostStack Test Program!\n" + \
381                             program_stdout
382             return (True, program_stdout)
383         return (False, json.dumps(program_json))
384
385     @staticmethod
386     def hoststack_test_program_defer_fail(server_defer_fail, client_defer_fail):
387         """Return True if either HostStack test program fail was deferred.
388
389         :param server_defer_fail: server no results value.
390         :param client_defer_fail: client no results value.
391         :type server_defer_fail: bool
392         :type client_defer_fail: bool
393         :rtype: bool
394         """
395         return server_defer_fail and client_defer_fail
396
397     @staticmethod
398     def log_vpp_hoststack_data(node):
399         """Retrieve and log VPP HostStack data.
400
401         :param node: DUT node.
402         :type node: dict
403         :raises RuntimeError: If node subtype is not a DUT or startup failed.
404         """
405
406         if node[u"type"] != u"DUT":
407             raise RuntimeError(u"Node type is not a DUT!")
408
409         PapiSocketExecutor.run_cli_cmd(node, u"show error")
410         PapiSocketExecutor.run_cli_cmd(node, u"show interface")