resources/libraries/python/HoststackUtil.py

   1 # Copyright (c) 2023 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Host Stack util library."""
  15 import json
  16 from time import sleep
  17 from robot.api import logger
  18
  19 from resources.libraries.python.Constants import Constants
  20 from resources.libraries.python.DUTSetup import DUTSetup
  21 from resources.libraries.python.model.ExportResult import (
  22     export_hoststack_results
  23 )
  24 from resources.libraries.python.PapiExecutor import PapiSocketExecutor
  25 from resources.libraries.python.ssh import exec_cmd, exec_cmd_no_error
  26
  27 class HoststackUtil():
  28     """Utilities for Host Stack tests."""
  29
  30     @staticmethod
  31     def get_vpp_echo_command(vpp_echo_attributes):
  32         """Construct the vpp_echo command using the specified attributes.
  33
  34         :param vpp_echo_attributes: vpp_echo test program attributes.
  35         :type vpp_echo_attributes: dict
  36         :returns: Command line components of the vpp_echo command
  37             'name' - program name
  38             'args' - command arguments.
  39         :rtype: dict
  40         """
  41         proto = vpp_echo_attributes[u"uri_protocol"]
  42         addr = vpp_echo_attributes[u"uri_ip4_addr"]
  43         port = vpp_echo_attributes[u"uri_port"]
  44         vpp_echo_cmd = {}
  45         vpp_echo_cmd[u"name"] = u"vpp_echo"
  46         vpp_echo_cmd[u"args"] = f"{vpp_echo_attributes[u'role']} " \
  47             f"socket-name {vpp_echo_attributes[u'vpp_api_socket']} " \
  48             f"{vpp_echo_attributes[u'json_output']} " \
  49             f"uri {proto}://{addr}/{port} " \
  50             f"nthreads {vpp_echo_attributes[u'nthreads']} " \
  51             f"mq-size {vpp_echo_attributes[u'mq_size']} " \
  52             f"nclients {vpp_echo_attributes[u'nclients']} " \
  53             f"quic-streams {vpp_echo_attributes[u'quic_streams']} " \
  54             f"time {vpp_echo_attributes[u'time']} " \
  55             f"fifo-size {vpp_echo_attributes[u'fifo_size']} " \
  56             f"TX={vpp_echo_attributes[u'tx_bytes']} " \
  57             f"RX={vpp_echo_attributes[u'rx_bytes']}"
  58         if vpp_echo_attributes[u"rx_results_diff"]:
  59             vpp_echo_cmd[u"args"] += u" rx-results-diff"
  60         if vpp_echo_attributes[u"tx_results_diff"]:
  61             vpp_echo_cmd[u"args"] += u" tx-results-diff"
  62         return vpp_echo_cmd
  63
  64     @staticmethod
  65     def get_iperf3_command(iperf3_attributes):
  66         """Construct the iperf3 command using the specified attributes.
  67
  68         :param iperf3_attributes: iperf3 test program attributes.
  69         :type iperf3_attributes: dict
  70         :returns: Command line components of the iperf3 command
  71             'env_vars' - environment variables
  72             'name' - program name
  73             'args' - command arguments.
  74         :rtype: dict
  75         """
  76         iperf3_cmd = {}
  77         iperf3_cmd[u"env_vars"] = f"VCL_CONFIG={Constants.REMOTE_FW_DIR}/" \
  78             f"{Constants.RESOURCES_TPL_VCL}/" \
  79             f"{iperf3_attributes[u'vcl_config']}"
  80         if iperf3_attributes[u"ld_preload"]:
  81             iperf3_cmd[u"env_vars"] += \
  82                 f" LD_PRELOAD={Constants.VCL_LDPRELOAD_LIBRARY}"
  83         if iperf3_attributes[u'transparent_tls']:
  84             iperf3_cmd[u"env_vars"] += u" LDP_ENV_TLS_TRANS=1"
  85
  86         json_results = u" --json" if iperf3_attributes[u'json'] else u""
  87         ip_address = f" {iperf3_attributes[u'ip_address']}" if u"ip_address" \
  88                      in iperf3_attributes else u""
  89         iperf3_cmd[u"name"] = u"iperf3"
  90         iperf3_cmd[u"args"] = f"--{iperf3_attributes[u'role']}{ip_address} " \
  91                               f"--interval 0{json_results} " \
  92                               f"--version{iperf3_attributes[u'ip_version']}"
  93
  94         if iperf3_attributes[u"role"] == u"server":
  95             iperf3_cmd[u"args"] += u" --one-off"
  96         else:
  97             iperf3_cmd[u"args"] += u" --get-server-output"
  98             if u"parallel" in iperf3_attributes:
  99                 iperf3_cmd[u"args"] += \
 100                     f" --parallel {iperf3_attributes[u'parallel']}"
 101             if u"time" in iperf3_attributes:
 102                 iperf3_cmd[u"args"] += \
 103                     f" --time {iperf3_attributes[u'time']}"
 104             if iperf3_attributes[u"udp"]:
 105                 iperf3_cmd[u"args"] += u" --udp"
 106                 iperf3_cmd[u"args"] += \
 107                     f" --bandwidth {iperf3_attributes[u'bandwidth']}"
 108             if iperf3_attributes[u"length"] > 0:
 109                 iperf3_cmd[u"args"] += \
 110                     f" --length {iperf3_attributes[u'length']}"
 111         return iperf3_cmd
 112
 113     @staticmethod
 114     def set_hoststack_quic_fifo_size(node, fifo_size):
 115         """Set the QUIC protocol fifo size.
 116
 117         :param node: Node to set the QUIC fifo size on.
 118         :param fifo_size: fifo size, passed to the quic set fifo-size command.
 119         :type node: dict
 120         :type fifo_size: str
 121         """
 122         cmd = f"quic set fifo-size {fifo_size}"
 123         PapiSocketExecutor.run_cli_cmd(node, cmd)
 124
 125     @staticmethod
 126     def set_hoststack_quic_crypto_engine(node, quic_crypto_engine,
 127                                          fail_on_error=False):
 128         """Set the Hoststack QUIC crypto engine on node
 129
 130         :param node: Node to enable/disable HostStack.
 131         :param quic_crypto_engine: type of crypto engine
 132         :type node: dict
 133         :type quic_crypto_engine: str
 134         """
 135         vpp_crypto_engines = {u"openssl", u"native", u"ipsecmb"}
 136         if quic_crypto_engine == u"nocrypto":
 137             logger.trace(u"No QUIC crypto engine.")
 138             return
 139
 140         if quic_crypto_engine in vpp_crypto_engines:
 141             cmds = [u"quic set crypto api vpp",
 142                     f"set crypto handler aes-128-gcm {quic_crypto_engine}",
 143                     f"set crypto handler aes-256-gcm {quic_crypto_engine}"]
 144         elif quic_crypto_engine == u"picotls":
 145             cmds = [u"quic set crypto api picotls"]
 146         else:
 147             raise ValueError(f"Unknown QUIC crypto_engine {quic_crypto_engine}")
 148
 149         for cmd in cmds:
 150             try:
 151                 PapiSocketExecutor.run_cli_cmd(node, cmd)
 152             except AssertionError:
 153                 if fail_on_error:
 154                     raise
 155
 156     @staticmethod
 157     def get_hoststack_test_program_logs(node, program):
 158         """Get HostStack test program stdout log.
 159
 160         :param node: DUT node.
 161         :param program: test program.
 162         :type node: dict
 163         :type program: dict
 164         """
 165         program_name = program[u"name"]
 166         cmd = f"sh -c \'cat /tmp/{program_name}_stdout.log\'"
 167         stdout_log, _ = exec_cmd_no_error(node, cmd, sudo=True, \
 168             message=f"Get {program_name} stdout log failed!")
 169
 170         cmd = f"sh -c \'cat /tmp/{program_name}_stderr.log\'"
 171         stderr_log, _ = exec_cmd_no_error(node, cmd, sudo=True, \
 172             message=f"Get {program_name} stderr log failed!")
 173         return stdout_log, stderr_log
 174
 175     @staticmethod
 176     def get_nginx_command(nginx_attributes, nginx_version, nginx_ins_dir):
 177         """Construct the NGINX command using the specified attributes.
 178
 179         :param nginx_attributes: NGINX test program attributes.
 180         :param nginx_version: NGINX version.
 181         :param nginx_ins_dir: NGINX install dir.
 182         :type nginx_attributes: dict
 183         :type nginx_version: str
 184         :type nginx_ins_dir: str
 185         :returns: Command line components of the NGINX command
 186             'env_vars' - environment variables
 187             'name' - program name
 188             'args' - command arguments.
 189             'path' - program path.
 190         :rtype: dict
 191         """
 192         nginx_cmd = dict()
 193         nginx_cmd[u"env_vars"] = f"VCL_CONFIG={Constants.REMOTE_FW_DIR}/" \
 194                                  f"{Constants.RESOURCES_TPL_VCL}/" \
 195                                  f"{nginx_attributes[u'vcl_config']}"
 196         if nginx_attributes[u"ld_preload"]:
 197             nginx_cmd[u"env_vars"] += \
 198                 f" LD_PRELOAD={Constants.VCL_LDPRELOAD_LIBRARY}"
 199         if nginx_attributes[u'transparent_tls']:
 200             nginx_cmd[u"env_vars"] += u" LDP_ENV_TLS_TRANS=1"
 201
 202         nginx_cmd[u"name"] = u"nginx"
 203         nginx_cmd[u"path"] = f"{nginx_ins_dir}nginx-{nginx_version}/sbin/"
 204         nginx_cmd[u"args"] = f"-c {nginx_ins_dir}/" \
 205                              f"nginx-{nginx_version}/conf/nginx.conf"
 206         return nginx_cmd
 207
 208     @staticmethod
 209     def start_hoststack_test_program(node, namespace, core_list, program):
 210         """Start the specified HostStack test program.
 211
 212         :param node: DUT node.
 213         :param namespace: Net Namespace to run program in.
 214         :param core_list: List of cpu's to pass to taskset to pin the test
 215             program to a different set of cores on the same numa node as VPP.
 216         :param program: Test program.
 217         :type node: dict
 218         :type namespace: str
 219         :type core_list: str
 220         :type program: dict
 221         :returns: Process ID
 222         :rtype: int
 223         :raises RuntimeError: If node subtype is not a DUT or startup failed.
 224         """
 225         if node[u"type"] != u"DUT":
 226             raise RuntimeError(u"Node type is not a DUT!")
 227
 228         program_name = program[u"name"]
 229         DUTSetup.kill_program(node, program_name, namespace)
 230
 231         if namespace == u"default":
 232             shell_cmd = u"sh -c"
 233         else:
 234             shell_cmd = f"ip netns exec {namespace} sh -c"
 235
 236         env_vars = f"{program[u'env_vars']} " if u"env_vars" in program else u""
 237         args = program[u"args"]
 238         program_path = program.get(u"path", u"")
 239         # NGINX used `worker_cpu_affinity` in configuration file
 240         taskset_cmd = u"" if program_name == u"nginx" else \
 241                                              f"taskset --cpu-list {core_list}"
 242         cmd = f"nohup {shell_cmd} \'{env_vars}{taskset_cmd} " \
 243               f"{program_path}{program_name} {args} >/tmp/{program_name}_" \
 244               f"stdout.log 2>/tmp/{program_name}_stderr.log &\'"
 245         try:
 246             exec_cmd_no_error(node, cmd, sudo=True)
 247             return DUTSetup.get_pid(node, program_name)[0]
 248         except RuntimeError:
 249             stdout_log, stderr_log = \
 250                 HoststackUtil.get_hoststack_test_program_logs(node,
 251                                                               program)
 252             raise RuntimeError(f"Start {program_name} failed!\nSTDERR:\n" \
 253                                f"{stderr_log}\nSTDOUT:\n{stdout_log}")
 254         return None
 255
 256     @staticmethod
 257     def stop_hoststack_test_program(node, program, pid):
 258         """Stop the specified Hoststack test program.
 259
 260         :param node: DUT node.
 261         :param program: Test program.
 262         :param pid: Process ID of test program.
 263         :type node: dict
 264         :type program: dict
 265         :type pid: int
 266         """
 267         program_name = program[u"name"]
 268         if program_name == u"nginx":
 269             cmd = u"nginx -s quit"
 270             errmsg = u"Quit nginx failed!"
 271         else:
 272             cmd = f'if [ -n "$(ps {pid} | grep {program_name})" ] ; ' \
 273                 f'then kill -s SIGTERM {pid}; fi'
 274             errmsg = f"Kill {program_name} ({pid}) failed!"
 275
 276         exec_cmd_no_error(node, cmd, message=errmsg, sudo=True)
 277
 278     @staticmethod
 279     def hoststack_test_program_finished(node, program_pid):
 280         """Wait for the specified HostStack test program process to complete.
 281
 282         :param node: DUT node.
 283         :param program_pid: test program pid.
 284         :type node: dict
 285         :type program_pid: str
 286         :raises RuntimeError: If node subtype is not a DUT.
 287         """
 288         if node[u"type"] != u"DUT":
 289             raise RuntimeError(u"Node type is not a DUT!")
 290
 291         cmd = f"sh -c 'strace -qqe trace=none -p {program_pid}'"
 292         exec_cmd(node, cmd, sudo=True)
 293         # Wait a bit for stdout/stderr to be flushed to log files
 294         sleep(1)
 295
 296     @staticmethod
 297     def analyze_hoststack_test_program_output(
 298             node, role, nsim_attr, program):
 299         """Gather HostStack test program output and check for errors.
 300
 301         The [defer_fail] return bool is used instead of failing immediately
 302         to allow the analysis of both the client and server instances of
 303         the test program for debugging a test failure.  When [defer_fail]
 304         is true, then the string returned is debug output instead of
 305         JSON formatted test program results.
 306
 307         :param node: DUT node.
 308         :param role: Role (client|server) of test program.
 309         :param nsim_attr: Network Simulation Attributes.
 310         :param program: Test program.
 311         :param program_args: List of test program args.
 312         :type node: dict
 313         :type role: str
 314         :type nsim_attr: dict
 315         :type program: dict
 316         :returns: tuple of [defer_fail] bool and either JSON formatted hoststack
 317             test program output or failure debug output.
 318         :rtype: bool, str
 319         :raises RuntimeError: If node subtype is not a DUT.
 320         """
 321         if node[u"type"] != u"DUT":
 322             raise RuntimeError(u"Node type is not a DUT!")
 323
 324         program_name = program[u"name"]
 325         program_stdout, program_stderr = \
 326             HoststackUtil.get_hoststack_test_program_logs(node, program)
 327         if len(program_stdout) == 0 and len(program_stderr) == 0:
 328             logger.trace(f"Retrying {program_name} log retrieval")
 329             program_stdout, program_stderr = \
 330                HoststackUtil.get_hoststack_test_program_logs(node, program)
 331
 332         env_vars = f"{program[u'env_vars']} " if u"env_vars" in program else u""
 333         program_cmd = f"{env_vars}{program_name} {program[u'args']}"
 334         test_results = f"Test Results of '{program_cmd}':\n"
 335
 336         if nsim_attr[u"output_nsim_enable"] or \
 337             nsim_attr[u"xc_nsim_enable"]:
 338             if nsim_attr[u"output_nsim_enable"]:
 339                 feature_name = u"output"
 340             else:
 341                 feature_name = u"cross-connect"
 342             test_results += \
 343                 f"NSIM({feature_name}): delay " \
 344                 f"{nsim_attr[u'delay_in_usec']} usecs, " \
 345                 f"avg-pkt-size {nsim_attr[u'average_packet_size']}, " \
 346                 f"bandwidth {nsim_attr[u'bw_in_bits_per_second']} " \
 347                 f"bits/sec, pkt-drop-rate {nsim_attr[u'packets_per_drop']} " \
 348                 f"pkts/drop\n"
 349
 350         test_results += \
 351             f"\n{role} VPP 'show errors' on host {node[u'host']}:\n" \
 352             f"{PapiSocketExecutor.run_cli_cmd(node, u'show error')}\n"
 353
 354         if u"error" in program_stderr.lower():
 355             test_results += f"ERROR DETECTED:\n{program_stderr}"
 356             return (True, test_results)
 357         if not program_stdout:
 358             test_results += f"\nNo {program} test data retrieved!\n"
 359             ls_stdout, _ = exec_cmd_no_error(node, u"ls -l /tmp/*.log",
 360                                              sudo=True)
 361             test_results += f"{ls_stdout}\n"
 362             return (True, test_results)
 363         if program[u"name"] == u"vpp_echo":
 364             if u"JSON stats" in program_stdout and \
 365                     u'"has_failed": "0"' in program_stdout:
 366                 json_start = program_stdout.find(u"{")
 367                 json_end = program_stdout.find(u',\n  "closing"')
 368                 json_results = f"{program_stdout[json_start:json_end]}\n}}"
 369                 program_json = json.loads(json_results)
 370                 export_hoststack_results(
 371                     bandwidth=program_json["rx_bits_per_second"],
 372                     duration=program_json["time"]
 373                 )
 374             else:
 375                 test_results += u"Invalid test data output!\n" + program_stdout
 376                 return (True, test_results)
 377         elif program[u"name"] == u"iperf3":
 378             test_results += program_stdout
 379             iperf3_json = json.loads(program_stdout)
 380             program_json = iperf3_json[u"intervals"][0][u"sum"]
 381             export_hoststack_results(
 382                 bandwidth=program_json["bits_per_second"],
 383                 duration=program_json["seconds"],
 384                 retransmits=program_json["retransmits"]
 385             )
 386         else:
 387             test_results += u"Unknown HostStack Test Program!\n" + \
 388                             program_stdout
 389             return (True, program_stdout)
 390         return (False, json.dumps(program_json))
 391
 392     @staticmethod
 393     def hoststack_test_program_defer_fail(server_defer_fail, client_defer_fail):
 394         """Return True if either HostStack test program fail was deferred.
 395
 396         :param server_defer_fail: server no results value.
 397         :param client_defer_fail: client no results value.
 398         :type server_defer_fail: bool
 399         :type client_defer_fail: bool
 400         :rtype: bool
 401         """
 402         return server_defer_fail and client_defer_fail
 403
 404     @staticmethod
 405     def log_vpp_hoststack_data(node):
 406         """Retrieve and log VPP HostStack data.
 407
 408         :param node: DUT node.
 409         :type node: dict
 410         :raises RuntimeError: If node subtype is not a DUT or startup failed.
 411         """
 412
 413         if node[u"type"] != u"DUT":
 414             raise RuntimeError(u"Node type is not a DUT!")
 415
 416         PapiSocketExecutor.run_cli_cmd(node, u"show error")
 417         PapiSocketExecutor.run_cli_cmd(node, u"show interface")