resources/libraries/python/HoststackUtil.py

   1 # Copyright (c) 2023 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Host Stack util library."""
  15 import json
  16 from time import sleep
  17 from robot.api import logger
  18
  19 from resources.libraries.python.Constants import Constants
  20 from resources.libraries.python.DUTSetup import DUTSetup
  21 from resources.libraries.python.model.ExportResult import (
  22     export_hoststack_results
  23 )
  24 from resources.libraries.python.PapiExecutor import PapiSocketExecutor
  25 from resources.libraries.python.ssh import exec_cmd, exec_cmd_no_error
  26
  27 class HoststackUtil():
  28     """Utilities for Host Stack tests."""
  29
  30     @staticmethod
  31     def get_vpp_echo_command(vpp_echo_attributes):
  32         """Construct the vpp_echo command using the specified attributes.
  33
  34         :param vpp_echo_attributes: vpp_echo test program attributes.
  35         :type vpp_echo_attributes: dict
  36         :returns: Command line components of the vpp_echo command
  37             'name' - program name
  38             'args' - command arguments.
  39         :rtype: dict
  40         """
  41         proto = vpp_echo_attributes[u"uri_protocol"]
  42         addr = vpp_echo_attributes[u"uri_ip4_addr"]
  43         port = vpp_echo_attributes[u"uri_port"]
  44         vpp_echo_cmd = {}
  45         vpp_echo_cmd[u"name"] = u"vpp_echo"
  46         vpp_echo_cmd[u"args"] = f"{vpp_echo_attributes[u'role']} " \
  47             f"socket-name {vpp_echo_attributes[u'app_api_socket']} " \
  48             f"{vpp_echo_attributes[u'json_output']} " \
  49             f"uri {proto}://{addr}/{port} " \
  50             f"nthreads {vpp_echo_attributes[u'nthreads']} " \
  51             f"mq-size {vpp_echo_attributes[u'mq_size']} " \
  52             f"nclients {vpp_echo_attributes[u'nclients']} " \
  53             f"quic-streams {vpp_echo_attributes[u'quic_streams']} " \
  54             f"time {vpp_echo_attributes[u'time']} " \
  55             f"fifo-size {vpp_echo_attributes[u'fifo_size']} " \
  56             f"TX={vpp_echo_attributes[u'tx_bytes']} " \
  57             f"RX={vpp_echo_attributes[u'rx_bytes']}"
  58         if vpp_echo_attributes[u"rx_results_diff"]:
  59             vpp_echo_cmd[u"args"] += u" rx-results-diff"
  60         if vpp_echo_attributes[u"tx_results_diff"]:
  61             vpp_echo_cmd[u"args"] += u" tx-results-diff"
  62         if vpp_echo_attributes[u"use_app_socket_api"]:
  63             vpp_echo_cmd[u"args"] += u" use-app-socket-api"
  64         return vpp_echo_cmd
  65
  66     @staticmethod
  67     def get_iperf3_command(iperf3_attributes):
  68         """Construct the iperf3 command using the specified attributes.
  69
  70         :param iperf3_attributes: iperf3 test program attributes.
  71         :type iperf3_attributes: dict
  72         :returns: Command line components of the iperf3 command
  73             'env_vars' - environment variables
  74             'name' - program name
  75             'args' - command arguments.
  76         :rtype: dict
  77         """
  78         iperf3_cmd = {}
  79         iperf3_cmd[u"env_vars"] = f"VCL_CONFIG={Constants.REMOTE_FW_DIR}/" \
  80             f"{Constants.RESOURCES_TPL_VCL}/" \
  81             f"{iperf3_attributes[u'vcl_config']}"
  82         if iperf3_attributes[u"ld_preload"]:
  83             iperf3_cmd[u"env_vars"] += \
  84                 f" LD_PRELOAD={Constants.VCL_LDPRELOAD_LIBRARY}"
  85         if iperf3_attributes[u'transparent_tls']:
  86             iperf3_cmd[u"env_vars"] += u" LDP_ENV_TLS_TRANS=1"
  87
  88         json_results = u" --json" if iperf3_attributes[u'json'] else u""
  89         ip_address = f" {iperf3_attributes[u'ip_address']}" if u"ip_address" \
  90                      in iperf3_attributes else u""
  91         iperf3_cmd[u"name"] = u"iperf3"
  92         iperf3_cmd[u"args"] = f"--{iperf3_attributes[u'role']}{ip_address} " \
  93                               f"--interval 0{json_results} " \
  94                               f"--version{iperf3_attributes[u'ip_version']}"
  95
  96         if iperf3_attributes[u"role"] == u"server":
  97             iperf3_cmd[u"args"] += u" --one-off"
  98         else:
  99             iperf3_cmd[u"args"] += u" --get-server-output"
 100             if u"parallel" in iperf3_attributes:
 101                 iperf3_cmd[u"args"] += \
 102                     f" --parallel {iperf3_attributes[u'parallel']}"
 103             if u"time" in iperf3_attributes:
 104                 iperf3_cmd[u"args"] += \
 105                     f" --time {iperf3_attributes[u'time']}"
 106             if iperf3_attributes[u"udp"]:
 107                 iperf3_cmd[u"args"] += u" --udp"
 108                 iperf3_cmd[u"args"] += \
 109                     f" --bandwidth {iperf3_attributes[u'bandwidth']}"
 110             if iperf3_attributes[u"length"] > 0:
 111                 iperf3_cmd[u"args"] += \
 112                     f" --length {iperf3_attributes[u'length']}"
 113         return iperf3_cmd
 114
 115     @staticmethod
 116     def set_hoststack_quic_fifo_size(node, fifo_size):
 117         """Set the QUIC protocol fifo size.
 118
 119         :param node: Node to set the QUIC fifo size on.
 120         :param fifo_size: fifo size, passed to the quic set fifo-size command.
 121         :type node: dict
 122         :type fifo_size: str
 123         """
 124         cmd = f"quic set fifo-size {fifo_size}"
 125         PapiSocketExecutor.run_cli_cmd(node, cmd)
 126
 127     @staticmethod
 128     def set_hoststack_quic_crypto_engine(node, quic_crypto_engine,
 129                                          fail_on_error=False):
 130         """Set the Hoststack QUIC crypto engine on node
 131
 132         :param node: Node to enable/disable HostStack.
 133         :param quic_crypto_engine: type of crypto engine
 134         :type node: dict
 135         :type quic_crypto_engine: str
 136         """
 137         vpp_crypto_engines = {u"openssl", u"native", u"ipsecmb"}
 138         if quic_crypto_engine == u"nocrypto":
 139             logger.trace(u"No QUIC crypto engine.")
 140             return
 141
 142         if quic_crypto_engine in vpp_crypto_engines:
 143             cmds = [u"quic set crypto api vpp",
 144                     f"set crypto handler aes-128-gcm {quic_crypto_engine}",
 145                     f"set crypto handler aes-256-gcm {quic_crypto_engine}"]
 146         elif quic_crypto_engine == u"picotls":
 147             cmds = [u"quic set crypto api picotls"]
 148         else:
 149             raise ValueError(f"Unknown QUIC crypto_engine {quic_crypto_engine}")
 150
 151         for cmd in cmds:
 152             try:
 153                 PapiSocketExecutor.run_cli_cmd(node, cmd)
 154             except AssertionError:
 155                 if fail_on_error:
 156                     raise
 157
 158     @staticmethod
 159     def _get_hoststack_test_program_logs(node, program_name):
 160         """Get HostStack test program stdout log.
 161
 162         :param node: DUT node.
 163         :param program_name: test program.
 164         :type node: dict
 165         :type program_name: str
 166         """
 167         cmd = f"sh -c \'cat /tmp/{program_name}_stdout.log\'"
 168         stdout_log, _ = exec_cmd_no_error(node, cmd, sudo=True, \
 169             message=f"Get {program_name} stdout log failed!")
 170
 171         cmd = f"sh -c \'cat /tmp/{program_name}_stderr.log\'"
 172         stderr_log, _ = exec_cmd_no_error(node, cmd, sudo=True, \
 173             message=f"Get {program_name} stderr log failed!")
 174
 175         return stdout_log, stderr_log
 176
 177     @staticmethod
 178     def get_hoststack_test_program_logs(node, program):
 179         """Get HostStack test program stdout log.
 180
 181         :param node: DUT node.
 182         :param program: test program.
 183         :type node: dict
 184         :type program: dict
 185         """
 186         program_name = program[u"name"]
 187         program_stdout_log, program_stderr_log = \
 188             HoststackUtil._get_hoststack_test_program_logs(node,
 189                                                            program_name)
 190         if len(program_stdout_log) == 0 and len(program_stderr_log) == 0:
 191             logger.trace(f"Retrying {program_name} log retrieval")
 192             program_stdout_log, program_stderr_log = \
 193                HoststackUtil._get_hoststack_test_program_logs(node,
 194                                                               program_name)
 195         return program_stdout_log, program_stderr_log
 196
 197     @staticmethod
 198     def get_nginx_command(nginx_attributes, nginx_version, nginx_ins_dir):
 199         """Construct the NGINX command using the specified attributes.
 200
 201         :param nginx_attributes: NGINX test program attributes.
 202         :param nginx_version: NGINX version.
 203         :param nginx_ins_dir: NGINX install dir.
 204         :type nginx_attributes: dict
 205         :type nginx_version: str
 206         :type nginx_ins_dir: str
 207         :returns: Command line components of the NGINX command
 208             'env_vars' - environment variables
 209             'name' - program name
 210             'args' - command arguments.
 211             'path' - program path.
 212         :rtype: dict
 213         """
 214         nginx_cmd = dict()
 215         nginx_cmd[u"env_vars"] = f"VCL_CONFIG={Constants.REMOTE_FW_DIR}/" \
 216                                  f"{Constants.RESOURCES_TPL_VCL}/" \
 217                                  f"{nginx_attributes[u'vcl_config']}"
 218         if nginx_attributes[u"ld_preload"]:
 219             nginx_cmd[u"env_vars"] += \
 220                 f" LD_PRELOAD={Constants.VCL_LDPRELOAD_LIBRARY}"
 221         if nginx_attributes[u'transparent_tls']:
 222             nginx_cmd[u"env_vars"] += u" LDP_ENV_TLS_TRANS=1"
 223
 224         nginx_cmd[u"name"] = u"nginx"
 225         nginx_cmd[u"path"] = f"{nginx_ins_dir}nginx-{nginx_version}/sbin/"
 226         nginx_cmd[u"args"] = f"-c {nginx_ins_dir}/" \
 227                              f"nginx-{nginx_version}/conf/nginx.conf"
 228         return nginx_cmd
 229
 230     @staticmethod
 231     def start_hoststack_test_program(node, namespace, core_list, program):
 232         """Start the specified HostStack test program.
 233
 234         :param node: DUT node.
 235         :param namespace: Net Namespace to run program in.
 236         :param core_list: List of cpu's to pass to taskset to pin the test
 237             program to a different set of cores on the same numa node as VPP.
 238         :param program: Test program.
 239         :type node: dict
 240         :type namespace: str
 241         :type core_list: str
 242         :type program: dict
 243         :returns: Process ID
 244         :rtype: int
 245         :raises RuntimeError: If node subtype is not a DUT or startup failed.
 246         """
 247         if node[u"type"] != u"DUT":
 248             raise RuntimeError(u"Node type is not a DUT!")
 249
 250         program_name = program[u"name"]
 251         DUTSetup.kill_program(node, program_name, namespace)
 252
 253         if namespace == u"default":
 254             shell_cmd = u"sh -c"
 255         else:
 256             shell_cmd = f"ip netns exec {namespace} sh -c"
 257
 258         env_vars = f"{program[u'env_vars']} " if u"env_vars" in program else u""
 259         args = program[u"args"]
 260         program_path = program.get(u"path", u"")
 261         # NGINX used `worker_cpu_affinity` in configuration file
 262         taskset_cmd = u"" if program_name == u"nginx" else \
 263                                              f"taskset --cpu-list {core_list}"
 264         cmd = f"nohup {shell_cmd} \'{env_vars}{taskset_cmd} " \
 265               f"{program_path}{program_name} {args} >/tmp/{program_name}_" \
 266               f"stdout.log 2>/tmp/{program_name}_stderr.log &\'"
 267         try:
 268             exec_cmd_no_error(node, cmd, sudo=True)
 269             return DUTSetup.get_pid(node, program_name)[0]
 270         except RuntimeError:
 271             stdout_log, stderr_log = \
 272                 HoststackUtil.get_hoststack_test_program_logs(node,
 273                                                               program)
 274             raise RuntimeError(f"Start {program_name} failed!\nSTDERR:\n" \
 275                                f"{stderr_log}\nSTDOUT:\n{stdout_log}")
 276         return None
 277
 278     @staticmethod
 279     def stop_hoststack_test_program(node, program, pid):
 280         """Stop the specified Hoststack test program.
 281
 282         :param node: DUT node.
 283         :param program: Test program.
 284         :param pid: Process ID of test program.
 285         :type node: dict
 286         :type program: dict
 287         :type pid: int
 288         """
 289         program_name = program[u"name"]
 290         if program_name == u"nginx":
 291             cmd = u"nginx -s quit"
 292             errmsg = u"Quit nginx failed!"
 293         else:
 294             cmd = f'if [ -n "$(ps {pid} | grep {program_name})" ] ; ' \
 295                 f'then kill -s SIGTERM {pid}; fi'
 296             errmsg = f"Kill {program_name} ({pid}) failed!"
 297
 298         exec_cmd_no_error(node, cmd, message=errmsg, sudo=True)
 299
 300     @staticmethod
 301     def hoststack_test_program_finished(node, program_pid, program,
 302                                         other_node, other_program):
 303         """Wait for the specified HostStack test program process to complete.
 304
 305         :param node: DUT node.
 306         :param program_pid: test program pid.
 307         :param program: test program
 308         :param other_node: DUT node of other hoststack program
 309         :param other_program: other test program
 310         :type node: dict
 311         :type program_pid: str
 312         :type program: dict
 313         :type other_node: dict
 314         :type other_program: dict
 315         :raises RuntimeError: If node subtype is not a DUT.
 316         """
 317         if node[u"type"] != u"DUT":
 318             raise RuntimeError(u"Node type is not a DUT!")
 319         if other_node[u"type"] != u"DUT":
 320             raise RuntimeError(u"Other node type is not a DUT!")
 321
 322         cmd = f"sh -c 'strace -qqe trace=none -p {program_pid}'"
 323         try:
 324             exec_cmd(node, cmd, sudo=True)
 325         except:
 326             sleep(180)
 327             if u"client" in program[u"args"]:
 328                 role = u"client"
 329             else:
 330                 role = u"server"
 331             program_stdout, program_stderr = \
 332                 HoststackUtil.get_hoststack_test_program_logs(node, program)
 333             if len(program_stdout) > 0:
 334                 logger.debug(f"{program[u'name']} {role} stdout log:\n"
 335                              f"{program_stdout}")
 336             else:
 337                 logger.debug(f"Empty {program[u'name']} {role} stdout log :(")
 338             if len(program_stderr) > 0:
 339                 logger.debug(f"{program[u'name']} stderr log:\n"
 340                              f"{program_stderr}")
 341             else:
 342                 logger.debug(f"Empty {program[u'name']} stderr log :(")
 343             if u"client" in other_program[u"args"]:
 344                 role = u"client"
 345             else:
 346                 role = u"server"
 347             program_stdout, program_stderr = \
 348                 HoststackUtil.get_hoststack_test_program_logs(other_node,
 349                                                               other_program)
 350             if len(program_stdout) > 0:
 351                 logger.debug(f"{other_program[u'name']} {role} stdout log:\n"
 352                              f"{program_stdout}")
 353             else:
 354                 logger.debug(f"Empty {other_program[u'name']} "
 355                              f"{role} stdout log :(")
 356             if len(program_stderr) > 0:
 357                 logger.debug(f"{other_program[u'name']} {role} stderr log:\n"
 358                              f"{program_stderr}")
 359             else:
 360                 logger.debug(f"Empty {other_program[u'name']} "
 361                              f"{role} stderr log :(")
 362             raise
 363         # Wait a bit for stdout/stderr to be flushed to log files
 364         sleep(1)
 365
 366     @staticmethod
 367     def analyze_hoststack_test_program_output(
 368             node, role, nsim_attr, program):
 369         """Gather HostStack test program output and check for errors.
 370
 371         The [defer_fail] return bool is used instead of failing immediately
 372         to allow the analysis of both the client and server instances of
 373         the test program for debugging a test failure.  When [defer_fail]
 374         is true, then the string returned is debug output instead of
 375         JSON formatted test program results.
 376
 377         :param node: DUT node.
 378         :param role: Role (client|server) of test program.
 379         :param nsim_attr: Network Simulation Attributes.
 380         :param program: Test program.
 381         :param program_args: List of test program args.
 382         :type node: dict
 383         :type role: str
 384         :type nsim_attr: dict
 385         :type program: dict
 386         :returns: tuple of [defer_fail] bool and either JSON formatted hoststack
 387             test program output or failure debug output.
 388         :rtype: bool, str
 389         :raises RuntimeError: If node subtype is not a DUT.
 390         """
 391         if node[u"type"] != u"DUT":
 392             raise RuntimeError(u"Node type is not a DUT!")
 393
 394         program_name = program[u"name"]
 395         program_stdout, program_stderr = \
 396             HoststackUtil.get_hoststack_test_program_logs(node, program)
 397
 398         env_vars = f"{program[u'env_vars']} " if u"env_vars" in program else u""
 399         program_cmd = f"{env_vars}{program_name} {program[u'args']}"
 400         test_results = f"Test Results of '{program_cmd}':\n"
 401
 402         if nsim_attr[u"output_nsim_enable"] or \
 403             nsim_attr[u"xc_nsim_enable"]:
 404             if nsim_attr[u"output_nsim_enable"]:
 405                 feature_name = u"output"
 406             else:
 407                 feature_name = u"cross-connect"
 408             test_results += \
 409                 f"NSIM({feature_name}): delay " \
 410                 f"{nsim_attr[u'delay_in_usec']} usecs, " \
 411                 f"avg-pkt-size {nsim_attr[u'average_packet_size']}, " \
 412                 f"bandwidth {nsim_attr[u'bw_in_bits_per_second']} " \
 413                 f"bits/sec, pkt-drop-rate {nsim_attr[u'packets_per_drop']} " \
 414                 f"pkts/drop\n"
 415
 416         test_results += \
 417             f"\n{role} VPP 'show errors' on host {node[u'host']}:\n" \
 418             f"{PapiSocketExecutor.run_cli_cmd(node, u'show error')}\n"
 419
 420         if u"error" in program_stderr.lower():
 421             test_results += f"ERROR DETECTED:\n{program_stderr}"
 422             return (True, test_results)
 423         if not program_stdout:
 424             test_results += f"\nNo {program} test data retrieved!\n"
 425             ls_stdout, _ = exec_cmd_no_error(node, u"ls -l /tmp/*.log",
 426                                              sudo=True)
 427             test_results += f"{ls_stdout}\n"
 428             return (True, test_results)
 429         if program[u"name"] == u"vpp_echo":
 430             if u"JSON stats" in program_stdout and \
 431                     u'"has_failed": "0"' in program_stdout:
 432                 json_start = program_stdout.find(u"{")
 433                 json_end = program_stdout.find(u',\n  "closing"')
 434                 json_results = f"{program_stdout[json_start:json_end]}\n}}"
 435                 program_json = json.loads(json_results)
 436                 export_hoststack_results(
 437                     bandwidth=program_json["rx_bits_per_second"],
 438                     duration=float(program_json["time"])
 439                 )
 440             else:
 441                 test_results += u"Invalid test data output!\n" + program_stdout
 442                 return (True, test_results)
 443         elif program[u"name"] == u"iperf3":
 444             test_results += program_stdout
 445             program_json = json.loads(program_stdout)[u"intervals"][0][u"sum"]
 446             try:
 447                 retransmits = program_json["retransmits"]
 448             except KeyError:
 449                 retransmits = None
 450             export_hoststack_results(
 451                 bandwidth=program_json["bits_per_second"],
 452                 duration=program_json["seconds"],
 453                 retransmits=retransmits
 454             )
 455         else:
 456             test_results += u"Unknown HostStack Test Program!\n" + \
 457                             program_stdout
 458             return (True, program_stdout)
 459         return (False, json.dumps(program_json))
 460
 461     @staticmethod
 462     def hoststack_test_program_defer_fail(server_defer_fail, client_defer_fail):
 463         """Return True if either HostStack test program fail was deferred.
 464
 465         :param server_defer_fail: server no results value.
 466         :param client_defer_fail: client no results value.
 467         :type server_defer_fail: bool
 468         :type client_defer_fail: bool
 469         :rtype: bool
 470         """
 471         return server_defer_fail and client_defer_fail
 472
 473     @staticmethod
 474     def log_vpp_hoststack_data(node):
 475         """Retrieve and log VPP HostStack data.
 476
 477         :param node: DUT node.
 478         :type node: dict
 479         :raises RuntimeError: If node subtype is not a DUT or startup failed.
 480         """
 481
 482         if node[u"type"] != u"DUT":
 483             raise RuntimeError(u"Node type is not a DUT!")
 484
 485         PapiSocketExecutor.run_cli_cmd(node, u"show error")
 486         PapiSocketExecutor.run_cli_cmd(node, u"show interface")