resources/libraries/python/HoststackUtil.py

   1 # Copyright (c) 2024 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Host Stack util library."""
  15 import json
  16 from time import sleep
  17 from robot.api import logger
  18
  19 from resources.libraries.python.Constants import Constants
  20 from resources.libraries.python.DUTSetup import DUTSetup
  21 from resources.libraries.python.model.ExportResult import (
  22     export_hoststack_results
  23 )
  24 from resources.libraries.python.PapiExecutor import PapiSocketExecutor
  25 from resources.libraries.python.ssh import exec_cmd, exec_cmd_no_error
  26
  27 class HoststackUtil():
  28     """Utilities for Host Stack tests."""
  29
  30     @staticmethod
  31     def get_vpp_echo_command(vpp_echo_attributes):
  32         """Construct the vpp_echo command using the specified attributes.
  33
  34         :param vpp_echo_attributes: vpp_echo test program attributes.
  35         :type vpp_echo_attributes: dict
  36         :returns: Command line components of the vpp_echo command
  37             'name' - program name
  38             'args' - command arguments.
  39         :rtype: dict
  40         """
  41         proto = vpp_echo_attributes[u"uri_protocol"]
  42         addr = vpp_echo_attributes[u"uri_ip4_addr"]
  43         port = vpp_echo_attributes[u"uri_port"]
  44         vpp_echo_cmd = {}
  45         vpp_echo_cmd[u"name"] = u"vpp_echo"
  46         vpp_echo_cmd[u"args"] = f"{vpp_echo_attributes[u'role']} " \
  47             f"socket-name {vpp_echo_attributes[u'app_api_socket']} " \
  48             f"{vpp_echo_attributes[u'json_output']} " \
  49             f"uri {proto}://{addr}/{port} " \
  50             f"nthreads {vpp_echo_attributes[u'nthreads']} " \
  51             f"mq-size {vpp_echo_attributes[u'mq_size']} " \
  52             f"nclients {vpp_echo_attributes[u'nclients']} " \
  53             f"quic-streams {vpp_echo_attributes[u'quic_streams']} " \
  54             f"time {vpp_echo_attributes[u'time']} " \
  55             f"fifo-size {vpp_echo_attributes[u'fifo_size']} " \
  56             f"TX={vpp_echo_attributes[u'tx_bytes']} " \
  57             f"RX={vpp_echo_attributes[u'rx_bytes']}"
  58         if vpp_echo_attributes[u"rx_results_diff"]:
  59             vpp_echo_cmd[u"args"] += u" rx-results-diff"
  60         if vpp_echo_attributes[u"tx_results_diff"]:
  61             vpp_echo_cmd[u"args"] += u" tx-results-diff"
  62         if vpp_echo_attributes[u"use_app_socket_api"]:
  63             vpp_echo_cmd[u"args"] += u" use-app-socket-api"
  64         return vpp_echo_cmd
  65
  66     @staticmethod
  67     def get_iperf3_command(iperf3_attributes):
  68         """Construct the iperf3 command using the specified attributes.
  69
  70         :param iperf3_attributes: iperf3 test program attributes.
  71         :type iperf3_attributes: dict
  72         :returns: Command line components of the iperf3 command
  73             'env_vars' - environment variables
  74             'name' - program name
  75             'args' - command arguments.
  76         :rtype: dict
  77         """
  78         iperf3_cmd = {}
  79         iperf3_cmd[u"env_vars"] = f"VCL_CONFIG={Constants.REMOTE_FW_DIR}/" \
  80             f"{Constants.RESOURCES_TPL_VCL}/" \
  81             f"{iperf3_attributes[u'vcl_config']}"
  82         if iperf3_attributes[u"ld_preload"]:
  83             iperf3_cmd[u"env_vars"] += \
  84                 f" LD_PRELOAD={Constants.VCL_LDPRELOAD_LIBRARY}"
  85         if iperf3_attributes[u'transparent_tls']:
  86             iperf3_cmd[u"env_vars"] += u" LDP_ENV_TLS_TRANS=1"
  87
  88         json_results = u" --json" if iperf3_attributes[u'json'] else u""
  89         ip_address = f" {iperf3_attributes[u'ip_address']}" if u"ip_address" \
  90                      in iperf3_attributes else u""
  91         iperf3_cmd[u"name"] = u"iperf3"
  92         iperf3_cmd[u"args"] = f"--{iperf3_attributes[u'role']}{ip_address} " \
  93                               f"--interval 0{json_results} " \
  94                               f"--version{iperf3_attributes[u'ip_version']}"
  95
  96         if iperf3_attributes[u"role"] == u"server":
  97             iperf3_cmd[u"args"] += u" --one-off"
  98         else:
  99             iperf3_cmd[u"args"] += u" --get-server-output"
 100             if u"parallel" in iperf3_attributes:
 101                 iperf3_cmd[u"args"] += \
 102                     f" --parallel {iperf3_attributes[u'parallel']}"
 103             if u"time" in iperf3_attributes:
 104                 iperf3_cmd[u"args"] += \
 105                     f" --time {iperf3_attributes[u'time']}"
 106             if iperf3_attributes[u"udp"]:
 107                 iperf3_cmd[u"args"] += u" --udp"
 108                 iperf3_cmd[u"args"] += \
 109                     f" --bandwidth {iperf3_attributes[u'bandwidth']}"
 110             if iperf3_attributes[u"length"] > 0:
 111                 iperf3_cmd[u"args"] += \
 112                     f" --length {iperf3_attributes[u'length']}"
 113         return iperf3_cmd
 114
 115     @staticmethod
 116     def set_hoststack_quic_fifo_size(node, fifo_size):
 117         """Set the QUIC protocol fifo size.
 118
 119         :param node: Node to set the QUIC fifo size on.
 120         :param fifo_size: fifo size, passed to the quic set fifo-size command.
 121         :type node: dict
 122         :type fifo_size: str
 123         """
 124         cmd = f"quic set fifo-size {fifo_size}"
 125         PapiSocketExecutor.run_cli_cmd(node, cmd)
 126
 127     @staticmethod
 128     def set_hoststack_quic_crypto_engine(node, quic_crypto_engine,
 129                                          fail_on_error=False):
 130         """Set the Hoststack QUIC crypto engine on node
 131
 132         :param node: Node to enable/disable HostStack.
 133         :param quic_crypto_engine: type of crypto engine
 134         :type node: dict
 135         :type quic_crypto_engine: str
 136         """
 137         vpp_crypto_engines = {u"openssl", u"native", u"ipsecmb"}
 138         if quic_crypto_engine == u"nocrypto":
 139             logger.trace(u"No QUIC crypto engine.")
 140             return
 141
 142         if quic_crypto_engine in vpp_crypto_engines:
 143             cmds = [u"quic set crypto api vpp",
 144                     f"set crypto handler aes-128-gcm {quic_crypto_engine}",
 145                     f"set crypto handler aes-256-gcm {quic_crypto_engine}"]
 146         elif quic_crypto_engine == u"picotls":
 147             cmds = [u"quic set crypto api picotls"]
 148         else:
 149             raise ValueError(f"Unknown QUIC crypto_engine {quic_crypto_engine}")
 150
 151         for cmd in cmds:
 152             try:
 153                 PapiSocketExecutor.run_cli_cmd(node, cmd)
 154             except AssertionError:
 155                 if fail_on_error:
 156                     raise
 157
 158     @staticmethod
 159     def _get_hoststack_test_program_logs(node, program_name):
 160         """Get HostStack test program stdout log.
 161
 162         :param node: DUT node.
 163         :param program_name: test program.
 164         :type node: dict
 165         :type program_name: str
 166         """
 167         cmd = f"sh -c \'cat /tmp/{program_name}_stdout.log\'"
 168         stdout_log, _ = exec_cmd_no_error(node, cmd, sudo=True, \
 169             message=f"Get {program_name} stdout log failed!")
 170
 171         cmd = f"sh -c \'cat /tmp/{program_name}_stderr.log\'"
 172         stderr_log, _ = exec_cmd_no_error(node, cmd, sudo=True, \
 173             message=f"Get {program_name} stderr log failed!")
 174
 175         return stdout_log, stderr_log
 176
 177     @staticmethod
 178     def get_hoststack_test_program_logs(node, program):
 179         """Get HostStack test program stdout log.
 180
 181         :param node: DUT node.
 182         :param program: test program.
 183         :type node: dict
 184         :type program: dict
 185         """
 186         program_name = program[u"name"]
 187         program_stdout_log, program_stderr_log = \
 188             HoststackUtil._get_hoststack_test_program_logs(node,
 189                                                            program_name)
 190         if len(program_stdout_log) == 0 and len(program_stderr_log) == 0:
 191             logger.trace(f"Retrying {program_name} log retrieval")
 192             program_stdout_log, program_stderr_log = \
 193                HoststackUtil._get_hoststack_test_program_logs(node,
 194                                                               program_name)
 195         return program_stdout_log, program_stderr_log
 196
 197     @staticmethod
 198     def get_nginx_command(nginx_attributes, nginx_version, nginx_ins_dir):
 199         """Construct the NGINX command using the specified attributes.
 200
 201         :param nginx_attributes: NGINX test program attributes.
 202         :param nginx_version: NGINX version.
 203         :param nginx_ins_dir: NGINX install dir.
 204         :type nginx_attributes: dict
 205         :type nginx_version: str
 206         :type nginx_ins_dir: str
 207         :returns: Command line components of the NGINX command
 208             'env_vars' - environment variables
 209             'name' - program name
 210             'args' - command arguments.
 211             'path' - program path.
 212         :rtype: dict
 213         """
 214         nginx_cmd = dict()
 215         nginx_cmd[u"env_vars"] = f"VCL_CONFIG={Constants.REMOTE_FW_DIR}/" \
 216                                  f"{Constants.RESOURCES_TPL_VCL}/" \
 217                                  f"{nginx_attributes[u'vcl_config']}"
 218         if nginx_attributes[u"ld_preload"]:
 219             nginx_cmd[u"env_vars"] += \
 220                 f" LD_PRELOAD={Constants.VCL_LDPRELOAD_LIBRARY}"
 221         if nginx_attributes[u'transparent_tls']:
 222             nginx_cmd[u"env_vars"] += u" LDP_ENV_TLS_TRANS=1"
 223
 224         nginx_cmd[u"name"] = u"nginx"
 225         nginx_cmd[u"path"] = f"{nginx_ins_dir}nginx-{nginx_version}/sbin/"
 226         nginx_cmd[u"args"] = f"-c {nginx_ins_dir}/" \
 227                              f"nginx-{nginx_version}/conf/nginx.conf"
 228         return nginx_cmd
 229
 230     @staticmethod
 231     def start_hoststack_test_program(node, namespace, core_list, program):
 232         """Start the specified HostStack test program.
 233
 234         :param node: DUT node.
 235         :param namespace: Net Namespace to run program in.
 236         :param core_list: List of cpu's to pass to taskset to pin the test
 237             program to a different set of cores on the same numa node as VPP.
 238         :param program: Test program.
 239         :type node: dict
 240         :type namespace: str
 241         :type core_list: str
 242         :type program: dict
 243         :returns: Process ID
 244         :rtype: int
 245         :raises RuntimeError: If node subtype is not a DUT or startup failed.
 246         """
 247         if node[u"type"] != u"DUT":
 248             raise RuntimeError(u"Node type is not a DUT!")
 249
 250         program_name = program[u"name"]
 251         DUTSetup.kill_program(node, program_name, namespace)
 252
 253         if namespace == u"default":
 254             shell_cmd = u"sh -c"
 255         else:
 256             shell_cmd = f"ip netns exec {namespace} sh -c"
 257
 258         env_vars = f"{program[u'env_vars']} " if u"env_vars" in program else u""
 259         args = program[u"args"]
 260         program_path = program.get(u"path", u"")
 261         # NGINX used `worker_cpu_affinity` in configuration file
 262         taskset_cmd = u"" if program_name == u"nginx" else \
 263                                              f"taskset --cpu-list {core_list}"
 264         cmd = f"nohup {shell_cmd} \'{env_vars}{taskset_cmd} " \
 265               f"{program_path}{program_name} {args} >/tmp/{program_name}_" \
 266               f"stdout.log 2>/tmp/{program_name}_stderr.log &\'"
 267         try:
 268             exec_cmd_no_error(node, cmd, sudo=True)
 269             return DUTSetup.get_pid(node, program_name)[0]
 270         except RuntimeError:
 271             stdout_log, stderr_log = \
 272                 HoststackUtil.get_hoststack_test_program_logs(node,
 273                                                               program)
 274             raise RuntimeError(f"Start {program_name} failed!\nSTDERR:\n" \
 275                                f"{stderr_log}\nSTDOUT:\n{stdout_log}")
 276         return None
 277
 278     @staticmethod
 279     def stop_hoststack_test_program(node, program, pid):
 280         """Stop the specified Hoststack test program.
 281
 282         :param node: DUT node.
 283         :param program: Test program.
 284         :param pid: Process ID of test program.
 285         :type node: dict
 286         :type program: dict
 287         :type pid: int
 288         """
 289         program_name = program[u"name"]
 290         if program_name == u"nginx":
 291             cmd = u"nginx -s quit"
 292             errmsg = u"Quit nginx failed!"
 293         else:
 294             cmd = f'if [ -n "$(ps {pid} | grep {program_name})" ] ; ' \
 295                 f'then kill -s SIGTERM {pid}; fi'
 296             errmsg = f"Kill {program_name} ({pid}) failed!"
 297
 298         exec_cmd_no_error(node, cmd, message=errmsg, sudo=True)
 299
 300     @staticmethod
 301     def hoststack_test_program_finished(node, program_pid, program,
 302                                         other_node, other_program):
 303         """Wait for the specified HostStack test program process to complete.
 304
 305         :param node: DUT node.
 306         :param program_pid: test program pid.
 307         :param program: test program
 308         :param other_node: DUT node of other hoststack program
 309         :param other_program: other test program
 310         :type node: dict
 311         :type program_pid: str
 312         :type program: dict
 313         :type other_node: dict
 314         :type other_program: dict
 315         :raises RuntimeError: If node subtype is not a DUT.
 316         """
 317         if node[u"type"] != u"DUT":
 318             raise RuntimeError(u"Node type is not a DUT!")
 319         if other_node[u"type"] != u"DUT":
 320             raise RuntimeError(u"Other node type is not a DUT!")
 321
 322         cmd = f"sh -c 'strace -qqe trace=none -p {program_pid}'"
 323         try:
 324             exec_cmd(node, cmd, sudo=True)
 325         except:
 326             sleep(180)
 327             if u"client" in program[u"args"]:
 328                 role = u"client"
 329             else:
 330                 role = u"server"
 331             program_stdout, program_stderr = \
 332                 HoststackUtil.get_hoststack_test_program_logs(node, program)
 333             if len(program_stdout) > 0:
 334                 logger.debug(f"{program[u'name']} {role} stdout log:\n"
 335                              f"{program_stdout}")
 336             else:
 337                 logger.debug(f"Empty {program[u'name']} {role} stdout log :(")
 338             if len(program_stderr) > 0:
 339                 logger.debug(f"{program[u'name']} stderr log:\n"
 340                              f"{program_stderr}")
 341             else:
 342                 logger.debug(f"Empty {program[u'name']} stderr log :(")
 343             if u"client" in other_program[u"args"]:
 344                 role = u"client"
 345             else:
 346                 role = u"server"
 347             program_stdout, program_stderr = \
 348                 HoststackUtil.get_hoststack_test_program_logs(other_node,
 349                                                               other_program)
 350             if len(program_stdout) > 0:
 351                 logger.debug(f"{other_program[u'name']} {role} stdout log:\n"
 352                              f"{program_stdout}")
 353             else:
 354                 logger.debug(f"Empty {other_program[u'name']} "
 355                              f"{role} stdout log :(")
 356             if len(program_stderr) > 0:
 357                 logger.debug(f"{other_program[u'name']} {role} stderr log:\n"
 358                              f"{program_stderr}")
 359             else:
 360                 logger.debug(f"Empty {other_program[u'name']} "
 361                              f"{role} stderr log :(")
 362             raise
 363         # Wait a bit for stdout/stderr to be flushed to log files
 364         sleep(1)
 365
 366     @staticmethod
 367     def analyze_hoststack_test_program_output(
 368             node, role, nsim_attr, program):
 369         """Gather HostStack test program output and check for errors.
 370
 371         The [defer_fail] return bool is used instead of failing immediately
 372         to allow the analysis of both the client and server instances of
 373         the test program for debugging a test failure.  When [defer_fail]
 374         is true, then the string returned is debug output instead of
 375         JSON formatted test program results.
 376
 377         :param node: DUT node.
 378         :param role: Role (client|server) of test program.
 379         :param nsim_attr: Network Simulation Attributes.
 380         :param program: Test program.
 381         :param program_args: List of test program args.
 382         :type node: dict
 383         :type role: str
 384         :type nsim_attr: dict
 385         :type program: dict
 386         :returns: tuple of [defer_fail] bool and either JSON formatted hoststack
 387             test program output or failure debug output.
 388         :rtype: bool, str
 389         :raises RuntimeError: If node subtype is not a DUT.
 390         """
 391         if node[u"type"] != u"DUT":
 392             raise RuntimeError(u"Node type is not a DUT!")
 393
 394         program_name = program[u"name"]
 395         program_stdout, program_stderr = \
 396             HoststackUtil.get_hoststack_test_program_logs(node, program)
 397
 398         env_vars = f"{program[u'env_vars']} " if u"env_vars" in program else u""
 399         program_cmd = f"{env_vars}{program_name} {program[u'args']}"
 400         test_results = f"Test Results of '{program_cmd}':\n"
 401
 402         if nsim_attr[u"output_nsim_enable"] or \
 403             nsim_attr[u"xc_nsim_enable"]:
 404             if nsim_attr[u"output_nsim_enable"]:
 405                 feature_name = u"output"
 406             else:
 407                 feature_name = u"cross-connect"
 408             test_results += \
 409                 f"NSIM({feature_name}): delay " \
 410                 f"{nsim_attr[u'delay_in_usec']} usecs, " \
 411                 f"avg-pkt-size {nsim_attr[u'average_packet_size']}, " \
 412                 f"bandwidth {nsim_attr[u'bw_in_bits_per_second']} " \
 413                 f"bits/sec, pkt-drop-rate {nsim_attr[u'packets_per_drop']} " \
 414                 f"pkts/drop\n"
 415
 416         if u"error" in program_stderr.lower():
 417             test_results += f"ERROR DETECTED:\n{program_stderr}"
 418             return (True, test_results)
 419         if not program_stdout:
 420             test_results += f"\nNo {program} test data retrieved!\n"
 421             ls_stdout, _ = exec_cmd_no_error(node, u"ls -l /tmp/*.log",
 422                                              sudo=True)
 423             test_results += f"{ls_stdout}\n"
 424             return (True, test_results)
 425         if program[u"name"] == u"vpp_echo":
 426             if u"JSON stats" in program_stdout and \
 427                     u'"has_failed": "0"' in program_stdout:
 428                 json_start = program_stdout.find(u"{")
 429                 json_end = program_stdout.find(u',\n  "closing"')
 430                 json_results = f"{program_stdout[json_start:json_end]}\n}}"
 431                 program_json = json.loads(json_results)
 432                 export_hoststack_results(
 433                     bandwidth=program_json["rx_bits_per_second"],
 434                     duration=float(program_json["time"])
 435                 )
 436             else:
 437                 test_results += u"Invalid test data output!\n" + program_stdout
 438                 return (True, test_results)
 439         elif program[u"name"] == u"iperf3":
 440             test_results += program_stdout
 441             program_json = json.loads(program_stdout)[u"intervals"][0][u"sum"]
 442             try:
 443                 retransmits = program_json["retransmits"]
 444             except KeyError:
 445                 retransmits = None
 446             export_hoststack_results(
 447                 bandwidth=program_json["bits_per_second"],
 448                 duration=program_json["seconds"],
 449                 retransmits=retransmits
 450             )
 451         else:
 452             test_results += u"Unknown HostStack Test Program!\n" + \
 453                             program_stdout
 454             return (True, program_stdout)
 455         return (False, json.dumps(program_json))
 456
 457     @staticmethod
 458     def hoststack_test_program_defer_fail(server_defer_fail, client_defer_fail):
 459         """Return True if either HostStack test program fail was deferred.
 460
 461         :param server_defer_fail: server no results value.
 462         :param client_defer_fail: client no results value.
 463         :type server_defer_fail: bool
 464         :type client_defer_fail: bool
 465         :rtype: bool
 466         """
 467         return server_defer_fail and client_defer_fail