resources/libraries/python/HoststackUtil.py

   1 # Copyright (c) 2021 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Host Stack util library."""
  15 import json
  16 from time import sleep
  17 from robot.api import logger
  18
  19 from resources.libraries.python.Constants import Constants
  20 from resources.libraries.python.ssh import exec_cmd, exec_cmd_no_error
  21 from resources.libraries.python.PapiExecutor import PapiSocketExecutor
  22 from resources.libraries.python.DUTSetup import DUTSetup
  23
  24 class HoststackUtil():
  25     """Utilities for Host Stack tests."""
  26
  27     @staticmethod
  28     def get_vpp_echo_command(vpp_echo_attributes):
  29         """Construct the vpp_echo command using the specified attributes.
  30
  31         :param vpp_echo_attributes: vpp_echo test program attributes.
  32         :type vpp_echo_attributes: dict
  33         :returns: Command line components of the vpp_echo command
  34             'name' - program name
  35             'args' - command arguments.
  36         :rtype: dict
  37         """
  38         proto = vpp_echo_attributes[u"uri_protocol"]
  39         addr = vpp_echo_attributes[u"uri_ip4_addr"]
  40         port = vpp_echo_attributes[u"uri_port"]
  41         vpp_echo_cmd = {}
  42         vpp_echo_cmd[u"name"] = u"vpp_echo"
  43         vpp_echo_cmd[u"args"] = f"{vpp_echo_attributes[u'role']} " \
  44             f"socket-name {vpp_echo_attributes[u'vpp_api_socket']} " \
  45             f"{vpp_echo_attributes[u'json_output']} " \
  46             f"uri {proto}://{addr}/{port} " \
  47             f"nthreads {vpp_echo_attributes[u'nthreads']} " \
  48             f"mq-size {vpp_echo_attributes[u'mq_size']} " \
  49             f"nclients {vpp_echo_attributes[u'nclients']} " \
  50             f"quic-streams {vpp_echo_attributes[u'quic_streams']} " \
  51             f"time {vpp_echo_attributes[u'time']} " \
  52             f"fifo-size {vpp_echo_attributes[u'fifo_size']} " \
  53             f"TX={vpp_echo_attributes[u'tx_bytes']} " \
  54             f"RX={vpp_echo_attributes[u'rx_bytes']}"
  55         if vpp_echo_attributes[u"rx_results_diff"]:
  56             vpp_echo_cmd[u"args"] += u" rx-results-diff"
  57         if vpp_echo_attributes[u"tx_results_diff"]:
  58             vpp_echo_cmd[u"args"] += u" tx-results-diff"
  59         return vpp_echo_cmd
  60
  61     @staticmethod
  62     def get_iperf3_command(iperf3_attributes):
  63         """Construct the iperf3 command using the specified attributes.
  64
  65         :param iperf3_attributes: iperf3 test program attributes.
  66         :type iperf3_attributes: dict
  67         :returns: Command line components of the iperf3 command
  68             'env_vars' - environment variables
  69             'name' - program name
  70             'args' - command arguments.
  71         :rtype: dict
  72         """
  73         iperf3_cmd = {}
  74         iperf3_cmd[u"env_vars"] = f"VCL_CONFIG={Constants.REMOTE_FW_DIR}/" \
  75             f"{Constants.RESOURCES_TPL_VCL}/" \
  76             f"{iperf3_attributes[u'vcl_config']}"
  77         if iperf3_attributes[u"ld_preload"]:
  78             iperf3_cmd[u"env_vars"] += \
  79                 f" LD_PRELOAD={Constants.VCL_LDPRELOAD_LIBRARY}"
  80         if iperf3_attributes[u'transparent_tls']:
  81             iperf3_cmd[u"env_vars"] += u" LDP_ENV_TLS_TRANS=1"
  82
  83         json_results = u" --json" if iperf3_attributes[u'json'] else u""
  84         ip_address = f" {iperf3_attributes[u'ip_address']}" if u"ip_address" \
  85                      in iperf3_attributes else u""
  86         iperf3_cmd[u"name"] = u"iperf3"
  87         iperf3_cmd[u"args"] = f"--{iperf3_attributes[u'role']}{ip_address} " \
  88                               f"--interval 0{json_results} " \
  89                               f"--version{iperf3_attributes[u'ip_version']}"
  90
  91         if iperf3_attributes[u"role"] == u"server":
  92             iperf3_cmd[u"args"] += u" --one-off"
  93         else:
  94             iperf3_cmd[u"args"] += u" --get-server-output"
  95             if u"parallel" in iperf3_attributes:
  96                 iperf3_cmd[u"args"] += \
  97                     f" --parallel {iperf3_attributes[u'parallel']}"
  98             if u"time" in iperf3_attributes:
  99                 iperf3_cmd[u"args"] += \
 100                     f" --time {iperf3_attributes[u'time']}"
 101             if iperf3_attributes[u"udp"]:
 102                 iperf3_cmd[u"args"] += u" --udp"
 103                 iperf3_cmd[u"args"] += \
 104                     f" --bandwidth {iperf3_attributes[u'bandwidth']}"
 105             if iperf3_attributes[u"length"] > 0:
 106                 iperf3_cmd[u"args"] += \
 107                     f" --length {iperf3_attributes[u'length']}"
 108         return iperf3_cmd
 109
 110     @staticmethod
 111     def set_hoststack_quic_fifo_size(node, fifo_size):
 112         """Set the QUIC protocol fifo size.
 113
 114         :param node: Node to set the QUIC fifo size on.
 115         :param fifo_size: fifo size, passed to the quic set fifo-size command.
 116         :type node: dict
 117         :type fifo_size: str
 118         """
 119         cmd = f"quic set fifo-size {fifo_size}"
 120         PapiSocketExecutor.run_cli_cmd(node, cmd)
 121
 122     @staticmethod
 123     def set_hoststack_quic_crypto_engine(node, quic_crypto_engine,
 124                                          fail_on_error=False):
 125         """Set the Hoststack QUIC crypto engine on node
 126
 127         :param node: Node to enable/disable HostStack.
 128         :param quic_crypto_engine: type of crypto engine
 129         :type node: dict
 130         :type quic_crypto_engine: str
 131         """
 132         vpp_crypto_engines = {u"openssl", u"native", u"ipsecmb"}
 133         if quic_crypto_engine == u"nocrypto":
 134             logger.trace(u"No QUIC crypto engine.")
 135             return
 136
 137         if quic_crypto_engine in vpp_crypto_engines:
 138             cmds = [u"quic set crypto api vpp",
 139                     f"set crypto handler aes-128-gcm {quic_crypto_engine}",
 140                     f"set crypto handler aes-256-gcm {quic_crypto_engine}"]
 141         elif quic_crypto_engine == u"picotls":
 142             cmds = [u"quic set crypto api picotls"]
 143         else:
 144             raise ValueError(f"Unknown QUIC crypto_engine {quic_crypto_engine}")
 145
 146         for cmd in cmds:
 147             try:
 148                 PapiSocketExecutor.run_cli_cmd(node, cmd)
 149             except AssertionError:
 150                 if fail_on_error:
 151                     raise
 152
 153     @staticmethod
 154     def get_hoststack_test_program_logs(node, program):
 155         """Get HostStack test program stdout log.
 156
 157         :param node: DUT node.
 158         :param program: test program.
 159         :type node: dict
 160         :type program: dict
 161         """
 162         program_name = program[u"name"]
 163         cmd = f"sh -c \'cat /tmp/{program_name}_stdout.log\'"
 164         stdout_log, _ = exec_cmd_no_error(node, cmd, sudo=True, \
 165             message=f"Get {program_name} stdout log failed!")
 166
 167         cmd = f"sh -c \'cat /tmp/{program_name}_stderr.log\'"
 168         stderr_log, _ = exec_cmd_no_error(node, cmd, sudo=True, \
 169             message=f"Get {program_name} stderr log failed!")
 170         return stdout_log, stderr_log
 171
 172     @staticmethod
 173     def get_nginx_command(nginx_attributes, nginx_version, nginx_ins_dir):
 174         """Construct the NGINX command using the specified attributes.
 175
 176         :param nginx_attributes: NGINX test program attributes.
 177         :param nginx_version: NGINX version.
 178         :param nginx_ins_dir: NGINX install dir.
 179         :type nginx_attributes: dict
 180         :type nginx_version: str
 181         :type nginx_ins_dir: str
 182         :returns: Command line components of the NGINX command
 183             'env_vars' - environment variables
 184             'name' - program name
 185             'args' - command arguments.
 186             'path' - program path.
 187         :rtype: dict
 188         """
 189         nginx_cmd = dict()
 190         nginx_cmd[u"env_vars"] = f"VCL_CONFIG={Constants.REMOTE_FW_DIR}/" \
 191                                  f"{Constants.RESOURCES_TPL_VCL}/" \
 192                                  f"{nginx_attributes[u'vcl_config']}"
 193         if nginx_attributes[u"ld_preload"]:
 194             nginx_cmd[u"env_vars"] += \
 195                 f" LD_PRELOAD={Constants.VCL_LDPRELOAD_LIBRARY}"
 196         if nginx_attributes[u'transparent_tls']:
 197             nginx_cmd[u"env_vars"] += u" LDP_ENV_TLS_TRANS=1"
 198
 199         nginx_cmd[u"name"] = u"nginx"
 200         nginx_cmd[u"path"] = f"{nginx_ins_dir}nginx-{nginx_version}/sbin/"
 201         nginx_cmd[u"args"] = f"-c {nginx_ins_dir}/" \
 202                              f"nginx-{nginx_version}/conf/nginx.conf"
 203         return nginx_cmd
 204
 205     @staticmethod
 206     def start_hoststack_test_program(node, namespace, core_list, program):
 207         """Start the specified HostStack test program.
 208
 209         :param node: DUT node.
 210         :param namespace: Net Namespace to run program in.
 211         :param core_list: List of cpu's to pass to taskset to pin the test
 212             program to a different set of cores on the same numa node as VPP.
 213         :param program: Test program.
 214         :type node: dict
 215         :type namespace: str
 216         :type core_list: str
 217         :type program: dict
 218         :returns: Process ID
 219         :rtype: int
 220         :raises RuntimeError: If node subtype is not a DUT or startup failed.
 221         """
 222         if node[u"type"] != u"DUT":
 223             raise RuntimeError(u"Node type is not a DUT!")
 224
 225         program_name = program[u"name"]
 226         DUTSetup.kill_program(node, program_name, namespace)
 227
 228         if namespace == u"default":
 229             shell_cmd = u"sh -c"
 230         else:
 231             shell_cmd = f"ip netns exec {namespace} sh -c"
 232
 233         env_vars = f"{program[u'env_vars']} " if u"env_vars" in program else u""
 234         args = program[u"args"]
 235         program_path = program.get(u"path", u"")
 236         # NGINX used `worker_cpu_affinity` in configuration file
 237         taskset_cmd = u"" if program_name == u"nginx" else \
 238                                              f"taskset --cpu-list {core_list}"
 239         cmd = f"nohup {shell_cmd} \'{env_vars}{taskset_cmd} " \
 240               f"{program_path}{program_name} {args} >/tmp/{program_name}_" \
 241               f"stdout.log 2>/tmp/{program_name}_stderr.log &\'"
 242         try:
 243             exec_cmd_no_error(node, cmd, sudo=True)
 244             return DUTSetup.get_pid(node, program_name)[0]
 245         except RuntimeError:
 246             stdout_log, stderr_log = \
 247                 HoststackUtil.get_hoststack_test_program_logs(node,
 248                                                               program)
 249             raise RuntimeError(f"Start {program_name} failed!\nSTDERR:\n" \
 250                                f"{stderr_log}\nSTDOUT:\n{stdout_log}")
 251         return None
 252
 253     @staticmethod
 254     def stop_hoststack_test_program(node, program, pid):
 255         """Stop the specified Hoststack test program.
 256
 257         :param node: DUT node.
 258         :param program: Test program.
 259         :param pid: Process ID of test program.
 260         :type node: dict
 261         :type program: dict
 262         :type pid: int
 263         """
 264         program_name = program[u"name"]
 265         if program_name == u"nginx":
 266             cmd = u"nginx -s quit"
 267             errmsg = u"Quit nginx failed!"
 268         else:
 269             cmd = f'if [ -n "$(ps {pid} | grep {program_name})" ] ; ' \
 270                 f'then kill -s SIGTERM {pid}; fi'
 271             errmsg = f"Kill {program_name} ({pid}) failed!"
 272
 273         exec_cmd_no_error(node, cmd, message=errmsg, sudo=True)
 274
 275     @staticmethod
 276     def hoststack_test_program_finished(node, program_pid):
 277         """Wait for the specified HostStack test program process to complete.
 278
 279         :param node: DUT node.
 280         :param program_pid: test program pid.
 281         :type node: dict
 282         :type program_pid: str
 283         :raises RuntimeError: If node subtype is not a DUT.
 284         """
 285         if node[u"type"] != u"DUT":
 286             raise RuntimeError(u"Node type is not a DUT!")
 287
 288         cmd = f"sh -c 'strace -qqe trace=none -p {program_pid}'"
 289         exec_cmd(node, cmd, sudo=True)
 290         # Wait a bit for stdout/stderr to be flushed to log files
 291         # TODO: see if sub-second sleep works e.g. sleep(0.1)
 292         sleep(1)
 293
 294     @staticmethod
 295     def analyze_hoststack_test_program_output(
 296             node, role, nsim_attr, program):
 297         """Gather HostStack test program output and check for errors.
 298
 299         The [defer_fail] return bool is used instead of failing immediately
 300         to allow the analysis of both the client and server instances of
 301         the test program for debugging a test failure.  When [defer_fail]
 302         is true, then the string returned is debug output instead of
 303         JSON formatted test program results.
 304
 305         :param node: DUT node.
 306         :param role: Role (client|server) of test program.
 307         :param nsim_attr: Network Simulation Attributes.
 308         :param program: Test program.
 309         :param program_args: List of test program args.
 310         :type node: dict
 311         :type role: str
 312         :type nsim_attr: dict
 313         :type program: dict
 314         :returns: tuple of [defer_fail] bool and either JSON formatted hoststack
 315             test program output or failure debug output.
 316         :rtype: bool, str
 317         :raises RuntimeError: If node subtype is not a DUT.
 318         """
 319         if node[u"type"] != u"DUT":
 320             raise RuntimeError(u"Node type is not a DUT!")
 321
 322         program_name = program[u"name"]
 323         program_stdout, program_stderr = \
 324             HoststackUtil.get_hoststack_test_program_logs(node, program)
 325         if len(program_stdout) == 0 and len(program_stderr) == 0:
 326             logger.trace(f"Retrying {program_name} log retrieval")
 327             program_stdout, program_stderr = \
 328                HoststackUtil.get_hoststack_test_program_logs(node, program)
 329
 330         env_vars = f"{program[u'env_vars']} " if u"env_vars" in program else u""
 331         program_cmd = f"{env_vars}{program_name} {program[u'args']}"
 332         test_results = f"Test Results of '{program_cmd}':\n"
 333
 334         if nsim_attr[u"output_nsim_enable"] or \
 335             nsim_attr[u"xc_nsim_enable"]:
 336             if nsim_attr[u"output_nsim_enable"]:
 337                 feature_name = u"output"
 338             else:
 339                 feature_name = u"cross-connect"
 340             test_results += \
 341                 f"NSIM({feature_name}): delay " \
 342                 f"{nsim_attr[u'delay_in_usec']} usecs, " \
 343                 f"avg-pkt-size {nsim_attr[u'average_packet_size']}, " \
 344                 f"bandwidth {nsim_attr[u'bw_in_bits_per_second']} " \
 345                 f"bits/sec, pkt-drop-rate {nsim_attr[u'packets_per_drop']} " \
 346                 f"pkts/drop\n"
 347
 348         # TODO: Incorporate show error stats into results analysis
 349         test_results += \
 350             f"\n{role} VPP 'show errors' on host {node[u'host']}:\n" \
 351             f"{PapiSocketExecutor.run_cli_cmd(node, u'show error')}\n"
 352
 353         if u"error" in program_stderr.lower():
 354             test_results += f"ERROR DETECTED:\n{program_stderr}"
 355             return (True, test_results)
 356         if not program_stdout:
 357             test_results += f"\nNo {program} test data retrieved!\n"
 358             ls_stdout, _ = exec_cmd_no_error(node, u"ls -l /tmp/*.log",
 359                                              sudo=True)
 360             test_results += f"{ls_stdout}\n"
 361             return (True, test_results)
 362         if program[u"name"] == u"vpp_echo":
 363             if u"JSON stats" in program_stdout and \
 364                     u'"has_failed": "0"' in program_stdout:
 365                 json_start = program_stdout.find(u"{")
 366                 #TODO: Fix parsing once vpp_echo produces valid
 367                 # JSON output. Truncate for now.
 368                 json_end = program_stdout.find(u',\n  "closing"')
 369                 json_results = f"{program_stdout[json_start:json_end]}\n}}"
 370                 program_json = json.loads(json_results)
 371             else:
 372                 test_results += u"Invalid test data output!\n" + program_stdout
 373                 return (True, test_results)
 374         elif program[u"name"] == u"iperf3":
 375             test_results += program_stdout
 376             iperf3_json = json.loads(program_stdout)
 377             program_json = iperf3_json[u"intervals"][0][u"sum"]
 378         else:
 379             test_results += u"Unknown HostStack Test Program!\n" + \
 380                             program_stdout
 381             return (True, program_stdout)
 382         return (False, json.dumps(program_json))
 383
 384     @staticmethod
 385     def hoststack_test_program_defer_fail(server_defer_fail, client_defer_fail):
 386         """Return True if either HostStack test program fail was deferred.
 387
 388         :param server_defer_fail: server no results value.
 389         :param client_defer_fail: client no results value.
 390         :type server_defer_fail: bool
 391         :type client_defer_fail: bool
 392         :rtype: bool
 393         """
 394         return server_defer_fail and client_defer_fail
 395
 396     @staticmethod
 397     def log_vpp_hoststack_data(node):
 398         """Retrieve and log VPP HostStack data.
 399
 400         :param node: DUT node.
 401         :type node: dict
 402         :raises RuntimeError: If node subtype is not a DUT or startup failed.
 403         """
 404
 405         if node[u"type"] != u"DUT":
 406             raise RuntimeError(u"Node type is not a DUT!")
 407
 408         PapiSocketExecutor.run_cli_cmd(node, u"show error")
 409         PapiSocketExecutor.run_cli_cmd(node, u"show interface")