resources/libraries/python/HoststackUtil.py

   1 # Copyright (c) 2021 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Host Stack util library."""
  15 import json
  16 from time import sleep
  17 from robot.api import logger
  18
  19 from resources.libraries.python.Constants import Constants
  20 from resources.libraries.python.ssh import exec_cmd, exec_cmd_no_error
  21 from resources.libraries.python.PapiExecutor import PapiSocketExecutor
  22 from resources.libraries.python.DUTSetup import DUTSetup
  23
  24 class HoststackUtil():
  25     """Utilities for Host Stack tests."""
  26
  27     @staticmethod
  28     def get_vpp_echo_command(vpp_echo_attributes):
  29         """Construct the vpp_echo command using the specified attributes.
  30
  31         :param vpp_echo_attributes: vpp_echo test program attributes.
  32         :type vpp_echo_attributes: dict
  33         :returns: Command line components of the vpp_echo command
  34             'name' - program name
  35             'args' - command arguments.
  36         :rtype: dict
  37         """
  38         proto = vpp_echo_attributes[u"uri_protocol"]
  39         addr = vpp_echo_attributes[u"uri_ip4_addr"]
  40         port = vpp_echo_attributes[u"uri_port"]
  41         vpp_echo_cmd = {}
  42         vpp_echo_cmd[u"name"] = u"vpp_echo"
  43         vpp_echo_cmd[u"args"] = f"{vpp_echo_attributes[u'role']} " \
  44             f"socket-name {vpp_echo_attributes[u'vpp_api_socket']} " \
  45             f"{vpp_echo_attributes[u'json_output']} " \
  46             f"uri {proto}://{addr}/{port} " \
  47             f"nthreads {vpp_echo_attributes[u'nthreads']} " \
  48             f"mq-size {vpp_echo_attributes[u'mq_size']} " \
  49             f"nclients {vpp_echo_attributes[u'nclients']} " \
  50             f"quic-streams {vpp_echo_attributes[u'quic_streams']} " \
  51             f"time {vpp_echo_attributes[u'time']} " \
  52             f"fifo-size {vpp_echo_attributes[u'fifo_size']} " \
  53             f"TX={vpp_echo_attributes[u'tx_bytes']} " \
  54             f"RX={vpp_echo_attributes[u'rx_bytes']}"
  55         if vpp_echo_attributes[u"rx_results_diff"]:
  56             vpp_echo_cmd[u"args"] += u" rx-results-diff"
  57         if vpp_echo_attributes[u"tx_results_diff"]:
  58             vpp_echo_cmd[u"args"] += u" tx-results-diff"
  59         return vpp_echo_cmd
  60
  61     @staticmethod
  62     def get_iperf3_command(iperf3_attributes):
  63         """Construct the iperf3 command using the specified attributes.
  64
  65         :param iperf3_attributes: iperf3 test program attributes.
  66         :type iperf3_attributes: dict
  67         :returns: Command line components of the iperf3 command
  68             'env_vars' - environment variables
  69             'name' - program name
  70             'args' - command arguments.
  71         :rtype: dict
  72         """
  73         iperf3_cmd = {}
  74         iperf3_cmd[u"env_vars"] = f"VCL_CONFIG={Constants.REMOTE_FW_DIR}/" \
  75             f"{Constants.RESOURCES_TPL_VCL}/" \
  76             f"{iperf3_attributes[u'vcl_config']}"
  77         if iperf3_attributes[u"ld_preload"]:
  78             iperf3_cmd[u"env_vars"] += \
  79                 f" LD_PRELOAD={Constants.VCL_LDPRELOAD_LIBRARY}"
  80         if iperf3_attributes[u'transparent_tls']:
  81             iperf3_cmd[u"env_vars"] += u" LDP_ENV_TLS_TRANS=1"
  82
  83         json_results = u" --json" if iperf3_attributes[u'json'] else u""
  84         ip_address = f" {iperf3_attributes[u'ip_address']}" if u"ip_address" \
  85                      in iperf3_attributes else u""
  86         iperf3_cmd[u"name"] = u"iperf3"
  87         # TODO: Use OptionString library.
  88         iperf3_cmd[u"args"] = f"--{iperf3_attributes[u'role']}{ip_address} " \
  89                               f"--interval 0{json_results} " \
  90                               f"--version{iperf3_attributes[u'ip_version']}"
  91
  92         if iperf3_attributes[u"role"] == u"server":
  93             iperf3_cmd[u"args"] += u" --one-off"
  94         else:
  95             iperf3_cmd[u"args"] += u" --get-server-output"
  96             if u"parallel" in iperf3_attributes:
  97                 iperf3_cmd[u"args"] += \
  98                     f" --parallel {iperf3_attributes[u'parallel']}"
  99             if u"time" in iperf3_attributes:
 100                 iperf3_cmd[u"args"] += \
 101                     f" --time {iperf3_attributes[u'time']}"
 102             if iperf3_attributes[u"udp"]:
 103                 iperf3_cmd[u"args"] += u" --udp"
 104                 iperf3_cmd[u"args"] += \
 105                     f" --bandwidth {iperf3_attributes[u'bandwidth']}"
 106             if iperf3_attributes[u"length"] > 0:
 107                 iperf3_cmd[u"args"] += \
 108                     f" --length {iperf3_attributes[u'length']}"
 109         return iperf3_cmd
 110
 111     @staticmethod
 112     def set_hoststack_quic_fifo_size(node, fifo_size):
 113         """Set the QUIC protocol fifo size.
 114
 115         :param node: Node to set the QUIC fifo size on.
 116         :param fifo_size: fifo size, passed to the quic set fifo-size command.
 117         :type node: dict
 118         :type fifo_size: str
 119         """
 120         cmd = f"quic set fifo-size {fifo_size}"
 121         PapiSocketExecutor.run_cli_cmd(node, cmd)
 122
 123     @staticmethod
 124     def set_hoststack_quic_crypto_engine(node, quic_crypto_engine,
 125                                          fail_on_error=False):
 126         """Set the Hoststack QUIC crypto engine on node
 127
 128         :param node: Node to enable/disable HostStack.
 129         :param quic_crypto_engine: type of crypto engine
 130         :type node: dict
 131         :type quic_crypto_engine: str
 132         """
 133         vpp_crypto_engines = {u"openssl", u"native", u"ipsecmb"}
 134         if quic_crypto_engine == u"nocrypto":
 135             logger.trace(u"No QUIC crypto engine.")
 136             return
 137
 138         if quic_crypto_engine in vpp_crypto_engines:
 139             cmds = [u"quic set crypto api vpp",
 140                     f"set crypto handler aes-128-gcm {quic_crypto_engine}",
 141                     f"set crypto handler aes-256-gcm {quic_crypto_engine}"]
 142         elif quic_crypto_engine == u"picotls":
 143             cmds = [u"quic set crypto api picotls"]
 144         else:
 145             raise ValueError(f"Unknown QUIC crypto_engine {quic_crypto_engine}")
 146
 147         for cmd in cmds:
 148             try:
 149                 PapiSocketExecutor.run_cli_cmd(node, cmd)
 150             except AssertionError:
 151                 if fail_on_error:
 152                     raise
 153
 154     @staticmethod
 155     def get_hoststack_test_program_logs(node, program):
 156         """Get HostStack test program stdout log.
 157
 158         :param node: DUT node.
 159         :param program: test program.
 160         :type node: dict
 161         :type program: dict
 162         """
 163         program_name = program[u"name"]
 164         cmd = f"sh -c \'cat /tmp/{program_name}_stdout.log\'"
 165         stdout_log, _ = exec_cmd_no_error(node, cmd, sudo=True, \
 166             message=f"Get {program_name} stdout log failed!")
 167
 168         cmd = f"sh -c \'cat /tmp/{program_name}_stderr.log\'"
 169         stderr_log, _ = exec_cmd_no_error(node, cmd, sudo=True, \
 170             message=f"Get {program_name} stderr log failed!")
 171         return stdout_log, stderr_log
 172
 173     @staticmethod
 174     def get_nginx_command(nginx_attributes, nginx_version, nginx_ins_dir):
 175         """Construct the NGINX command using the specified attributes.
 176
 177         :param nginx_attributes: NGINX test program attributes.
 178         :param nginx_version: NGINX version.
 179         :param nginx_ins_dir: NGINX install dir.
 180         :type nginx_attributes: dict
 181         :type nginx_version: str
 182         :type nginx_ins_dir: str
 183         :returns: Command line components of the NGINX command
 184             'env_vars' - environment variables
 185             'name' - program name
 186             'args' - command arguments.
 187             'path' - program path.
 188         :rtype: dict
 189         """
 190         nginx_cmd = dict()
 191         nginx_cmd[u"env_vars"] = f"VCL_CONFIG={Constants.REMOTE_FW_DIR}/" \
 192                                  f"{Constants.RESOURCES_TPL_VCL}/" \
 193                                  f"{nginx_attributes[u'vcl_config']}"
 194         if nginx_attributes[u"ld_preload"]:
 195             nginx_cmd[u"env_vars"] += \
 196                 f" LD_PRELOAD={Constants.VCL_LDPRELOAD_LIBRARY}"
 197         if nginx_attributes[u'transparent_tls']:
 198             nginx_cmd[u"env_vars"] += u" LDP_ENV_TLS_TRANS=1"
 199
 200         nginx_cmd[u"name"] = u"nginx"
 201         nginx_cmd[u"path"] = f"{nginx_ins_dir}nginx-{nginx_version}/sbin/"
 202         nginx_cmd[u"args"] = f"-c {nginx_ins_dir}/" \
 203                              f"nginx-{nginx_version}/conf/nginx.conf"
 204         return nginx_cmd
 205
 206     @staticmethod
 207     def start_hoststack_test_program(node, namespace, core_list, program):
 208         """Start the specified HostStack test program.
 209
 210         :param node: DUT node.
 211         :param namespace: Net Namespace to run program in.
 212         :param core_list: List of cpu's to pass to taskset to pin the test
 213             program to a different set of cores on the same numa node as VPP.
 214         :param program: Test program.
 215         :type node: dict
 216         :type namespace: str
 217         :type core_list: str
 218         :type program: dict
 219         :returns: Process ID
 220         :rtype: int
 221         :raises RuntimeError: If node subtype is not a DUT or startup failed.
 222         """
 223         if node[u"type"] != u"DUT":
 224             raise RuntimeError(u"Node type is not a DUT!")
 225
 226         program_name = program[u"name"]
 227         DUTSetup.kill_program(node, program_name, namespace)
 228
 229         if namespace == u"default":
 230             shell_cmd = u"sh -c"
 231         else:
 232             shell_cmd = f"ip netns exec {namespace} sh -c"
 233
 234         env_vars = f"{program[u'env_vars']} " if u"env_vars" in program else u""
 235         args = program[u"args"]
 236         program_path = program.get(u"path", u"")
 237         # NGINX used `worker_cpu_affinity` in configuration file
 238         taskset_cmd = u"" if program_name == u"nginx" else \
 239                                              f"taskset --cpu-list {core_list}"
 240         cmd = f"nohup {shell_cmd} \'{env_vars}{taskset_cmd} " \
 241               f"{program_path}{program_name} {args} >/tmp/{program_name}_" \
 242               f"stdout.log 2>/tmp/{program_name}_stderr.log &\'"
 243         try:
 244             exec_cmd_no_error(node, cmd, sudo=True)
 245             return DUTSetup.get_pid(node, program_name)[0]
 246         except RuntimeError:
 247             stdout_log, stderr_log = \
 248                 HoststackUtil.get_hoststack_test_program_logs(node,
 249                                                               program)
 250             raise RuntimeError(f"Start {program_name} failed!\nSTDERR:\n" \
 251                                f"{stderr_log}\nSTDOUT:\n{stdout_log}")
 252         return None
 253
 254     @staticmethod
 255     def stop_hoststack_test_program(node, program, pid):
 256         """Stop the specified Hoststack test program.
 257
 258         :param node: DUT node.
 259         :param program: Test program.
 260         :param pid: Process ID of test program.
 261         :type node: dict
 262         :type program: dict
 263         :type pid: int
 264         """
 265         program_name = program[u"name"]
 266         if program_name == u"nginx":
 267             cmd = u"nginx -s quit"
 268             errmsg = u"Quit nginx failed!"
 269         else:
 270             cmd = f'if [ -n "$(ps {pid} | grep {program_name})" ] ; ' \
 271                 f'then kill -s SIGTERM {pid}; fi'
 272             errmsg = f"Kill {program_name} ({pid}) failed!"
 273
 274         exec_cmd_no_error(node, cmd, message=errmsg, sudo=True)
 275
 276     @staticmethod
 277     def hoststack_test_program_finished(node, program_pid):
 278         """Wait for the specified HostStack test program process to complete.
 279
 280         :param node: DUT node.
 281         :param program_pid: test program pid.
 282         :type node: dict
 283         :type program_pid: str
 284         :raises RuntimeError: If node subtype is not a DUT.
 285         """
 286         if node[u"type"] != u"DUT":
 287             raise RuntimeError(u"Node type is not a DUT!")
 288
 289         cmd = f"sh -c 'strace -qqe trace=none -p {program_pid}'"
 290         exec_cmd(node, cmd, sudo=True)
 291         # Wait a bit for stdout/stderr to be flushed to log files
 292         # TODO: see if sub-second sleep works e.g. sleep(0.1)
 293         sleep(1)
 294
 295     @staticmethod
 296     def analyze_hoststack_test_program_output(
 297             node, role, nsim_attr, program):
 298         """Gather HostStack test program output and check for errors.
 299
 300         The [defer_fail] return bool is used instead of failing immediately
 301         to allow the analysis of both the client and server instances of
 302         the test program for debugging a test failure.  When [defer_fail]
 303         is true, then the string returned is debug output instead of
 304         JSON formatted test program results.
 305
 306         :param node: DUT node.
 307         :param role: Role (client|server) of test program.
 308         :param nsim_attr: Network Simulation Attributes.
 309         :param program: Test program.
 310         :param program_args: List of test program args.
 311         :type node: dict
 312         :type role: str
 313         :type nsim_attr: dict
 314         :type program: dict
 315         :returns: tuple of [defer_fail] bool and either JSON formatted hoststack
 316             test program output or failure debug output.
 317         :rtype: bool, str
 318         :raises RuntimeError: If node subtype is not a DUT.
 319         """
 320         if node[u"type"] != u"DUT":
 321             raise RuntimeError(u"Node type is not a DUT!")
 322
 323         program_name = program[u"name"]
 324         program_stdout, program_stderr = \
 325             HoststackUtil.get_hoststack_test_program_logs(node, program)
 326         if len(program_stdout) == 0 and len(program_stderr) == 0:
 327             logger.trace(f"Retrying {program_name} log retrieval")
 328             program_stdout, program_stderr = \
 329                HoststackUtil.get_hoststack_test_program_logs(node, program)
 330
 331         env_vars = f"{program[u'env_vars']} " if u"env_vars" in program else u""
 332         program_cmd = f"{env_vars}{program_name} {program[u'args']}"
 333         test_results = f"Test Results of '{program_cmd}':\n"
 334
 335         if nsim_attr[u"output_nsim_enable"] or \
 336             nsim_attr[u"xc_nsim_enable"]:
 337             if nsim_attr[u"output_nsim_enable"]:
 338                 feature_name = u"output"
 339             else:
 340                 feature_name = u"cross-connect"
 341             test_results += \
 342                 f"NSIM({feature_name}): delay " \
 343                 f"{nsim_attr[u'delay_in_usec']} usecs, " \
 344                 f"avg-pkt-size {nsim_attr[u'average_packet_size']}, " \
 345                 f"bandwidth {nsim_attr[u'bw_in_bits_per_second']} " \
 346                 f"bits/sec, pkt-drop-rate {nsim_attr[u'packets_per_drop']} " \
 347                 f"pkts/drop\n"
 348
 349         # TODO: Incorporate show error stats into results analysis
 350         test_results += \
 351             f"\n{role} VPP 'show errors' on host {node[u'host']}:\n" \
 352             f"{PapiSocketExecutor.run_cli_cmd(node, u'show error')}\n"
 353
 354         if u"error" in program_stderr.lower():
 355             test_results += f"ERROR DETECTED:\n{program_stderr}"
 356             return (True, test_results)
 357         if not program_stdout:
 358             test_results += f"\nNo {program} test data retrieved!\n"
 359             ls_stdout, _ = exec_cmd_no_error(node, u"ls -l /tmp/*.log",
 360                                              sudo=True)
 361             test_results += f"{ls_stdout}\n"
 362             return (True, test_results)
 363         if program[u"name"] == u"vpp_echo":
 364             if u"JSON stats" in program_stdout and \
 365                     u'"has_failed": "0"' in program_stdout:
 366                 json_start = program_stdout.find(u"{")
 367                 #TODO: Fix parsing once vpp_echo produces valid
 368                 # JSON output. Truncate for now.
 369                 json_end = program_stdout.find(u',\n  "closing"')
 370                 json_results = f"{program_stdout[json_start:json_end]}\n}}"
 371                 program_json = json.loads(json_results)
 372             else:
 373                 test_results += u"Invalid test data output!\n" + program_stdout
 374                 return (True, test_results)
 375         elif program[u"name"] == u"iperf3":
 376             test_results += program_stdout
 377             iperf3_json = json.loads(program_stdout)
 378             program_json = iperf3_json[u"intervals"][0][u"sum"]
 379         else:
 380             test_results += u"Unknown HostStack Test Program!\n" + \
 381                             program_stdout
 382             return (True, program_stdout)
 383         return (False, json.dumps(program_json))
 384
 385     @staticmethod
 386     def hoststack_test_program_defer_fail(server_defer_fail, client_defer_fail):
 387         """Return True if either HostStack test program fail was deferred.
 388
 389         :param server_defer_fail: server no results value.
 390         :param client_defer_fail: client no results value.
 391         :type server_defer_fail: bool
 392         :type client_defer_fail: bool
 393         :rtype: bool
 394         """
 395         return server_defer_fail and client_defer_fail
 396
 397     @staticmethod
 398     def log_vpp_hoststack_data(node):
 399         """Retrieve and log VPP HostStack data.
 400
 401         :param node: DUT node.
 402         :type node: dict
 403         :raises RuntimeError: If node subtype is not a DUT or startup failed.
 404         """
 405
 406         if node[u"type"] != u"DUT":
 407             raise RuntimeError(u"Node type is not a DUT!")
 408
 409         PapiSocketExecutor.run_cli_cmd(node, u"show error")
 410         PapiSocketExecutor.run_cli_cmd(node, u"show interface")