resources/libraries/python/HoststackUtil.py

   1 # Copyright (c) 2021 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Host Stack util library."""
  15 import json
  16 from time import sleep
  17 from robot.api import logger
  18
  19 from resources.libraries.python.Constants import Constants
  20 from resources.libraries.python.ssh import exec_cmd, exec_cmd_no_error
  21 from resources.libraries.python.PapiExecutor import PapiSocketExecutor
  22 from resources.libraries.python.DUTSetup import DUTSetup
  23
  24 class HoststackUtil():
  25     """Utilities for Host Stack tests."""
  26
  27     @staticmethod
  28     def get_vpp_echo_command(vpp_echo_attributes):
  29         """Construct the vpp_echo command using the specified attributes.
  30
  31         :param vpp_echo_attributes: vpp_echo test program attributes.
  32         :type vpp_echo_attributes: dict
  33         :returns: Command line components of the vpp_echo command
  34             'name' - program name
  35             'args' - command arguments.
  36         :rtype: dict
  37         """
  38         # TODO: Use a python class instead of dictionary for the return type
  39         proto = vpp_echo_attributes[u"uri_protocol"]
  40         addr = vpp_echo_attributes[u"uri_ip4_addr"]
  41         port = vpp_echo_attributes[u"uri_port"]
  42         vpp_echo_cmd = {}
  43         vpp_echo_cmd[u"name"] = u"vpp_echo"
  44         vpp_echo_cmd[u"args"] = f"{vpp_echo_attributes[u'role']} " \
  45             f"socket-name {vpp_echo_attributes[u'vpp_api_socket']} " \
  46             f"{vpp_echo_attributes[u'json_output']} " \
  47             f"uri {proto}://{addr}/{port} " \
  48             f"nthreads {vpp_echo_attributes[u'nthreads']} " \
  49             f"mq-size {vpp_echo_attributes[u'mq_size']} " \
  50             f"nclients {vpp_echo_attributes[u'nclients']} " \
  51             f"quic-streams {vpp_echo_attributes[u'quic_streams']} " \
  52             f"time {vpp_echo_attributes[u'time']} " \
  53             f"fifo-size {vpp_echo_attributes[u'fifo_size']} " \
  54             f"TX={vpp_echo_attributes[u'tx_bytes']} " \
  55             f"RX={vpp_echo_attributes[u'rx_bytes']}"
  56         if vpp_echo_attributes[u"rx_results_diff"]:
  57             vpp_echo_cmd[u"args"] += u" rx-results-diff"
  58         if vpp_echo_attributes[u"tx_results_diff"]:
  59             vpp_echo_cmd[u"args"] += u" tx-results-diff"
  60         return vpp_echo_cmd
  61
  62     @staticmethod
  63     def get_iperf3_command(iperf3_attributes):
  64         """Construct the iperf3 command using the specified attributes.
  65
  66         :param iperf3_attributes: iperf3 test program attributes.
  67         :type iperf3_attributes: dict
  68         :returns: Command line components of the iperf3 command
  69             'env_vars' - environment variables
  70             'name' - program name
  71             'args' - command arguments.
  72         :rtype: dict
  73         """
  74         # TODO: Use a python class instead of dictionary for the return type
  75         iperf3_cmd = {}
  76         iperf3_cmd[u"env_vars"] = f"VCL_CONFIG={Constants.REMOTE_FW_DIR}/" \
  77             f"{Constants.RESOURCES_TPL_VCL}/" \
  78             f"{iperf3_attributes[u'vcl_config']}"
  79         if iperf3_attributes[u"ld_preload"]:
  80             iperf3_cmd[u"env_vars"] += \
  81                 f" LD_PRELOAD={Constants.VCL_LDPRELOAD_LIBRARY}"
  82         if iperf3_attributes[u'transparent_tls']:
  83             iperf3_cmd[u"env_vars"] += u" LDP_ENV_TLS_TRANS=1"
  84
  85         json_results = u" --json" if iperf3_attributes[u'json'] else u""
  86         ip_address = f" {iperf3_attributes[u'ip_address']}" if u"ip_address" \
  87                      in iperf3_attributes else u""
  88         iperf3_cmd[u"name"] = u"iperf3"
  89         iperf3_cmd[u"args"] = f"--{iperf3_attributes[u'role']}{ip_address} " \
  90                               f"--interval 0{json_results} " \
  91                               f"--version{iperf3_attributes[u'ip_version']}"
  92
  93         if iperf3_attributes[u"role"] == u"server":
  94             iperf3_cmd[u"args"] += u" --one-off"
  95         else:
  96             iperf3_cmd[u"args"] += u" --get-server-output"
  97             if u"parallel" in iperf3_attributes:
  98                 iperf3_cmd[u"args"] += \
  99                     f" --parallel {iperf3_attributes[u'parallel']}"
 100             if u"time" in iperf3_attributes:
 101                 iperf3_cmd[u"args"] += \
 102                     f" --time {iperf3_attributes[u'time']}"
 103         return iperf3_cmd
 104
 105     @staticmethod
 106     def set_hoststack_quic_fifo_size(node, fifo_size):
 107         """Set the QUIC protocol fifo size.
 108
 109         :param node: Node to set the QUIC fifo size on.
 110         :param fifo_size: fifo size, passed to the quic set fifo-size command.
 111         :type node: dict
 112         :type fifo_size: str
 113         """
 114         cmd = f"quic set fifo-size {fifo_size}"
 115         PapiSocketExecutor.run_cli_cmd(node, cmd)
 116
 117     @staticmethod
 118     def set_hoststack_quic_crypto_engine(node, quic_crypto_engine,
 119                                          fail_on_error=False):
 120         """Set the Hoststack QUIC crypto engine on node
 121
 122         :param node: Node to enable/disable HostStack.
 123         :param quic_crypto_engine: type of crypto engine
 124         :type node: dict
 125         :type quic_crypto_engine: str
 126         """
 127         vpp_crypto_engines = {u"openssl", u"native", u"ipsecmb"}
 128         if quic_crypto_engine == u"nocrypto":
 129             logger.trace(u"No QUIC crypto engine.")
 130             return
 131
 132         if quic_crypto_engine in vpp_crypto_engines:
 133             cmds = [u"quic set crypto api vpp",
 134                     f"set crypto handler aes-128-gcm {quic_crypto_engine}",
 135                     f"set crypto handler aes-256-gcm {quic_crypto_engine}"]
 136         elif quic_crypto_engine == u"picotls":
 137             cmds = [u"quic set crypto api picotls"]
 138         else:
 139             raise ValueError(f"Unknown QUIC crypto_engine {quic_crypto_engine}")
 140
 141         for cmd in cmds:
 142             try:
 143                 PapiSocketExecutor.run_cli_cmd(node, cmd)
 144             except AssertionError:
 145                 if fail_on_error:
 146                     raise
 147
 148     @staticmethod
 149     def get_hoststack_test_program_logs(node, program):
 150         """Get HostStack test program stdout log.
 151
 152         :param node: DUT node.
 153         :param program: test program.
 154         :type node: dict
 155         :type program: dict
 156         """
 157         program_name = program[u"name"]
 158         cmd = f"sh -c \'cat /tmp/{program_name}_stdout.log\'"
 159         stdout_log, _ = exec_cmd_no_error(node, cmd, sudo=True, \
 160             message=f"Get {program_name} stdout log failed!")
 161
 162         cmd = f"sh -c \'cat /tmp/{program_name}_stderr.log\'"
 163         stderr_log, _ = exec_cmd_no_error(node, cmd, sudo=True, \
 164             message=f"Get {program_name} stderr log failed!")
 165         return stdout_log, stderr_log
 166
 167     @staticmethod
 168     def get_nginx_command(nginx_attributes, nginx_version, nginx_ins_dir):
 169         """Construct the NGINX command using the specified attributes.
 170
 171         :param nginx_attributes: NGINX test program attributes.
 172         :param nginx_version: NGINX version.
 173         :param nginx_ins_dir: NGINX install dir.
 174         :type nginx_attributes: dict
 175         :type nginx_version: str
 176         :type nginx_ins_dir: str
 177         :returns: Command line components of the NGINX command
 178             'env_vars' - environment variables
 179             'name' - program name
 180             'args' - command arguments.
 181             'path' - program path.
 182         :rtype: dict
 183         """
 184         nginx_cmd = dict()
 185         nginx_cmd[u"env_vars"] = f"VCL_CONFIG={Constants.REMOTE_FW_DIR}/" \
 186                                  f"{Constants.RESOURCES_TPL_VCL}/" \
 187                                  f"{nginx_attributes[u'vcl_config']}"
 188         if nginx_attributes[u"ld_preload"]:
 189             nginx_cmd[u"env_vars"] += \
 190                 f" LD_PRELOAD={Constants.VCL_LDPRELOAD_LIBRARY}"
 191         if nginx_attributes[u'transparent_tls']:
 192             nginx_cmd[u"env_vars"] += u" LDP_ENV_TLS_TRANS=1"
 193
 194         nginx_cmd[u"name"] = u"nginx"
 195         nginx_cmd[u"path"] = f"{nginx_ins_dir}nginx-{nginx_version}/sbin/"
 196         nginx_cmd[u"args"] = f"-c {nginx_ins_dir}/" \
 197                              f"nginx-{nginx_version}/conf/nginx.conf"
 198         return nginx_cmd
 199
 200     @staticmethod
 201     def start_hoststack_test_program(node, namespace, core_list, program):
 202         """Start the specified HostStack test program.
 203
 204         :param node: DUT node.
 205         :param namespace: Net Namespace to run program in.
 206         :param core_list: List of cpu's to pass to taskset to pin the test
 207             program to a different set of cores on the same numa node as VPP.
 208         :param program: Test program.
 209         :type node: dict
 210         :type namespace: str
 211         :type core_list: str
 212         :type program: dict
 213         :returns: Process ID
 214         :rtype: int
 215         :raises RuntimeError: If node subtype is not a DUT or startup failed.
 216         """
 217         if node[u"type"] != u"DUT":
 218             raise RuntimeError(u"Node type is not a DUT!")
 219
 220         program_name = program[u"name"]
 221         DUTSetup.kill_program(node, program_name, namespace)
 222
 223         if namespace == u"default":
 224             shell_cmd = u"sh -c"
 225         else:
 226             shell_cmd = f"ip netns exec {namespace} sh -c"
 227
 228         env_vars = f"{program[u'env_vars']} " if u"env_vars" in program else u""
 229         args = program[u"args"]
 230         program_path = program.get(u"path", u"")
 231         # NGINX used `worker_cpu_affinity` in configuration file
 232         taskset_cmd = u"" if program_name == u"nginx" else \
 233                                              f"taskset --cpu-list {core_list}"
 234         cmd = f"nohup {shell_cmd} \'{env_vars}{taskset_cmd} " \
 235               f"{program_path}{program_name} {args} >/tmp/{program_name}_" \
 236               f"stdout.log 2>/tmp/{program_name}_stderr.log &\'"
 237         try:
 238             exec_cmd_no_error(node, cmd, sudo=True)
 239             return DUTSetup.get_pid(node, program_name)[0]
 240         except RuntimeError:
 241             stdout_log, stderr_log = \
 242                 HoststackUtil.get_hoststack_test_program_logs(node,
 243                                                               program)
 244             raise RuntimeError(f"Start {program_name} failed!\nSTDERR:\n" \
 245                                f"{stderr_log}\nSTDOUT:\n{stdout_log}")
 246         return None
 247
 248     @staticmethod
 249     def stop_hoststack_test_program(node, program, pid):
 250         """Stop the specified Hoststack test program.
 251
 252         :param node: DUT node.
 253         :param program: Test program.
 254         :param pid: Process ID of test program.
 255         :type node: dict
 256         :type program: dict
 257         :type pid: int
 258         """
 259         program_name = program[u"name"]
 260         if program_name == u"nginx":
 261             cmd = u"nginx -s quit"
 262             errmsg = u"Quit nginx failed!"
 263         else:
 264             cmd = f'if [ -n "$(ps {pid} | grep {program_name})" ] ; ' \
 265                 f'then kill -s SIGTERM {pid}; fi'
 266             errmsg = f"Kill {program_name} ({pid}) failed!"
 267
 268         exec_cmd_no_error(node, cmd, message=errmsg, sudo=True)
 269
 270     @staticmethod
 271     def hoststack_test_program_finished(node, program_pid):
 272         """Wait for the specified HostStack test program process to complete.
 273
 274         :param node: DUT node.
 275         :param program_pid: test program pid.
 276         :type node: dict
 277         :type program_pid: str
 278         :raises RuntimeError: If node subtype is not a DUT.
 279         """
 280         if node[u"type"] != u"DUT":
 281             raise RuntimeError(u"Node type is not a DUT!")
 282
 283         cmd = f"sh -c 'strace -qqe trace=none -p {program_pid}'"
 284         exec_cmd(node, cmd, sudo=True)
 285         # Wait a bit for stdout/stderr to be flushed to log files
 286         # TODO: see if sub-second sleep works e.g. sleep(0.1)
 287         sleep(1)
 288
 289     @staticmethod
 290     def analyze_hoststack_test_program_output(
 291             node, role, nsim_attr, program):
 292         """Gather HostStack test program output and check for errors.
 293
 294         The [defer_fail] return bool is used instead of failing immediately
 295         to allow the analysis of both the client and server instances of
 296         the test program for debugging a test failure.  When [defer_fail]
 297         is true, then the string returned is debug output instead of
 298         JSON formatted test program results.
 299
 300         :param node: DUT node.
 301         :param role: Role (client|server) of test program.
 302         :param nsim_attr: Network Simulation Attributes.
 303         :param program: Test program.
 304         :param program_args: List of test program args.
 305         :type node: dict
 306         :type role: str
 307         :type nsim_attr: dict
 308         :type program: dict
 309         :returns: tuple of [defer_fail] bool and either JSON formatted hoststack
 310             test program output or failure debug output.
 311         :rtype: bool, str
 312         :raises RuntimeError: If node subtype is not a DUT.
 313         """
 314         if node[u"type"] != u"DUT":
 315             raise RuntimeError(u"Node type is not a DUT!")
 316
 317         program_name = program[u"name"]
 318         program_stdout, program_stderr = \
 319             HoststackUtil.get_hoststack_test_program_logs(node, program)
 320         if len(program_stdout) == 0 and len(program_stderr) == 0:
 321             logger.trace(f"Retrying {program_name} log retrieval")
 322             program_stdout, program_stderr = \
 323                HoststackUtil.get_hoststack_test_program_logs(node, program)
 324
 325         env_vars = f"{program[u'env_vars']} " if u"env_vars" in program else u""
 326         program_cmd = f"{env_vars}{program_name} {program[u'args']}"
 327         test_results = f"Test Results of '{program_cmd}':\n"
 328
 329         if nsim_attr[u"output_nsim_enable"] or \
 330             nsim_attr[u"xc_nsim_enable"]:
 331             if nsim_attr[u"output_nsim_enable"]:
 332                 feature_name = u"output"
 333             else:
 334                 feature_name = u"cross-connect"
 335             test_results += \
 336                 f"NSIM({feature_name}): delay " \
 337                 f"{nsim_attr[u'delay_in_usec']} usecs, " \
 338                 f"avg-pkt-size {nsim_attr[u'average_packet_size']}, " \
 339                 f"bandwidth {nsim_attr[u'bw_in_bits_per_second']} " \
 340                 f"bits/sec, pkt-drop-rate {nsim_attr[u'packets_per_drop']} " \
 341                 f"pkts/drop\n"
 342
 343         # TODO: Incorporate show error stats into results analysis
 344         test_results += \
 345             f"\n{role} VPP 'show errors' on host {node[u'host']}:\n" \
 346             f"{PapiSocketExecutor.run_cli_cmd(node, u'show error')}\n"
 347
 348         if u"error" in program_stderr.lower():
 349             test_results += f"ERROR DETECTED:\n{program_stderr}"
 350             return (True, test_results)
 351         if not program_stdout:
 352             test_results += f"\nNo {program} test data retrieved!\n"
 353             ls_stdout, _ = exec_cmd_no_error(node, u"ls -l /tmp/*.log",
 354                                              sudo=True)
 355             test_results += f"{ls_stdout}\n"
 356             return (True, test_results)
 357         if program[u"name"] == u"vpp_echo":
 358             if u"JSON stats" in program_stdout and \
 359                     u'"has_failed": "0"' in program_stdout:
 360                 json_start = program_stdout.find(u"{")
 361                 #TODO: Fix parsing once vpp_echo produces valid
 362                 # JSON output. Truncate for now.
 363                 json_end = program_stdout.find(u',\n  "closing"')
 364                 json_results = f"{program_stdout[json_start:json_end]}\n}}"
 365                 program_json = json.loads(json_results)
 366             else:
 367                 test_results += u"Invalid test data output!\n" + program_stdout
 368                 return (True, test_results)
 369         elif program[u"name"] == u"iperf3":
 370             test_results += program_stdout
 371             iperf3_json = json.loads(program_stdout)
 372             program_json = iperf3_json[u"intervals"][0][u"sum"]
 373         else:
 374             test_results += u"Unknown HostStack Test Program!\n" + \
 375                             program_stdout
 376             return (True, program_stdout)
 377         return (False, json.dumps(program_json))
 378
 379     @staticmethod
 380     def hoststack_test_program_defer_fail(server_defer_fail, client_defer_fail):
 381         """Return True if either HostStack test program fail was deferred.
 382
 383         :param server_defer_fail: server no results value.
 384         :param client_defer_fail: client no results value.
 385         :type server_defer_fail: bool
 386         :type client_defer_fail: bool
 387         :rtype: bool
 388         """
 389         return server_defer_fail and client_defer_fail
 390
 391     @staticmethod
 392     def log_vpp_hoststack_data(node):
 393         """Retrieve and log VPP HostStack data.
 394
 395         :param node: DUT node.
 396         :type node: dict
 397         :raises RuntimeError: If node subtype is not a DUT or startup failed.
 398         """
 399
 400         if node[u"type"] != u"DUT":
 401             raise RuntimeError(u"Node type is not a DUT!")
 402
 403         PapiSocketExecutor.run_cli_cmd(node, u"show error")
 404         PapiSocketExecutor.run_cli_cmd(node, u"show interface")