resources/libraries/python/TrafficGenerator.py

   1 # Copyright (c) 2020 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Performance testing traffic generator library."""
  15
  16 import time
  17
  18 from robot.api import logger
  19 from robot.libraries.BuiltIn import BuiltIn
  20
  21 from .Constants import Constants
  22 from .CpuUtils import CpuUtils
  23 from .DropRateSearch import DropRateSearch
  24 from .MLRsearch.AbstractMeasurer import AbstractMeasurer
  25 from .MLRsearch.MultipleLossRatioSearch import MultipleLossRatioSearch
  26 from .MLRsearch.ReceiveRateMeasurement import ReceiveRateMeasurement
  27 from .PLRsearch.PLRsearch import PLRsearch
  28 from .OptionString import OptionString
  29 from .ssh import exec_cmd_no_error, exec_cmd
  30 from .topology import NodeType
  31 from .topology import NodeSubTypeTG
  32 from .topology import Topology
  33
  34 __all__ = [u"TGDropRateSearchImpl", u"TrafficGenerator", u"OptimizedSearch"]
  35
  36
  37 def check_subtype(node):
  38     """Return supported subtype of given node, or raise an exception.
  39
  40     Currently only one subtype is supported,
  41     but we want our code to be ready for other ones.
  42
  43     :param node: Topology node to check. Can be None.
  44     :type node: dict or NoneType
  45     :returns: Subtype detected.
  46     :rtype: NodeSubTypeTG
  47     :raises RuntimeError: If node is not supported, message explains how.
  48     """
  49     if node.get(u"type") is None:
  50         msg = u"Node type is not defined"
  51     elif node[u"type"] != NodeType.TG:
  52         msg = f"Node type is {node[u'type']!r}, not a TG"
  53     elif node.get(u"subtype") is None:
  54         msg = u"TG subtype is not defined"
  55     elif node[u"subtype"] != NodeSubTypeTG.TREX:
  56         msg = f"TG subtype {node[u'subtype']!r} is not supported"
  57     else:
  58         return NodeSubTypeTG.TREX
  59     raise RuntimeError(msg)
  60
  61
  62 class TGDropRateSearchImpl(DropRateSearch):
  63     """Drop Rate Search implementation."""
  64
  65     # def __init__(self):
  66     #     super(TGDropRateSearchImpl, self).__init__()
  67
  68     def measure_loss(
  69             self, rate, frame_size, loss_acceptance, loss_acceptance_type,
  70             traffic_profile, skip_warmup=False):
  71         """Runs the traffic and evaluate the measured results.
  72
  73         :param rate: Offered traffic load.
  74         :param frame_size: Size of frame.
  75         :param loss_acceptance: Permitted drop ratio or frames count.
  76         :param loss_acceptance_type: Type of permitted loss.
  77         :param traffic_profile: Module name as a traffic profile identifier.
  78             See GPL/traffic_profiles/trex for implemented modules.
  79         :param skip_warmup: Start TRex without warmup traffic if true.
  80         :type rate: float
  81         :type frame_size: str
  82         :type loss_acceptance: float
  83         :type loss_acceptance_type: LossAcceptanceType
  84         :type traffic_profile: str
  85         :type skip_warmup: bool
  86         :returns: Drop threshold exceeded? (True/False)
  87         :rtype: bool
  88         :raises NotImplementedError: If TG is not supported.
  89         :raises RuntimeError: If TG is not specified.
  90         """
  91         # we need instance of TrafficGenerator instantiated by Robot Framework
  92         # to be able to use trex_stl-*()
  93         tg_instance = BuiltIn().get_library_instance(
  94             u"resources.libraries.python.TrafficGenerator"
  95         )
  96         subtype = check_subtype(tg_instance.node)
  97         if subtype == NodeSubTypeTG.TREX:
  98             unit_rate = str(rate) + self.get_rate_type_str()
  99             if skip_warmup:
 100                 tg_instance.trex_stl_start_remote_exec(
 101                     self.get_duration(), unit_rate, frame_size, traffic_profile,
 102                     warmup_time=0.0
 103                 )
 104             else:
 105                 tg_instance.trex_stl_start_remote_exec(
 106                     self.get_duration(), unit_rate, frame_size, traffic_profile
 107                 )
 108             loss = tg_instance.get_loss()
 109             sent = tg_instance.get_sent()
 110             if self.loss_acceptance_type_is_percentage():
 111                 loss = (float(loss) / float(sent)) * 100
 112             logger.trace(
 113                 f"comparing: {loss} < {loss_acceptance} {loss_acceptance_type}"
 114             )
 115             return float(loss) <= float(loss_acceptance)
 116         return False
 117
 118     def get_latency(self):
 119         """Returns min/avg/max latency.
 120
 121         :returns: Latency stats.
 122         :rtype: list
 123         """
 124         tg_instance = BuiltIn().get_library_instance(
 125             u"resources.libraries.python.TrafficGenerator"
 126         )
 127         return tg_instance.get_latency_int()
 128
 129
 130 class TrexMode:
 131     """Defines mode of T-Rex traffic generator."""
 132     # Advanced stateful mode
 133     ASTF = u"ASTF"
 134     # Stateless mode
 135     STL = u"STL"
 136
 137
 138 # TODO: Pylint says too-many-instance-attributes.
 139 class TrafficGenerator(AbstractMeasurer):
 140     """Traffic Generator."""
 141
 142     # TODO: Remove "trex" from lines which could work with other TGs.
 143
 144     # Use one instance of TrafficGenerator for all tests in test suite
 145     ROBOT_LIBRARY_SCOPE = u"TEST SUITE"
 146
 147     def __init__(self):
 148         self._node = None
 149         self._mode = None
 150         # TG interface order mapping
 151         self._ifaces_reordered = False
 152         # Result holding fields, to be removed.
 153         self._result = None
 154         self._loss = None
 155         self._sent = None
 156         self._latency = None
 157         self._received = None
 158         self._approximated_rate = None
 159         self._approximated_duration = None
 160         self._l7_data = None
 161         # Measurement input fields, needed for async stop result.
 162         self._start_time = None
 163         self._rate = None
 164         # Other input parameters, not knowable from measure() signature.
 165         self.frame_size = None
 166         self.traffic_profile = None
 167         self.warmup_time = None
 168         self.traffic_directions = None
 169         self.negative_loss = None
 170         self.use_latency = None
 171         # Transient data needed for async measurements.
 172         self._xstats = (None, None)
 173         # TODO: Rename "xstats" to something opaque, so T-Rex is not privileged?
 174
 175     @property
 176     def node(self):
 177         """Getter.
 178
 179         :returns: Traffic generator node.
 180         :rtype: dict
 181         """
 182         return self._node
 183
 184     def get_loss(self):
 185         """Return number of lost packets.
 186
 187         :returns: Number of lost packets.
 188         :rtype: str
 189         """
 190         return self._loss
 191
 192     def get_sent(self):
 193         """Return number of sent packets.
 194
 195         :returns: Number of sent packets.
 196         :rtype: str
 197         """
 198         return self._sent
 199
 200     def get_received(self):
 201         """Return number of received packets.
 202
 203         :returns: Number of received packets.
 204         :rtype: str
 205         """
 206         return self._received
 207
 208     def get_latency_int(self):
 209         """Return rounded min/avg/max latency.
 210
 211         :returns: Latency stats.
 212         :rtype: list
 213         """
 214         return self._latency
 215
 216     def get_approximated_rate(self):
 217         """Return approximated rate computed as ratio of transmitted packets
 218         over duration of trial.
 219
 220         :returns: Approximated rate.
 221         :rtype: str
 222         """
 223         return self._approximated_rate
 224
 225     def get_l7_data(self):
 226         """Return L7 data.
 227
 228         :returns: Number of received packets.
 229         :rtype: dict
 230         """
 231         return self._l7_data
 232
 233     def check_mode(self, expected_mode):
 234         """Check TG mode.
 235
 236         :param expected_mode: Expected traffic generator mode.
 237         :type expected_mode: object
 238         :raises RuntimeError: In case of unexpected TG mode.
 239         """
 240         if self._mode == expected_mode:
 241             return
 242         raise RuntimeError(
 243             f"{self._node[u'subtype']} not running in {expected_mode} mode!"
 244         )
 245
 246     # TODO: pylint says disable=too-many-locals.
 247     # A fix is developed in https://gerrit.fd.io/r/c/csit/+/22221
 248     def initialize_traffic_generator(
 249             self, tg_node, tg_if1, tg_if2, tg_if1_adj_node, tg_if1_adj_if,
 250             tg_if2_adj_node, tg_if2_adj_if, osi_layer, tg_if1_dst_mac=None,
 251             tg_if2_dst_mac=None):
 252         """TG initialization.
 253
 254         TODO: Document why do we need (and how do we use) _ifaces_reordered.
 255
 256         :param tg_node: Traffic generator node.
 257         :param tg_if1: TG - name of first interface.
 258         :param tg_if2: TG - name of second interface.
 259         :param tg_if1_adj_node: TG if1 adjecent node.
 260         :param tg_if1_adj_if: TG if1 adjecent interface.
 261         :param tg_if2_adj_node: TG if2 adjecent node.
 262         :param tg_if2_adj_if: TG if2 adjecent interface.
 263         :param osi_layer: 'L2', 'L3' or 'L7' - OSI Layer testing type.
 264         :param tg_if1_dst_mac: Interface 1 destination MAC address.
 265         :param tg_if2_dst_mac: Interface 2 destination MAC address.
 266         :type tg_node: dict
 267         :type tg_if1: str
 268         :type tg_if2: str
 269         :type tg_if1_adj_node: dict
 270         :type tg_if1_adj_if: str
 271         :type tg_if2_adj_node: dict
 272         :type tg_if2_adj_if: str
 273         :type osi_layer: str
 274         :type tg_if1_dst_mac: str
 275         :type tg_if2_dst_mac: str
 276         :returns: nothing
 277         :raises RuntimeError: In case of issue during initialization.
 278         """
 279         subtype = check_subtype(tg_node)
 280         if subtype == NodeSubTypeTG.TREX:
 281             self._node = tg_node
 282             self._mode = TrexMode.ASTF if osi_layer == u"L7" else TrexMode.STL
 283             if1 = dict()
 284             if2 = dict()
 285             if1[u"pci"] = Topology().get_interface_pci_addr(self._node, tg_if1)
 286             if2[u"pci"] = Topology().get_interface_pci_addr(self._node, tg_if2)
 287             if1[u"addr"] = Topology().get_interface_mac(self._node, tg_if1)
 288             if2[u"addr"] = Topology().get_interface_mac(self._node, tg_if2)
 289
 290             if osi_layer == u"L2":
 291                 if1[u"adj_addr"] = if2[u"addr"]
 292                 if2[u"adj_addr"] = if1[u"addr"]
 293             elif osi_layer in (u"L3", u"L7"):
 294                 if1[u"adj_addr"] = Topology().get_interface_mac(
 295                     tg_if1_adj_node, tg_if1_adj_if
 296                 )
 297                 if2[u"adj_addr"] = Topology().get_interface_mac(
 298                     tg_if2_adj_node, tg_if2_adj_if
 299                 )
 300             else:
 301                 raise ValueError(u"Unknown OSI layer!")
 302
 303             # in case of switched environment we can override MAC addresses
 304             if tg_if1_dst_mac is not None and tg_if2_dst_mac is not None:
 305                 if1[u"adj_addr"] = tg_if1_dst_mac
 306                 if2[u"adj_addr"] = tg_if2_dst_mac
 307
 308             if min(if1[u"pci"], if2[u"pci"]) != if1[u"pci"]:
 309                 if1, if2 = if2, if1
 310                 self._ifaces_reordered = True
 311
 312             master_thread_id, latency_thread_id, socket, threads = \
 313                 CpuUtils.get_affinity_trex(
 314                     self._node, tg_if1, tg_if2,
 315                     tg_dtc=Constants.TREX_CORE_COUNT)
 316
 317             if osi_layer in (u"L2", u"L3", u"L7"):
 318                 exec_cmd_no_error(
 319                     self._node,
 320                     f"sh -c 'cat << EOF > /etc/trex_cfg.yaml\n"
 321                     f"- version: 2\n"
 322                     f"  c: {len(threads)}\n"
 323                     f"  limit_memory: {Constants.TREX_LIMIT_MEMORY}\n"
 324                     f"  interfaces: [\"{if1[u'pci']}\",\"{if2[u'pci']}\"]\n"
 325                     f"  port_info:\n"
 326                     f"      - dest_mac: \'{if1[u'adj_addr']}\'\n"
 327                     f"        src_mac: \'{if1[u'addr']}\'\n"
 328                     f"      - dest_mac: \'{if2[u'adj_addr']}\'\n"
 329                     f"        src_mac: \'{if2[u'addr']}\'\n"
 330                     f"  platform :\n"
 331                     f"      master_thread_id: {master_thread_id}\n"
 332                     f"      latency_thread_id: {latency_thread_id}\n"
 333                     f"      dual_if:\n"
 334                     f"          - socket: {socket}\n"
 335                     f"            threads: {threads}\n"
 336                     f"EOF'",
 337                     sudo=True, message=u"T-Rex config generation!"
 338                 )
 339             else:
 340                 raise ValueError(u"Unknown OSI layer!")
 341
 342             TrafficGenerator.startup_trex(
 343                 self._node, osi_layer, subtype=subtype
 344             )
 345
 346     @staticmethod
 347     def startup_trex(tg_node, osi_layer, subtype=None):
 348         """Startup sequence for the TRex traffic generator.
 349
 350         :param tg_node: Traffic generator node.
 351         :param osi_layer: 'L2', 'L3' or 'L7' - OSI Layer testing type.
 352         :param subtype: Traffic generator sub-type.
 353         :type tg_node: dict
 354         :type osi_layer: str
 355         :type subtype: NodeSubTypeTG
 356         :raises RuntimeError: If T-Rex startup failed.
 357         :raises ValueError: If OSI layer is not supported.
 358         """
 359         if not subtype:
 360             subtype = check_subtype(tg_node)
 361         if subtype == NodeSubTypeTG.TREX:
 362             for _ in range(0, 3):
 363                 # Kill TRex only if it is already running.
 364                 cmd = u"sh -c \"pgrep t-rex && pkill t-rex && sleep 3 || true\""
 365                 exec_cmd_no_error(
 366                     tg_node, cmd, sudo=True, message=u"Kill TRex failed!"
 367                 )
 368
 369                 # Configure TRex.
 370                 ports = ''
 371                 for port in tg_node[u"interfaces"].values():
 372                     ports += f" {port.get(u'pci_address')}"
 373
 374                 cmd = f"sh -c \"cd {Constants.TREX_INSTALL_DIR}/scripts/ && " \
 375                     f"./dpdk_nic_bind.py -u {ports} || true\""
 376                 exec_cmd_no_error(
 377                     tg_node, cmd, sudo=True,
 378                     message=u"Unbind PCI ports from driver failed!"
 379                 )
 380
 381                 # Start TRex.
 382                 cd_cmd = f"cd '{Constants.TREX_INSTALL_DIR}/scripts/'"
 383                 trex_cmd = OptionString([u"nohup", u"./t-rex-64"])
 384                 trex_cmd.add(u"-i")
 385                 trex_cmd.add(u"--prefix $(hostname)")
 386                 trex_cmd.add(u"--hdrh")
 387                 trex_cmd.add(u"--no-scapy-server")
 388                 trex_cmd.add_if(u"--astf", osi_layer == u"L7")
 389                 # OptionString does not create double space if extra is empty.
 390                 trex_cmd.add(f"{Constants.TREX_EXTRA_CMDLINE}")
 391                 inner_command = f"{cd_cmd} && {trex_cmd} > /tmp/trex.log 2>&1 &"
 392                 cmd = f"sh -c \"{inner_command}\" > /dev/null"
 393                 try:
 394                     exec_cmd_no_error(tg_node, cmd, sudo=True)
 395                 except RuntimeError:
 396                     cmd = u"sh -c \"cat /tmp/trex.log\""
 397                     exec_cmd_no_error(
 398                         tg_node, cmd, sudo=True,
 399                         message=u"Get TRex logs failed!"
 400                     )
 401                     raise RuntimeError(u"Start TRex failed!")
 402
 403                 # Test T-Rex API responsiveness.
 404                 cmd = u"python3"
 405                 cmd += f" {Constants.REMOTE_FW_DIR}/GPL/tools/trex/"
 406                 if osi_layer in (u"L2", u"L3"):
 407                     cmd += f"trex_stl_assert.py"
 408                 elif osi_layer == u"L7":
 409                     cmd += f"trex_astf_assert.py"
 410                 else:
 411                     raise ValueError(u"Unknown OSI layer!")
 412                 try:
 413                     exec_cmd_no_error(
 414                         tg_node, cmd, sudo=True,
 415                         message=u"T-Rex API is not responding!", retries=20
 416                     )
 417                 except RuntimeError:
 418                     continue
 419                 return
 420             # After max retries TRex is still not responding to API critical
 421             # error occurred.
 422             exec_cmd(tg_node, u"cat /tmp/trex.log", sudo=True)
 423             raise RuntimeError(u"Start T-Rex failed after multiple retries!")
 424
 425     @staticmethod
 426     def is_trex_running(node):
 427         """Check if T-Rex is running using pidof.
 428
 429         :param node: Traffic generator node.
 430         :type node: dict
 431         :returns: True if T-Rex is running otherwise False.
 432         :rtype: bool
 433         """
 434         ret, _, _ = exec_cmd(node, u"pgrep t-rex", sudo=True)
 435         return bool(int(ret) == 0)
 436
 437     @staticmethod
 438     def teardown_traffic_generator(node):
 439         """TG teardown.
 440
 441         :param node: Traffic generator node.
 442         :type node: dict
 443         :returns: nothing
 444         :raises RuntimeError: If node type is not a TG,
 445             or if T-Rex teardown fails.
 446         """
 447         subtype = check_subtype(node)
 448         if subtype == NodeSubTypeTG.TREX:
 449             exec_cmd_no_error(
 450                 node,
 451                 u"sh -c "
 452                 u"\"if pgrep t-rex; then sudo pkill t-rex && sleep 3; fi\"",
 453                 sudo=False,
 454                 message=u"T-Rex kill failed!"
 455             )
 456
 457     def _parse_traffic_results(self, stdout):
 458         """Parse stdout of scripts into fields of self.
 459
 460         Block of code to reuse, by sync start, or stop after async.
 461
 462         :param stdout: Text containing the standard output.
 463         :type stdout: str
 464         """
 465         subtype = check_subtype(self._node)
 466         if subtype == NodeSubTypeTG.TREX:
 467             # Last line from console output
 468             line = stdout.splitlines()[-1]
 469             results = line.split(",")
 470             if results[-1] == u" ":
 471                 results.remove(u" ")
 472             self._result = dict()
 473             for result in results:
 474                 key, value = result.split(u"=", maxsplit=1)
 475                 self._result[key.strip()] = value
 476             logger.info(f"TrafficGen results:\n{self._result}")
 477             self._received = self._result.get(u"total_received")
 478             self._sent = self._result.get(u"total_sent")
 479             self._loss = self._result.get(u"frame_loss")
 480             self._approximated_duration = \
 481                 self._result.get(u"approximated_duration")
 482             self._approximated_rate = self._result.get(u"approximated_rate")
 483             self._latency = list()
 484             self._latency.append(self._result.get(u"latency_stream_0(usec)"))
 485             self._latency.append(self._result.get(u"latency_stream_1(usec)"))
 486             if self._mode == TrexMode.ASTF:
 487                 self._l7_data = dict()
 488                 self._l7_data[u"client"] = dict()
 489                 self._l7_data[u"client"][u"active_flows"] = \
 490                     self._result.get(u"client_active_flows")
 491                 self._l7_data[u"client"][u"established_flows"] = \
 492                     self._result.get(u"client_established_flows")
 493                 self._l7_data[u"server"] = dict()
 494                 self._l7_data[u"server"][u"active_flows"] = \
 495                     self._result.get(u"server_active_flows")
 496                 self._l7_data[u"server"][u"established_flows"] = \
 497                     self._result.get(u"server_established_flows")
 498                 if u"udp" in self.traffic_profile:
 499                     self._l7_data[u"client"][u"udp"] = dict()
 500                     self._l7_data[u"client"][u"udp"][u"established_flows"] = \
 501                         self._result.get(u"client_udp_connects")
 502                     self._l7_data[u"client"][u"udp"][u"closed_flows"] = \
 503                         self._result.get(u"client_udp_closed")
 504                     self._l7_data[u"server"][u"udp"] = dict()
 505                     self._l7_data[u"server"][u"udp"][u"accepted_flows"] = \
 506                         self._result.get(u"server_udp_accepts")
 507                     self._l7_data[u"server"][u"udp"][u"closed_flows"] = \
 508                         self._result.get(u"server_udp_closed")
 509                 elif u"tcp" in self.traffic_profile:
 510                     self._l7_data[u"client"][u"tcp"] = dict()
 511                     self._l7_data[u"client"][u"tcp"][u"initiated_flows"] = \
 512                         self._result.get(u"client_tcp_connect_inits")
 513                     self._l7_data[u"client"][u"tcp"][u"established_flows"] = \
 514                         self._result.get(u"client_tcp_connects")
 515                     self._l7_data[u"client"][u"tcp"][u"closed_flows"] = \
 516                         self._result.get(u"client_tcp_closed")
 517                     self._l7_data[u"server"][u"tcp"] = dict()
 518                     self._l7_data[u"server"][u"tcp"][u"accepted_flows"] = \
 519                         self._result.get(u"server_tcp_accepts")
 520                     self._l7_data[u"server"][u"tcp"][u"established_flows"] = \
 521                         self._result.get(u"server_tcp_connects")
 522                     self._l7_data[u"server"][u"tcp"][u"closed_flows"] = \
 523                         self._result.get(u"server_tcp_closed")
 524
 525     def trex_astf_stop_remote_exec(self, node):
 526         """Execute T-Rex ASTF script on remote node over ssh to stop running
 527         traffic.
 528
 529         Internal state is updated with measurement results.
 530
 531         :param node: T-Rex generator node.
 532         :type node: dict
 533         :raises RuntimeError: If stop traffic script fails.
 534         """
 535         command_line = OptionString().add(u"python3")
 536         dirname = f"{Constants.REMOTE_FW_DIR}/GPL/tools/trex"
 537         command_line.add(f"'{dirname}/trex_astf_stop.py'")
 538         command_line.change_prefix(u"--")
 539         for index, value in enumerate(self._xstats):
 540             if value is not None:
 541                 value = value.replace(u"'", u"\"")
 542                 command_line.add_equals(f"xstat{index}", f"'{value}'")
 543         stdout, _ = exec_cmd_no_error(
 544             node, command_line,
 545             message=u"T-Rex ASTF runtime error!"
 546         )
 547         self._parse_traffic_results(stdout)
 548
 549     def trex_stl_stop_remote_exec(self, node):
 550         """Execute T-Rex STL script on remote node over ssh to stop running
 551         traffic.
 552
 553         Internal state is updated with measurement results.
 554
 555         :param node: T-Rex generator node.
 556         :type node: dict
 557         :raises RuntimeError: If stop traffic script fails.
 558         """
 559         command_line = OptionString().add(u"python3")
 560         dirname = f"{Constants.REMOTE_FW_DIR}/GPL/tools/trex"
 561         command_line.add(f"'{dirname}/trex_stl_stop.py'")
 562         command_line.change_prefix(u"--")
 563         for index, value in enumerate(self._xstats):
 564             if value is not None:
 565                 value = value.replace(u"'", u"\"")
 566                 command_line.add_equals(f"xstat{index}", f"'{value}'")
 567         stdout, _ = exec_cmd_no_error(
 568             node, command_line,
 569             message=u"T-Rex STL runtime error!"
 570         )
 571         self._parse_traffic_results(stdout)
 572
 573     def stop_traffic_on_tg(self):
 574         """Stop all traffic on TG.
 575
 576         :returns: Structure containing the result of the measurement.
 577         :rtype: ReceiveRateMeasurement
 578         :raises ValueError: If TG traffic profile is not supported.
 579         """
 580         subtype = check_subtype(self._node)
 581         if subtype == NodeSubTypeTG.TREX:
 582             if u"trex-astf" in self.traffic_profile:
 583                 self.trex_astf_stop_remote_exec(self._node)
 584             elif u"trex-sl" in self.traffic_profile:
 585                 self.trex_stl_stop_remote_exec(self._node)
 586             else:
 587                 raise ValueError(u"Unsupported T-Rex traffic profile!")
 588
 589         return self.get_measurement_result()
 590
 591     def trex_astf_start_remote_exec(
 592             self, duration, mult, frame_size, traffic_profile, async_call=False,
 593             latency=True, warmup_time=5.0, traffic_directions=2, tx_port=0,
 594             rx_port=1):
 595         """Execute T-Rex ASTF script on remote node over ssh to start running
 596         traffic.
 597
 598         In sync mode, measurement results are stored internally.
 599         In async mode, initial data including xstats are stored internally.
 600
 601         :param duration: Time expresed in seconds for how long to send traffic.
 602         :param mult: Traffic rate expressed with units (pps, %)
 603         :param frame_size: L2 frame size to send (without padding and IPG).
 604         :param traffic_profile: Module name as a traffic profile identifier.
 605             See GPL/traffic_profiles/trex for implemented modules.
 606         :param async_call: If enabled then don't wait for all incoming traffic.
 607         :param latency: With latency measurement.
 608         :param warmup_time: Warmup time period.
 609         :param traffic_directions: Traffic is bi- (2) or uni- (1) directional.
 610             Default: 2
 611         :param tx_port: Traffic generator transmit port for first flow.
 612             Default: 0
 613         :param rx_port: Traffic generator receive port for first flow.
 614             Default: 1
 615         :type duration: float
 616         :type mult: int
 617         :type frame_size: str
 618         :type traffic_profile: str
 619         :type async_call: bool
 620         :type latency: bool
 621         :type warmup_time: float
 622         :type traffic_directions: int
 623         :type tx_port: int
 624         :type rx_port: int
 625         :raises RuntimeError: In case of T-Rex driver issue.
 626         """
 627         self.check_mode(TrexMode.ASTF)
 628         p_0, p_1 = (rx_port, tx_port) if self._ifaces_reordered \
 629             else (tx_port, rx_port)
 630         if not isinstance(duration, (float, int)):
 631             duration = float(duration)
 632         if not isinstance(warmup_time, (float, int)):
 633             warmup_time = float(warmup_time)
 634
 635         command_line = OptionString().add(u"python3")
 636         dirname = f"{Constants.REMOTE_FW_DIR}/GPL/tools/trex"
 637         command_line.add(f"'{dirname}/trex_astf_profile.py'")
 638         command_line.change_prefix(u"--")
 639         dirname = f"{Constants.REMOTE_FW_DIR}/GPL/traffic_profiles/trex"
 640         command_line.add_with_value(
 641             u"profile", f"'{dirname}/{traffic_profile}.py'"
 642         )
 643         command_line.add_with_value(u"duration", f"{duration!r}")
 644         command_line.add_with_value(u"frame_size", frame_size)
 645         command_line.add_with_value(u"mult", int(mult))
 646         command_line.add_with_value(u"warmup_time", f"{warmup_time!r}")
 647         command_line.add_with_value(u"port_0", p_0)
 648         command_line.add_with_value(u"port_1", p_1)
 649         command_line.add_with_value(u"traffic_directions", traffic_directions)
 650         command_line.add_if(u"async_start", async_call)
 651         command_line.add_if(u"latency", latency)
 652         command_line.add_if(u"force", Constants.TREX_SEND_FORCE)
 653
 654         stdout, _ = exec_cmd_no_error(
 655             self._node, command_line,
 656             timeout=int(duration) + 600 if u"tcp" in self.traffic_profile
 657             else 60,
 658             message=u"T-Rex ASTF runtime error!"
 659         )
 660
 661         self.traffic_directions = traffic_directions
 662         if async_call:
 663             # no result
 664             self._start_time = time.time()
 665             self._rate = float(mult)
 666             self._received = None
 667             self._sent = None
 668             self._loss = None
 669             self._latency = None
 670             xstats = [None, None]
 671             self._l7_data[u"client"] = dict()
 672             self._l7_data[u"client"][u"active_flows"] = None
 673             self._l7_data[u"client"][u"established_flows"] = None
 674             self._l7_data[u"server"] = dict()
 675             self._l7_data[u"server"][u"active_flows"] = None
 676             self._l7_data[u"server"][u"established_flows"] = None
 677             if u"udp" in self.traffic_profile:
 678                 self._l7_data[u"client"][u"udp"] = dict()
 679                 self._l7_data[u"client"][u"udp"][u"established_flows"] = None
 680                 self._l7_data[u"client"][u"udp"][u"closed_flows"] = None
 681                 self._l7_data[u"server"][u"udp"] = dict()
 682                 self._l7_data[u"server"][u"udp"][u"accepted_flows"] = None
 683                 self._l7_data[u"server"][u"udp"][u"closed_flows"] = None
 684             elif u"tcp" in self.traffic_profile:
 685                 self._l7_data[u"client"][u"tcp"] = dict()
 686                 self._l7_data[u"client"][u"tcp"][u"initiated_flows"] = None
 687                 self._l7_data[u"client"][u"tcp"][u"established_flows"] = None
 688                 self._l7_data[u"client"][u"tcp"][u"closed_flows"] = None
 689                 self._l7_data[u"server"][u"tcp"] = dict()
 690                 self._l7_data[u"server"][u"tcp"][u"accepted_flows"] = None
 691                 self._l7_data[u"server"][u"tcp"][u"established_flows"] = None
 692                 self._l7_data[u"server"][u"tcp"][u"closed_flows"] = None
 693             else:
 694                 logger.warn(u"Unsupported T-Rex ASTF traffic profile!")
 695             index = 0
 696             for line in stdout.splitlines():
 697                 if f"Xstats snapshot {index}: " in line:
 698                     xstats[index] = line[19:]
 699                     index += 1
 700                 if index == 2:
 701                     break
 702             self._xstats = tuple(xstats)
 703         else:
 704             self._parse_traffic_results(stdout)
 705             self._start_time = None
 706             self._rate = None
 707
 708     def trex_stl_start_remote_exec(
 709             self, duration, rate, frame_size, traffic_profile, async_call=False,
 710             latency=False, warmup_time=5.0, traffic_directions=2, tx_port=0,
 711             rx_port=1):
 712         """Execute T-Rex STL script on remote node over ssh to start running
 713         traffic.
 714
 715         In sync mode, measurement results are stored internally.
 716         In async mode, initial data including xstats are stored internally.
 717
 718         :param duration: Time expressed in seconds for how long to send traffic.
 719         :param rate: Traffic rate expressed with units (pps, %)
 720         :param frame_size: L2 frame size to send (without padding and IPG).
 721         :param traffic_profile: Module name as a traffic profile identifier.
 722             See GPL/traffic_profiles/trex for implemented modules.
 723         :param async_call: If enabled then don't wait for all incoming traffic.
 724         :param latency: With latency measurement.
 725         :param warmup_time: Warmup time period.
 726         :param traffic_directions: Traffic is bi- (2) or uni- (1) directional.
 727             Default: 2
 728         :param tx_port: Traffic generator transmit port for first flow.
 729             Default: 0
 730         :param rx_port: Traffic generator receive port for first flow.
 731             Default: 1
 732         :type duration: float
 733         :type rate: str
 734         :type frame_size: str
 735         :type traffic_profile: str
 736         :type async_call: bool
 737         :type latency: bool
 738         :type warmup_time: float
 739         :type traffic_directions: int
 740         :type tx_port: int
 741         :type rx_port: int
 742         :raises RuntimeError: In case of T-Rex driver issue.
 743         """
 744         self.check_mode(TrexMode.STL)
 745         p_0, p_1 = (rx_port, tx_port) if self._ifaces_reordered \
 746             else (tx_port, rx_port)
 747         if not isinstance(duration, (float, int)):
 748             duration = float(duration)
 749         if not isinstance(warmup_time, (float, int)):
 750             warmup_time = float(warmup_time)
 751
 752         command_line = OptionString().add(u"python3")
 753         dirname = f"{Constants.REMOTE_FW_DIR}/GPL/tools/trex"
 754         command_line.add(f"'{dirname}/trex_stl_profile.py'")
 755         command_line.change_prefix(u"--")
 756         dirname = f"{Constants.REMOTE_FW_DIR}/GPL/traffic_profiles/trex"
 757         command_line.add_with_value(
 758             u"profile", f"'{dirname}/{traffic_profile}.py'"
 759         )
 760         command_line.add_with_value(u"duration", f"{duration!r}")
 761         command_line.add_with_value(u"frame_size", frame_size)
 762         command_line.add_with_value(u"rate", f"{rate!r}")
 763         command_line.add_with_value(u"warmup_time", f"{warmup_time!r}")
 764         command_line.add_with_value(u"port_0", p_0)
 765         command_line.add_with_value(u"port_1", p_1)
 766         command_line.add_with_value(u"traffic_directions", traffic_directions)
 767         command_line.add_if(u"async_start", async_call)
 768         command_line.add_if(u"latency", latency)
 769         command_line.add_if(u"force", Constants.TREX_SEND_FORCE)
 770
 771         stdout, _ = exec_cmd_no_error(
 772             self._node, command_line, timeout=int(duration) + 60,
 773             message=u"T-Rex STL runtime error"
 774         )
 775
 776         self.traffic_directions = traffic_directions
 777         if async_call:
 778             # no result
 779             self._start_time = time.time()
 780             self._rate = float(rate[:-3]) if u"pps" in rate else float(rate)
 781             self._received = None
 782             self._sent = None
 783             self._loss = None
 784             self._latency = None
 785
 786             xstats = [None, None]
 787             index = 0
 788             for line in stdout.splitlines():
 789                 if f"Xstats snapshot {index}: " in line:
 790                     xstats[index] = line[19:]
 791                     index += 1
 792                 if index == 2:
 793                     break
 794             self._xstats = tuple(xstats)
 795         else:
 796             self._parse_traffic_results(stdout)
 797             self._start_time = None
 798             self._rate = None
 799
 800     def send_traffic_on_tg(
 801             self, duration, rate, frame_size, traffic_profile, warmup_time=5,
 802             async_call=False, latency=False, traffic_directions=2, tx_port=0,
 803             rx_port=1):
 804         """Send traffic from all configured interfaces on TG.
 805
 806         In async mode, xstats is stored internally,
 807         to enable getting correct result when stopping the traffic.
 808         In both modes, stdout is returned,
 809         but _parse_traffic_results only works in sync output.
 810
 811         Note that bidirectional traffic also contains flows
 812         transmitted from rx_port and received in tx_port.
 813         But some tests use asymmetric traffic, so those arguments are relevant.
 814
 815         Also note that traffic generator uses DPDK driver which might
 816         reorder port numbers based on wiring and PCI numbering.
 817         This method handles that, so argument values are invariant,
 818         but you can see swapped valued in debug logs.
 819
 820         :param duration: Duration of test traffic generation in seconds.
 821         :param rate: Traffic rate.
 822             - T-Rex stateless mode => Offered load per interface in pps,
 823             - T-Rex advanced stateful mode => multiplier of profile CPS.
 824         :param frame_size: Frame size (L2) in Bytes.
 825         :param traffic_profile: Module name as a traffic profile identifier.
 826             See GPL/traffic_profiles/trex for implemented modules.
 827         :param warmup_time: Warmup phase in seconds.
 828         :param async_call: Async mode.
 829         :param latency: With latency measurement.
 830         :param traffic_directions: Traffic is bi- (2) or uni- (1) directional.
 831             Default: 2
 832         :param tx_port: Traffic generator transmit port for first flow.
 833             Default: 0
 834         :param rx_port: Traffic generator receive port for first flow.
 835             Default: 1
 836         :type duration: float
 837         :type rate: float
 838         :type frame_size: str
 839         :type traffic_profile: str
 840         :type warmup_time: float
 841         :type async_call: bool
 842         :type latency: bool
 843         :type traffic_directions: int
 844         :type tx_port: int
 845         :type rx_port: int
 846         :returns: TG results.
 847         :rtype: str
 848         :raises ValueError: If TG traffic profile is not supported.
 849         """
 850         subtype = check_subtype(self._node)
 851         if subtype == NodeSubTypeTG.TREX:
 852             self.set_rate_provider_defaults(
 853                 frame_size, traffic_profile,
 854                 traffic_directions=traffic_directions)
 855             if u"trex-astf" in self.traffic_profile:
 856                 self.trex_astf_start_remote_exec(
 857                     duration, int(rate), frame_size, traffic_profile,
 858                     async_call, latency, warmup_time, traffic_directions,
 859                     tx_port, rx_port
 860                 )
 861             # TODO: rename all t-rex stateless profiles to use 'trex-stl'
 862             elif u"trex-sl" in self.traffic_profile:
 863                 unit_rate_str = str(rate) + u"pps"
 864                 self.trex_stl_start_remote_exec(
 865                     duration, unit_rate_str, frame_size, traffic_profile,
 866                     async_call, latency, warmup_time, traffic_directions,
 867                     tx_port, rx_port
 868                 )
 869             else:
 870                 raise ValueError(u"Unsupported T-Rex traffic profile!")
 871
 872         return self._result
 873
 874     def no_traffic_loss_occurred(self):
 875         """Fail if loss occurred in traffic run.
 876
 877         :returns: nothing
 878         :raises Exception: If loss occured.
 879         """
 880         if self._loss is None:
 881             raise RuntimeError(u"The traffic generation has not been issued")
 882         if self._loss != u"0":
 883             raise RuntimeError(f"Traffic loss occurred: {self._loss}")
 884
 885     def fail_if_no_traffic_forwarded(self):
 886         """Fail if no traffic forwarded.
 887
 888         :returns: nothing
 889         :raises Exception: If no traffic forwarded.
 890         """
 891         if self._received is None:
 892             raise RuntimeError(u"The traffic generation has not been issued")
 893         if self._received == u"0":
 894             raise RuntimeError(u"No traffic forwarded")
 895
 896     def partial_traffic_loss_accepted(
 897             self, loss_acceptance, loss_acceptance_type):
 898         """Fail if loss is higher then accepted in traffic run.
 899
 900         :param loss_acceptance: Permitted drop ratio or frames count.
 901         :param loss_acceptance_type: Type of permitted loss.
 902         :type loss_acceptance: float
 903         :type loss_acceptance_type: LossAcceptanceType
 904         :returns: nothing
 905         :raises Exception: If loss is above acceptance criteria.
 906         """
 907         if self._loss is None:
 908             raise Exception(u"The traffic generation has not been issued")
 909
 910         if loss_acceptance_type == u"percentage":
 911             loss = (float(self._loss) / float(self._sent)) * 100
 912         elif loss_acceptance_type == u"frames":
 913             loss = float(self._loss)
 914         else:
 915             raise Exception(u"Loss acceptance type not supported")
 916
 917         if loss > float(loss_acceptance):
 918             raise Exception(
 919                 f"Traffic loss {loss} above loss acceptance: {loss_acceptance}"
 920             )
 921
 922     def set_rate_provider_defaults(
 923             self, frame_size, traffic_profile, warmup_time=0.0,
 924             traffic_directions=2, negative_loss=True, latency=False):
 925         """Store values accessed by measure().
 926
 927         :param frame_size: Frame size identifier or value [B].
 928         :param traffic_profile: Module name as a traffic profile identifier.
 929             See GPL/traffic_profiles/trex for implemented modules.
 930         :param warmup_time: Traffic duration before measurement starts [s].
 931         :param traffic_directions: Traffic is bi- (2) or uni- (1) directional.
 932             Default: 2
 933         :param negative_loss: If false, negative loss is reported as zero loss.
 934         :param latency: Whether to measure latency during the trial.
 935             Default: False.
 936         :type frame_size: str or int
 937         :type traffic_profile: str
 938         :type warmup_time: float
 939         :type traffic_directions: int
 940         :type negative_loss: bool
 941         :type latency: bool
 942         """
 943         self.frame_size = frame_size
 944         self.traffic_profile = str(traffic_profile)
 945         self.warmup_time = float(warmup_time)
 946         self.traffic_directions = traffic_directions
 947         self.negative_loss = negative_loss
 948         self.use_latency = latency
 949
 950     def get_measurement_result(self, duration=None, transmit_rate=None):
 951         """Return the result of last measurement as ReceiveRateMeasurement.
 952
 953         Separate function, as measurements can end either by time
 954         or by explicit call, this is the common block at the end.
 955
 956         TODO: Fail on running or already reported measurement.
 957
 958         :param duration: Measurement duration [s] if known beforehand.
 959             For explicitly stopped measurement it is estimated.
 960         :param transmit_rate: Target aggregate transmit rate [pps].
 961             If not given, computed assuming it was bidirectional.
 962         :type duration: float or NoneType
 963         :type transmit_rate: float or NoneType
 964         :returns: Structure containing the result of the measurement.
 965         :rtype: ReceiveRateMeasurement
 966         """
 967         if duration is None:
 968             duration = time.time() - self._start_time
 969             self._start_time = None
 970         if transmit_rate is None:
 971             transmit_rate = self._rate * self.traffic_directions
 972         transmit_count = int(self.get_sent())
 973         loss_count = int(self.get_loss())
 974         if loss_count < 0 and not self.negative_loss:
 975             loss_count = 0
 976         measurement = ReceiveRateMeasurement(
 977             duration, transmit_rate, transmit_count, loss_count
 978         )
 979         measurement.latency = self.get_latency_int()
 980         return measurement
 981
 982     def measure(self, duration, transmit_rate):
 983         """Run trial measurement, parse and return aggregate results.
 984
 985         Aggregate means sum over traffic directions.
 986
 987         :param duration: Trial duration [s].
 988         :param transmit_rate: Target aggregate transmit rate [pps] / Connections
 989         per second (CPS) for UDP/TCP flows.
 990         :type duration: float
 991         :type transmit_rate: float
 992         :returns: Structure containing the result of the measurement.
 993         :rtype: ReceiveRateMeasurement
 994         :raises RuntimeError: If TG is not set or if node is not TG
 995             or if subtype is not specified.
 996         :raises NotImplementedError: If TG is not supported.
 997         """
 998         duration = float(duration)
 999         # TG needs target Tr per stream, but reports aggregate Tx and Dx.
1000         unit_rate_int = transmit_rate / float(self.traffic_directions)
1001         self.send_traffic_on_tg(
1002             duration, unit_rate_int, self.frame_size, self.traffic_profile,
1003             warmup_time=self.warmup_time, latency=self.use_latency,
1004             traffic_directions=self.traffic_directions
1005         )
1006         return self.get_measurement_result(duration, transmit_rate)
1007
1008
1009 class OptimizedSearch:
1010     """Class to be imported as Robot Library, containing search keywords.
1011
1012     Aside of setting up measurer and forwarding arguments,
1013     the main business is to translate min/max rate from unidir to aggregate.
1014     """
1015
1016     @staticmethod
1017     def perform_optimized_ndrpdr_search(
1018             frame_size, traffic_profile, minimum_transmit_rate,
1019             maximum_transmit_rate, packet_loss_ratio=0.005,
1020             final_relative_width=0.005, final_trial_duration=30.0,
1021             initial_trial_duration=1.0, number_of_intermediate_phases=2,
1022             timeout=720.0, doublings=1, traffic_directions=2):
1023         """Setup initialized TG, perform optimized search, return intervals.
1024
1025         :param frame_size: Frame size identifier or value [B].
1026         :param traffic_profile: Module name as a traffic profile identifier.
1027             See GPL/traffic_profiles/trex for implemented modules.
1028         :param minimum_transmit_rate: Minimal uni-directional
1029             target transmit rate [pps].
1030         :param maximum_transmit_rate: Maximal uni-directional
1031             target transmit rate [pps].
1032         :param packet_loss_ratio: Fraction of packets lost, for PDR [1].
1033         :param final_relative_width: Final lower bound transmit rate
1034             cannot be more distant that this multiple of upper bound [1].
1035         :param final_trial_duration: Trial duration for the final phase [s].
1036         :param initial_trial_duration: Trial duration for the initial phase
1037             and also for the first intermediate phase [s].
1038         :param number_of_intermediate_phases: Number of intermediate phases
1039             to perform before the final phase [1].
1040         :param timeout: The search will fail itself when not finished
1041             before this overall time [s].
1042         :param doublings: How many doublings to do in external search step.
1043             Default 1 is suitable for fairly stable tests,
1044             less stable tests might get better overal duration with 2 or more.
1045         :param traffic_directions: Traffic is bi- (2) or uni- (1) directional.
1046             Default: 2
1047         :type frame_size: str or int
1048         :type traffic_profile: str
1049         :type minimum_transmit_rate: float
1050         :type maximum_transmit_rate: float
1051         :type packet_loss_ratio: float
1052         :type final_relative_width: float
1053         :type final_trial_duration: float
1054         :type initial_trial_duration: float
1055         :type number_of_intermediate_phases: int
1056         :type timeout: float
1057         :type doublings: int
1058         :type traffic_directions: int
1059         :returns: Structure containing narrowed down NDR and PDR intervals
1060             and their measurements.
1061         :rtype: NdrPdrResult
1062         :raises RuntimeError: If total duration is larger than timeout.
1063         """
1064         minimum_transmit_rate *= traffic_directions
1065         maximum_transmit_rate *= traffic_directions
1066         # we need instance of TrafficGenerator instantiated by Robot Framework
1067         # to be able to use trex_stl-*()
1068         tg_instance = BuiltIn().get_library_instance(
1069             u"resources.libraries.python.TrafficGenerator"
1070         )
1071         tg_instance.set_rate_provider_defaults(
1072             frame_size, traffic_profile, traffic_directions=traffic_directions)
1073         algorithm = MultipleLossRatioSearch(
1074             measurer=tg_instance, final_trial_duration=final_trial_duration,
1075             final_relative_width=final_relative_width,
1076             number_of_intermediate_phases=number_of_intermediate_phases,
1077             initial_trial_duration=initial_trial_duration, timeout=timeout,
1078             doublings=doublings
1079         )
1080         result = algorithm.narrow_down_ndr_and_pdr(
1081             minimum_transmit_rate, maximum_transmit_rate, packet_loss_ratio
1082         )
1083         return result
1084
1085     @staticmethod
1086     def perform_soak_search(
1087             frame_size, traffic_profile, minimum_transmit_rate,
1088             maximum_transmit_rate, plr_target=1e-7, tdpt=0.1,
1089             initial_count=50, timeout=1800.0, trace_enabled=False,
1090             traffic_directions=2, latency=False):
1091         """Setup initialized TG, perform soak search, return avg and stdev.
1092
1093         :param frame_size: Frame size identifier or value [B].
1094         :param traffic_profile: Module name as a traffic profile identifier.
1095             See GPL/traffic_profiles/trex for implemented modules.
1096         :param minimum_transmit_rate: Minimal uni-directional
1097             target transmit rate [pps].
1098         :param maximum_transmit_rate: Maximal uni-directional
1099             target transmit rate [pps].
1100         :param plr_target: Fraction of packets lost to achieve [1].
1101         :param tdpt: Trial duration per trial.
1102             The algorithm linearly increases trial duration with trial number,
1103             this is the increment between succesive trials, in seconds.
1104         :param initial_count: Offset to apply before the first trial.
1105             For example initial_count=50 makes first trial to be 51*tdpt long.
1106             This is needed because initial "search" phase of integrator
1107             takes significant time even without any trial results.
1108         :param timeout: The search will stop after this overall time [s].
1109         :param trace_enabled: True if trace enabled else False.
1110         :param traffic_directions: Traffic is bi- (2) or uni- (1) directional.
1111             Default: 2
1112         :param latency: Whether to measure latency during the trial.
1113             Default: False.
1114         :type frame_size: str or int
1115         :type traffic_profile: str
1116         :type minimum_transmit_rate: float
1117         :type maximum_transmit_rate: float
1118         :type plr_target: float
1119         :type initial_count: int
1120         :type timeout: float
1121         :type trace_enabled: bool
1122         :type traffic_directions: int
1123         :type latency: bool
1124         :returns: Average and stdev of estimated aggregate rate giving PLR.
1125         :rtype: 2-tuple of float
1126         """
1127         minimum_transmit_rate *= traffic_directions
1128         maximum_transmit_rate *= traffic_directions
1129         tg_instance = BuiltIn().get_library_instance(
1130             u"resources.libraries.python.TrafficGenerator"
1131         )
1132         tg_instance.set_rate_provider_defaults(
1133             frame_size, traffic_profile, traffic_directions=traffic_directions,
1134             negative_loss=False, latency=latency)
1135         algorithm = PLRsearch(
1136             measurer=tg_instance, trial_duration_per_trial=tdpt,
1137             packet_loss_ratio_target=plr_target,
1138             trial_number_offset=initial_count, timeout=timeout,
1139             trace_enabled=trace_enabled
1140         )
1141         result = algorithm.search(minimum_transmit_rate, maximum_transmit_rate)
1142         return result