resources/libraries/python/TrafficGenerator.py

   1 # Copyright (c) 2020 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Performance testing traffic generator library."""
  15
  16 import time
  17
  18 from robot.api import logger
  19 from robot.libraries.BuiltIn import BuiltIn
  20
  21 from .Constants import Constants
  22 from .CpuUtils import CpuUtils
  23 from .DropRateSearch import DropRateSearch
  24 from .MLRsearch.AbstractMeasurer import AbstractMeasurer
  25 from .MLRsearch.MultipleLossRatioSearch import MultipleLossRatioSearch
  26 from .MLRsearch.ReceiveRateMeasurement import ReceiveRateMeasurement
  27 from .PLRsearch.PLRsearch import PLRsearch
  28 from .OptionString import OptionString
  29 from .ssh import exec_cmd_no_error, exec_cmd
  30 from .topology import NodeType
  31 from .topology import NodeSubTypeTG
  32 from .topology import Topology
  33
  34 __all__ = [u"TGDropRateSearchImpl", u"TrafficGenerator", u"OptimizedSearch"]
  35
  36
  37 def check_subtype(node):
  38     """Return supported subtype of given node, or raise an exception.
  39
  40     Currently only one subtype is supported,
  41     but we want our code to be ready for other ones.
  42
  43     :param node: Topology node to check. Can be None.
  44     :type node: dict or NoneType
  45     :returns: Subtype detected.
  46     :rtype: NodeSubTypeTG
  47     :raises RuntimeError: If node is not supported, message explains how.
  48     """
  49     if node.get(u"type") is None:
  50         msg = u"Node type is not defined"
  51     elif node[u"type"] != NodeType.TG:
  52         msg = f"Node type is {node[u'type']!r}, not a TG"
  53     elif node.get(u"subtype") is None:
  54         msg = u"TG subtype is not defined"
  55     elif node[u"subtype"] != NodeSubTypeTG.TREX:
  56         msg = f"TG subtype {node[u'subtype']!r} is not supported"
  57     else:
  58         return NodeSubTypeTG.TREX
  59     raise RuntimeError(msg)
  60
  61
  62 class TGDropRateSearchImpl(DropRateSearch):
  63     """Drop Rate Search implementation."""
  64
  65     # def __init__(self):
  66     #     super(TGDropRateSearchImpl, self).__init__()
  67
  68     def measure_loss(
  69             self, rate, frame_size, loss_acceptance, loss_acceptance_type,
  70             traffic_profile, skip_warmup=False):
  71         """Runs the traffic and evaluate the measured results.
  72
  73         :param rate: Offered traffic load.
  74         :param frame_size: Size of frame.
  75         :param loss_acceptance: Permitted drop ratio or frames count.
  76         :param loss_acceptance_type: Type of permitted loss.
  77         :param traffic_profile: Module name as a traffic profile identifier.
  78             See GPL/traffic_profiles/trex for implemented modules.
  79         :param skip_warmup: Start TRex without warmup traffic if true.
  80         :type rate: float
  81         :type frame_size: str
  82         :type loss_acceptance: float
  83         :type loss_acceptance_type: LossAcceptanceType
  84         :type traffic_profile: str
  85         :type skip_warmup: bool
  86         :returns: Drop threshold exceeded? (True/False)
  87         :rtype: bool
  88         :raises NotImplementedError: If TG is not supported.
  89         :raises RuntimeError: If TG is not specified.
  90         """
  91         # we need instance of TrafficGenerator instantiated by Robot Framework
  92         # to be able to use trex_stl-*()
  93         tg_instance = BuiltIn().get_library_instance(
  94             u"resources.libraries.python.TrafficGenerator"
  95         )
  96         subtype = check_subtype(tg_instance.node)
  97         if subtype == NodeSubTypeTG.TREX:
  98             unit_rate = str(rate) + self.get_rate_type_str()
  99             if skip_warmup:
 100                 tg_instance.trex_stl_start_remote_exec(
 101                     self.get_duration(), unit_rate, frame_size, traffic_profile,
 102                     warmup_time=0.0
 103                 )
 104             else:
 105                 tg_instance.trex_stl_start_remote_exec(
 106                     self.get_duration(), unit_rate, frame_size, traffic_profile
 107                 )
 108             loss = tg_instance.get_loss()
 109             sent = tg_instance.get_sent()
 110             if self.loss_acceptance_type_is_percentage():
 111                 loss = (float(loss) / float(sent)) * 100
 112             logger.trace(
 113                 f"comparing: {loss} < {loss_acceptance} {loss_acceptance_type}"
 114             )
 115             return float(loss) <= float(loss_acceptance)
 116         return False
 117
 118     def get_latency(self):
 119         """Returns min/avg/max latency.
 120
 121         :returns: Latency stats.
 122         :rtype: list
 123         """
 124         tg_instance = BuiltIn().get_library_instance(
 125             u"resources.libraries.python.TrafficGenerator"
 126         )
 127         return tg_instance.get_latency_int()
 128
 129
 130 class TrexMode:
 131     """Defines mode of T-Rex traffic generator."""
 132     # Advanced stateful mode
 133     ASTF = u"ASTF"
 134     # Stateless mode
 135     STL = u"STL"
 136
 137
 138 # TODO: Pylint says too-many-instance-attributes.
 139 class TrafficGenerator(AbstractMeasurer):
 140     """Traffic Generator."""
 141
 142     # TODO: Remove "trex" from lines which could work with other TGs.
 143
 144     # Use one instance of TrafficGenerator for all tests in test suite
 145     ROBOT_LIBRARY_SCOPE = u"TEST SUITE"
 146
 147     def __init__(self):
 148         self._node = None
 149         self._mode = None
 150         # TG interface order mapping
 151         self._ifaces_reordered = False
 152         # Result holding fields, to be removed.
 153         self._result = None
 154         self._loss = None
 155         self._sent = None
 156         self._latency = None
 157         self._received = None
 158         self._approximated_rate = None
 159         self._approximated_duration = None
 160         self._l7_data = None
 161         # Measurement input fields, needed for async stop result.
 162         self._start_time = None
 163         self._rate = None
 164         # Other input parameters, not knowable from measure() signature.
 165         self.frame_size = None
 166         self.traffic_profile = None
 167         self.warmup_time = None
 168         self.traffic_directions = None
 169         self.negative_loss = None
 170         # Transient data needed for async measurements.
 171         self._xstats = (None, None)
 172         # TODO: Rename "xstats" to something opaque, so T-Rex is not privileged?
 173
 174     @property
 175     def node(self):
 176         """Getter.
 177
 178         :returns: Traffic generator node.
 179         :rtype: dict
 180         """
 181         return self._node
 182
 183     def get_loss(self):
 184         """Return number of lost packets.
 185
 186         :returns: Number of lost packets.
 187         :rtype: str
 188         """
 189         return self._loss
 190
 191     def get_sent(self):
 192         """Return number of sent packets.
 193
 194         :returns: Number of sent packets.
 195         :rtype: str
 196         """
 197         return self._sent
 198
 199     def get_received(self):
 200         """Return number of received packets.
 201
 202         :returns: Number of received packets.
 203         :rtype: str
 204         """
 205         return self._received
 206
 207     def get_latency_int(self):
 208         """Return rounded min/avg/max latency.
 209
 210         :returns: Latency stats.
 211         :rtype: list
 212         """
 213         return self._latency
 214
 215     def get_approximated_rate(self):
 216         """Return approximated rate computed as ratio of transmitted packets
 217         over duration of trial.
 218
 219         :returns: Approximated rate.
 220         :rtype: str
 221         """
 222         return self._approximated_rate
 223
 224     def get_l7_data(self):
 225         """Return L7 data.
 226
 227         :returns: Number of received packets.
 228         :rtype: dict
 229         """
 230         return self._l7_data
 231
 232     def check_mode(self, expected_mode):
 233         """Check TG mode.
 234
 235         :param expected_mode: Expected traffic generator mode.
 236         :type expected_mode: object
 237         :raises RuntimeError: In case of unexpected TG mode.
 238         """
 239         if self._mode == expected_mode:
 240             return
 241         raise RuntimeError(
 242             f"{self._node[u'subtype']} not running in {expected_mode} mode!"
 243         )
 244
 245     # TODO: pylint says disable=too-many-locals.
 246     # A fix is developed in https://gerrit.fd.io/r/c/csit/+/22221
 247     def initialize_traffic_generator(
 248             self, tg_node, tg_if1, tg_if2, tg_if1_adj_node, tg_if1_adj_if,
 249             tg_if2_adj_node, tg_if2_adj_if, osi_layer, tg_if1_dst_mac=None,
 250             tg_if2_dst_mac=None):
 251         """TG initialization.
 252
 253         TODO: Document why do we need (and how do we use) _ifaces_reordered.
 254
 255         :param tg_node: Traffic generator node.
 256         :param tg_if1: TG - name of first interface.
 257         :param tg_if2: TG - name of second interface.
 258         :param tg_if1_adj_node: TG if1 adjecent node.
 259         :param tg_if1_adj_if: TG if1 adjecent interface.
 260         :param tg_if2_adj_node: TG if2 adjecent node.
 261         :param tg_if2_adj_if: TG if2 adjecent interface.
 262         :param osi_layer: 'L2', 'L3' or 'L7' - OSI Layer testing type.
 263         :param tg_if1_dst_mac: Interface 1 destination MAC address.
 264         :param tg_if2_dst_mac: Interface 2 destination MAC address.
 265         :type tg_node: dict
 266         :type tg_if1: str
 267         :type tg_if2: str
 268         :type tg_if1_adj_node: dict
 269         :type tg_if1_adj_if: str
 270         :type tg_if2_adj_node: dict
 271         :type tg_if2_adj_if: str
 272         :type osi_layer: str
 273         :type tg_if1_dst_mac: str
 274         :type tg_if2_dst_mac: str
 275         :returns: nothing
 276         :raises RuntimeError: In case of issue during initialization.
 277         """
 278         subtype = check_subtype(tg_node)
 279         if subtype == NodeSubTypeTG.TREX:
 280             self._node = tg_node
 281             self._mode = TrexMode.ASTF if osi_layer == u"L7" else TrexMode.STL
 282             if1 = dict()
 283             if2 = dict()
 284             if1[u"pci"] = Topology().get_interface_pci_addr(self._node, tg_if1)
 285             if2[u"pci"] = Topology().get_interface_pci_addr(self._node, tg_if2)
 286             if1[u"addr"] = Topology().get_interface_mac(self._node, tg_if1)
 287             if2[u"addr"] = Topology().get_interface_mac(self._node, tg_if2)
 288
 289             if osi_layer == u"L2":
 290                 if1[u"adj_addr"] = if2[u"addr"]
 291                 if2[u"adj_addr"] = if1[u"addr"]
 292             elif osi_layer in (u"L3", u"L7"):
 293                 if1[u"adj_addr"] = Topology().get_interface_mac(
 294                     tg_if1_adj_node, tg_if1_adj_if
 295                 )
 296                 if2[u"adj_addr"] = Topology().get_interface_mac(
 297                     tg_if2_adj_node, tg_if2_adj_if
 298                 )
 299             else:
 300                 raise ValueError(u"Unknown OSI layer!")
 301
 302             # in case of switched environment we can override MAC addresses
 303             if tg_if1_dst_mac is not None and tg_if2_dst_mac is not None:
 304                 if1[u"adj_addr"] = tg_if1_dst_mac
 305                 if2[u"adj_addr"] = tg_if2_dst_mac
 306
 307             if min(if1[u"pci"], if2[u"pci"]) != if1[u"pci"]:
 308                 if1, if2 = if2, if1
 309                 self._ifaces_reordered = True
 310
 311             master_thread_id, latency_thread_id, socket, threads = \
 312                 CpuUtils.get_affinity_trex(
 313                     self._node, tg_if1, tg_if2,
 314                     tg_dtc=Constants.TREX_CORE_COUNT)
 315
 316             if osi_layer in (u"L2", u"L3", u"L7"):
 317                 exec_cmd_no_error(
 318                     self._node,
 319                     f"sh -c 'cat << EOF > /etc/trex_cfg.yaml\n"
 320                     f"- version: 2\n"
 321                     f"  c: {len(threads)}\n"
 322                     f"  limit_memory: {Constants.TREX_LIMIT_MEMORY}\n"
 323                     f"  interfaces: [\"{if1[u'pci']}\",\"{if2[u'pci']}\"]\n"
 324                     f"  port_info:\n"
 325                     f"      - dest_mac: \'{if1[u'adj_addr']}\'\n"
 326                     f"        src_mac: \'{if1[u'addr']}\'\n"
 327                     f"      - dest_mac: \'{if2[u'adj_addr']}\'\n"
 328                     f"        src_mac: \'{if2[u'addr']}\'\n"
 329                     f"  platform :\n"
 330                     f"      master_thread_id: {master_thread_id}\n"
 331                     f"      latency_thread_id: {latency_thread_id}\n"
 332                     f"      dual_if:\n"
 333                     f"          - socket: {socket}\n"
 334                     f"            threads: {threads}\n"
 335                     f"EOF'",
 336                     sudo=True, message=u"T-Rex config generation!"
 337                 )
 338             else:
 339                 raise ValueError(u"Unknown OSI layer!")
 340
 341             TrafficGenerator.startup_trex(
 342                 self._node, osi_layer, subtype=subtype
 343             )
 344
 345     @staticmethod
 346     def startup_trex(tg_node, osi_layer, subtype=None):
 347         """Startup sequence for the TRex traffic generator.
 348
 349         :param tg_node: Traffic generator node.
 350         :param osi_layer: 'L2', 'L3' or 'L7' - OSI Layer testing type.
 351         :param subtype: Traffic generator sub-type.
 352         :type tg_node: dict
 353         :type osi_layer: str
 354         :type subtype: NodeSubTypeTG
 355         :raises RuntimeError: If T-Rex startup failed.
 356         :raises ValueError: If OSI layer is not supported.
 357         """
 358         if not subtype:
 359             subtype = check_subtype(tg_node)
 360         if subtype == NodeSubTypeTG.TREX:
 361             for _ in range(0, 3):
 362                 # Kill TRex only if it is already running.
 363                 cmd = u"sh -c \"pgrep t-rex && pkill t-rex && sleep 3 || true\""
 364                 exec_cmd_no_error(
 365                     tg_node, cmd, sudo=True, message=u"Kill TRex failed!"
 366                 )
 367
 368                 # Configure TRex.
 369                 ports = ''
 370                 for port in tg_node[u"interfaces"].values():
 371                     ports += f" {port.get(u'pci_address')}"
 372
 373                 cmd = f"sh -c \"cd {Constants.TREX_INSTALL_DIR}/scripts/ && " \
 374                     f"./dpdk_nic_bind.py -u {ports} || true\""
 375                 exec_cmd_no_error(
 376                     tg_node, cmd, sudo=True,
 377                     message=u"Unbind PCI ports from driver failed!"
 378                 )
 379
 380                 # Start TRex.
 381                 cd_cmd = f"cd '{Constants.TREX_INSTALL_DIR}/scripts/'"
 382                 trex_cmd = OptionString([u"nohup", u"./t-rex-64"])
 383                 trex_cmd.add(u"-i")
 384                 trex_cmd.add(u"--prefix $(hostname)")
 385                 trex_cmd.add(u"--hdrh")
 386                 trex_cmd.add(u"--no-scapy-server")
 387                 trex_cmd.add_if(u"--astf", osi_layer == u"L7")
 388                 # OptionString does not create double space if extra is empty.
 389                 trex_cmd.add(f"{Constants.TREX_EXTRA_CMDLINE}")
 390                 inner_command = f"{cd_cmd} && {trex_cmd} > /tmp/trex.log 2>&1 &"
 391                 cmd = f"sh -c \"{inner_command}\" > /dev/null"
 392                 try:
 393                     exec_cmd_no_error(tg_node, cmd, sudo=True)
 394                 except RuntimeError:
 395                     cmd = u"sh -c \"cat /tmp/trex.log\""
 396                     exec_cmd_no_error(
 397                         tg_node, cmd, sudo=True,
 398                         message=u"Get TRex logs failed!"
 399                     )
 400                     raise RuntimeError(u"Start TRex failed!")
 401
 402                 # Test T-Rex API responsiveness.
 403                 cmd = u"python3"
 404                 cmd += f" {Constants.REMOTE_FW_DIR}/GPL/tools/trex/"
 405                 if osi_layer in (u"L2", u"L3"):
 406                     cmd += f"trex_stl_assert.py"
 407                 elif osi_layer == u"L7":
 408                     cmd += f"trex_astf_assert.py"
 409                 else:
 410                     raise ValueError(u"Unknown OSI layer!")
 411                 try:
 412                     exec_cmd_no_error(
 413                         tg_node, cmd, sudo=True,
 414                         message=u"T-Rex API is not responding!", retries=20
 415                     )
 416                 except RuntimeError:
 417                     continue
 418                 return
 419             # After max retries TRex is still not responding to API critical
 420             # error occurred.
 421             exec_cmd(tg_node, u"cat /tmp/trex.log", sudo=True)
 422             raise RuntimeError(u"Start T-Rex failed after multiple retries!")
 423
 424     @staticmethod
 425     def is_trex_running(node):
 426         """Check if T-Rex is running using pidof.
 427
 428         :param node: Traffic generator node.
 429         :type node: dict
 430         :returns: True if T-Rex is running otherwise False.
 431         :rtype: bool
 432         """
 433         ret, _, _ = exec_cmd(node, u"pgrep t-rex", sudo=True)
 434         return bool(int(ret) == 0)
 435
 436     @staticmethod
 437     def teardown_traffic_generator(node):
 438         """TG teardown.
 439
 440         :param node: Traffic generator node.
 441         :type node: dict
 442         :returns: nothing
 443         :raises RuntimeError: If node type is not a TG,
 444             or if T-Rex teardown fails.
 445         """
 446         subtype = check_subtype(node)
 447         if subtype == NodeSubTypeTG.TREX:
 448             exec_cmd_no_error(
 449                 node,
 450                 u"sh -c "
 451                 u"\"if pgrep t-rex; then sudo pkill t-rex && sleep 3; fi\"",
 452                 sudo=False,
 453                 message=u"T-Rex kill failed!"
 454             )
 455
 456     def _parse_traffic_results(self, stdout):
 457         """Parse stdout of scripts into fields of self.
 458
 459         Block of code to reuse, by sync start, or stop after async.
 460
 461         :param stdout: Text containing the standard output.
 462         :type stdout: str
 463         """
 464         subtype = check_subtype(self._node)
 465         if subtype == NodeSubTypeTG.TREX:
 466             # Last line from console output
 467             line = stdout.splitlines()[-1]
 468             results = line.split(",")
 469             if results[-1] == u" ":
 470                 results.remove(u" ")
 471             self._result = dict()
 472             for result in results:
 473                 key, value = result.split(u"=", maxsplit=1)
 474                 self._result[key.strip()] = value
 475             logger.info(f"TrafficGen results:\n{self._result}")
 476             self._received = self._result.get(u"total_received")
 477             self._sent = self._result.get(u"total_sent")
 478             self._loss = self._result.get(u"frame_loss")
 479             self._approximated_duration = \
 480                 self._result.get(u"approximated_duration")
 481             self._approximated_rate = self._result.get(u"approximated_rate")
 482             self._latency = list()
 483             self._latency.append(self._result.get(u"latency_stream_0(usec)"))
 484             self._latency.append(self._result.get(u"latency_stream_1(usec)"))
 485             if self._mode == TrexMode.ASTF:
 486                 self._l7_data = dict()
 487                 self._l7_data[u"client"] = dict()
 488                 self._l7_data[u"client"][u"active_flows"] = \
 489                     self._result.get(u"client_active_flows")
 490                 self._l7_data[u"client"][u"established_flows"] = \
 491                     self._result.get(u"client_established_flows")
 492                 self._l7_data[u"server"] = dict()
 493                 self._l7_data[u"server"][u"active_flows"] = \
 494                     self._result.get(u"server_active_flows")
 495                 self._l7_data[u"server"][u"established_flows"] = \
 496                     self._result.get(u"server_established_flows")
 497                 if u"udp" in self.traffic_profile:
 498                     self._l7_data[u"client"][u"udp"] = dict()
 499                     self._l7_data[u"client"][u"udp"][u"established_flows"] = \
 500                         self._result.get(u"client_udp_connects")
 501                     self._l7_data[u"client"][u"udp"][u"closed_flows"] = \
 502                         self._result.get(u"client_udp_closed")
 503                     self._l7_data[u"server"][u"udp"] = dict()
 504                     self._l7_data[u"server"][u"udp"][u"accepted_flows"] = \
 505                         self._result.get(u"server_udp_accepts")
 506                     self._l7_data[u"server"][u"udp"][u"closed_flows"] = \
 507                         self._result.get(u"server_udp_closed")
 508                 elif u"tcp" in self.traffic_profile:
 509                     self._l7_data[u"client"][u"tcp"] = dict()
 510                     self._l7_data[u"client"][u"tcp"][u"initiated_flows"] = \
 511                         self._result.get(u"client_tcp_connect_inits")
 512                     self._l7_data[u"client"][u"tcp"][u"established_flows"] = \
 513                         self._result.get(u"client_tcp_connects")
 514                     self._l7_data[u"client"][u"tcp"][u"closed_flows"] = \
 515                         self._result.get(u"client_tcp_closed")
 516                     self._l7_data[u"server"][u"tcp"] = dict()
 517                     self._l7_data[u"server"][u"tcp"][u"accepted_flows"] = \
 518                         self._result.get(u"server_tcp_accepts")
 519                     self._l7_data[u"server"][u"tcp"][u"established_flows"] = \
 520                         self._result.get(u"server_tcp_connects")
 521                     self._l7_data[u"server"][u"tcp"][u"closed_flows"] = \
 522                         self._result.get(u"server_tcp_closed")
 523
 524     def trex_astf_stop_remote_exec(self, node):
 525         """Execute T-Rex ASTF script on remote node over ssh to stop running
 526         traffic.
 527
 528         Internal state is updated with measurement results.
 529
 530         :param node: T-Rex generator node.
 531         :type node: dict
 532         :raises RuntimeError: If stop traffic script fails.
 533         """
 534         command_line = OptionString().add(u"python3")
 535         dirname = f"{Constants.REMOTE_FW_DIR}/GPL/tools/trex"
 536         command_line.add(f"'{dirname}/trex_astf_stop.py'")
 537         command_line.change_prefix(u"--")
 538         for index, value in enumerate(self._xstats):
 539             if value is not None:
 540                 value = value.replace(u"'", u"\"")
 541                 command_line.add_equals(f"xstat{index}", f"'{value}'")
 542         stdout, _ = exec_cmd_no_error(
 543             node, command_line,
 544             message=u"T-Rex ASTF runtime error!"
 545         )
 546         self._parse_traffic_results(stdout)
 547
 548     def trex_stl_stop_remote_exec(self, node):
 549         """Execute T-Rex STL script on remote node over ssh to stop running
 550         traffic.
 551
 552         Internal state is updated with measurement results.
 553
 554         :param node: T-Rex generator node.
 555         :type node: dict
 556         :raises RuntimeError: If stop traffic script fails.
 557         """
 558         command_line = OptionString().add(u"python3")
 559         dirname = f"{Constants.REMOTE_FW_DIR}/GPL/tools/trex"
 560         command_line.add(f"'{dirname}/trex_stl_stop.py'")
 561         command_line.change_prefix(u"--")
 562         for index, value in enumerate(self._xstats):
 563             if value is not None:
 564                 value = value.replace(u"'", u"\"")
 565                 command_line.add_equals(f"xstat{index}", f"'{value}'")
 566         stdout, _ = exec_cmd_no_error(
 567             node, command_line,
 568             message=u"T-Rex STL runtime error!"
 569         )
 570         self._parse_traffic_results(stdout)
 571
 572     def stop_traffic_on_tg(self):
 573         """Stop all traffic on TG.
 574
 575         :returns: Structure containing the result of the measurement.
 576         :rtype: ReceiveRateMeasurement
 577         :raises ValueError: If TG traffic profile is not supported.
 578         """
 579         subtype = check_subtype(self._node)
 580         if subtype == NodeSubTypeTG.TREX:
 581             if u"trex-astf" in self.traffic_profile:
 582                 self.trex_astf_stop_remote_exec(self._node)
 583             elif u"trex-sl" in self.traffic_profile:
 584                 self.trex_stl_stop_remote_exec(self._node)
 585             else:
 586                 raise ValueError(u"Unsupported T-Rex traffic profile!")
 587
 588         return self.get_measurement_result()
 589
 590     def trex_astf_start_remote_exec(
 591             self, duration, mult, frame_size, traffic_profile, async_call=False,
 592             latency=True, warmup_time=5.0, traffic_directions=2, tx_port=0,
 593             rx_port=1):
 594         """Execute T-Rex ASTF script on remote node over ssh to start running
 595         traffic.
 596
 597         In sync mode, measurement results are stored internally.
 598         In async mode, initial data including xstats are stored internally.
 599
 600         :param duration: Time expresed in seconds for how long to send traffic.
 601         :param mult: Traffic rate expressed with units (pps, %)
 602         :param frame_size: L2 frame size to send (without padding and IPG).
 603         :param traffic_profile: Module name as a traffic profile identifier.
 604             See GPL/traffic_profiles/trex for implemented modules.
 605         :param async_call: If enabled then don't wait for all incoming traffic.
 606         :param latency: With latency measurement.
 607         :param warmup_time: Warmup time period.
 608         :param traffic_directions: Traffic is bi- (2) or uni- (1) directional.
 609             Default: 2
 610         :param tx_port: Traffic generator transmit port for first flow.
 611             Default: 0
 612         :param rx_port: Traffic generator receive port for first flow.
 613             Default: 1
 614         :type duration: float
 615         :type mult: int
 616         :type frame_size: str
 617         :type traffic_profile: str
 618         :type async_call: bool
 619         :type latency: bool
 620         :type warmup_time: float
 621         :type traffic_directions: int
 622         :type tx_port: int
 623         :type rx_port: int
 624         :raises RuntimeError: In case of T-Rex driver issue.
 625         """
 626         self.check_mode(TrexMode.ASTF)
 627         p_0, p_1 = (rx_port, tx_port) if self._ifaces_reordered \
 628             else (tx_port, rx_port)
 629         if not isinstance(duration, (float, int)):
 630             duration = float(duration)
 631         if not isinstance(warmup_time, (float, int)):
 632             warmup_time = float(warmup_time)
 633
 634         command_line = OptionString().add(u"python3")
 635         dirname = f"{Constants.REMOTE_FW_DIR}/GPL/tools/trex"
 636         command_line.add(f"'{dirname}/trex_astf_profile.py'")
 637         command_line.change_prefix(u"--")
 638         dirname = f"{Constants.REMOTE_FW_DIR}/GPL/traffic_profiles/trex"
 639         command_line.add_with_value(
 640             u"profile", f"'{dirname}/{traffic_profile}.py'"
 641         )
 642         command_line.add_with_value(u"duration", f"{duration!r}")
 643         command_line.add_with_value(u"frame_size", frame_size)
 644         command_line.add_with_value(u"mult", int(mult))
 645         command_line.add_with_value(u"warmup_time", f"{warmup_time!r}")
 646         command_line.add_with_value(u"port_0", p_0)
 647         command_line.add_with_value(u"port_1", p_1)
 648         command_line.add_with_value(u"traffic_directions", traffic_directions)
 649         command_line.add_if(u"async_start", async_call)
 650         command_line.add_if(u"latency", latency)
 651         command_line.add_if(u"force", Constants.TREX_SEND_FORCE)
 652
 653         stdout, _ = exec_cmd_no_error(
 654             self._node, command_line,
 655             timeout=int(duration) + 600 if u"tcp" in self.traffic_profile
 656             else 60,
 657             message=u"T-Rex ASTF runtime error!"
 658         )
 659
 660         self.traffic_directions = traffic_directions
 661         if async_call:
 662             # no result
 663             self._start_time = time.time()
 664             self._rate = float(mult)
 665             self._received = None
 666             self._sent = None
 667             self._loss = None
 668             self._latency = None
 669             xstats = [None, None]
 670             self._l7_data[u"client"] = dict()
 671             self._l7_data[u"client"][u"active_flows"] = None
 672             self._l7_data[u"client"][u"established_flows"] = None
 673             self._l7_data[u"server"] = dict()
 674             self._l7_data[u"server"][u"active_flows"] = None
 675             self._l7_data[u"server"][u"established_flows"] = None
 676             if u"udp" in self.traffic_profile:
 677                 self._l7_data[u"client"][u"udp"] = dict()
 678                 self._l7_data[u"client"][u"udp"][u"established_flows"] = None
 679                 self._l7_data[u"client"][u"udp"][u"closed_flows"] = None
 680                 self._l7_data[u"server"][u"udp"] = dict()
 681                 self._l7_data[u"server"][u"udp"][u"accepted_flows"] = None
 682                 self._l7_data[u"server"][u"udp"][u"closed_flows"] = None
 683             elif u"tcp" in self.traffic_profile:
 684                 self._l7_data[u"client"][u"tcp"] = dict()
 685                 self._l7_data[u"client"][u"tcp"][u"initiated_flows"] = None
 686                 self._l7_data[u"client"][u"tcp"][u"established_flows"] = None
 687                 self._l7_data[u"client"][u"tcp"][u"closed_flows"] = None
 688                 self._l7_data[u"server"][u"tcp"] = dict()
 689                 self._l7_data[u"server"][u"tcp"][u"accepted_flows"] = None
 690                 self._l7_data[u"server"][u"tcp"][u"established_flows"] = None
 691                 self._l7_data[u"server"][u"tcp"][u"closed_flows"] = None
 692             else:
 693                 logger.warn(u"Unsupported T-Rex ASTF traffic profile!")
 694             index = 0
 695             for line in stdout.splitlines():
 696                 if f"Xstats snapshot {index}: " in line:
 697                     xstats[index] = line[19:]
 698                     index += 1
 699                 if index == 2:
 700                     break
 701             self._xstats = tuple(xstats)
 702         else:
 703             self._parse_traffic_results(stdout)
 704             self._start_time = None
 705             self._rate = None
 706
 707     def trex_stl_start_remote_exec(
 708             self, duration, rate, frame_size, traffic_profile, async_call=False,
 709             latency=True, warmup_time=5.0, traffic_directions=2, tx_port=0,
 710             rx_port=1):
 711         """Execute T-Rex STL script on remote node over ssh to start running
 712         traffic.
 713
 714         In sync mode, measurement results are stored internally.
 715         In async mode, initial data including xstats are stored internally.
 716
 717         :param duration: Time expressed in seconds for how long to send traffic.
 718         :param rate: Traffic rate expressed with units (pps, %)
 719         :param frame_size: L2 frame size to send (without padding and IPG).
 720         :param traffic_profile: Module name as a traffic profile identifier.
 721             See GPL/traffic_profiles/trex for implemented modules.
 722         :param async_call: If enabled then don't wait for all incoming traffic.
 723         :param latency: With latency measurement.
 724         :param warmup_time: Warmup time period.
 725         :param traffic_directions: Traffic is bi- (2) or uni- (1) directional.
 726             Default: 2
 727         :param tx_port: Traffic generator transmit port for first flow.
 728             Default: 0
 729         :param rx_port: Traffic generator receive port for first flow.
 730             Default: 1
 731         :type duration: float
 732         :type rate: str
 733         :type frame_size: str
 734         :type traffic_profile: str
 735         :type async_call: bool
 736         :type latency: bool
 737         :type warmup_time: float
 738         :type traffic_directions: int
 739         :type tx_port: int
 740         :type rx_port: int
 741         :raises RuntimeError: In case of T-Rex driver issue.
 742         """
 743         self.check_mode(TrexMode.STL)
 744         p_0, p_1 = (rx_port, tx_port) if self._ifaces_reordered \
 745             else (tx_port, rx_port)
 746         if not isinstance(duration, (float, int)):
 747             duration = float(duration)
 748         if not isinstance(warmup_time, (float, int)):
 749             warmup_time = float(warmup_time)
 750
 751         command_line = OptionString().add(u"python3")
 752         dirname = f"{Constants.REMOTE_FW_DIR}/GPL/tools/trex"
 753         command_line.add(f"'{dirname}/trex_stl_profile.py'")
 754         command_line.change_prefix(u"--")
 755         dirname = f"{Constants.REMOTE_FW_DIR}/GPL/traffic_profiles/trex"
 756         command_line.add_with_value(
 757             u"profile", f"'{dirname}/{traffic_profile}.py'"
 758         )
 759         command_line.add_with_value(u"duration", f"{duration!r}")
 760         command_line.add_with_value(u"frame_size", frame_size)
 761         command_line.add_with_value(u"rate", f"{rate!r}")
 762         command_line.add_with_value(u"warmup_time", f"{warmup_time!r}")
 763         command_line.add_with_value(u"port_0", p_0)
 764         command_line.add_with_value(u"port_1", p_1)
 765         command_line.add_with_value(u"traffic_directions", traffic_directions)
 766         command_line.add_if(u"async_start", async_call)
 767         command_line.add_if(u"latency", latency)
 768         command_line.add_if(u"force", Constants.TREX_SEND_FORCE)
 769
 770         stdout, _ = exec_cmd_no_error(
 771             self._node, command_line, timeout=int(duration) + 60,
 772             message=u"T-Rex STL runtime error"
 773         )
 774
 775         self.traffic_directions = traffic_directions
 776         if async_call:
 777             # no result
 778             self._start_time = time.time()
 779             self._rate = float(rate[:-3]) if u"pps" in rate else float(rate)
 780             self._received = None
 781             self._sent = None
 782             self._loss = None
 783             self._latency = None
 784
 785             xstats = [None, None]
 786             index = 0
 787             for line in stdout.splitlines():
 788                 if f"Xstats snapshot {index}: " in line:
 789                     xstats[index] = line[19:]
 790                     index += 1
 791                 if index == 2:
 792                     break
 793             self._xstats = tuple(xstats)
 794         else:
 795             self._parse_traffic_results(stdout)
 796             self._start_time = None
 797             self._rate = None
 798
 799     def send_traffic_on_tg(
 800             self, duration, rate, frame_size, traffic_profile, warmup_time=5,
 801             async_call=False, latency=True, traffic_directions=2, tx_port=0,
 802             rx_port=1):
 803         """Send traffic from all configured interfaces on TG.
 804
 805         In async mode, xstats is stored internally,
 806         to enable getting correct result when stopping the traffic.
 807         In both modes, stdout is returned,
 808         but _parse_traffic_results only works in sync output.
 809
 810         Note that bidirectional traffic also contains flows
 811         transmitted from rx_port and received in tx_port.
 812         But some tests use asymmetric traffic, so those arguments are relevant.
 813
 814         Also note that traffic generator uses DPDK driver which might
 815         reorder port numbers based on wiring and PCI numbering.
 816         This method handles that, so argument values are invariant,
 817         but you can see swapped valued in debug logs.
 818
 819         :param duration: Duration of test traffic generation in seconds.
 820         :param rate: Traffic rate.
 821             - T-Rex stateless mode => Offered load per interface in pps,
 822             - T-Rex advanced stateful mode => multiplier of profile CPS.
 823         :param frame_size: Frame size (L2) in Bytes.
 824         :param traffic_profile: Module name as a traffic profile identifier.
 825             See GPL/traffic_profiles/trex for implemented modules.
 826         :param warmup_time: Warmup phase in seconds.
 827         :param async_call: Async mode.
 828         :param latency: With latency measurement.
 829         :param traffic_directions: Traffic is bi- (2) or uni- (1) directional.
 830             Default: 2
 831         :param tx_port: Traffic generator transmit port for first flow.
 832             Default: 0
 833         :param rx_port: Traffic generator receive port for first flow.
 834             Default: 1
 835         :type duration: float
 836         :type rate: float
 837         :type frame_size: str
 838         :type traffic_profile: str
 839         :type warmup_time: float
 840         :type async_call: bool
 841         :type latency: bool
 842         :type traffic_directions: int
 843         :type tx_port: int
 844         :type rx_port: int
 845         :returns: TG results.
 846         :rtype: str
 847         :raises ValueError: If TG traffic profile is not supported.
 848         """
 849         subtype = check_subtype(self._node)
 850         if subtype == NodeSubTypeTG.TREX:
 851             self.set_rate_provider_defaults(
 852                 frame_size, traffic_profile,
 853                 traffic_directions=traffic_directions)
 854             if u"trex-astf" in self.traffic_profile:
 855                 self.trex_astf_start_remote_exec(
 856                     duration, int(rate), frame_size, traffic_profile,
 857                     async_call, latency, warmup_time, traffic_directions,
 858                     tx_port, rx_port
 859                 )
 860             # TODO: rename all t-rex stateless profiles to use 'trex-stl'
 861             elif u"trex-sl" in self.traffic_profile:
 862                 unit_rate_str = str(rate) + u"pps"
 863                 self.trex_stl_start_remote_exec(
 864                     duration, unit_rate_str, frame_size, traffic_profile,
 865                     async_call, latency, warmup_time, traffic_directions,
 866                     tx_port, rx_port
 867                 )
 868             else:
 869                 raise ValueError(u"Unsupported T-Rex traffic profile!")
 870
 871         return self._result
 872
 873     def no_traffic_loss_occurred(self):
 874         """Fail if loss occurred in traffic run.
 875
 876         :returns: nothing
 877         :raises Exception: If loss occured.
 878         """
 879         if self._loss is None:
 880             raise RuntimeError(u"The traffic generation has not been issued")
 881         if self._loss != u"0":
 882             raise RuntimeError(f"Traffic loss occurred: {self._loss}")
 883
 884     def fail_if_no_traffic_forwarded(self):
 885         """Fail if no traffic forwarded.
 886
 887         :returns: nothing
 888         :raises Exception: If no traffic forwarded.
 889         """
 890         if self._received is None:
 891             raise RuntimeError(u"The traffic generation has not been issued")
 892         if self._received == u"0":
 893             raise RuntimeError(u"No traffic forwarded")
 894
 895     def partial_traffic_loss_accepted(
 896             self, loss_acceptance, loss_acceptance_type):
 897         """Fail if loss is higher then accepted in traffic run.
 898
 899         :param loss_acceptance: Permitted drop ratio or frames count.
 900         :param loss_acceptance_type: Type of permitted loss.
 901         :type loss_acceptance: float
 902         :type loss_acceptance_type: LossAcceptanceType
 903         :returns: nothing
 904         :raises Exception: If loss is above acceptance criteria.
 905         """
 906         if self._loss is None:
 907             raise Exception(u"The traffic generation has not been issued")
 908
 909         if loss_acceptance_type == u"percentage":
 910             loss = (float(self._loss) / float(self._sent)) * 100
 911         elif loss_acceptance_type == u"frames":
 912             loss = float(self._loss)
 913         else:
 914             raise Exception(u"Loss acceptance type not supported")
 915
 916         if loss > float(loss_acceptance):
 917             raise Exception(
 918                 f"Traffic loss {loss} above loss acceptance: {loss_acceptance}"
 919             )
 920
 921     def set_rate_provider_defaults(
 922             self, frame_size, traffic_profile, warmup_time=0.0,
 923             traffic_directions=2, negative_loss=True):
 924         """Store values accessed by measure().
 925
 926         :param frame_size: Frame size identifier or value [B].
 927         :param traffic_profile: Module name as a traffic profile identifier.
 928             See GPL/traffic_profiles/trex for implemented modules.
 929         :param warmup_time: Traffic duration before measurement starts [s].
 930         :param traffic_directions: Traffic is bi- (2) or uni- (1) directional.
 931             Default: 2
 932         :param negative_loss: If false, negative loss is reported as zero loss.
 933         :type frame_size: str or int
 934         :type traffic_profile: str
 935         :type warmup_time: float
 936         :type traffic_directions: int
 937         :type negative_loss: bool
 938         """
 939         self.frame_size = frame_size
 940         self.traffic_profile = str(traffic_profile)
 941         self.warmup_time = float(warmup_time)
 942         self.traffic_directions = traffic_directions
 943         self.negative_loss = negative_loss
 944
 945     def get_measurement_result(self, duration=None, transmit_rate=None):
 946         """Return the result of last measurement as ReceiveRateMeasurement.
 947
 948         Separate function, as measurements can end either by time
 949         or by explicit call, this is the common block at the end.
 950
 951         TODO: Fail on running or already reported measurement.
 952
 953         :param duration: Measurement duration [s] if known beforehand.
 954             For explicitly stopped measurement it is estimated.
 955         :param transmit_rate: Target aggregate transmit rate [pps].
 956             If not given, computed assuming it was bidirectional.
 957         :type duration: float or NoneType
 958         :type transmit_rate: float or NoneType
 959         :returns: Structure containing the result of the measurement.
 960         :rtype: ReceiveRateMeasurement
 961         """
 962         if duration is None:
 963             duration = time.time() - self._start_time
 964             self._start_time = None
 965         if transmit_rate is None:
 966             transmit_rate = self._rate * self.traffic_directions
 967         transmit_count = int(self.get_sent())
 968         loss_count = int(self.get_loss())
 969         if loss_count < 0 and not self.negative_loss:
 970             loss_count = 0
 971         measurement = ReceiveRateMeasurement(
 972             duration, transmit_rate, transmit_count, loss_count
 973         )
 974         measurement.latency = self.get_latency_int()
 975         return measurement
 976
 977     def measure(self, duration, transmit_rate):
 978         """Run trial measurement, parse and return aggregate results.
 979
 980         Aggregate means sum over traffic directions.
 981
 982         :param duration: Trial duration [s].
 983         :param transmit_rate: Target aggregate transmit rate [pps] / Connections
 984         per second (CPS) for UDP/TCP flows.
 985         :type duration: float
 986         :type transmit_rate: float
 987         :returns: Structure containing the result of the measurement.
 988         :rtype: ReceiveRateMeasurement
 989         :raises RuntimeError: If TG is not set or if node is not TG
 990             or if subtype is not specified.
 991         :raises NotImplementedError: If TG is not supported.
 992         """
 993         duration = float(duration)
 994         # TG needs target Tr per stream, but reports aggregate Tx and Dx.
 995         unit_rate_int = transmit_rate / float(self.traffic_directions)
 996         self.send_traffic_on_tg(
 997             duration, unit_rate_int, self.frame_size, self.traffic_profile,
 998             warmup_time=self.warmup_time, latency=True,
 999             traffic_directions=self.traffic_directions
1000         )
1001         return self.get_measurement_result(duration, transmit_rate)
1002
1003
1004 class OptimizedSearch:
1005     """Class to be imported as Robot Library, containing search keywords.
1006
1007     Aside of setting up measurer and forwarding arguments,
1008     the main business is to translate min/max rate from unidir to aggregate.
1009     """
1010
1011     @staticmethod
1012     def perform_optimized_ndrpdr_search(
1013             frame_size, traffic_profile, minimum_transmit_rate,
1014             maximum_transmit_rate, packet_loss_ratio=0.005,
1015             final_relative_width=0.005, final_trial_duration=30.0,
1016             initial_trial_duration=1.0, number_of_intermediate_phases=2,
1017             timeout=720.0, doublings=1, traffic_directions=2):
1018         """Setup initialized TG, perform optimized search, return intervals.
1019
1020         :param frame_size: Frame size identifier or value [B].
1021         :param traffic_profile: Module name as a traffic profile identifier.
1022             See GPL/traffic_profiles/trex for implemented modules.
1023         :param minimum_transmit_rate: Minimal uni-directional
1024             target transmit rate [pps].
1025         :param maximum_transmit_rate: Maximal uni-directional
1026             target transmit rate [pps].
1027         :param packet_loss_ratio: Fraction of packets lost, for PDR [1].
1028         :param final_relative_width: Final lower bound transmit rate
1029             cannot be more distant that this multiple of upper bound [1].
1030         :param final_trial_duration: Trial duration for the final phase [s].
1031         :param initial_trial_duration: Trial duration for the initial phase
1032             and also for the first intermediate phase [s].
1033         :param number_of_intermediate_phases: Number of intermediate phases
1034             to perform before the final phase [1].
1035         :param timeout: The search will fail itself when not finished
1036             before this overall time [s].
1037         :param doublings: How many doublings to do in external search step.
1038             Default 1 is suitable for fairly stable tests,
1039             less stable tests might get better overal duration with 2 or more.
1040         :param traffic_directions: Traffic is bi- (2) or uni- (1) directional.
1041             Default: 2
1042         :type frame_size: str or int
1043         :type traffic_profile: str
1044         :type minimum_transmit_rate: float
1045         :type maximum_transmit_rate: float
1046         :type packet_loss_ratio: float
1047         :type final_relative_width: float
1048         :type final_trial_duration: float
1049         :type initial_trial_duration: float
1050         :type number_of_intermediate_phases: int
1051         :type timeout: float
1052         :type doublings: int
1053         :type traffic_directions: int
1054         :returns: Structure containing narrowed down NDR and PDR intervals
1055             and their measurements.
1056         :rtype: NdrPdrResult
1057         :raises RuntimeError: If total duration is larger than timeout.
1058         """
1059         minimum_transmit_rate *= traffic_directions
1060         maximum_transmit_rate *= traffic_directions
1061         # we need instance of TrafficGenerator instantiated by Robot Framework
1062         # to be able to use trex_stl-*()
1063         tg_instance = BuiltIn().get_library_instance(
1064             u"resources.libraries.python.TrafficGenerator"
1065         )
1066         tg_instance.set_rate_provider_defaults(
1067             frame_size, traffic_profile, traffic_directions=traffic_directions)
1068         algorithm = MultipleLossRatioSearch(
1069             measurer=tg_instance, final_trial_duration=final_trial_duration,
1070             final_relative_width=final_relative_width,
1071             number_of_intermediate_phases=number_of_intermediate_phases,
1072             initial_trial_duration=initial_trial_duration, timeout=timeout,
1073             doublings=doublings
1074         )
1075         result = algorithm.narrow_down_ndr_and_pdr(
1076             minimum_transmit_rate, maximum_transmit_rate, packet_loss_ratio
1077         )
1078         return result
1079
1080     @staticmethod
1081     def perform_soak_search(
1082             frame_size, traffic_profile, minimum_transmit_rate,
1083             maximum_transmit_rate, plr_target=1e-7, tdpt=0.1,
1084             initial_count=50, timeout=1800.0, trace_enabled=False,
1085             traffic_directions=2):
1086         """Setup initialized TG, perform soak search, return avg and stdev.
1087
1088         :param frame_size: Frame size identifier or value [B].
1089         :param traffic_profile: Module name as a traffic profile identifier.
1090             See GPL/traffic_profiles/trex for implemented modules.
1091         :param minimum_transmit_rate: Minimal uni-directional
1092             target transmit rate [pps].
1093         :param maximum_transmit_rate: Maximal uni-directional
1094             target transmit rate [pps].
1095         :param plr_target: Fraction of packets lost to achieve [1].
1096         :param tdpt: Trial duration per trial.
1097             The algorithm linearly increases trial duration with trial number,
1098             this is the increment between succesive trials, in seconds.
1099         :param initial_count: Offset to apply before the first trial.
1100             For example initial_count=50 makes first trial to be 51*tdpt long.
1101             This is needed because initial "search" phase of integrator
1102             takes significant time even without any trial results.
1103         :param timeout: The search will stop after this overall time [s].
1104         :param trace_enabled: True if trace enabled else False.
1105         :param traffic_directions: Traffic is bi- (2) or uni- (1) directional.
1106             Default: 2
1107         :type frame_size: str or int
1108         :type traffic_profile: str
1109         :type minimum_transmit_rate: float
1110         :type maximum_transmit_rate: float
1111         :type plr_target: float
1112         :type initial_count: int
1113         :type timeout: float
1114         :type trace_enabled: bool
1115         :type traffic_directions: int
1116         :returns: Average and stdev of estimated aggregate rate giving PLR.
1117         :rtype: 2-tuple of float
1118         """
1119         minimum_transmit_rate *= traffic_directions
1120         maximum_transmit_rate *= traffic_directions
1121         tg_instance = BuiltIn().get_library_instance(
1122             u"resources.libraries.python.TrafficGenerator"
1123         )
1124         tg_instance.set_rate_provider_defaults(
1125             frame_size, traffic_profile, traffic_directions=traffic_directions,
1126             negative_loss=False)
1127         algorithm = PLRsearch(
1128             measurer=tg_instance, trial_duration_per_trial=tdpt,
1129             packet_loss_ratio_target=plr_target,
1130             trial_number_offset=initial_count, timeout=timeout,
1131             trace_enabled=trace_enabled
1132         )
1133         result = algorithm.search(minimum_transmit_rate, maximum_transmit_rate)
1134         return result