resources/libraries/python/TrafficGenerator.py

   1 # Copyright (c) 2022 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Performance testing traffic generator library."""
  15
  16 import math
  17 import time
  18
  19 from robot.api import logger
  20 from robot.libraries.BuiltIn import BuiltIn
  21
  22 from .Constants import Constants
  23 from .CpuUtils import CpuUtils
  24 from .DropRateSearch import DropRateSearch
  25 from .MLRsearch.AbstractMeasurer import AbstractMeasurer
  26 from .MLRsearch.MultipleLossRatioSearch import MultipleLossRatioSearch
  27 from .MLRsearch.ReceiveRateMeasurement import ReceiveRateMeasurement
  28 from .PLRsearch.PLRsearch import PLRsearch
  29 from .OptionString import OptionString
  30 from .ssh import exec_cmd_no_error, exec_cmd
  31 from .topology import NodeType
  32 from .topology import NodeSubTypeTG
  33 from .topology import Topology
  34
  35 __all__ = [u"TGDropRateSearchImpl", u"TrafficGenerator", u"OptimizedSearch"]
  36
  37
  38 def check_subtype(node):
  39     """Return supported subtype of given node, or raise an exception.
  40
  41     Currently only one subtype is supported,
  42     but we want our code to be ready for other ones.
  43
  44     :param node: Topology node to check. Can be None.
  45     :type node: dict or NoneType
  46     :returns: Subtype detected.
  47     :rtype: NodeSubTypeTG
  48     :raises RuntimeError: If node is not supported, message explains how.
  49     """
  50     if node.get(u"type") is None:
  51         msg = u"Node type is not defined"
  52     elif node[u"type"] != NodeType.TG:
  53         msg = f"Node type is {node[u'type']!r}, not a TG"
  54     elif node.get(u"subtype") is None:
  55         msg = u"TG subtype is not defined"
  56     elif node[u"subtype"] != NodeSubTypeTG.TREX:
  57         msg = f"TG subtype {node[u'subtype']!r} is not supported"
  58     else:
  59         return NodeSubTypeTG.TREX
  60     raise RuntimeError(msg)
  61
  62
  63 class TGDropRateSearchImpl(DropRateSearch):
  64     """Drop Rate Search implementation."""
  65
  66     # def __init__(self):
  67     #     super(TGDropRateSearchImpl, self).__init__()
  68
  69     def measure_loss(
  70             self, rate, frame_size, loss_acceptance, loss_acceptance_type,
  71             traffic_profile):
  72         """Runs the traffic and evaluate the measured results.
  73
  74         :param rate: Offered traffic load.
  75         :param frame_size: Size of frame.
  76         :param loss_acceptance: Permitted drop ratio or frames count.
  77         :param loss_acceptance_type: Type of permitted loss.
  78         :param traffic_profile: Module name as a traffic profile identifier.
  79             See GPL/traffic_profiles/trex for implemented modules.
  80         :type rate: float
  81         :type frame_size: str
  82         :type loss_acceptance: float
  83         :type loss_acceptance_type: LossAcceptanceType
  84         :type traffic_profile: str
  85         :returns: Drop threshold exceeded? (True/False)
  86         :rtype: bool
  87         :raises NotImplementedError: If TG is not supported.
  88         :raises RuntimeError: If TG is not specified.
  89         """
  90         # we need instance of TrafficGenerator instantiated by Robot Framework
  91         # to be able to use trex_stl-*()
  92         tg_instance = BuiltIn().get_library_instance(
  93             u"resources.libraries.python.TrafficGenerator"
  94         )
  95         subtype = check_subtype(tg_instance.node)
  96         if subtype == NodeSubTypeTG.TREX:
  97             unit_rate = str(rate) + self.get_rate_type_str()
  98             tg_instance.trex_stl_start_remote_exec(
  99                 self.get_duration(), unit_rate, frame_size, traffic_profile
 100             )
 101             loss = tg_instance.get_loss()
 102             sent = tg_instance.get_sent()
 103             if self.loss_acceptance_type_is_percentage():
 104                 loss = (float(loss) / float(sent)) * 100
 105             logger.trace(
 106                 f"comparing: {loss} < {loss_acceptance} {loss_acceptance_type}"
 107             )
 108             return float(loss) <= float(loss_acceptance)
 109         return False
 110
 111     def get_latency(self):
 112         """Returns min/avg/max latency.
 113
 114         :returns: Latency stats.
 115         :rtype: list
 116         """
 117         tg_instance = BuiltIn().get_library_instance(
 118             u"resources.libraries.python.TrafficGenerator"
 119         )
 120         return tg_instance.get_latency_int()
 121
 122
 123 class TrexMode:
 124     """Defines mode of T-Rex traffic generator."""
 125     # Advanced stateful mode
 126     ASTF = u"ASTF"
 127     # Stateless mode
 128     STL = u"STL"
 129
 130
 131 # TODO: Pylint says too-many-instance-attributes.
 132 class TrafficGenerator(AbstractMeasurer):
 133     """Traffic Generator."""
 134
 135     # TODO: Remove "trex" from lines which could work with other TGs.
 136
 137     # Use one instance of TrafficGenerator for all tests in test suite
 138     ROBOT_LIBRARY_SCOPE = u"TEST SUITE"
 139
 140     def __init__(self):
 141         # TODO: Separate into few dataclasses/dicts.
 142         #       Pylint dislikes large unstructured state, and it is right.
 143         self._node = None
 144         self._mode = None
 145         # TG interface order mapping
 146         self._ifaces_reordered = False
 147         # Result holding fields, to be removed.
 148         self._result = None
 149         self._loss = None
 150         self._sent = None
 151         self._latency = None
 152         self._received = None
 153         self._approximated_rate = None
 154         self._approximated_duration = None
 155         self._l7_data = None
 156         # Measurement input fields, needed for async stop result.
 157         self._start_time = None
 158         self._stop_time = None
 159         self._rate = None
 160         self._target_duration = None
 161         self._duration = None
 162         # Other input parameters, not knowable from measure() signature.
 163         self.frame_size = None
 164         self.traffic_profile = None
 165         self.traffic_directions = None
 166         self.negative_loss = None
 167         self.use_latency = None
 168         self.ppta = None
 169         self.resetter = None
 170         self.transaction_scale = None
 171         self.transaction_duration = None
 172         self.sleep_till_duration = None
 173         self.transaction_type = None
 174         self.duration_limit = None
 175         self.ramp_up_start = None
 176         self.ramp_up_stop = None
 177         self.ramp_up_rate = None
 178         self.ramp_up_duration = None
 179         self.state_timeout = None
 180         # Transient data needed for async measurements.
 181         self._xstats = (None, None)
 182         # TODO: Rename "xstats" to something opaque, so T-Rex is not privileged?
 183
 184     @property
 185     def node(self):
 186         """Getter.
 187
 188         :returns: Traffic generator node.
 189         :rtype: dict
 190         """
 191         return self._node
 192
 193     def get_loss(self):
 194         """Return number of lost packets.
 195
 196         :returns: Number of lost packets.
 197         :rtype: str
 198         """
 199         return self._loss
 200
 201     def get_sent(self):
 202         """Return number of sent packets.
 203
 204         :returns: Number of sent packets.
 205         :rtype: str
 206         """
 207         return self._sent
 208
 209     def get_received(self):
 210         """Return number of received packets.
 211
 212         :returns: Number of received packets.
 213         :rtype: str
 214         """
 215         return self._received
 216
 217     def get_latency_int(self):
 218         """Return rounded min/avg/max latency.
 219
 220         :returns: Latency stats.
 221         :rtype: list
 222         """
 223         return self._latency
 224
 225     def get_approximated_rate(self):
 226         """Return approximated rate computed as ratio of transmitted packets
 227         over duration of trial.
 228
 229         :returns: Approximated rate.
 230         :rtype: str
 231         """
 232         return self._approximated_rate
 233
 234     def get_l7_data(self):
 235         """Return L7 data.
 236
 237         :returns: Number of received packets.
 238         :rtype: dict
 239         """
 240         return self._l7_data
 241
 242     def check_mode(self, expected_mode):
 243         """Check TG mode.
 244
 245         :param expected_mode: Expected traffic generator mode.
 246         :type expected_mode: object
 247         :raises RuntimeError: In case of unexpected TG mode.
 248         """
 249         if self._mode == expected_mode:
 250             return
 251         raise RuntimeError(
 252             f"{self._node[u'subtype']} not running in {expected_mode} mode!"
 253         )
 254
 255     # TODO: pylint says disable=too-many-locals.
 256     def initialize_traffic_generator(
 257             self, tg_node, tg_if1, tg_if2, tg_if1_adj_node, tg_if1_adj_if,
 258             tg_if2_adj_node, tg_if2_adj_if, osi_layer, tg_if1_dst_mac=None,
 259             tg_if2_dst_mac=None):
 260         """TG initialization.
 261
 262         TODO: Document why do we need (and how do we use) _ifaces_reordered.
 263
 264         :param tg_node: Traffic generator node.
 265         :param tg_if1: TG - name of first interface.
 266         :param tg_if2: TG - name of second interface.
 267         :param tg_if1_adj_node: TG if1 adjecent node.
 268         :param tg_if1_adj_if: TG if1 adjecent interface.
 269         :param tg_if2_adj_node: TG if2 adjecent node.
 270         :param tg_if2_adj_if: TG if2 adjecent interface.
 271         :param osi_layer: 'L2', 'L3' or 'L7' - OSI Layer testing type.
 272         :param tg_if1_dst_mac: Interface 1 destination MAC address.
 273         :param tg_if2_dst_mac: Interface 2 destination MAC address.
 274         :type tg_node: dict
 275         :type tg_if1: str
 276         :type tg_if2: str
 277         :type tg_if1_adj_node: dict
 278         :type tg_if1_adj_if: str
 279         :type tg_if2_adj_node: dict
 280         :type tg_if2_adj_if: str
 281         :type osi_layer: str
 282         :type tg_if1_dst_mac: str
 283         :type tg_if2_dst_mac: str
 284         :returns: nothing
 285         :raises RuntimeError: In case of issue during initialization.
 286         """
 287         subtype = check_subtype(tg_node)
 288         if subtype == NodeSubTypeTG.TREX:
 289             self._node = tg_node
 290             self._mode = TrexMode.ASTF if osi_layer == u"L7" else TrexMode.STL
 291             if1 = dict()
 292             if2 = dict()
 293             if1[u"pci"] = Topology().get_interface_pci_addr(self._node, tg_if1)
 294             if2[u"pci"] = Topology().get_interface_pci_addr(self._node, tg_if2)
 295             if1[u"addr"] = Topology().get_interface_mac(self._node, tg_if1)
 296             if2[u"addr"] = Topology().get_interface_mac(self._node, tg_if2)
 297
 298             if osi_layer == u"L2":
 299                 if1[u"adj_addr"] = if2[u"addr"]
 300                 if2[u"adj_addr"] = if1[u"addr"]
 301             elif osi_layer in (u"L3", u"L7"):
 302                 if1[u"adj_addr"] = Topology().get_interface_mac(
 303                     tg_if1_adj_node, tg_if1_adj_if
 304                 )
 305                 if2[u"adj_addr"] = Topology().get_interface_mac(
 306                     tg_if2_adj_node, tg_if2_adj_if
 307                 )
 308             else:
 309                 raise ValueError(u"Unknown OSI layer!")
 310
 311             # in case of switched environment we can override MAC addresses
 312             if tg_if1_dst_mac is not None and tg_if2_dst_mac is not None:
 313                 if1[u"adj_addr"] = tg_if1_dst_mac
 314                 if2[u"adj_addr"] = tg_if2_dst_mac
 315
 316             if min(if1[u"pci"], if2[u"pci"]) != if1[u"pci"]:
 317                 if1, if2 = if2, if1
 318                 self._ifaces_reordered = True
 319
 320             master_thread_id, latency_thread_id, socket, threads = \
 321                 CpuUtils.get_affinity_trex(
 322                     self._node, tg_if1, tg_if2,
 323                     tg_dtc=Constants.TREX_CORE_COUNT)
 324
 325             if osi_layer in (u"L2", u"L3", u"L7"):
 326                 exec_cmd_no_error(
 327                     self._node,
 328                     f"sh -c 'cat << EOF > /etc/trex_cfg.yaml\n"
 329                     f"- version: 2\n"
 330                     f"  c: {len(threads)}\n"
 331                     f"  limit_memory: {Constants.TREX_LIMIT_MEMORY}\n"
 332                     f"  interfaces: [\"{if1[u'pci']}\",\"{if2[u'pci']}\"]\n"
 333                     f"  port_info:\n"
 334                     f"      - dest_mac: \'{if1[u'adj_addr']}\'\n"
 335                     f"        src_mac: \'{if1[u'addr']}\'\n"
 336                     f"      - dest_mac: \'{if2[u'adj_addr']}\'\n"
 337                     f"        src_mac: \'{if2[u'addr']}\'\n"
 338                     f"  platform :\n"
 339                     f"      master_thread_id: {master_thread_id}\n"
 340                     f"      latency_thread_id: {latency_thread_id}\n"
 341                     f"      dual_if:\n"
 342                     f"          - socket: {socket}\n"
 343                     f"            threads: {threads}\n"
 344                     f"EOF'",
 345                     sudo=True, message=u"T-Rex config generation!"
 346                 )
 347
 348                 if Constants.TREX_RX_DESCRIPTORS_COUNT != 0:
 349                     exec_cmd_no_error(
 350                         self._node,
 351                         f"sh -c 'cat << EOF >> /etc/trex_cfg.yaml\n"
 352                         f"  rx_desc: {Constants.TREX_RX_DESCRIPTORS_COUNT}\n"
 353                         f"EOF'",
 354                         sudo=True, message=u"T-Rex rx_desc modification!"
 355                     )
 356
 357                 if Constants.TREX_TX_DESCRIPTORS_COUNT != 0:
 358                     exec_cmd_no_error(
 359                         self._node,
 360                         f"sh -c 'cat << EOF >> /etc/trex_cfg.yaml\n"
 361                         f"  tx_desc: {Constants.TREX_TX_DESCRIPTORS_COUNT}\n"
 362                         f"EOF'",
 363                         sudo=True, message=u"T-Rex tx_desc modification!"
 364                     )
 365             else:
 366                 raise ValueError(u"Unknown OSI layer!")
 367
 368             TrafficGenerator.startup_trex(
 369                 self._node, osi_layer, subtype=subtype
 370             )
 371
 372     @staticmethod
 373     def startup_trex(tg_node, osi_layer, subtype=None):
 374         """Startup sequence for the TRex traffic generator.
 375
 376         :param tg_node: Traffic generator node.
 377         :param osi_layer: 'L2', 'L3' or 'L7' - OSI Layer testing type.
 378         :param subtype: Traffic generator sub-type.
 379         :type tg_node: dict
 380         :type osi_layer: str
 381         :type subtype: NodeSubTypeTG
 382         :raises RuntimeError: If T-Rex startup failed.
 383         :raises ValueError: If OSI layer is not supported.
 384         """
 385         if not subtype:
 386             subtype = check_subtype(tg_node)
 387         if subtype == NodeSubTypeTG.TREX:
 388             for _ in range(0, 3):
 389                 # Kill TRex only if it is already running.
 390                 cmd = u"sh -c \"pgrep t-rex && pkill t-rex && sleep 3 || true\""
 391                 exec_cmd_no_error(
 392                     tg_node, cmd, sudo=True, message=u"Kill TRex failed!"
 393                 )
 394
 395                 # Configure TRex.
 396                 ports = ''
 397                 for port in tg_node[u"interfaces"].values():
 398                     if u'Mellanox' not in port.get(u'model'):
 399                         ports += f" {port.get(u'pci_address')}"
 400
 401                 cmd = f"sh -c \"cd {Constants.TREX_INSTALL_DIR}/scripts/ && " \
 402                     f"./dpdk_nic_bind.py -u {ports} || true\""
 403                 exec_cmd_no_error(
 404                     tg_node, cmd, sudo=True,
 405                     message=u"Unbind PCI ports from driver failed!"
 406                 )
 407
 408                 # Start TRex.
 409                 cd_cmd = f"cd '{Constants.TREX_INSTALL_DIR}/scripts/'"
 410                 trex_cmd = OptionString([u"nohup", u"./t-rex-64"])
 411                 trex_cmd.add(u"-i")
 412                 trex_cmd.add(u"--prefix $(hostname)")
 413                 trex_cmd.add(u"--hdrh")
 414                 trex_cmd.add(u"--no-scapy-server")
 415                 trex_cmd.add_if(u"--astf", osi_layer == u"L7")
 416                 # OptionString does not create double space if extra is empty.
 417                 trex_cmd.add(f"{Constants.TREX_EXTRA_CMDLINE}")
 418                 inner_command = f"{cd_cmd} && {trex_cmd} > /tmp/trex.log 2>&1 &"
 419                 cmd = f"sh -c \"{inner_command}\" > /dev/null"
 420                 try:
 421                     exec_cmd_no_error(tg_node, cmd, sudo=True)
 422                 except RuntimeError:
 423                     cmd = u"sh -c \"cat /tmp/trex.log\""
 424                     exec_cmd_no_error(
 425                         tg_node, cmd, sudo=True,
 426                         message=u"Get TRex logs failed!"
 427                     )
 428                     raise RuntimeError(u"Start TRex failed!")
 429
 430                 # Test T-Rex API responsiveness.
 431                 cmd = f"python3 {Constants.REMOTE_FW_DIR}/GPL/tools/trex/"
 432                 if osi_layer in (u"L2", u"L3"):
 433                     cmd += u"trex_stl_assert.py"
 434                 elif osi_layer == u"L7":
 435                     cmd += u"trex_astf_assert.py"
 436                 else:
 437                     raise ValueError(u"Unknown OSI layer!")
 438                 try:
 439                     exec_cmd_no_error(
 440                         tg_node, cmd, sudo=True,
 441                         message=u"T-Rex API is not responding!", retries=20
 442                     )
 443                 except RuntimeError:
 444                     continue
 445                 return
 446             # After max retries TRex is still not responding to API critical
 447             # error occurred.
 448             exec_cmd(tg_node, u"cat /tmp/trex.log", sudo=True)
 449             raise RuntimeError(u"Start T-Rex failed after multiple retries!")
 450
 451     @staticmethod
 452     def is_trex_running(node):
 453         """Check if T-Rex is running using pidof.
 454
 455         :param node: Traffic generator node.
 456         :type node: dict
 457         :returns: True if T-Rex is running otherwise False.
 458         :rtype: bool
 459         """
 460         ret, _, _ = exec_cmd(node, u"pgrep t-rex", sudo=True)
 461         return bool(int(ret) == 0)
 462
 463     @staticmethod
 464     def teardown_traffic_generator(node):
 465         """TG teardown.
 466
 467         :param node: Traffic generator node.
 468         :type node: dict
 469         :returns: nothing
 470         :raises RuntimeError: If node type is not a TG,
 471             or if T-Rex teardown fails.
 472         """
 473         subtype = check_subtype(node)
 474         if subtype == NodeSubTypeTG.TREX:
 475             exec_cmd_no_error(
 476                 node,
 477                 u"sh -c "
 478                 u"\"if pgrep t-rex; then sudo pkill t-rex && sleep 3; fi\"",
 479                 sudo=False,
 480                 message=u"T-Rex kill failed!"
 481             )
 482
 483     def trex_astf_stop_remote_exec(self, node):
 484         """Execute T-Rex ASTF script on remote node over ssh to stop running
 485         traffic.
 486
 487         Internal state is updated with measurement results.
 488
 489         :param node: T-Rex generator node.
 490         :type node: dict
 491         :raises RuntimeError: If stop traffic script fails.
 492         """
 493         command_line = OptionString().add(u"python3")
 494         dirname = f"{Constants.REMOTE_FW_DIR}/GPL/tools/trex"
 495         command_line.add(f"'{dirname}/trex_astf_stop.py'")
 496         command_line.change_prefix(u"--")
 497         for index, value in enumerate(self._xstats):
 498             if value is not None:
 499                 value = value.replace(u"'", u"\"")
 500                 command_line.add_equals(f"xstat{index}", f"'{value}'")
 501         stdout, _ = exec_cmd_no_error(
 502             node, command_line,
 503             message=u"T-Rex ASTF runtime error!"
 504         )
 505         self._parse_traffic_results(stdout)
 506
 507     def trex_stl_stop_remote_exec(self, node):
 508         """Execute T-Rex STL script on remote node over ssh to stop running
 509         traffic.
 510
 511         Internal state is updated with measurement results.
 512
 513         :param node: T-Rex generator node.
 514         :type node: dict
 515         :raises RuntimeError: If stop traffic script fails.
 516         """
 517         command_line = OptionString().add(u"python3")
 518         dirname = f"{Constants.REMOTE_FW_DIR}/GPL/tools/trex"
 519         command_line.add(f"'{dirname}/trex_stl_stop.py'")
 520         command_line.change_prefix(u"--")
 521         for index, value in enumerate(self._xstats):
 522             if value is not None:
 523                 value = value.replace(u"'", u"\"")
 524                 command_line.add_equals(f"xstat{index}", f"'{value}'")
 525         stdout, _ = exec_cmd_no_error(
 526             node, command_line,
 527             message=u"T-Rex STL runtime error!"
 528         )
 529         self._parse_traffic_results(stdout)
 530
 531     def stop_traffic_on_tg(self):
 532         """Stop all traffic on TG.
 533
 534         :returns: Structure containing the result of the measurement.
 535         :rtype: ReceiveRateMeasurement
 536         :raises ValueError: If TG traffic profile is not supported.
 537         """
 538         subtype = check_subtype(self._node)
 539         if subtype != NodeSubTypeTG.TREX:
 540             raise ValueError(f"Unsupported TG subtype: {subtype!r}")
 541         if u"trex-astf" in self.traffic_profile:
 542             self.trex_astf_stop_remote_exec(self._node)
 543         elif u"trex-stl" in self.traffic_profile:
 544             self.trex_stl_stop_remote_exec(self._node)
 545         else:
 546             raise ValueError(u"Unsupported T-Rex traffic profile!")
 547         self._stop_time = time.monotonic()
 548
 549         return self._get_measurement_result()
 550
 551     def _compute_duration(self, duration, multiplier):
 552         """Compute duration for profile driver.
 553
 554         The final result is influenced by transaction scale and duration limit.
 555         It is assumed a higher level function has already set those to self.
 556         The duration argument is the target value from search point of view,
 557         before the overrides are applied here.
 558
 559         Minus one (signalling async traffic start) is kept.
 560
 561         Completeness flag is also included. Duration limited or async trials
 562         are not considered complete for ramp-up purposes.
 563
 564         :param duration: Time expressed in seconds for how long to send traffic.
 565         :param multiplier: Traffic rate in transactions per second.
 566         :type duration: float
 567         :type multiplier: float
 568         :returns: New duration and whether it was a complete ramp-up candidate.
 569         :rtype: float, bool
 570         """
 571         if duration < 0.0:
 572             # Keep the async -1.
 573             return duration, False
 574         computed_duration = duration
 575         if self.transaction_scale:
 576             computed_duration = self.transaction_scale / multiplier
 577             # Log the computed duration,
 578             # so we can compare with what telemetry suggests
 579             # the real duration was.
 580             logger.debug(f"Expected duration {computed_duration}")
 581         if not self.duration_limit:
 582             return computed_duration, True
 583         limited_duration = min(computed_duration, self.duration_limit)
 584         return limited_duration, (limited_duration == computed_duration)
 585
 586     def trex_astf_start_remote_exec(
 587             self, duration, multiplier, async_call=False):
 588         """Execute T-Rex ASTF script on remote node over ssh to start running
 589         traffic.
 590
 591         In sync mode, measurement results are stored internally.
 592         In async mode, initial data including xstats are stored internally.
 593
 594         This method contains the logic to compute duration as maximum time
 595         if transaction_scale is nonzero.
 596         The transaction_scale argument defines (limits) how many transactions
 597         will be started in total. As that amount of transaction can take
 598         considerable time (sometimes due to explicit delays in the profile),
 599         the real time a trial needs to finish is computed here. For now,
 600         in that case the duration argument is ignored, assuming it comes
 601         from ASTF-unaware search algorithm. The overall time a single
 602         transaction needs is given in parameter transaction_duration,
 603         it includes both explicit delays and implicit time it takes
 604         to transfer data (or whatever the transaction does).
 605
 606         Currently it is observed TRex does not start the ASTF traffic
 607         immediately, an ad-hoc constant is added to the computed duration
 608         to compensate for that.
 609
 610         If transaction_scale is zero, duration is not recomputed.
 611         It is assumed the subsequent result parsing gets the real duration
 612         if the traffic stops sooner for any reason.
 613
 614         Currently, it is assumed traffic profile defines a single transaction.
 615         To avoid heavy logic here, the input rate is expected to be in
 616         transactions per second, as that directly translates to TRex multiplier,
 617         (assuming the profile does not override the default cps value of one).
 618
 619         :param duration: Time expressed in seconds for how long to send traffic.
 620         :param multiplier: Traffic rate in transactions per second.
 621         :param async_call: If enabled then don't wait for all incoming traffic.
 622         :type duration: float
 623         :type multiplier: int
 624         :type async_call: bool
 625         :raises RuntimeError: In case of T-Rex driver issue.
 626         """
 627         self.check_mode(TrexMode.ASTF)
 628         p_0, p_1 = (1, 0) if self._ifaces_reordered else (0, 1)
 629         if not isinstance(duration, (float, int)):
 630             duration = float(duration)
 631
 632         # TODO: Refactor the code so duration is computed only once,
 633         # and both the initial and the computed durations are logged.
 634         computed_duration, _ = self._compute_duration(duration, multiplier)
 635
 636         command_line = OptionString().add(u"python3")
 637         dirname = f"{Constants.REMOTE_FW_DIR}/GPL/tools/trex"
 638         command_line.add(f"'{dirname}/trex_astf_profile.py'")
 639         command_line.change_prefix(u"--")
 640         dirname = f"{Constants.REMOTE_FW_DIR}/GPL/traffic_profiles/trex"
 641         command_line.add_with_value(
 642             u"profile", f"'{dirname}/{self.traffic_profile}.py'"
 643         )
 644         command_line.add_with_value(u"duration", f"{computed_duration!r}")
 645         command_line.add_with_value(u"frame_size", self.frame_size)
 646         command_line.add_with_value(
 647             u"n_data_frames", Constants.ASTF_N_DATA_FRAMES
 648         )
 649         command_line.add_with_value(u"multiplier", multiplier)
 650         command_line.add_with_value(u"port_0", p_0)
 651         command_line.add_with_value(u"port_1", p_1)
 652         command_line.add_with_value(
 653             u"traffic_directions", self.traffic_directions
 654         )
 655         command_line.add_if(u"async_start", async_call)
 656         command_line.add_if(u"latency", self.use_latency)
 657         command_line.add_if(u"force", Constants.TREX_SEND_FORCE)
 658         command_line.add_with_value(
 659             u"delay", Constants.PERF_TRIAL_ASTF_DELAY
 660         )
 661
 662         self._start_time = time.monotonic()
 663         self._rate = multiplier
 664         stdout, _ = exec_cmd_no_error(
 665             self._node, command_line, timeout=computed_duration + 10.0,
 666             message=u"T-Rex ASTF runtime error!"
 667         )
 668
 669         if async_call:
 670             # no result
 671             self._target_duration = None
 672             self._duration = None
 673             self._received = None
 674             self._sent = None
 675             self._loss = None
 676             self._latency = None
 677             xstats = [None, None]
 678             self._l7_data = dict()
 679             self._l7_data[u"client"] = dict()
 680             self._l7_data[u"client"][u"active_flows"] = None
 681             self._l7_data[u"client"][u"established_flows"] = None
 682             self._l7_data[u"client"][u"traffic_duration"] = None
 683             self._l7_data[u"server"] = dict()
 684             self._l7_data[u"server"][u"active_flows"] = None
 685             self._l7_data[u"server"][u"established_flows"] = None
 686             self._l7_data[u"server"][u"traffic_duration"] = None
 687             if u"udp" in self.traffic_profile:
 688                 self._l7_data[u"client"][u"udp"] = dict()
 689                 self._l7_data[u"client"][u"udp"][u"connects"] = None
 690                 self._l7_data[u"client"][u"udp"][u"closed_flows"] = None
 691                 self._l7_data[u"client"][u"udp"][u"err_cwf"] = None
 692                 self._l7_data[u"server"][u"udp"] = dict()
 693                 self._l7_data[u"server"][u"udp"][u"accepted_flows"] = None
 694                 self._l7_data[u"server"][u"udp"][u"closed_flows"] = None
 695             elif u"tcp" in self.traffic_profile:
 696                 self._l7_data[u"client"][u"tcp"] = dict()
 697                 self._l7_data[u"client"][u"tcp"][u"initiated_flows"] = None
 698                 self._l7_data[u"client"][u"tcp"][u"connects"] = None
 699                 self._l7_data[u"client"][u"tcp"][u"closed_flows"] = None
 700                 self._l7_data[u"client"][u"tcp"][u"connattempt"] = None
 701                 self._l7_data[u"server"][u"tcp"] = dict()
 702                 self._l7_data[u"server"][u"tcp"][u"accepted_flows"] = None
 703                 self._l7_data[u"server"][u"tcp"][u"connects"] = None
 704                 self._l7_data[u"server"][u"tcp"][u"closed_flows"] = None
 705             else:
 706                 logger.warn(u"Unsupported T-Rex ASTF traffic profile!")
 707             index = 0
 708             for line in stdout.splitlines():
 709                 if f"Xstats snapshot {index}: " in line:
 710                     xstats[index] = line[19:]
 711                     index += 1
 712                 if index == 2:
 713                     break
 714             self._xstats = tuple(xstats)
 715         else:
 716             self._target_duration = duration
 717             self._duration = computed_duration
 718             self._parse_traffic_results(stdout)
 719
 720     def trex_stl_start_remote_exec(self, duration, rate, async_call=False):
 721         """Execute T-Rex STL script on remote node over ssh to start running
 722         traffic.
 723
 724         In sync mode, measurement results are stored internally.
 725         In async mode, initial data including xstats are stored internally.
 726
 727         Mode-unaware code (e.g. in search algorithms) works with transactions.
 728         To keep the logic simple, multiplier is set to that value.
 729         As bidirectional traffic profiles send packets in both directions,
 730         they are treated as transactions with two packets (one per direction).
 731
 732         :param duration: Time expressed in seconds for how long to send traffic.
 733         :param rate: Traffic rate in transactions per second.
 734         :param async_call: If enabled then don't wait for all incoming traffic.
 735         :type duration: float
 736         :type rate: str
 737         :type async_call: bool
 738         :raises RuntimeError: In case of T-Rex driver issue.
 739         """
 740         self.check_mode(TrexMode.STL)
 741         p_0, p_1 = (1, 0) if self._ifaces_reordered else (0, 1)
 742         if not isinstance(duration, (float, int)):
 743             duration = float(duration)
 744
 745         # TODO: Refactor the code so duration is computed only once,
 746         # and both the initial and the computed durations are logged.
 747         duration, _ = self._compute_duration(duration=duration, multiplier=rate)
 748
 749         command_line = OptionString().add(u"python3")
 750         dirname = f"{Constants.REMOTE_FW_DIR}/GPL/tools/trex"
 751         command_line.add(f"'{dirname}/trex_stl_profile.py'")
 752         command_line.change_prefix(u"--")
 753         dirname = f"{Constants.REMOTE_FW_DIR}/GPL/traffic_profiles/trex"
 754         command_line.add_with_value(
 755             u"profile", f"'{dirname}/{self.traffic_profile}.py'"
 756         )
 757         command_line.add_with_value(u"duration", f"{duration!r}")
 758         command_line.add_with_value(u"frame_size", self.frame_size)
 759         command_line.add_with_value(u"rate", f"{rate!r}")
 760         command_line.add_with_value(u"port_0", p_0)
 761         command_line.add_with_value(u"port_1", p_1)
 762         command_line.add_with_value(
 763             u"traffic_directions", self.traffic_directions
 764         )
 765         command_line.add_if(u"async_start", async_call)
 766         command_line.add_if(u"latency", self.use_latency)
 767         command_line.add_if(u"force", Constants.TREX_SEND_FORCE)
 768         command_line.add_with_value(u"delay", Constants.PERF_TRIAL_STL_DELAY)
 769
 770         # TODO: This is ugly. Handle parsing better.
 771         self._start_time = time.monotonic()
 772         self._rate = float(rate[:-3]) if u"pps" in rate else float(rate)
 773         stdout, _ = exec_cmd_no_error(
 774             self._node, command_line, timeout=int(duration) + 60,
 775             message=u"T-Rex STL runtime error"
 776         )
 777
 778         if async_call:
 779             # no result
 780             self._target_duration = None
 781             self._duration = None
 782             self._received = None
 783             self._sent = None
 784             self._loss = None
 785             self._latency = None
 786
 787             xstats = [None, None]
 788             index = 0
 789             for line in stdout.splitlines():
 790                 if f"Xstats snapshot {index}: " in line:
 791                     xstats[index] = line[19:]
 792                     index += 1
 793                 if index == 2:
 794                     break
 795             self._xstats = tuple(xstats)
 796         else:
 797             self._target_duration = duration
 798             self._duration = duration
 799             self._parse_traffic_results(stdout)
 800
 801     def send_traffic_on_tg(
 802             self,
 803             duration,
 804             rate,
 805             frame_size,
 806             traffic_profile,
 807             async_call=False,
 808             ppta=1,
 809             traffic_directions=2,
 810             transaction_duration=0.0,
 811             transaction_scale=0,
 812             transaction_type=u"packet",
 813             duration_limit=0.0,
 814             use_latency=False,
 815             ramp_up_rate=None,
 816             ramp_up_duration=None,
 817             state_timeout=240.0,
 818             ramp_up_only=False,
 819         ):
 820         """Send traffic from all configured interfaces on TG.
 821
 822         In async mode, xstats is stored internally,
 823         to enable getting correct result when stopping the traffic.
 824         In both modes, stdout is returned,
 825         but _parse_traffic_results only works in sync output.
 826
 827         Note that traffic generator uses DPDK driver which might
 828         reorder port numbers based on wiring and PCI numbering.
 829         This method handles that, so argument values are invariant,
 830         but you can see swapped valued in debug logs.
 831
 832         When transaction_scale is specified, the duration value is ignored
 833         and the needed time is computed. For cases where this results in
 834         to too long measurement (e.g. teardown trial with small rate),
 835         duration_limit is applied (of non-zero), so the trial is stopped sooner.
 836
 837         Bidirectional STL profiles are treated as transactions with two packets.
 838
 839         The return value is None for async.
 840
 841         :param duration: Duration of test traffic generation in seconds.
 842         :param rate: Traffic rate in transactions per second.
 843         :param frame_size: Frame size (L2) in Bytes.
 844         :param traffic_profile: Module name as a traffic profile identifier.
 845             See GPL/traffic_profiles/trex for implemented modules.
 846         :param async_call: Async mode.
 847         :param ppta: Packets per transaction, aggregated over directions.
 848             Needed for udp_pps which does not have a good transaction counter,
 849             so we need to compute expected number of packets.
 850             Default: 1.
 851         :param traffic_directions: Traffic is bi- (2) or uni- (1) directional.
 852             Default: 2
 853         :param transaction_duration: Total expected time to close transaction.
 854         :param transaction_scale: Number of transactions to perform.
 855             0 (default) means unlimited.
 856         :param transaction_type: An identifier specifying which counters
 857             and formulas to use when computing attempted and failed
 858             transactions. Default: "packet".
 859         :param duration_limit: Zero or maximum limit for computed (or given)
 860             duration.
 861         :param use_latency: Whether to measure latency during the trial.
 862             Default: False.
 863         :param ramp_up_rate: Rate to use in ramp-up trials [pps].
 864         :param ramp_up_duration: Duration of ramp-up trials [s].
 865         :param state_timeout: Time of life of DUT state [s].
 866         :param ramp_up_only: If true, do not perform main trial measurement.
 867         :type duration: float
 868         :type rate: float
 869         :type frame_size: str
 870         :type traffic_profile: str
 871         :type async_call: bool
 872         :type ppta: int
 873         :type traffic_directions: int
 874         :type transaction_duration: float
 875         :type transaction_scale: int
 876         :type transaction_type: str
 877         :type duration_limit: float
 878         :type use_latency: bool
 879         :type ramp_up_rate: float
 880         :type ramp_up_duration: float
 881         :type state_timeout: float
 882         :type ramp_up_only: bool
 883         :returns: TG results.
 884         :rtype: ReceiveRateMeasurement or None
 885         :raises ValueError: If TG traffic profile is not supported.
 886         """
 887         self.set_rate_provider_defaults(
 888             frame_size=frame_size,
 889             traffic_profile=traffic_profile,
 890             ppta=ppta,
 891             traffic_directions=traffic_directions,
 892             transaction_duration=transaction_duration,
 893             transaction_scale=transaction_scale,
 894             transaction_type=transaction_type,
 895             duration_limit=duration_limit,
 896             use_latency=use_latency,
 897             ramp_up_rate=ramp_up_rate,
 898             ramp_up_duration=ramp_up_duration,
 899             state_timeout=state_timeout,
 900         )
 901         return self._send_traffic_on_tg_with_ramp_up(
 902             duration=duration,
 903             rate=rate,
 904             async_call=async_call,
 905             ramp_up_only=ramp_up_only,
 906         )
 907
 908     def _send_traffic_on_tg_internal(
 909             self, duration, rate, async_call=False):
 910         """Send traffic from all configured interfaces on TG.
 911
 912         This is an internal function, it assumes set_rate_provider_defaults
 913         has been called to remember most values.
 914         The reason why need to remember various values is that
 915         the traffic can be asynchronous, and parsing needs those values.
 916         The reason why this is is a separate function from the one
 917         which calls set_rate_provider_defaults is that some search algorithms
 918         need to specify their own values, and we do not want the measure call
 919         to overwrite them with defaults.
 920
 921         This function is used both for automated ramp-up trials
 922         and for explicitly called trials.
 923
 924         :param duration: Duration of test traffic generation in seconds.
 925         :param rate: Traffic rate in transactions per second.
 926         :param async_call: Async mode.
 927         :type duration: float
 928         :type rate: float
 929         :type async_call: bool
 930         :returns: TG results.
 931         :rtype: ReceiveRateMeasurement or None
 932         :raises ValueError: If TG traffic profile is not supported.
 933         """
 934         subtype = check_subtype(self._node)
 935         if subtype == NodeSubTypeTG.TREX:
 936             if u"trex-astf" in self.traffic_profile:
 937                 self.trex_astf_start_remote_exec(
 938                     duration, float(rate), async_call
 939                 )
 940             elif u"trex-stl" in self.traffic_profile:
 941                 unit_rate_str = str(rate) + u"pps"
 942                 # TODO: Suport transaction_scale et al?
 943                 self.trex_stl_start_remote_exec(
 944                     duration, unit_rate_str, async_call
 945                 )
 946             else:
 947                 raise ValueError(u"Unsupported T-Rex traffic profile!")
 948
 949         return None if async_call else self._get_measurement_result()
 950
 951     def _send_traffic_on_tg_with_ramp_up(
 952             self, duration, rate, async_call=False, ramp_up_only=False):
 953         """Send traffic from all interfaces on TG, maybe after ramp-up.
 954
 955         This is an internal function, it assumes set_rate_provider_defaults
 956         has been called to remember most values.
 957         The reason why need to remember various values is that
 958         the traffic can be asynchronous, and parsing needs those values.
 959         The reason why this is a separate function from the one
 960         which calls set_rate_provider_defaults is that some search algorithms
 961         need to specify their own values, and we do not want the measure call
 962         to overwrite them with defaults.
 963
 964         If ramp-up tracking is detected, a computation is performed,
 965         and if state timeout is near, trial at ramp-up rate and duration
 966         is inserted before the main trial measurement.
 967
 968         The ramp_up_only parameter forces a ramp-up without immediate
 969         trial measurement, which is useful in case self remembers
 970         a previous ramp-up trial that belongs to a different test (phase).
 971
 972         Return None if trial is async or ramp-up only.
 973
 974         :param duration: Duration of test traffic generation in seconds.
 975         :param rate: Traffic rate in transactions per second.
 976         :param async_call: Async mode.
 977         :param ramp_up_only: If true, do not perform main trial measurement.
 978         :type duration: float
 979         :type rate: float
 980         :type async_call: bool
 981         :type ramp_up_only: bool
 982         :returns: TG results.
 983         :rtype: ReceiveRateMeasurement or None
 984         :raises ValueError: If TG traffic profile is not supported.
 985         """
 986         complete = False
 987         if self.ramp_up_rate:
 988             # Figure out whether we need to insert a ramp-up trial.
 989             # TODO: Give up on async_call=True?
 990             if ramp_up_only or self.ramp_up_start is None:
 991                 # We never ramped up yet (at least not in this test case).
 992                 ramp_up_needed = True
 993             else:
 994                 # We ramped up before, but maybe it was too long ago.
 995                 # Adding a constant overhead to be safe.
 996                 time_now = time.monotonic() + 1.0
 997                 computed_duration, complete = self._compute_duration(
 998                     duration=duration,
 999                     multiplier=rate,
1000                 )
1001                 # There are two conditions for inserting ramp-up.
1002                 # If early sessions are expiring already,
1003                 # or if late sessions are to expire before measurement is over.
1004                 ramp_up_start_delay = time_now - self.ramp_up_start
1005                 ramp_up_stop_delay = time_now - self.ramp_up_stop
1006                 ramp_up_stop_delay += computed_duration
1007                 bigger_delay = max(ramp_up_start_delay, ramp_up_stop_delay)
1008                 # Final boolean decision.
1009                 ramp_up_needed = (bigger_delay >= self.state_timeout)
1010             if ramp_up_needed:
1011                 logger.debug(
1012                     u"State may time out during next real trial, "
1013                     u"inserting a ramp-up trial."
1014                 )
1015                 self.ramp_up_start = time.monotonic()
1016                 self._send_traffic_on_tg_internal(
1017                     duration=self.ramp_up_duration,
1018                     rate=self.ramp_up_rate,
1019                     async_call=async_call,
1020                 )
1021                 self.ramp_up_stop = time.monotonic()
1022                 logger.debug(u"Ramp-up done.")
1023             else:
1024                 logger.debug(
1025                     u"State will probably not time out during next real trial, "
1026                     u"no ramp-up trial needed just yet."
1027                 )
1028         if ramp_up_only:
1029             return None
1030         trial_start = time.monotonic()
1031         result = self._send_traffic_on_tg_internal(
1032             duration=duration,
1033             rate=rate,
1034             async_call=async_call,
1035         )
1036         trial_end = time.monotonic()
1037         if self.ramp_up_rate:
1038             # Optimization: No loss acts as a good ramp-up, if it was complete.
1039             if complete and result is not None and result.loss_count == 0:
1040                 logger.debug(u"Good trial acts as a ramp-up")
1041                 self.ramp_up_start = trial_start
1042                 self.ramp_up_stop = trial_end
1043             else:
1044                 logger.debug(u"Loss or incomplete, does not act as a ramp-up.")
1045         return result
1046
1047     def no_traffic_loss_occurred(self):
1048         """Fail if loss occurred in traffic run.
1049
1050         :returns: nothing
1051         :raises Exception: If loss occured.
1052         """
1053         if self._loss is None:
1054             raise RuntimeError(u"The traffic generation has not been issued")
1055         if self._loss != u"0":
1056             raise RuntimeError(f"Traffic loss occurred: {self._loss}")
1057
1058     def fail_if_no_traffic_forwarded(self):
1059         """Fail if no traffic forwarded.
1060
1061         TODO: Check number of passed transactions instead.
1062
1063         :returns: nothing
1064         :raises Exception: If no traffic forwarded.
1065         """
1066         if self._received is None:
1067             raise RuntimeError(u"The traffic generation has not been issued")
1068         if self._received == 0:
1069             raise RuntimeError(u"No traffic forwarded")
1070
1071     def partial_traffic_loss_accepted(
1072             self, loss_acceptance, loss_acceptance_type):
1073         """Fail if loss is higher then accepted in traffic run.
1074
1075         :param loss_acceptance: Permitted drop ratio or frames count.
1076         :param loss_acceptance_type: Type of permitted loss.
1077         :type loss_acceptance: float
1078         :type loss_acceptance_type: LossAcceptanceType
1079         :returns: nothing
1080         :raises Exception: If loss is above acceptance criteria.
1081         """
1082         if self._loss is None:
1083             raise Exception(u"The traffic generation has not been issued")
1084
1085         if loss_acceptance_type == u"percentage":
1086             loss = (float(self._loss) / float(self._sent)) * 100
1087         elif loss_acceptance_type == u"frames":
1088             loss = float(self._loss)
1089         else:
1090             raise Exception(u"Loss acceptance type not supported")
1091
1092         if loss > float(loss_acceptance):
1093             raise Exception(
1094                 f"Traffic loss {loss} above loss acceptance: {loss_acceptance}"
1095             )
1096
1097     def _parse_traffic_results(self, stdout):
1098         """Parse stdout of scripts into fields of self.
1099
1100         Block of code to reuse, by sync start, or stop after async.
1101
1102         :param stdout: Text containing the standard output.
1103         :type stdout: str
1104         """
1105         subtype = check_subtype(self._node)
1106         if subtype == NodeSubTypeTG.TREX:
1107             # Last line from console output
1108             line = stdout.splitlines()[-1]
1109             results = line.split(u";")
1110             if results[-1] in (u" ", u""):
1111                 results.pop(-1)
1112             self._result = dict()
1113             for result in results:
1114                 key, value = result.split(u"=", maxsplit=1)
1115                 self._result[key.strip()] = value
1116             logger.info(f"TrafficGen results:\n{self._result}")
1117             self._received = int(self._result.get(u"total_received"), 0)
1118             self._sent = int(self._result.get(u"total_sent", 0))
1119             self._loss = int(self._result.get(u"frame_loss", 0))
1120             self._approximated_duration = \
1121                 self._result.get(u"approximated_duration", 0.0)
1122             if u"manual" not in str(self._approximated_duration):
1123                 self._approximated_duration = float(self._approximated_duration)
1124             self._latency = list()
1125             self._latency.append(self._result.get(u"latency_stream_0(usec)"))
1126             self._latency.append(self._result.get(u"latency_stream_1(usec)"))
1127             if self._mode == TrexMode.ASTF:
1128                 self._l7_data = dict()
1129                 self._l7_data[u"client"] = dict()
1130                 self._l7_data[u"client"][u"sent"] = \
1131                     int(self._result.get(u"client_sent", 0))
1132                 self._l7_data[u"client"][u"received"] = \
1133                     int(self._result.get(u"client_received", 0))
1134                 self._l7_data[u"client"][u"active_flows"] = \
1135                     int(self._result.get(u"client_active_flows", 0))
1136                 self._l7_data[u"client"][u"established_flows"] = \
1137                     int(self._result.get(u"client_established_flows", 0))
1138                 self._l7_data[u"client"][u"traffic_duration"] = \
1139                     float(self._result.get(u"client_traffic_duration", 0.0))
1140                 self._l7_data[u"client"][u"err_rx_throttled"] = \
1141                     int(self._result.get(u"client_err_rx_throttled", 0))
1142                 self._l7_data[u"client"][u"err_c_nf_throttled"] = \
1143                     int(self._result.get(u"client_err_nf_throttled", 0))
1144                 self._l7_data[u"client"][u"err_flow_overflow"] = \
1145                     int(self._result.get(u"client_err_flow_overflow", 0))
1146                 self._l7_data[u"server"] = dict()
1147                 self._l7_data[u"server"][u"active_flows"] = \
1148                     int(self._result.get(u"server_active_flows", 0))
1149                 self._l7_data[u"server"][u"established_flows"] = \
1150                     int(self._result.get(u"server_established_flows", 0))
1151                 self._l7_data[u"server"][u"traffic_duration"] = \
1152                     float(self._result.get(u"server_traffic_duration", 0.0))
1153                 self._l7_data[u"server"][u"err_rx_throttled"] = \
1154                     int(self._result.get(u"client_err_rx_throttled", 0))
1155                 if u"udp" in self.traffic_profile:
1156                     self._l7_data[u"client"][u"udp"] = dict()
1157                     self._l7_data[u"client"][u"udp"][u"connects"] = \
1158                         int(self._result.get(u"client_udp_connects", 0))
1159                     self._l7_data[u"client"][u"udp"][u"closed_flows"] = \
1160                         int(self._result.get(u"client_udp_closed", 0))
1161                     self._l7_data[u"client"][u"udp"][u"tx_bytes"] = \
1162                         int(self._result.get(u"client_udp_tx_bytes", 0))
1163                     self._l7_data[u"client"][u"udp"][u"rx_bytes"] = \
1164                         int(self._result.get(u"client_udp_rx_bytes", 0))
1165                     self._l7_data[u"client"][u"udp"][u"tx_packets"] = \
1166                         int(self._result.get(u"client_udp_tx_packets", 0))
1167                     self._l7_data[u"client"][u"udp"][u"rx_packets"] = \
1168                         int(self._result.get(u"client_udp_rx_packets", 0))
1169                     self._l7_data[u"client"][u"udp"][u"keep_drops"] = \
1170                         int(self._result.get(u"client_udp_keep_drops", 0))
1171                     self._l7_data[u"client"][u"udp"][u"err_cwf"] = \
1172                         int(self._result.get(u"client_err_cwf", 0))
1173                     self._l7_data[u"server"][u"udp"] = dict()
1174                     self._l7_data[u"server"][u"udp"][u"accepted_flows"] = \
1175                         int(self._result.get(u"server_udp_accepts", 0))
1176                     self._l7_data[u"server"][u"udp"][u"closed_flows"] = \
1177                         int(self._result.get(u"server_udp_closed", 0))
1178                     self._l7_data[u"server"][u"udp"][u"tx_bytes"] = \
1179                         int(self._result.get(u"server_udp_tx_bytes", 0))
1180                     self._l7_data[u"server"][u"udp"][u"rx_bytes"] = \
1181                         int(self._result.get(u"server_udp_rx_bytes", 0))
1182                     self._l7_data[u"server"][u"udp"][u"tx_packets"] = \
1183                         int(self._result.get(u"server_udp_tx_packets", 0))
1184                     self._l7_data[u"server"][u"udp"][u"rx_packets"] = \
1185                         int(self._result.get(u"server_udp_rx_packets", 0))
1186                 elif u"tcp" in self.traffic_profile:
1187                     self._l7_data[u"client"][u"tcp"] = dict()
1188                     self._l7_data[u"client"][u"tcp"][u"initiated_flows"] = \
1189                         int(self._result.get(u"client_tcp_connect_inits", 0))
1190                     self._l7_data[u"client"][u"tcp"][u"connects"] = \
1191                         int(self._result.get(u"client_tcp_connects", 0))
1192                     self._l7_data[u"client"][u"tcp"][u"closed_flows"] = \
1193                         int(self._result.get(u"client_tcp_closed", 0))
1194                     self._l7_data[u"client"][u"tcp"][u"connattempt"] = \
1195                         int(self._result.get(u"client_tcp_connattempt", 0))
1196                     self._l7_data[u"client"][u"tcp"][u"tx_bytes"] = \
1197                         int(self._result.get(u"client_tcp_tx_bytes", 0))
1198                     self._l7_data[u"client"][u"tcp"][u"rx_bytes"] = \
1199                         int(self._result.get(u"client_tcp_rx_bytes", 0))
1200                     self._l7_data[u"server"][u"tcp"] = dict()
1201                     self._l7_data[u"server"][u"tcp"][u"accepted_flows"] = \
1202                         int(self._result.get(u"server_tcp_accepts", 0))
1203                     self._l7_data[u"server"][u"tcp"][u"connects"] = \
1204                         int(self._result.get(u"server_tcp_connects", 0))
1205                     self._l7_data[u"server"][u"tcp"][u"closed_flows"] = \
1206                         int(self._result.get(u"server_tcp_closed", 0))
1207                     self._l7_data[u"server"][u"tcp"][u"tx_bytes"] = \
1208                         int(self._result.get(u"server_tcp_tx_bytes", 0))
1209                     self._l7_data[u"server"][u"tcp"][u"rx_bytes"] = \
1210                         int(self._result.get(u"server_tcp_rx_bytes", 0))
1211
1212     def _get_measurement_result(self):
1213         """Return the result of last measurement as ReceiveRateMeasurement.
1214
1215         Separate function, as measurements can end either by time
1216         or by explicit call, this is the common block at the end.
1217
1218         The target_tr field of ReceiveRateMeasurement is in
1219         transactions per second. Transmit count and loss count units
1220         depend on the transaction type. Usually they are in transactions
1221         per second, or aggregated packets per second.
1222
1223         TODO: Fail on running or already reported measurement.
1224
1225         :returns: Structure containing the result of the measurement.
1226         :rtype: ReceiveRateMeasurement
1227         """
1228         try:
1229             # Client duration seems to include a setup period
1230             # where TRex does not send any packets yet.
1231             # Server duration does not include it.
1232             server_data = self._l7_data[u"server"]
1233             approximated_duration = float(server_data[u"traffic_duration"])
1234         except (KeyError, AttributeError, ValueError, TypeError):
1235             approximated_duration = None
1236         try:
1237             if not approximated_duration:
1238                 approximated_duration = float(self._approximated_duration)
1239         except ValueError:  # "manual"
1240             approximated_duration = None
1241         if not approximated_duration:
1242             if self._duration and self._duration > 0:
1243                 # Known recomputed or target duration.
1244                 approximated_duration = self._duration
1245             else:
1246                 # It was an explicit stop.
1247                 if not self._stop_time:
1248                     raise RuntimeError(u"Unable to determine duration.")
1249                 approximated_duration = self._stop_time - self._start_time
1250         target_duration = self._target_duration
1251         if not target_duration:
1252             target_duration = approximated_duration
1253         transmit_rate = self._rate
1254         unsent = 0
1255         if self.transaction_type == u"packet":
1256             partial_attempt_count = self._sent
1257             packet_rate = transmit_rate * self.ppta
1258             # We have a float. TRex way of rounding it is not obvious.
1259             # The biggest source of mismatch is Inter Stream Gap.
1260             # So the code tolerates 10 usec of missing packets.
1261             expected_attempt_count = (target_duration - 1e-5) * packet_rate
1262             expected_attempt_count = math.ceil(expected_attempt_count)
1263             # TRex can send more.
1264             expected_attempt_count = max(expected_attempt_count, self._sent)
1265             unsent = expected_attempt_count - self._sent
1266             pass_count = self._received
1267             fail_count = expected_attempt_count - pass_count
1268         elif self.transaction_type == u"udp_cps":
1269             if not self.transaction_scale:
1270                 raise RuntimeError(u"Add support for no-limit udp_cps.")
1271             partial_attempt_count = self._l7_data[u"client"][u"sent"]
1272             # We do not care whether TG is slow, it should have attempted all.
1273             expected_attempt_count = self.transaction_scale
1274             unsent = expected_attempt_count - partial_attempt_count
1275             pass_count = self._l7_data[u"client"][u"received"]
1276             fail_count = expected_attempt_count - pass_count
1277         elif self.transaction_type == u"tcp_cps":
1278             if not self.transaction_scale:
1279                 raise RuntimeError(u"Add support for no-limit tcp_cps.")
1280             ctca = self._l7_data[u"client"][u"tcp"][u"connattempt"]
1281             partial_attempt_count = ctca
1282             # We do not care whether TG is slow, it should have attempted all.
1283             expected_attempt_count = self.transaction_scale
1284             unsent = expected_attempt_count - partial_attempt_count
1285             # From TCP point of view, server/connects counts full connections,
1286             # but we are testing NAT session so client/connects counts that
1287             # (half connections from TCP point of view).
1288             pass_count = self._l7_data[u"client"][u"tcp"][u"connects"]
1289             fail_count = expected_attempt_count - pass_count
1290         elif self.transaction_type == u"udp_pps":
1291             if not self.transaction_scale:
1292                 raise RuntimeError(u"Add support for no-limit udp_pps.")
1293             partial_attempt_count = self._sent
1294             expected_attempt_count = self.transaction_scale * self.ppta
1295             unsent = expected_attempt_count - self._sent
1296             fail_count = self._loss + unsent
1297         elif self.transaction_type == u"tcp_pps":
1298             if not self.transaction_scale:
1299                 raise RuntimeError(u"Add support for no-limit tcp_pps.")
1300             partial_attempt_count = self._sent
1301             expected_attempt_count = self.transaction_scale * self.ppta
1302             # One loss-like scenario happens when TRex receives all packets
1303             # on L2 level, but is not fast enough to process them all
1304             # at L7 level, which leads to retransmissions.
1305             # Those manifest as opackets larger than expected.
1306             # A simple workaround is to add absolute difference.
1307             # Probability of retransmissions exactly cancelling
1308             # packets unsent due to duration stretching is quite low.
1309             unsent = abs(expected_attempt_count - self._sent)
1310             fail_count = self._loss + unsent
1311         else:
1312             raise RuntimeError(f"Unknown parsing {self.transaction_type!r}")
1313         if unsent and isinstance(self._approximated_duration, float):
1314             # Do not report unsent for "manual".
1315             logger.debug(f"Unsent packets/transactions: {unsent}")
1316         if fail_count < 0 and not self.negative_loss:
1317             fail_count = 0
1318         measurement = ReceiveRateMeasurement(
1319             duration=target_duration,
1320             target_tr=transmit_rate,
1321             transmit_count=expected_attempt_count,
1322             loss_count=fail_count,
1323             approximated_duration=approximated_duration,
1324             partial_transmit_count=partial_attempt_count,
1325         )
1326         measurement.latency = self.get_latency_int()
1327         return measurement
1328
1329     def measure(self, duration, transmit_rate):
1330         """Run trial measurement, parse and return results.
1331
1332         The input rate is for transactions. Stateles bidirectional traffic
1333         is understood as sequence of (asynchronous) transactions,
1334         two packets each.
1335
1336         The result units depend on test type, generally
1337         the count either transactions or packets (aggregated over directions).
1338
1339         Optionally, this method sleeps if measurement finished before
1340         the time specified as duration.
1341
1342         :param duration: Trial duration [s].
1343         :param transmit_rate: Target rate in transactions per second.
1344         :type duration: float
1345         :type transmit_rate: float
1346         :returns: Structure containing the result of the measurement.
1347         :rtype: ReceiveRateMeasurement
1348         :raises RuntimeError: If TG is not set or if node is not TG
1349             or if subtype is not specified.
1350         :raises NotImplementedError: If TG is not supported.
1351         """
1352         duration = float(duration)
1353         time_start = time.monotonic()
1354         time_stop = time_start + duration
1355         if self.resetter:
1356             self.resetter()
1357         result = self._send_traffic_on_tg_with_ramp_up(
1358             duration=duration,
1359             rate=transmit_rate,
1360             async_call=False,
1361         )
1362         logger.debug(f"trial measurement result: {result!r}")
1363         # In PLRsearch, computation needs the specified time to complete.
1364         if self.sleep_till_duration:
1365             sleeptime = time_stop - time.monotonic()
1366             if sleeptime > 0.0:
1367                 # TODO: Sometimes we have time to do additional trials here,
1368                 # adapt PLRsearch to accept all the results.
1369                 time.sleep(sleeptime)
1370         return result
1371
1372     def set_rate_provider_defaults(
1373             self,
1374             frame_size,
1375             traffic_profile,
1376             ppta=1,
1377             resetter=None,
1378             traffic_directions=2,
1379             transaction_duration=0.0,
1380             transaction_scale=0,
1381             transaction_type=u"packet",
1382             duration_limit=0.0,
1383             negative_loss=True,
1384             sleep_till_duration=False,
1385             use_latency=False,
1386             ramp_up_rate=None,
1387             ramp_up_duration=None,
1388             state_timeout=240.0,
1389         ):
1390         """Store values accessed by measure().
1391
1392         :param frame_size: Frame size identifier or value [B].
1393         :param traffic_profile: Module name as a traffic profile identifier.
1394             See GPL/traffic_profiles/trex for implemented modules.
1395         :param ppta: Packets per transaction, aggregated over directions.
1396             Needed for udp_pps which does not have a good transaction counter,
1397             so we need to compute expected number of packets.
1398             Default: 1.
1399         :param resetter: Callable to reset DUT state for repeated trials.
1400         :param traffic_directions: Traffic from packet counting point of view
1401             is bi- (2) or uni- (1) directional.
1402             Default: 2
1403         :param transaction_duration: Total expected time to close transaction.
1404         :param transaction_scale: Number of transactions to perform.
1405             0 (default) means unlimited.
1406         :param transaction_type: An identifier specifying which counters
1407             and formulas to use when computing attempted and failed
1408             transactions. Default: "packet".
1409             TODO: Does this also specify parsing for the measured duration?
1410         :param duration_limit: Zero or maximum limit for computed (or given)
1411             duration.
1412         :param negative_loss: If false, negative loss is reported as zero loss.
1413         :param sleep_till_duration: If true and measurement returned faster,
1414             sleep until it matches duration. Needed for PLRsearch.
1415         :param use_latency: Whether to measure latency during the trial.
1416             Default: False.
1417         :param ramp_up_rate: Rate to use in ramp-up trials [pps].
1418         :param ramp_up_duration: Duration of ramp-up trials [s].
1419         :param state_timeout: Time of life of DUT state [s].
1420         :type frame_size: str or int
1421         :type traffic_profile: str
1422         :type ppta: int
1423         :type resetter: Optional[Callable[[], None]]
1424         :type traffic_directions: int
1425         :type transaction_duration: float
1426         :type transaction_scale: int
1427         :type transaction_type: str
1428         :type duration_limit: float
1429         :type negative_loss: bool
1430         :type sleep_till_duration: bool
1431         :type use_latency: bool
1432         :type ramp_up_rate: float
1433         :type ramp_up_duration: float
1434         :type state_timeout: float
1435         """
1436         self.frame_size = frame_size
1437         self.traffic_profile = str(traffic_profile)
1438         self.resetter = resetter
1439         self.ppta = ppta
1440         self.traffic_directions = int(traffic_directions)
1441         self.transaction_duration = float(transaction_duration)
1442         self.transaction_scale = int(transaction_scale)
1443         self.transaction_type = str(transaction_type)
1444         self.duration_limit = float(duration_limit)
1445         self.negative_loss = bool(negative_loss)
1446         self.sleep_till_duration = bool(sleep_till_duration)
1447         self.use_latency = bool(use_latency)
1448         self.ramp_up_rate = float(ramp_up_rate)
1449         self.ramp_up_duration = float(ramp_up_duration)
1450         self.state_timeout = float(state_timeout)
1451
1452
1453 class OptimizedSearch:
1454     """Class to be imported as Robot Library, containing search keywords.
1455
1456     Aside of setting up measurer and forwarding arguments,
1457     the main business is to translate min/max rate from unidir to aggregated.
1458     """
1459
1460     @staticmethod
1461     def perform_optimized_ndrpdr_search(
1462             frame_size,
1463             traffic_profile,
1464             minimum_transmit_rate,
1465             maximum_transmit_rate,
1466             packet_loss_ratio=0.005,
1467             final_relative_width=0.005,
1468             final_trial_duration=30.0,
1469             initial_trial_duration=1.0,
1470             number_of_intermediate_phases=2,
1471             timeout=1200.0,
1472             ppta=1,
1473             resetter=None,
1474             traffic_directions=2,
1475             transaction_duration=0.0,
1476             transaction_scale=0,
1477             transaction_type=u"packet",
1478             use_latency=False,
1479             ramp_up_rate=None,
1480             ramp_up_duration=None,
1481             state_timeout=240.0,
1482             expansion_coefficient=4.0,
1483     ):
1484         """Setup initialized TG, perform optimized search, return intervals.
1485
1486         If transaction_scale is nonzero, all init and non-init trial durations
1487         are set to 1.0 (as they do not affect the real trial duration)
1488         and zero intermediate phases are used.
1489         This way no re-measurement happens.
1490         Warmup has to be handled via resetter or ramp-up mechanisms.
1491
1492         :param frame_size: Frame size identifier or value [B].
1493         :param traffic_profile: Module name as a traffic profile identifier.
1494             See GPL/traffic_profiles/trex for implemented modules.
1495         :param minimum_transmit_rate: Minimal load in transactions per second.
1496         :param maximum_transmit_rate: Maximal load in transactions per second.
1497         :param packet_loss_ratio: Ratio of packets lost, for PDR [1].
1498         :param final_relative_width: Final lower bound transmit rate
1499             cannot be more distant that this multiple of upper bound [1].
1500         :param final_trial_duration: Trial duration for the final phase [s].
1501         :param initial_trial_duration: Trial duration for the initial phase
1502             and also for the first intermediate phase [s].
1503         :param number_of_intermediate_phases: Number of intermediate phases
1504             to perform before the final phase [1].
1505         :param timeout: The search will fail itself when not finished
1506             before this overall time [s].
1507         :param ppta: Packets per transaction, aggregated over directions.
1508             Needed for udp_pps which does not have a good transaction counter,
1509             so we need to compute expected number of packets.
1510             Default: 1.
1511         :param resetter: Callable to reset DUT state for repeated trials.
1512         :param traffic_directions: Traffic is bi- (2) or uni- (1) directional.
1513             Default: 2
1514         :param transaction_duration: Total expected time to close transaction.
1515         :param transaction_scale: Number of transactions to perform.
1516             0 (default) means unlimited.
1517         :param transaction_type: An identifier specifying which counters
1518             and formulas to use when computing attempted and failed
1519             transactions. Default: "packet".
1520         :param use_latency: Whether to measure latency during the trial.
1521             Default: False.
1522         :param ramp_up_rate: Rate to use in ramp-up trials [pps].
1523         :param ramp_up_duration: Duration of ramp-up trials [s].
1524         :param state_timeout: Time of life of DUT state [s].
1525         :param expansion_coefficient: In external search multiply width by this.
1526         :type frame_size: str or int
1527         :type traffic_profile: str
1528         :type minimum_transmit_rate: float
1529         :type maximum_transmit_rate: float
1530         :type packet_loss_ratio: float
1531         :type final_relative_width: float
1532         :type final_trial_duration: float
1533         :type initial_trial_duration: float
1534         :type number_of_intermediate_phases: int
1535         :type timeout: float
1536         :type ppta: int
1537         :type resetter: Optional[Callable[[], None]]
1538         :type traffic_directions: int
1539         :type transaction_duration: float
1540         :type transaction_scale: int
1541         :type transaction_type: str
1542         :type use_latency: bool
1543         :type ramp_up_rate: float
1544         :type ramp_up_duration: float
1545         :type state_timeout: float
1546         :type expansion_coefficient: float
1547         :returns: Structure containing narrowed down NDR and PDR intervals
1548             and their measurements.
1549         :rtype: List[Receiverateinterval]
1550         :raises RuntimeError: If total duration is larger than timeout.
1551         """
1552         # we need instance of TrafficGenerator instantiated by Robot Framework
1553         # to be able to use trex_stl-*()
1554         tg_instance = BuiltIn().get_library_instance(
1555             u"resources.libraries.python.TrafficGenerator"
1556         )
1557         # Overrides for fixed transaction amount.
1558         # TODO: Move to robot code? We have two call sites, so this saves space,
1559         #       even though this is surprising for log readers.
1560         if transaction_scale:
1561             initial_trial_duration = 1.0
1562             final_trial_duration = 1.0
1563             number_of_intermediate_phases = 0
1564             timeout += transaction_scale * 3e-4
1565         tg_instance.set_rate_provider_defaults(
1566             frame_size=frame_size,
1567             traffic_profile=traffic_profile,
1568             sleep_till_duration=False,
1569             ppta=ppta,
1570             resetter=resetter,
1571             traffic_directions=traffic_directions,
1572             transaction_duration=transaction_duration,
1573             transaction_scale=transaction_scale,
1574             transaction_type=transaction_type,
1575             use_latency=use_latency,
1576             ramp_up_rate=ramp_up_rate,
1577             ramp_up_duration=ramp_up_duration,
1578             state_timeout=state_timeout,
1579         )
1580         algorithm = MultipleLossRatioSearch(
1581             measurer=tg_instance,
1582             final_trial_duration=final_trial_duration,
1583             final_relative_width=final_relative_width,
1584             number_of_intermediate_phases=number_of_intermediate_phases,
1585             initial_trial_duration=initial_trial_duration,
1586             timeout=timeout,
1587             debug=logger.debug,
1588             expansion_coefficient=expansion_coefficient,
1589         )
1590         if packet_loss_ratio:
1591             packet_loss_ratios = [0.0, packet_loss_ratio]
1592         else:
1593             # Happens in reconf tests.
1594             packet_loss_ratios = [packet_loss_ratio]
1595         results = algorithm.narrow_down_intervals(
1596             min_rate=minimum_transmit_rate,
1597             max_rate=maximum_transmit_rate,
1598             packet_loss_ratios=packet_loss_ratios,
1599         )
1600         return results
1601
1602     @staticmethod
1603     def perform_soak_search(
1604             frame_size,
1605             traffic_profile,
1606             minimum_transmit_rate,
1607             maximum_transmit_rate,
1608             plr_target=1e-7,
1609             tdpt=0.1,
1610             initial_count=50,
1611             timeout=7200.0,
1612             ppta=1,
1613             resetter=None,
1614             trace_enabled=False,
1615             traffic_directions=2,
1616             transaction_duration=0.0,
1617             transaction_scale=0,
1618             transaction_type=u"packet",
1619             use_latency=False,
1620             ramp_up_rate=None,
1621             ramp_up_duration=None,
1622             state_timeout=240.0,
1623     ):
1624         """Setup initialized TG, perform soak search, return avg and stdev.
1625
1626         :param frame_size: Frame size identifier or value [B].
1627         :param traffic_profile: Module name as a traffic profile identifier.
1628             See GPL/traffic_profiles/trex for implemented modules.
1629         :param minimum_transmit_rate: Minimal load in transactions per second.
1630         :param maximum_transmit_rate: Maximal load in transactions per second.
1631         :param plr_target: Ratio of packets lost to achieve [1].
1632         :param tdpt: Trial duration per trial.
1633             The algorithm linearly increases trial duration with trial number,
1634             this is the increment between succesive trials, in seconds.
1635         :param initial_count: Offset to apply before the first trial.
1636             For example initial_count=50 makes first trial to be 51*tdpt long.
1637             This is needed because initial "search" phase of integrator
1638             takes significant time even without any trial results.
1639         :param timeout: The search will stop after this overall time [s].
1640         :param ppta: Packets per transaction, aggregated over directions.
1641             Needed for udp_pps which does not have a good transaction counter,
1642             so we need to compute expected number of packets.
1643             Default: 1.
1644         :param resetter: Callable to reset DUT state for repeated trials.
1645         :param trace_enabled: True if trace enabled else False.
1646             This is very verbose tracing on numeric computations,
1647             do not use in production.
1648             Default: False
1649         :param traffic_directions: Traffic is bi- (2) or uni- (1) directional.
1650             Default: 2
1651         :param transaction_duration: Total expected time to close transaction.
1652         :param transaction_scale: Number of transactions to perform.
1653             0 (default) means unlimited.
1654         :param transaction_type: An identifier specifying which counters
1655             and formulas to use when computing attempted and failed
1656             transactions. Default: "packet".
1657         :param use_latency: Whether to measure latency during the trial.
1658             Default: False.
1659         :param ramp_up_rate: Rate to use in ramp-up trials [pps].
1660         :param ramp_up_duration: Duration of ramp-up trials [s].
1661         :param state_timeout: Time of life of DUT state [s].
1662         :type frame_size: str or int
1663         :type traffic_profile: str
1664         :type minimum_transmit_rate: float
1665         :type maximum_transmit_rate: float
1666         :type plr_target: float
1667         :type initial_count: int
1668         :type timeout: float
1669         :type ppta: int
1670         :type resetter: Optional[Callable[[], None]]
1671         :type trace_enabled: bool
1672         :type traffic_directions: int
1673         :type transaction_duration: float
1674         :type transaction_scale: int
1675         :type transaction_type: str
1676         :type use_latency: bool
1677         :type ramp_up_rate: float
1678         :type ramp_up_duration: float
1679         :type state_timeout: float
1680         :returns: Average and stdev of estimated aggregated rate giving PLR.
1681         :rtype: 2-tuple of float
1682         """
1683         tg_instance = BuiltIn().get_library_instance(
1684             u"resources.libraries.python.TrafficGenerator"
1685         )
1686         # Overrides for fixed transaction amount.
1687         # TODO: Move to robot code? We have a single call site
1688         #       but MLRsearch has two and we want the two to be used similarly.
1689         if transaction_scale:
1690             # TODO: What is a good value for max scale?
1691             # TODO: Scale the timeout with transaction scale.
1692             timeout = 7200.0
1693         tg_instance.set_rate_provider_defaults(
1694             frame_size=frame_size,
1695             traffic_profile=traffic_profile,
1696             negative_loss=False,
1697             sleep_till_duration=True,
1698             ppta=ppta,
1699             resetter=resetter,
1700             traffic_directions=traffic_directions,
1701             transaction_duration=transaction_duration,
1702             transaction_scale=transaction_scale,
1703             transaction_type=transaction_type,
1704             use_latency=use_latency,
1705             ramp_up_rate=ramp_up_rate,
1706             ramp_up_duration=ramp_up_duration,
1707             state_timeout=state_timeout,
1708         )
1709         algorithm = PLRsearch(
1710             measurer=tg_instance,
1711             trial_duration_per_trial=tdpt,
1712             packet_loss_ratio_target=plr_target,
1713             trial_number_offset=initial_count,
1714             timeout=timeout,
1715             trace_enabled=trace_enabled,
1716         )
1717         result = algorithm.search(
1718             min_rate=minimum_transmit_rate,
1719             max_rate=maximum_transmit_rate,
1720         )
1721         return result