resources/libraries/python/TrafficGenerator.py

   1 # Copyright (c) 2021 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Performance testing traffic generator library."""
  15
  16 import time
  17
  18 from robot.api import logger
  19 from robot.libraries.BuiltIn import BuiltIn
  20
  21 from .Constants import Constants
  22 from .CpuUtils import CpuUtils
  23 from .DropRateSearch import DropRateSearch
  24 from .MLRsearch.AbstractMeasurer import AbstractMeasurer
  25 from .MLRsearch.MultipleLossRatioSearch import MultipleLossRatioSearch
  26 from .MLRsearch.ReceiveRateMeasurement import ReceiveRateMeasurement
  27 from .PLRsearch.PLRsearch import PLRsearch
  28 from .OptionString import OptionString
  29 from .ssh import exec_cmd_no_error, exec_cmd
  30 from .topology import NodeType
  31 from .topology import NodeSubTypeTG
  32 from .topology import Topology
  33
  34 __all__ = [u"TGDropRateSearchImpl", u"TrafficGenerator", u"OptimizedSearch"]
  35
  36
  37 def check_subtype(node):
  38     """Return supported subtype of given node, or raise an exception.
  39
  40     Currently only one subtype is supported,
  41     but we want our code to be ready for other ones.
  42
  43     :param node: Topology node to check. Can be None.
  44     :type node: dict or NoneType
  45     :returns: Subtype detected.
  46     :rtype: NodeSubTypeTG
  47     :raises RuntimeError: If node is not supported, message explains how.
  48     """
  49     if node.get(u"type") is None:
  50         msg = u"Node type is not defined"
  51     elif node[u"type"] != NodeType.TG:
  52         msg = f"Node type is {node[u'type']!r}, not a TG"
  53     elif node.get(u"subtype") is None:
  54         msg = u"TG subtype is not defined"
  55     elif node[u"subtype"] != NodeSubTypeTG.TREX:
  56         msg = f"TG subtype {node[u'subtype']!r} is not supported"
  57     else:
  58         return NodeSubTypeTG.TREX
  59     raise RuntimeError(msg)
  60
  61
  62 class TGDropRateSearchImpl(DropRateSearch):
  63     """Drop Rate Search implementation."""
  64
  65     # def __init__(self):
  66     #     super(TGDropRateSearchImpl, self).__init__()
  67
  68     def measure_loss(
  69             self, rate, frame_size, loss_acceptance, loss_acceptance_type,
  70             traffic_profile):
  71         """Runs the traffic and evaluate the measured results.
  72
  73         :param rate: Offered traffic load.
  74         :param frame_size: Size of frame.
  75         :param loss_acceptance: Permitted drop ratio or frames count.
  76         :param loss_acceptance_type: Type of permitted loss.
  77         :param traffic_profile: Module name as a traffic profile identifier.
  78             See GPL/traffic_profiles/trex for implemented modules.
  79         :type rate: float
  80         :type frame_size: str
  81         :type loss_acceptance: float
  82         :type loss_acceptance_type: LossAcceptanceType
  83         :type traffic_profile: str
  84         :returns: Drop threshold exceeded? (True/False)
  85         :rtype: bool
  86         :raises NotImplementedError: If TG is not supported.
  87         :raises RuntimeError: If TG is not specified.
  88         """
  89         # we need instance of TrafficGenerator instantiated by Robot Framework
  90         # to be able to use trex_stl-*()
  91         tg_instance = BuiltIn().get_library_instance(
  92             u"resources.libraries.python.TrafficGenerator"
  93         )
  94         subtype = check_subtype(tg_instance.node)
  95         if subtype == NodeSubTypeTG.TREX:
  96             unit_rate = str(rate) + self.get_rate_type_str()
  97             tg_instance.trex_stl_start_remote_exec(
  98                 self.get_duration(), unit_rate, frame_size, traffic_profile
  99             )
 100             loss = tg_instance.get_loss()
 101             sent = tg_instance.get_sent()
 102             if self.loss_acceptance_type_is_percentage():
 103                 loss = (float(loss) / float(sent)) * 100
 104             logger.trace(
 105                 f"comparing: {loss} < {loss_acceptance} {loss_acceptance_type}"
 106             )
 107             return float(loss) <= float(loss_acceptance)
 108         return False
 109
 110     def get_latency(self):
 111         """Returns min/avg/max latency.
 112
 113         :returns: Latency stats.
 114         :rtype: list
 115         """
 116         tg_instance = BuiltIn().get_library_instance(
 117             u"resources.libraries.python.TrafficGenerator"
 118         )
 119         return tg_instance.get_latency_int()
 120
 121
 122 class TrexMode:
 123     """Defines mode of T-Rex traffic generator."""
 124     # Advanced stateful mode
 125     ASTF = u"ASTF"
 126     # Stateless mode
 127     STL = u"STL"
 128
 129
 130 # TODO: Pylint says too-many-instance-attributes.
 131 class TrafficGenerator(AbstractMeasurer):
 132     """Traffic Generator."""
 133
 134     # TODO: Remove "trex" from lines which could work with other TGs.
 135
 136     # Use one instance of TrafficGenerator for all tests in test suite
 137     ROBOT_LIBRARY_SCOPE = u"TEST SUITE"
 138
 139     def __init__(self):
 140         # TODO: Separate into few dataclasses/dicts.
 141         #       Pylint dislikes large unstructured state, and it is right.
 142         self._node = None
 143         self._mode = None
 144         # TG interface order mapping
 145         self._ifaces_reordered = False
 146         # Result holding fields, to be removed.
 147         self._result = None
 148         self._loss = None
 149         self._sent = None
 150         self._latency = None
 151         self._received = None
 152         self._approximated_rate = None
 153         self._approximated_duration = None
 154         self._l7_data = None
 155         # Measurement input fields, needed for async stop result.
 156         self._start_time = None
 157         self._stop_time = None
 158         self._rate = None
 159         self._target_duration = None
 160         self._duration = None
 161         # Other input parameters, not knowable from measure() signature.
 162         self.frame_size = None
 163         self.traffic_profile = None
 164         self.traffic_directions = None
 165         self.negative_loss = None
 166         self.use_latency = None
 167         self.ppta = None
 168         self.resetter = None
 169         self.transaction_scale = None
 170         self.transaction_duration = None
 171         self.sleep_till_duration = None
 172         self.transaction_type = None
 173         self.duration_limit = None
 174         self.ramp_up_start = None
 175         self.ramp_up_stop = None
 176         self.ramp_up_rate = None
 177         self.ramp_up_duration = None
 178         self.state_timeout = None
 179         # Transient data needed for async measurements.
 180         self._xstats = (None, None)
 181         # TODO: Rename "xstats" to something opaque, so T-Rex is not privileged?
 182
 183     @property
 184     def node(self):
 185         """Getter.
 186
 187         :returns: Traffic generator node.
 188         :rtype: dict
 189         """
 190         return self._node
 191
 192     def get_loss(self):
 193         """Return number of lost packets.
 194
 195         :returns: Number of lost packets.
 196         :rtype: str
 197         """
 198         return self._loss
 199
 200     def get_sent(self):
 201         """Return number of sent packets.
 202
 203         :returns: Number of sent packets.
 204         :rtype: str
 205         """
 206         return self._sent
 207
 208     def get_received(self):
 209         """Return number of received packets.
 210
 211         :returns: Number of received packets.
 212         :rtype: str
 213         """
 214         return self._received
 215
 216     def get_latency_int(self):
 217         """Return rounded min/avg/max latency.
 218
 219         :returns: Latency stats.
 220         :rtype: list
 221         """
 222         return self._latency
 223
 224     def get_approximated_rate(self):
 225         """Return approximated rate computed as ratio of transmitted packets
 226         over duration of trial.
 227
 228         :returns: Approximated rate.
 229         :rtype: str
 230         """
 231         return self._approximated_rate
 232
 233     def get_l7_data(self):
 234         """Return L7 data.
 235
 236         :returns: Number of received packets.
 237         :rtype: dict
 238         """
 239         return self._l7_data
 240
 241     def check_mode(self, expected_mode):
 242         """Check TG mode.
 243
 244         :param expected_mode: Expected traffic generator mode.
 245         :type expected_mode: object
 246         :raises RuntimeError: In case of unexpected TG mode.
 247         """
 248         if self._mode == expected_mode:
 249             return
 250         raise RuntimeError(
 251             f"{self._node[u'subtype']} not running in {expected_mode} mode!"
 252         )
 253
 254     # TODO: pylint says disable=too-many-locals.
 255     def initialize_traffic_generator(
 256             self, tg_node, tg_if1, tg_if2, tg_if1_adj_node, tg_if1_adj_if,
 257             tg_if2_adj_node, tg_if2_adj_if, osi_layer, tg_if1_dst_mac=None,
 258             tg_if2_dst_mac=None):
 259         """TG initialization.
 260
 261         TODO: Document why do we need (and how do we use) _ifaces_reordered.
 262
 263         :param tg_node: Traffic generator node.
 264         :param tg_if1: TG - name of first interface.
 265         :param tg_if2: TG - name of second interface.
 266         :param tg_if1_adj_node: TG if1 adjecent node.
 267         :param tg_if1_adj_if: TG if1 adjecent interface.
 268         :param tg_if2_adj_node: TG if2 adjecent node.
 269         :param tg_if2_adj_if: TG if2 adjecent interface.
 270         :param osi_layer: 'L2', 'L3' or 'L7' - OSI Layer testing type.
 271         :param tg_if1_dst_mac: Interface 1 destination MAC address.
 272         :param tg_if2_dst_mac: Interface 2 destination MAC address.
 273         :type tg_node: dict
 274         :type tg_if1: str
 275         :type tg_if2: str
 276         :type tg_if1_adj_node: dict
 277         :type tg_if1_adj_if: str
 278         :type tg_if2_adj_node: dict
 279         :type tg_if2_adj_if: str
 280         :type osi_layer: str
 281         :type tg_if1_dst_mac: str
 282         :type tg_if2_dst_mac: str
 283         :returns: nothing
 284         :raises RuntimeError: In case of issue during initialization.
 285         """
 286         subtype = check_subtype(tg_node)
 287         if subtype == NodeSubTypeTG.TREX:
 288             self._node = tg_node
 289             self._mode = TrexMode.ASTF if osi_layer == u"L7" else TrexMode.STL
 290             if1 = dict()
 291             if2 = dict()
 292             if1[u"pci"] = Topology().get_interface_pci_addr(self._node, tg_if1)
 293             if2[u"pci"] = Topology().get_interface_pci_addr(self._node, tg_if2)
 294             if1[u"addr"] = Topology().get_interface_mac(self._node, tg_if1)
 295             if2[u"addr"] = Topology().get_interface_mac(self._node, tg_if2)
 296
 297             if osi_layer == u"L2":
 298                 if1[u"adj_addr"] = if2[u"addr"]
 299                 if2[u"adj_addr"] = if1[u"addr"]
 300             elif osi_layer in (u"L3", u"L7"):
 301                 if1[u"adj_addr"] = Topology().get_interface_mac(
 302                     tg_if1_adj_node, tg_if1_adj_if
 303                 )
 304                 if2[u"adj_addr"] = Topology().get_interface_mac(
 305                     tg_if2_adj_node, tg_if2_adj_if
 306                 )
 307             else:
 308                 raise ValueError(u"Unknown OSI layer!")
 309
 310             # in case of switched environment we can override MAC addresses
 311             if tg_if1_dst_mac is not None and tg_if2_dst_mac is not None:
 312                 if1[u"adj_addr"] = tg_if1_dst_mac
 313                 if2[u"adj_addr"] = tg_if2_dst_mac
 314
 315             if min(if1[u"pci"], if2[u"pci"]) != if1[u"pci"]:
 316                 if1, if2 = if2, if1
 317                 self._ifaces_reordered = True
 318
 319             master_thread_id, latency_thread_id, socket, threads = \
 320                 CpuUtils.get_affinity_trex(
 321                     self._node, tg_if1, tg_if2,
 322                     tg_dtc=Constants.TREX_CORE_COUNT)
 323
 324             if osi_layer in (u"L2", u"L3", u"L7"):
 325                 exec_cmd_no_error(
 326                     self._node,
 327                     f"sh -c 'cat << EOF > /etc/trex_cfg.yaml\n"
 328                     f"- version: 2\n"
 329                     f"  c: {len(threads)}\n"
 330                     f"  limit_memory: {Constants.TREX_LIMIT_MEMORY}\n"
 331                     f"  interfaces: [\"{if1[u'pci']}\",\"{if2[u'pci']}\"]\n"
 332                     f"  port_info:\n"
 333                     f"      - dest_mac: \'{if1[u'adj_addr']}\'\n"
 334                     f"        src_mac: \'{if1[u'addr']}\'\n"
 335                     f"      - dest_mac: \'{if2[u'adj_addr']}\'\n"
 336                     f"        src_mac: \'{if2[u'addr']}\'\n"
 337                     f"  platform :\n"
 338                     f"      master_thread_id: {master_thread_id}\n"
 339                     f"      latency_thread_id: {latency_thread_id}\n"
 340                     f"      dual_if:\n"
 341                     f"          - socket: {socket}\n"
 342                     f"            threads: {threads}\n"
 343                     f"EOF'",
 344                     sudo=True, message=u"T-Rex config generation!"
 345                 )
 346
 347                 if Constants.TREX_RX_DESCRIPTORS_COUNT != 0:
 348                     exec_cmd_no_error(
 349                         self._node,
 350                         f"sh -c 'cat << EOF >> /etc/trex_cfg.yaml\n"
 351                         f"  rx_desc: {Constants.TREX_RX_DESCRIPTORS_COUNT}\n"
 352                         f"EOF'",
 353                         sudo=True, message=u"T-Rex rx_desc modification!"
 354                     )
 355
 356                 if Constants.TREX_TX_DESCRIPTORS_COUNT != 0:
 357                     exec_cmd_no_error(
 358                         self._node,
 359                         f"sh -c 'cat << EOF >> /etc/trex_cfg.yaml\n"
 360                         f"  tx_desc: {Constants.TREX_TX_DESCRIPTORS_COUNT}\n"
 361                         f"EOF'",
 362                         sudo=True, message=u"T-Rex tx_desc modification!"
 363                     )
 364             else:
 365                 raise ValueError(u"Unknown OSI layer!")
 366
 367             TrafficGenerator.startup_trex(
 368                 self._node, osi_layer, subtype=subtype
 369             )
 370
 371     @staticmethod
 372     def startup_trex(tg_node, osi_layer, subtype=None):
 373         """Startup sequence for the TRex traffic generator.
 374
 375         :param tg_node: Traffic generator node.
 376         :param osi_layer: 'L2', 'L3' or 'L7' - OSI Layer testing type.
 377         :param subtype: Traffic generator sub-type.
 378         :type tg_node: dict
 379         :type osi_layer: str
 380         :type subtype: NodeSubTypeTG
 381         :raises RuntimeError: If T-Rex startup failed.
 382         :raises ValueError: If OSI layer is not supported.
 383         """
 384         if not subtype:
 385             subtype = check_subtype(tg_node)
 386         if subtype == NodeSubTypeTG.TREX:
 387             for _ in range(0, 3):
 388                 # Kill TRex only if it is already running.
 389                 cmd = u"sh -c \"pgrep t-rex && pkill t-rex && sleep 3 || true\""
 390                 exec_cmd_no_error(
 391                     tg_node, cmd, sudo=True, message=u"Kill TRex failed!"
 392                 )
 393
 394                 # Configure TRex.
 395                 ports = ''
 396                 for port in tg_node[u"interfaces"].values():
 397                     if u'Mellanox' not in port.get(u'model'):
 398                         ports += f" {port.get(u'pci_address')}"
 399
 400                 cmd = f"sh -c \"cd {Constants.TREX_INSTALL_DIR}/scripts/ && " \
 401                     f"./dpdk_nic_bind.py -u {ports} || true\""
 402                 exec_cmd_no_error(
 403                     tg_node, cmd, sudo=True,
 404                     message=u"Unbind PCI ports from driver failed!"
 405                 )
 406
 407                 # Start TRex.
 408                 cd_cmd = f"cd '{Constants.TREX_INSTALL_DIR}/scripts/'"
 409                 trex_cmd = OptionString([u"nohup", u"./t-rex-64"])
 410                 trex_cmd.add(u"-i")
 411                 trex_cmd.add(u"--prefix $(hostname)")
 412                 trex_cmd.add(u"--hdrh")
 413                 trex_cmd.add(u"--no-scapy-server")
 414                 trex_cmd.add_if(u"--astf", osi_layer == u"L7")
 415                 # OptionString does not create double space if extra is empty.
 416                 trex_cmd.add(f"{Constants.TREX_EXTRA_CMDLINE}")
 417                 inner_command = f"{cd_cmd} && {trex_cmd} > /tmp/trex.log 2>&1 &"
 418                 cmd = f"sh -c \"{inner_command}\" > /dev/null"
 419                 try:
 420                     exec_cmd_no_error(tg_node, cmd, sudo=True)
 421                 except RuntimeError:
 422                     cmd = u"sh -c \"cat /tmp/trex.log\""
 423                     exec_cmd_no_error(
 424                         tg_node, cmd, sudo=True,
 425                         message=u"Get TRex logs failed!"
 426                     )
 427                     raise RuntimeError(u"Start TRex failed!")
 428
 429                 # Test T-Rex API responsiveness.
 430                 cmd = f"python3 {Constants.REMOTE_FW_DIR}/GPL/tools/trex/"
 431                 if osi_layer in (u"L2", u"L3"):
 432                     cmd += u"trex_stl_assert.py"
 433                 elif osi_layer == u"L7":
 434                     cmd += u"trex_astf_assert.py"
 435                 else:
 436                     raise ValueError(u"Unknown OSI layer!")
 437                 try:
 438                     exec_cmd_no_error(
 439                         tg_node, cmd, sudo=True,
 440                         message=u"T-Rex API is not responding!", retries=20
 441                     )
 442                 except RuntimeError:
 443                     continue
 444                 return
 445             # After max retries TRex is still not responding to API critical
 446             # error occurred.
 447             exec_cmd(tg_node, u"cat /tmp/trex.log", sudo=True)
 448             raise RuntimeError(u"Start T-Rex failed after multiple retries!")
 449
 450     @staticmethod
 451     def is_trex_running(node):
 452         """Check if T-Rex is running using pidof.
 453
 454         :param node: Traffic generator node.
 455         :type node: dict
 456         :returns: True if T-Rex is running otherwise False.
 457         :rtype: bool
 458         """
 459         ret, _, _ = exec_cmd(node, u"pgrep t-rex", sudo=True)
 460         return bool(int(ret) == 0)
 461
 462     @staticmethod
 463     def teardown_traffic_generator(node):
 464         """TG teardown.
 465
 466         :param node: Traffic generator node.
 467         :type node: dict
 468         :returns: nothing
 469         :raises RuntimeError: If node type is not a TG,
 470             or if T-Rex teardown fails.
 471         """
 472         subtype = check_subtype(node)
 473         if subtype == NodeSubTypeTG.TREX:
 474             exec_cmd_no_error(
 475                 node,
 476                 u"sh -c "
 477                 u"\"if pgrep t-rex; then sudo pkill t-rex && sleep 3; fi\"",
 478                 sudo=False,
 479                 message=u"T-Rex kill failed!"
 480             )
 481
 482     def trex_astf_stop_remote_exec(self, node):
 483         """Execute T-Rex ASTF script on remote node over ssh to stop running
 484         traffic.
 485
 486         Internal state is updated with measurement results.
 487
 488         :param node: T-Rex generator node.
 489         :type node: dict
 490         :raises RuntimeError: If stop traffic script fails.
 491         """
 492         command_line = OptionString().add(u"python3")
 493         dirname = f"{Constants.REMOTE_FW_DIR}/GPL/tools/trex"
 494         command_line.add(f"'{dirname}/trex_astf_stop.py'")
 495         command_line.change_prefix(u"--")
 496         for index, value in enumerate(self._xstats):
 497             if value is not None:
 498                 value = value.replace(u"'", u"\"")
 499                 command_line.add_equals(f"xstat{index}", f"'{value}'")
 500         stdout, _ = exec_cmd_no_error(
 501             node, command_line,
 502             message=u"T-Rex ASTF runtime error!"
 503         )
 504         self._parse_traffic_results(stdout)
 505
 506     def trex_stl_stop_remote_exec(self, node):
 507         """Execute T-Rex STL script on remote node over ssh to stop running
 508         traffic.
 509
 510         Internal state is updated with measurement results.
 511
 512         :param node: T-Rex generator node.
 513         :type node: dict
 514         :raises RuntimeError: If stop traffic script fails.
 515         """
 516         command_line = OptionString().add(u"python3")
 517         dirname = f"{Constants.REMOTE_FW_DIR}/GPL/tools/trex"
 518         command_line.add(f"'{dirname}/trex_stl_stop.py'")
 519         command_line.change_prefix(u"--")
 520         for index, value in enumerate(self._xstats):
 521             if value is not None:
 522                 value = value.replace(u"'", u"\"")
 523                 command_line.add_equals(f"xstat{index}", f"'{value}'")
 524         stdout, _ = exec_cmd_no_error(
 525             node, command_line,
 526             message=u"T-Rex STL runtime error!"
 527         )
 528         self._parse_traffic_results(stdout)
 529
 530     def stop_traffic_on_tg(self):
 531         """Stop all traffic on TG.
 532
 533         :returns: Structure containing the result of the measurement.
 534         :rtype: ReceiveRateMeasurement
 535         :raises ValueError: If TG traffic profile is not supported.
 536         """
 537         subtype = check_subtype(self._node)
 538         if subtype != NodeSubTypeTG.TREX:
 539             raise ValueError(f"Unsupported TG subtype: {subtype!r}")
 540         if u"trex-astf" in self.traffic_profile:
 541             self.trex_astf_stop_remote_exec(self._node)
 542         elif u"trex-stl" in self.traffic_profile:
 543             self.trex_stl_stop_remote_exec(self._node)
 544         else:
 545             raise ValueError(u"Unsupported T-Rex traffic profile!")
 546         self._stop_time = time.monotonic()
 547
 548         return self._get_measurement_result()
 549
 550     def _compute_duration(self, duration, multiplier):
 551         """Compute duration for profile driver.
 552
 553         The final result is influenced by transaction scale and duration limit.
 554         It is assumed a higher level function has already set those to self.
 555         The duration argument is the target value from search point of view,
 556         before the overrides are applied here.
 557
 558         Minus one (signalling async traffic start) is kept.
 559
 560         Completeness flag is also included. Duration limited or async trials
 561         are not considered complete for ramp-up purposes.
 562
 563         :param duration: Time expressed in seconds for how long to send traffic.
 564         :param multiplier: Traffic rate in transactions per second.
 565         :type duration: float
 566         :type multiplier: float
 567         :returns: New duration and whether it was a complete ramp-up candidate.
 568         :rtype: float, bool
 569         """
 570         if duration < 0.0:
 571             # Keep the async -1.
 572             return duration, False
 573         computed_duration = duration
 574         if self.transaction_scale:
 575             computed_duration = self.transaction_scale / multiplier
 576             # Log the computed duration,
 577             # so we can compare with what telemetry suggests
 578             # the real duration was.
 579             logger.debug(f"Expected duration {computed_duration}")
 580             computed_duration += 0.1115
 581         if not self.duration_limit:
 582             return computed_duration, True
 583         limited_duration = min(computed_duration, self.duration_limit)
 584         return limited_duration, (limited_duration == computed_duration)
 585
 586     def trex_astf_start_remote_exec(
 587             self, duration, multiplier, async_call=False):
 588         """Execute T-Rex ASTF script on remote node over ssh to start running
 589         traffic.
 590
 591         In sync mode, measurement results are stored internally.
 592         In async mode, initial data including xstats are stored internally.
 593
 594         This method contains the logic to compute duration as maximum time
 595         if transaction_scale is nonzero.
 596         The transaction_scale argument defines (limits) how many transactions
 597         will be started in total. As that amount of transaction can take
 598         considerable time (sometimes due to explicit delays in the profile),
 599         the real time a trial needs to finish is computed here. For now,
 600         in that case the duration argument is ignored, assuming it comes
 601         from ASTF-unaware search algorithm. The overall time a single
 602         transaction needs is given in parameter transaction_duration,
 603         it includes both explicit delays and implicit time it takes
 604         to transfer data (or whatever the transaction does).
 605
 606         Currently it is observed TRex does not start the ASTF traffic
 607         immediately, an ad-hoc constant is added to the computed duration
 608         to compensate for that.
 609
 610         If transaction_scale is zero, duration is not recomputed.
 611         It is assumed the subsequent result parsing gets the real duration
 612         if the traffic stops sooner for any reason.
 613
 614         Currently, it is assumed traffic profile defines a single transaction.
 615         To avoid heavy logic here, the input rate is expected to be in
 616         transactions per second, as that directly translates to TRex multiplier,
 617         (assuming the profile does not override the default cps value of one).
 618
 619         :param duration: Time expressed in seconds for how long to send traffic.
 620         :param multiplier: Traffic rate in transactions per second.
 621         :param async_call: If enabled then don't wait for all incoming traffic.
 622         :type duration: float
 623         :type multiplier: int
 624         :type async_call: bool
 625         :raises RuntimeError: In case of T-Rex driver issue.
 626         """
 627         self.check_mode(TrexMode.ASTF)
 628         p_0, p_1 = (1, 0) if self._ifaces_reordered else (0, 1)
 629         if not isinstance(duration, (float, int)):
 630             duration = float(duration)
 631
 632         # TODO: Refactor the code so duration is computed only once,
 633         # and both the initial and the computed durations are logged.
 634         computed_duration, _ = self._compute_duration(duration, multiplier)
 635
 636         command_line = OptionString().add(u"python3")
 637         dirname = f"{Constants.REMOTE_FW_DIR}/GPL/tools/trex"
 638         command_line.add(f"'{dirname}/trex_astf_profile.py'")
 639         command_line.change_prefix(u"--")
 640         dirname = f"{Constants.REMOTE_FW_DIR}/GPL/traffic_profiles/trex"
 641         command_line.add_with_value(
 642             u"profile", f"'{dirname}/{self.traffic_profile}.py'"
 643         )
 644         command_line.add_with_value(u"duration", f"{computed_duration!r}")
 645         command_line.add_with_value(u"frame_size", self.frame_size)
 646         command_line.add_with_value(u"multiplier", multiplier)
 647         command_line.add_with_value(u"port_0", p_0)
 648         command_line.add_with_value(u"port_1", p_1)
 649         command_line.add_with_value(
 650             u"traffic_directions", self.traffic_directions
 651         )
 652         command_line.add_if(u"async_start", async_call)
 653         command_line.add_if(u"latency", self.use_latency)
 654         command_line.add_if(u"force", Constants.TREX_SEND_FORCE)
 655
 656         self._start_time = time.monotonic()
 657         self._rate = multiplier
 658         stdout, _ = exec_cmd_no_error(
 659             self._node, command_line, timeout=computed_duration + 10.0,
 660             message=u"T-Rex ASTF runtime error!"
 661         )
 662
 663         if async_call:
 664             # no result
 665             self._target_duration = None
 666             self._duration = None
 667             self._received = None
 668             self._sent = None
 669             self._loss = None
 670             self._latency = None
 671             xstats = [None, None]
 672             self._l7_data = dict()
 673             self._l7_data[u"client"] = dict()
 674             self._l7_data[u"client"][u"active_flows"] = None
 675             self._l7_data[u"client"][u"established_flows"] = None
 676             self._l7_data[u"client"][u"traffic_duration"] = None
 677             self._l7_data[u"server"] = dict()
 678             self._l7_data[u"server"][u"active_flows"] = None
 679             self._l7_data[u"server"][u"established_flows"] = None
 680             self._l7_data[u"server"][u"traffic_duration"] = None
 681             if u"udp" in self.traffic_profile:
 682                 self._l7_data[u"client"][u"udp"] = dict()
 683                 self._l7_data[u"client"][u"udp"][u"connects"] = None
 684                 self._l7_data[u"client"][u"udp"][u"closed_flows"] = None
 685                 self._l7_data[u"client"][u"udp"][u"err_cwf"] = None
 686                 self._l7_data[u"server"][u"udp"] = dict()
 687                 self._l7_data[u"server"][u"udp"][u"accepted_flows"] = None
 688                 self._l7_data[u"server"][u"udp"][u"closed_flows"] = None
 689             elif u"tcp" in self.traffic_profile:
 690                 self._l7_data[u"client"][u"tcp"] = dict()
 691                 self._l7_data[u"client"][u"tcp"][u"initiated_flows"] = None
 692                 self._l7_data[u"client"][u"tcp"][u"connects"] = None
 693                 self._l7_data[u"client"][u"tcp"][u"closed_flows"] = None
 694                 self._l7_data[u"client"][u"tcp"][u"connattempt"] = None
 695                 self._l7_data[u"server"][u"tcp"] = dict()
 696                 self._l7_data[u"server"][u"tcp"][u"accepted_flows"] = None
 697                 self._l7_data[u"server"][u"tcp"][u"connects"] = None
 698                 self._l7_data[u"server"][u"tcp"][u"closed_flows"] = None
 699             else:
 700                 logger.warn(u"Unsupported T-Rex ASTF traffic profile!")
 701             index = 0
 702             for line in stdout.splitlines():
 703                 if f"Xstats snapshot {index}: " in line:
 704                     xstats[index] = line[19:]
 705                     index += 1
 706                 if index == 2:
 707                     break
 708             self._xstats = tuple(xstats)
 709         else:
 710             self._target_duration = duration
 711             self._duration = computed_duration
 712             self._parse_traffic_results(stdout)
 713
 714     def trex_stl_start_remote_exec(self, duration, rate, async_call=False):
 715         """Execute T-Rex STL script on remote node over ssh to start running
 716         traffic.
 717
 718         In sync mode, measurement results are stored internally.
 719         In async mode, initial data including xstats are stored internally.
 720
 721         Mode-unaware code (e.g. in search algorithms) works with transactions.
 722         To keep the logic simple, multiplier is set to that value.
 723         As bidirectional traffic profiles send packets in both directions,
 724         they are treated as transactions with two packets (one per direction).
 725
 726         :param duration: Time expressed in seconds for how long to send traffic.
 727         :param rate: Traffic rate in transactions per second.
 728         :param async_call: If enabled then don't wait for all incoming traffic.
 729         :type duration: float
 730         :type rate: str
 731         :type async_call: bool
 732         :raises RuntimeError: In case of T-Rex driver issue.
 733         """
 734         self.check_mode(TrexMode.STL)
 735         p_0, p_1 = (1, 0) if self._ifaces_reordered else (0, 1)
 736         if not isinstance(duration, (float, int)):
 737             duration = float(duration)
 738
 739         # TODO: Refactor the code so duration is computed only once,
 740         # and both the initial and the computed durations are logged.
 741         duration, _ = self._compute_duration(duration=duration, multiplier=rate)
 742
 743         command_line = OptionString().add(u"python3")
 744         dirname = f"{Constants.REMOTE_FW_DIR}/GPL/tools/trex"
 745         command_line.add(f"'{dirname}/trex_stl_profile.py'")
 746         command_line.change_prefix(u"--")
 747         dirname = f"{Constants.REMOTE_FW_DIR}/GPL/traffic_profiles/trex"
 748         command_line.add_with_value(
 749             u"profile", f"'{dirname}/{self.traffic_profile}.py'"
 750         )
 751         command_line.add_with_value(u"duration", f"{duration!r}")
 752         command_line.add_with_value(u"frame_size", self.frame_size)
 753         command_line.add_with_value(u"rate", f"{rate!r}")
 754         command_line.add_with_value(u"port_0", p_0)
 755         command_line.add_with_value(u"port_1", p_1)
 756         command_line.add_with_value(
 757             u"traffic_directions", self.traffic_directions
 758         )
 759         command_line.add_if(u"async_start", async_call)
 760         command_line.add_if(u"latency", self.use_latency)
 761         command_line.add_if(u"force", Constants.TREX_SEND_FORCE)
 762
 763         # TODO: This is ugly. Handle parsing better.
 764         self._start_time = time.monotonic()
 765         self._rate = float(rate[:-3]) if u"pps" in rate else float(rate)
 766         stdout, _ = exec_cmd_no_error(
 767             self._node, command_line, timeout=int(duration) + 60,
 768             message=u"T-Rex STL runtime error"
 769         )
 770
 771         if async_call:
 772             # no result
 773             self._target_duration = None
 774             self._duration = None
 775             self._received = None
 776             self._sent = None
 777             self._loss = None
 778             self._latency = None
 779
 780             xstats = [None, None]
 781             index = 0
 782             for line in stdout.splitlines():
 783                 if f"Xstats snapshot {index}: " in line:
 784                     xstats[index] = line[19:]
 785                     index += 1
 786                 if index == 2:
 787                     break
 788             self._xstats = tuple(xstats)
 789         else:
 790             self._target_duration = duration
 791             self._duration = duration
 792             self._parse_traffic_results(stdout)
 793
 794     def send_traffic_on_tg(
 795             self,
 796             duration,
 797             rate,
 798             frame_size,
 799             traffic_profile,
 800             async_call=False,
 801             ppta=1,
 802             traffic_directions=2,
 803             transaction_duration=0.0,
 804             transaction_scale=0,
 805             transaction_type=u"packet",
 806             duration_limit=0.0,
 807             use_latency=False,
 808             ramp_up_rate=None,
 809             ramp_up_duration=None,
 810             state_timeout=300.0,
 811             ramp_up_only=False,
 812         ):
 813         """Send traffic from all configured interfaces on TG.
 814
 815         In async mode, xstats is stored internally,
 816         to enable getting correct result when stopping the traffic.
 817         In both modes, stdout is returned,
 818         but _parse_traffic_results only works in sync output.
 819
 820         Note that traffic generator uses DPDK driver which might
 821         reorder port numbers based on wiring and PCI numbering.
 822         This method handles that, so argument values are invariant,
 823         but you can see swapped valued in debug logs.
 824
 825         When transaction_scale is specified, the duration value is ignored
 826         and the needed time is computed. For cases where this results in
 827         to too long measurement (e.g. teardown trial with small rate),
 828         duration_limit is applied (of non-zero), so the trial is stopped sooner.
 829
 830         Bidirectional STL profiles are treated as transactions with two packets.
 831
 832         The return value is None for async.
 833
 834         :param duration: Duration of test traffic generation in seconds.
 835         :param rate: Traffic rate in transactions per second.
 836         :param frame_size: Frame size (L2) in Bytes.
 837         :param traffic_profile: Module name as a traffic profile identifier.
 838             See GPL/traffic_profiles/trex for implemented modules.
 839         :param async_call: Async mode.
 840         :param ppta: Packets per transaction, aggregated over directions.
 841             Needed for udp_pps which does not have a good transaction counter,
 842             so we need to compute expected number of packets.
 843             Default: 1.
 844         :param traffic_directions: Traffic is bi- (2) or uni- (1) directional.
 845             Default: 2
 846         :param transaction_duration: Total expected time to close transaction.
 847         :param transaction_scale: Number of transactions to perform.
 848             0 (default) means unlimited.
 849         :param transaction_type: An identifier specifying which counters
 850             and formulas to use when computing attempted and failed
 851             transactions. Default: "packet".
 852         :param duration_limit: Zero or maximum limit for computed (or given)
 853             duration.
 854         :param use_latency: Whether to measure latency during the trial.
 855             Default: False.
 856         :param ramp_up_rate: Rate to use in ramp-up trials [pps].
 857         :param ramp_up_duration: Duration of ramp-up trials [s].
 858         :param state_timeout: Time of life of DUT state [s].
 859         :param ramp_up_only: If true, do not perform main trial measurement.
 860         :type duration: float
 861         :type rate: float
 862         :type frame_size: str
 863         :type traffic_profile: str
 864         :type async_call: bool
 865         :type ppta: int
 866         :type traffic_directions: int
 867         :type transaction_duration: float
 868         :type transaction_scale: int
 869         :type transaction_type: str
 870         :type duration_limit: float
 871         :type use_latency: bool
 872         :type ramp_up_rate: float
 873         :type ramp_up_duration: float
 874         :type state_timeout: float
 875         :type ramp_up_only: bool
 876         :returns: TG results.
 877         :rtype: ReceiveRateMeasurement or None
 878         :raises ValueError: If TG traffic profile is not supported.
 879         """
 880         self.set_rate_provider_defaults(
 881             frame_size=frame_size,
 882             traffic_profile=traffic_profile,
 883             ppta=ppta,
 884             traffic_directions=traffic_directions,
 885             transaction_duration=transaction_duration,
 886             transaction_scale=transaction_scale,
 887             transaction_type=transaction_type,
 888             duration_limit=duration_limit,
 889             use_latency=use_latency,
 890             ramp_up_rate=ramp_up_rate,
 891             ramp_up_duration=ramp_up_duration,
 892             state_timeout=state_timeout,
 893         )
 894         return self._send_traffic_on_tg_with_ramp_up(
 895             duration=duration,
 896             rate=rate,
 897             async_call=async_call,
 898             ramp_up_only=ramp_up_only,
 899         )
 900
 901     def _send_traffic_on_tg_internal(
 902             self, duration, rate, async_call=False):
 903         """Send traffic from all configured interfaces on TG.
 904
 905         This is an internal function, it assumes set_rate_provider_defaults
 906         has been called to remember most values.
 907         The reason why need to remember various values is that
 908         the traffic can be asynchronous, and parsing needs those values.
 909         The reason why this is is a separate function from the one
 910         which calls set_rate_provider_defaults is that some search algorithms
 911         need to specify their own values, and we do not want the measure call
 912         to overwrite them with defaults.
 913
 914         This function is used both for automated ramp-up trials
 915         and for explicitly called trials.
 916
 917         :param duration: Duration of test traffic generation in seconds.
 918         :param rate: Traffic rate in transactions per second.
 919         :param async_call: Async mode.
 920         :type duration: float
 921         :type rate: float
 922         :type async_call: bool
 923         :returns: TG results.
 924         :rtype: ReceiveRateMeasurement or None
 925         :raises ValueError: If TG traffic profile is not supported.
 926         """
 927         subtype = check_subtype(self._node)
 928         if subtype == NodeSubTypeTG.TREX:
 929             if u"trex-astf" in self.traffic_profile:
 930                 self.trex_astf_start_remote_exec(
 931                     duration, float(rate), async_call
 932                 )
 933             elif u"trex-stl" in self.traffic_profile:
 934                 unit_rate_str = str(rate) + u"pps"
 935                 # TODO: Suport transaction_scale et al?
 936                 self.trex_stl_start_remote_exec(
 937                     duration, unit_rate_str, async_call
 938                 )
 939             else:
 940                 raise ValueError(u"Unsupported T-Rex traffic profile!")
 941
 942         return None if async_call else self._get_measurement_result()
 943
 944     def _send_traffic_on_tg_with_ramp_up(
 945             self, duration, rate, async_call=False, ramp_up_only=False):
 946         """Send traffic from all interfaces on TG, maybe after ramp-up.
 947
 948         This is an internal function, it assumes set_rate_provider_defaults
 949         has been called to remember most values.
 950         The reason why need to remember various values is that
 951         the traffic can be asynchronous, and parsing needs those values.
 952         The reason why this is a separate function from the one
 953         which calls set_rate_provider_defaults is that some search algorithms
 954         need to specify their own values, and we do not want the measure call
 955         to overwrite them with defaults.
 956
 957         If ramp-up tracking is detected, a computation is performed,
 958         and if state timeout is near, trial at ramp-up rate and duration
 959         is inserted before the main trial measurement.
 960
 961         The ramp_up_only parameter forces a ramp-up without immediate
 962         trial measurement, which is useful in case self remembers
 963         a previous ramp-up trial that belongs to a different test (phase).
 964
 965         Return None if trial is async or ramp-up only.
 966
 967         :param duration: Duration of test traffic generation in seconds.
 968         :param rate: Traffic rate in transactions per second.
 969         :param async_call: Async mode.
 970         :param ramp_up_only: If true, do not perform main trial measurement.
 971         :type duration: float
 972         :type rate: float
 973         :type async_call: bool
 974         :type ramp_up_only: bool
 975         :returns: TG results.
 976         :rtype: ReceiveRateMeasurement or None
 977         :raises ValueError: If TG traffic profile is not supported.
 978         """
 979         complete = False
 980         if self.ramp_up_rate:
 981             # Figure out whether we need to insert a ramp-up trial.
 982             # TODO: Give up on async_call=True?
 983             if ramp_up_only or self.ramp_up_start is None:
 984                 # We never ramped up yet (at least not in this test case).
 985                 ramp_up_needed = True
 986             else:
 987                 # We ramped up before, but maybe it was too long ago.
 988                 # Adding a constant overhead to be safe.
 989                 time_now = time.monotonic() + 1.0
 990                 computed_duration, complete = self._compute_duration(
 991                     duration=duration,
 992                     multiplier=rate,
 993                 )
 994                 # There are two conditions for inserting ramp-up.
 995                 # If early sessions are expiring already,
 996                 # or if late sessions are to expire before measurement is over.
 997                 ramp_up_start_delay = time_now - self.ramp_up_start
 998                 ramp_up_stop_delay = time_now - self.ramp_up_stop
 999                 ramp_up_stop_delay += computed_duration
1000                 bigger_delay = max(ramp_up_start_delay, ramp_up_stop_delay)
1001                 # Final boolean decision.
1002                 ramp_up_needed = (bigger_delay >= self.state_timeout)
1003             if ramp_up_needed:
1004                 logger.debug(
1005                     u"State may time out during next real trial, "
1006                     u"inserting a ramp-up trial."
1007                 )
1008                 self.ramp_up_start = time.monotonic()
1009                 self._send_traffic_on_tg_internal(
1010                     duration=self.ramp_up_duration,
1011                     rate=self.ramp_up_rate,
1012                     async_call=async_call,
1013                 )
1014                 self.ramp_up_stop = time.monotonic()
1015                 logger.debug(u"Ramp-up done.")
1016             else:
1017                 logger.debug(
1018                     u"State will probably not time out during next real trial, "
1019                     u"no ramp-up trial needed just yet."
1020                 )
1021         if ramp_up_only:
1022             return None
1023         trial_start = time.monotonic()
1024         result = self._send_traffic_on_tg_internal(
1025             duration=duration,
1026             rate=rate,
1027             async_call=async_call,
1028         )
1029         trial_end = time.monotonic()
1030         if self.ramp_up_rate:
1031             # Optimization: No loss acts as a good ramp-up, if it was complete.
1032             if complete and result is not None and result.loss_count == 0:
1033                 logger.debug(u"Good trial acts as a ramp-up")
1034                 self.ramp_up_start = trial_start
1035                 self.ramp_up_stop = trial_end
1036             else:
1037                 logger.debug(u"Loss or incomplete, does not act as a ramp-up.")
1038         return result
1039
1040     def no_traffic_loss_occurred(self):
1041         """Fail if loss occurred in traffic run.
1042
1043         :returns: nothing
1044         :raises Exception: If loss occured.
1045         """
1046         if self._loss is None:
1047             raise RuntimeError(u"The traffic generation has not been issued")
1048         if self._loss != u"0":
1049             raise RuntimeError(f"Traffic loss occurred: {self._loss}")
1050
1051     def fail_if_no_traffic_forwarded(self):
1052         """Fail if no traffic forwarded.
1053
1054         TODO: Check number of passed transactions instead.
1055
1056         :returns: nothing
1057         :raises Exception: If no traffic forwarded.
1058         """
1059         if self._received is None:
1060             raise RuntimeError(u"The traffic generation has not been issued")
1061         if self._received == 0:
1062             raise RuntimeError(u"No traffic forwarded")
1063
1064     def partial_traffic_loss_accepted(
1065             self, loss_acceptance, loss_acceptance_type):
1066         """Fail if loss is higher then accepted in traffic run.
1067
1068         :param loss_acceptance: Permitted drop ratio or frames count.
1069         :param loss_acceptance_type: Type of permitted loss.
1070         :type loss_acceptance: float
1071         :type loss_acceptance_type: LossAcceptanceType
1072         :returns: nothing
1073         :raises Exception: If loss is above acceptance criteria.
1074         """
1075         if self._loss is None:
1076             raise Exception(u"The traffic generation has not been issued")
1077
1078         if loss_acceptance_type == u"percentage":
1079             loss = (float(self._loss) / float(self._sent)) * 100
1080         elif loss_acceptance_type == u"frames":
1081             loss = float(self._loss)
1082         else:
1083             raise Exception(u"Loss acceptance type not supported")
1084
1085         if loss > float(loss_acceptance):
1086             raise Exception(
1087                 f"Traffic loss {loss} above loss acceptance: {loss_acceptance}"
1088             )
1089
1090     def _parse_traffic_results(self, stdout):
1091         """Parse stdout of scripts into fields of self.
1092
1093         Block of code to reuse, by sync start, or stop after async.
1094
1095         :param stdout: Text containing the standard output.
1096         :type stdout: str
1097         """
1098         subtype = check_subtype(self._node)
1099         if subtype == NodeSubTypeTG.TREX:
1100             # Last line from console output
1101             line = stdout.splitlines()[-1]
1102             results = line.split(u";")
1103             if results[-1] in (u" ", u""):
1104                 results.pop(-1)
1105             self._result = dict()
1106             for result in results:
1107                 key, value = result.split(u"=", maxsplit=1)
1108                 self._result[key.strip()] = value
1109             logger.info(f"TrafficGen results:\n{self._result}")
1110             self._received = int(self._result.get(u"total_received"), 0)
1111             self._sent = int(self._result.get(u"total_sent", 0))
1112             self._loss = int(self._result.get(u"frame_loss", 0))
1113             self._approximated_duration = \
1114                 self._result.get(u"approximated_duration", 0.0)
1115             if u"manual" not in str(self._approximated_duration):
1116                 self._approximated_duration = float(self._approximated_duration)
1117             self._latency = list()
1118             self._latency.append(self._result.get(u"latency_stream_0(usec)"))
1119             self._latency.append(self._result.get(u"latency_stream_1(usec)"))
1120             if self._mode == TrexMode.ASTF:
1121                 self._l7_data = dict()
1122                 self._l7_data[u"client"] = dict()
1123                 self._l7_data[u"client"][u"sent"] = \
1124                     int(self._result.get(u"client_sent", 0))
1125                 self._l7_data[u"client"][u"received"] = \
1126                     int(self._result.get(u"client_received", 0))
1127                 self._l7_data[u"client"][u"active_flows"] = \
1128                     int(self._result.get(u"client_active_flows", 0))
1129                 self._l7_data[u"client"][u"established_flows"] = \
1130                     int(self._result.get(u"client_established_flows", 0))
1131                 self._l7_data[u"client"][u"traffic_duration"] = \
1132                     float(self._result.get(u"client_traffic_duration", 0.0))
1133                 self._l7_data[u"client"][u"err_rx_throttled"] = \
1134                     int(self._result.get(u"client_err_rx_throttled", 0))
1135                 self._l7_data[u"client"][u"err_c_nf_throttled"] = \
1136                     int(self._result.get(u"client_err_nf_throttled", 0))
1137                 self._l7_data[u"client"][u"err_flow_overflow"] = \
1138                     int(self._result.get(u"client_err_flow_overflow", 0))
1139                 self._l7_data[u"server"] = dict()
1140                 self._l7_data[u"server"][u"active_flows"] = \
1141                     int(self._result.get(u"server_active_flows", 0))
1142                 self._l7_data[u"server"][u"established_flows"] = \
1143                     int(self._result.get(u"server_established_flows", 0))
1144                 self._l7_data[u"server"][u"traffic_duration"] = \
1145                     float(self._result.get(u"server_traffic_duration", 0.0))
1146                 self._l7_data[u"server"][u"err_rx_throttled"] = \
1147                     int(self._result.get(u"client_err_rx_throttled", 0))
1148                 if u"udp" in self.traffic_profile:
1149                     self._l7_data[u"client"][u"udp"] = dict()
1150                     self._l7_data[u"client"][u"udp"][u"connects"] = \
1151                         int(self._result.get(u"client_udp_connects", 0))
1152                     self._l7_data[u"client"][u"udp"][u"closed_flows"] = \
1153                         int(self._result.get(u"client_udp_closed", 0))
1154                     self._l7_data[u"client"][u"udp"][u"tx_bytes"] = \
1155                         int(self._result.get(u"client_udp_tx_bytes", 0))
1156                     self._l7_data[u"client"][u"udp"][u"rx_bytes"] = \
1157                         int(self._result.get(u"client_udp_rx_bytes", 0))
1158                     self._l7_data[u"client"][u"udp"][u"tx_packets"] = \
1159                         int(self._result.get(u"client_udp_tx_packets", 0))
1160                     self._l7_data[u"client"][u"udp"][u"rx_packets"] = \
1161                         int(self._result.get(u"client_udp_rx_packets", 0))
1162                     self._l7_data[u"client"][u"udp"][u"keep_drops"] = \
1163                         int(self._result.get(u"client_udp_keep_drops", 0))
1164                     self._l7_data[u"client"][u"udp"][u"err_cwf"] = \
1165                         int(self._result.get(u"client_err_cwf", 0))
1166                     self._l7_data[u"server"][u"udp"] = dict()
1167                     self._l7_data[u"server"][u"udp"][u"accepted_flows"] = \
1168                         int(self._result.get(u"server_udp_accepts", 0))
1169                     self._l7_data[u"server"][u"udp"][u"closed_flows"] = \
1170                         int(self._result.get(u"server_udp_closed", 0))
1171                     self._l7_data[u"server"][u"udp"][u"tx_bytes"] = \
1172                         int(self._result.get(u"server_udp_tx_bytes", 0))
1173                     self._l7_data[u"server"][u"udp"][u"rx_bytes"] = \
1174                         int(self._result.get(u"server_udp_rx_bytes", 0))
1175                     self._l7_data[u"server"][u"udp"][u"tx_packets"] = \
1176                         int(self._result.get(u"server_udp_tx_packets", 0))
1177                     self._l7_data[u"server"][u"udp"][u"rx_packets"] = \
1178                         int(self._result.get(u"server_udp_rx_packets", 0))
1179                 elif u"tcp" in self.traffic_profile:
1180                     self._l7_data[u"client"][u"tcp"] = dict()
1181                     self._l7_data[u"client"][u"tcp"][u"initiated_flows"] = \
1182                         int(self._result.get(u"client_tcp_connect_inits", 0))
1183                     self._l7_data[u"client"][u"tcp"][u"connects"] = \
1184                         int(self._result.get(u"client_tcp_connects", 0))
1185                     self._l7_data[u"client"][u"tcp"][u"closed_flows"] = \
1186                         int(self._result.get(u"client_tcp_closed", 0))
1187                     self._l7_data[u"client"][u"tcp"][u"connattempt"] = \
1188                         int(self._result.get(u"client_tcp_connattempt", 0))
1189                     self._l7_data[u"client"][u"tcp"][u"tx_bytes"] = \
1190                         int(self._result.get(u"client_tcp_tx_bytes", 0))
1191                     self._l7_data[u"client"][u"tcp"][u"rx_bytes"] = \
1192                         int(self._result.get(u"client_tcp_rx_bytes", 0))
1193                     self._l7_data[u"server"][u"tcp"] = dict()
1194                     self._l7_data[u"server"][u"tcp"][u"accepted_flows"] = \
1195                         int(self._result.get(u"server_tcp_accepts", 0))
1196                     self._l7_data[u"server"][u"tcp"][u"connects"] = \
1197                         int(self._result.get(u"server_tcp_connects", 0))
1198                     self._l7_data[u"server"][u"tcp"][u"closed_flows"] = \
1199                         int(self._result.get(u"server_tcp_closed", 0))
1200                     self._l7_data[u"server"][u"tcp"][u"tx_bytes"] = \
1201                         int(self._result.get(u"server_tcp_tx_bytes", 0))
1202                     self._l7_data[u"server"][u"tcp"][u"rx_bytes"] = \
1203                         int(self._result.get(u"server_tcp_rx_bytes", 0))
1204
1205     def _get_measurement_result(self):
1206         """Return the result of last measurement as ReceiveRateMeasurement.
1207
1208         Separate function, as measurements can end either by time
1209         or by explicit call, this is the common block at the end.
1210
1211         The target_tr field of ReceiveRateMeasurement is in
1212         transactions per second. Transmit count and loss count units
1213         depend on the transaction type. Usually they are in transactions
1214         per second, or aggregate packets per second.
1215
1216         TODO: Fail on running or already reported measurement.
1217
1218         :returns: Structure containing the result of the measurement.
1219         :rtype: ReceiveRateMeasurement
1220         """
1221         try:
1222             # Client duration seems to include a setup period
1223             # where TRex does not send any packets yet.
1224             # Server duration does not include it.
1225             server_data = self._l7_data[u"server"]
1226             approximated_duration = float(server_data[u"traffic_duration"])
1227         except (KeyError, AttributeError, ValueError, TypeError):
1228             approximated_duration = None
1229         try:
1230             if not approximated_duration:
1231                 approximated_duration = float(self._approximated_duration)
1232         except ValueError:  # "manual"
1233             approximated_duration = None
1234         if not approximated_duration:
1235             if self._duration and self._duration > 0:
1236                 # Known recomputed or target duration.
1237                 approximated_duration = self._duration
1238             else:
1239                 # It was an explicit stop.
1240                 if not self._stop_time:
1241                     raise RuntimeError(u"Unable to determine duration.")
1242                 approximated_duration = self._stop_time - self._start_time
1243         target_duration = self._target_duration
1244         if not target_duration:
1245             target_duration = approximated_duration
1246         transmit_rate = self._rate
1247         if self.transaction_type == u"packet":
1248             partial_attempt_count = self._sent
1249             expected_attempt_count = self._sent
1250             fail_count = self._loss
1251         elif self.transaction_type == u"udp_cps":
1252             if not self.transaction_scale:
1253                 raise RuntimeError(u"Add support for no-limit udp_cps.")
1254             partial_attempt_count = self._l7_data[u"client"][u"sent"]
1255             # We do not care whether TG is slow, it should have attempted all.
1256             expected_attempt_count = self.transaction_scale
1257             pass_count = self._l7_data[u"client"][u"received"]
1258             fail_count = expected_attempt_count - pass_count
1259         elif self.transaction_type == u"tcp_cps":
1260             if not self.transaction_scale:
1261                 raise RuntimeError(u"Add support for no-limit tcp_cps.")
1262             ctca = self._l7_data[u"client"][u"tcp"][u"connattempt"]
1263             partial_attempt_count = ctca
1264             # We do not care whether TG is slow, it should have attempted all.
1265             expected_attempt_count = self.transaction_scale
1266             # From TCP point of view, server/connects counts full connections,
1267             # but we are testing NAT session so client/connects counts that
1268             # (half connections from TCP point of view).
1269             pass_count = self._l7_data[u"client"][u"tcp"][u"connects"]
1270             fail_count = expected_attempt_count - pass_count
1271         elif self.transaction_type == u"udp_pps":
1272             if not self.transaction_scale:
1273                 raise RuntimeError(u"Add support for no-limit udp_pps.")
1274             partial_attempt_count = self._sent
1275             expected_attempt_count = self.transaction_scale * self.ppta
1276             fail_count = self._loss + (expected_attempt_count - self._sent)
1277         elif self.transaction_type == u"tcp_pps":
1278             if not self.transaction_scale:
1279                 raise RuntimeError(u"Add support for no-limit tcp_pps.")
1280             partial_attempt_count = self._sent
1281             expected_attempt_count = self.transaction_scale * self.ppta
1282             # One loss-like scenario happens when TRex receives all packets
1283             # on L2 level, but is not fast enough to process them all
1284             # at L7 level, which leads to retransmissions.
1285             # Those manifest as opackets larger than expected.
1286             # A simple workaround is to add absolute difference.
1287             # Probability of retransmissions exactly cancelling
1288             # packets unsent due to duration stretching is quite low.
1289             fail_count = self._loss + abs(expected_attempt_count - self._sent)
1290         else:
1291             raise RuntimeError(f"Unknown parsing {self.transaction_type!r}")
1292         if fail_count < 0 and not self.negative_loss:
1293             fail_count = 0
1294         measurement = ReceiveRateMeasurement(
1295             duration=target_duration,
1296             target_tr=transmit_rate,
1297             transmit_count=expected_attempt_count,
1298             loss_count=fail_count,
1299             approximated_duration=approximated_duration,
1300             partial_transmit_count=partial_attempt_count,
1301         )
1302         measurement.latency = self.get_latency_int()
1303         return measurement
1304
1305     def measure(self, duration, transmit_rate):
1306         """Run trial measurement, parse and return results.
1307
1308         The input rate is for transactions. Stateles bidirectional traffic
1309         is understood as sequence of (asynchronous) transactions,
1310         two packets each.
1311
1312         The result units depend on test type, generally
1313         the count either transactions or packets (aggregated over directions).
1314
1315         Optionally, this method sleeps if measurement finished before
1316         the time specified as duration.
1317
1318         :param duration: Trial duration [s].
1319         :param transmit_rate: Target rate in transactions per second.
1320         :type duration: float
1321         :type transmit_rate: float
1322         :returns: Structure containing the result of the measurement.
1323         :rtype: ReceiveRateMeasurement
1324         :raises RuntimeError: If TG is not set or if node is not TG
1325             or if subtype is not specified.
1326         :raises NotImplementedError: If TG is not supported.
1327         """
1328         duration = float(duration)
1329         time_start = time.monotonic()
1330         time_stop = time_start + duration
1331         if self.resetter:
1332             self.resetter()
1333         result = self._send_traffic_on_tg_with_ramp_up(
1334             duration=duration,
1335             rate=transmit_rate,
1336             async_call=False,
1337         )
1338         logger.debug(f"trial measurement result: {result!r}")
1339         # In PLRsearch, computation needs the specified time to complete.
1340         if self.sleep_till_duration:
1341             sleeptime = time_stop - time.monotonic()
1342             if sleeptime > 0.0:
1343                 # TODO: Sometimes we have time to do additional trials here,
1344                 # adapt PLRsearch to accept all the results.
1345                 time.sleep(sleeptime)
1346         return result
1347
1348     def set_rate_provider_defaults(
1349             self,
1350             frame_size,
1351             traffic_profile,
1352             ppta=1,
1353             resetter=None,
1354             traffic_directions=2,
1355             transaction_duration=0.0,
1356             transaction_scale=0,
1357             transaction_type=u"packet",
1358             duration_limit=0.0,
1359             negative_loss=True,
1360             sleep_till_duration=False,
1361             use_latency=False,
1362             ramp_up_rate=None,
1363             ramp_up_duration=None,
1364             state_timeout=300.0,
1365         ):
1366         """Store values accessed by measure().
1367
1368         :param frame_size: Frame size identifier or value [B].
1369         :param traffic_profile: Module name as a traffic profile identifier.
1370             See GPL/traffic_profiles/trex for implemented modules.
1371         :param ppta: Packets per transaction, aggregated over directions.
1372             Needed for udp_pps which does not have a good transaction counter,
1373             so we need to compute expected number of packets.
1374             Default: 1.
1375         :param resetter: Callable to reset DUT state for repeated trials.
1376         :param traffic_directions: Traffic from packet counting point of view
1377             is bi- (2) or uni- (1) directional.
1378             Default: 2
1379         :param transaction_duration: Total expected time to close transaction.
1380         :param transaction_scale: Number of transactions to perform.
1381             0 (default) means unlimited.
1382         :param transaction_type: An identifier specifying which counters
1383             and formulas to use when computing attempted and failed
1384             transactions. Default: "packet".
1385             TODO: Does this also specify parsing for the measured duration?
1386         :param duration_limit: Zero or maximum limit for computed (or given)
1387             duration.
1388         :param negative_loss: If false, negative loss is reported as zero loss.
1389         :param sleep_till_duration: If true and measurement returned faster,
1390             sleep until it matches duration. Needed for PLRsearch.
1391         :param use_latency: Whether to measure latency during the trial.
1392             Default: False.
1393         :param ramp_up_rate: Rate to use in ramp-up trials [pps].
1394         :param ramp_up_duration: Duration of ramp-up trials [s].
1395         :param state_timeout: Time of life of DUT state [s].
1396         :type frame_size: str or int
1397         :type traffic_profile: str
1398         :type ppta: int
1399         :type resetter: Optional[Callable[[], None]]
1400         :type traffic_directions: int
1401         :type transaction_duration: float
1402         :type transaction_scale: int
1403         :type transaction_type: str
1404         :type duration_limit: float
1405         :type negative_loss: bool
1406         :type sleep_till_duration: bool
1407         :type use_latency: bool
1408         :type ramp_up_rate: float
1409         :type ramp_up_duration: float
1410         :type state_timeout: float
1411         """
1412         self.frame_size = frame_size
1413         self.traffic_profile = str(traffic_profile)
1414         self.resetter = resetter
1415         self.ppta = ppta
1416         self.traffic_directions = int(traffic_directions)
1417         self.transaction_duration = float(transaction_duration)
1418         self.transaction_scale = int(transaction_scale)
1419         self.transaction_type = str(transaction_type)
1420         self.duration_limit = float(duration_limit)
1421         self.negative_loss = bool(negative_loss)
1422         self.sleep_till_duration = bool(sleep_till_duration)
1423         self.use_latency = bool(use_latency)
1424         self.ramp_up_rate = float(ramp_up_rate)
1425         self.ramp_up_duration = float(ramp_up_duration)
1426         self.state_timeout = float(state_timeout)
1427
1428
1429 class OptimizedSearch:
1430     """Class to be imported as Robot Library, containing search keywords.
1431
1432     Aside of setting up measurer and forwarding arguments,
1433     the main business is to translate min/max rate from unidir to aggregate.
1434     """
1435
1436     @staticmethod
1437     def perform_optimized_ndrpdr_search(
1438             frame_size,
1439             traffic_profile,
1440             minimum_transmit_rate,
1441             maximum_transmit_rate,
1442             packet_loss_ratio=0.005,
1443             final_relative_width=0.005,
1444             final_trial_duration=30.0,
1445             initial_trial_duration=1.0,
1446             number_of_intermediate_phases=2,
1447             timeout=720.0,
1448             ppta=1,
1449             resetter=None,
1450             traffic_directions=2,
1451             transaction_duration=0.0,
1452             transaction_scale=0,
1453             transaction_type=u"packet",
1454             use_latency=False,
1455             ramp_up_rate=None,
1456             ramp_up_duration=None,
1457             state_timeout=300.0,
1458             expansion_coefficient=4.0,
1459     ):
1460         """Setup initialized TG, perform optimized search, return intervals.
1461
1462         If transaction_scale is nonzero, all init and non-init trial durations
1463         are set to 1.0 (as they do not affect the real trial duration)
1464         and zero intermediate phases are used.
1465         This way no re-measurement happens.
1466         Warmup has to be handled via resetter or ramp-up mechanisms.
1467
1468         :param frame_size: Frame size identifier or value [B].
1469         :param traffic_profile: Module name as a traffic profile identifier.
1470             See GPL/traffic_profiles/trex for implemented modules.
1471         :param minimum_transmit_rate: Minimal load in transactions per second.
1472         :param maximum_transmit_rate: Maximal load in transactions per second.
1473         :param packet_loss_ratio: Ratio of packets lost, for PDR [1].
1474         :param final_relative_width: Final lower bound transmit rate
1475             cannot be more distant that this multiple of upper bound [1].
1476         :param final_trial_duration: Trial duration for the final phase [s].
1477         :param initial_trial_duration: Trial duration for the initial phase
1478             and also for the first intermediate phase [s].
1479         :param number_of_intermediate_phases: Number of intermediate phases
1480             to perform before the final phase [1].
1481         :param timeout: The search will fail itself when not finished
1482             before this overall time [s].
1483         :param ppta: Packets per transaction, aggregated over directions.
1484             Needed for udp_pps which does not have a good transaction counter,
1485             so we need to compute expected number of packets.
1486             Default: 1.
1487         :param resetter: Callable to reset DUT state for repeated trials.
1488         :param traffic_directions: Traffic is bi- (2) or uni- (1) directional.
1489             Default: 2
1490         :param transaction_duration: Total expected time to close transaction.
1491         :param transaction_scale: Number of transactions to perform.
1492             0 (default) means unlimited.
1493         :param transaction_type: An identifier specifying which counters
1494             and formulas to use when computing attempted and failed
1495             transactions. Default: "packet".
1496         :param use_latency: Whether to measure latency during the trial.
1497             Default: False.
1498         :param ramp_up_rate: Rate to use in ramp-up trials [pps].
1499         :param ramp_up_duration: Duration of ramp-up trials [s].
1500         :param state_timeout: Time of life of DUT state [s].
1501         :param expansion_coefficient: In external search multiply width by this.
1502         :type frame_size: str or int
1503         :type traffic_profile: str
1504         :type minimum_transmit_rate: float
1505         :type maximum_transmit_rate: float
1506         :type packet_loss_ratio: float
1507         :type final_relative_width: float
1508         :type final_trial_duration: float
1509         :type initial_trial_duration: float
1510         :type number_of_intermediate_phases: int
1511         :type timeout: float
1512         :type ppta: int
1513         :type resetter: Optional[Callable[[], None]]
1514         :type traffic_directions: int
1515         :type transaction_duration: float
1516         :type transaction_scale: int
1517         :type transaction_type: str
1518         :type use_latency: bool
1519         :type ramp_up_rate: float
1520         :type ramp_up_duration: float
1521         :type state_timeout: float
1522         :type expansion_coefficient: float
1523         :returns: Structure containing narrowed down NDR and PDR intervals
1524             and their measurements.
1525         :rtype: List[Receiverateinterval]
1526         :raises RuntimeError: If total duration is larger than timeout.
1527         """
1528         # we need instance of TrafficGenerator instantiated by Robot Framework
1529         # to be able to use trex_stl-*()
1530         tg_instance = BuiltIn().get_library_instance(
1531             u"resources.libraries.python.TrafficGenerator"
1532         )
1533         # Overrides for fixed transaction amount.
1534         # TODO: Move to robot code? We have two call sites, so this saves space,
1535         #       even though this is surprising for log readers.
1536         if transaction_scale:
1537             initial_trial_duration = 1.0
1538             final_trial_duration = 1.0
1539             number_of_intermediate_phases = 0
1540             timeout += transaction_scale * 3e-4
1541         tg_instance.set_rate_provider_defaults(
1542             frame_size=frame_size,
1543             traffic_profile=traffic_profile,
1544             sleep_till_duration=False,
1545             ppta=ppta,
1546             resetter=resetter,
1547             traffic_directions=traffic_directions,
1548             transaction_duration=transaction_duration,
1549             transaction_scale=transaction_scale,
1550             transaction_type=transaction_type,
1551             use_latency=use_latency,
1552             ramp_up_rate=ramp_up_rate,
1553             ramp_up_duration=ramp_up_duration,
1554             state_timeout=state_timeout,
1555         )
1556         algorithm = MultipleLossRatioSearch(
1557             measurer=tg_instance,
1558             final_trial_duration=final_trial_duration,
1559             final_relative_width=final_relative_width,
1560             number_of_intermediate_phases=number_of_intermediate_phases,
1561             initial_trial_duration=initial_trial_duration,
1562             timeout=timeout,
1563             debug=logger.debug,
1564             expansion_coefficient=expansion_coefficient,
1565         )
1566         if packet_loss_ratio:
1567             packet_loss_ratios = [0.0, packet_loss_ratio]
1568         else:
1569             # Happens in reconf tests.
1570             packet_loss_ratios = [packet_loss_ratio]
1571         results = algorithm.narrow_down_intervals(
1572             min_rate=minimum_transmit_rate,
1573             max_rate=maximum_transmit_rate,
1574             packet_loss_ratios=packet_loss_ratios,
1575         )
1576         return results
1577
1578     @staticmethod
1579     def perform_soak_search(
1580             frame_size,
1581             traffic_profile,
1582             minimum_transmit_rate,
1583             maximum_transmit_rate,
1584             plr_target=1e-7,
1585             tdpt=0.1,
1586             initial_count=50,
1587             timeout=7200.0,
1588             ppta=1,
1589             resetter=None,
1590             trace_enabled=False,
1591             traffic_directions=2,
1592             transaction_duration=0.0,
1593             transaction_scale=0,
1594             transaction_type=u"packet",
1595             use_latency=False,
1596             ramp_up_rate=None,
1597             ramp_up_duration=None,
1598             state_timeout=300.0,
1599     ):
1600         """Setup initialized TG, perform soak search, return avg and stdev.
1601
1602         :param frame_size: Frame size identifier or value [B].
1603         :param traffic_profile: Module name as a traffic profile identifier.
1604             See GPL/traffic_profiles/trex for implemented modules.
1605         :param minimum_transmit_rate: Minimal load in transactions per second.
1606         :param maximum_transmit_rate: Maximal load in transactions per second.
1607         :param plr_target: Ratio of packets lost to achieve [1].
1608         :param tdpt: Trial duration per trial.
1609             The algorithm linearly increases trial duration with trial number,
1610             this is the increment between succesive trials, in seconds.
1611         :param initial_count: Offset to apply before the first trial.
1612             For example initial_count=50 makes first trial to be 51*tdpt long.
1613             This is needed because initial "search" phase of integrator
1614             takes significant time even without any trial results.
1615         :param timeout: The search will stop after this overall time [s].
1616         :param ppta: Packets per transaction, aggregated over directions.
1617             Needed for udp_pps which does not have a good transaction counter,
1618             so we need to compute expected number of packets.
1619             Default: 1.
1620         :param resetter: Callable to reset DUT state for repeated trials.
1621         :param trace_enabled: True if trace enabled else False.
1622             This is very verbose tracing on numeric computations,
1623             do not use in production.
1624             Default: False
1625         :param traffic_directions: Traffic is bi- (2) or uni- (1) directional.
1626             Default: 2
1627         :param transaction_duration: Total expected time to close transaction.
1628         :param transaction_scale: Number of transactions to perform.
1629             0 (default) means unlimited.
1630         :param transaction_type: An identifier specifying which counters
1631             and formulas to use when computing attempted and failed
1632             transactions. Default: "packet".
1633         :param use_latency: Whether to measure latency during the trial.
1634             Default: False.
1635         :param ramp_up_rate: Rate to use in ramp-up trials [pps].
1636         :param ramp_up_duration: Duration of ramp-up trials [s].
1637         :param state_timeout: Time of life of DUT state [s].
1638         :type frame_size: str or int
1639         :type traffic_profile: str
1640         :type minimum_transmit_rate: float
1641         :type maximum_transmit_rate: float
1642         :type plr_target: float
1643         :type initial_count: int
1644         :type timeout: float
1645         :type ppta: int
1646         :type resetter: Optional[Callable[[], None]]
1647         :type trace_enabled: bool
1648         :type traffic_directions: int
1649         :type transaction_duration: float
1650         :type transaction_scale: int
1651         :type transaction_type: str
1652         :type use_latency: bool
1653         :type ramp_up_rate: float
1654         :type ramp_up_duration: float
1655         :type state_timeout: float
1656         :returns: Average and stdev of estimated aggregate rate giving PLR.
1657         :rtype: 2-tuple of float
1658         """
1659         tg_instance = BuiltIn().get_library_instance(
1660             u"resources.libraries.python.TrafficGenerator"
1661         )
1662         # Overrides for fixed transaction amount.
1663         # TODO: Move to robot code? We have a single call site
1664         #       but MLRsearch has two and we want the two to be used similarly.
1665         if transaction_scale:
1666             # TODO: What is a good value for max scale?
1667             # TODO: Scale the timeout with transaction scale.
1668             timeout = 7200.0
1669         tg_instance.set_rate_provider_defaults(
1670             frame_size=frame_size,
1671             traffic_profile=traffic_profile,
1672             negative_loss=False,
1673             sleep_till_duration=True,
1674             ppta=ppta,
1675             resetter=resetter,
1676             traffic_directions=traffic_directions,
1677             transaction_duration=transaction_duration,
1678             transaction_scale=transaction_scale,
1679             transaction_type=transaction_type,
1680             use_latency=use_latency,
1681             ramp_up_rate=ramp_up_rate,
1682             ramp_up_duration=ramp_up_duration,
1683             state_timeout=state_timeout,
1684         )
1685         algorithm = PLRsearch(
1686             measurer=tg_instance,
1687             trial_duration_per_trial=tdpt,
1688             packet_loss_ratio_target=plr_target,
1689             trial_number_offset=initial_count,
1690             timeout=timeout,
1691             trace_enabled=trace_enabled,
1692         )
1693         result = algorithm.search(
1694             min_rate=minimum_transmit_rate,
1695             max_rate=maximum_transmit_rate,
1696         )
1697         return result