Add: Use containers for shared TG

[csit.git] / resources / libraries / python / TrafficGenerator.py

diff --git a/resources/libraries/python/TrafficGenerator.py b/resources/libraries/python/TrafficGenerator.py
index 36a8385..e93da04 100644 (file)
--- a/resources/libraries/python/TrafficGenerator.py
+++ b/resources/libraries/python/TrafficGenerator.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2018 Cisco and/or its affiliates.
+# Copyright (c) 2019 Cisco and/or its affiliates.

 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:


@@ -13,22 +13,50 @@
 
 """Performance testing traffic generator library."""
 
 
 """Performance testing traffic generator library."""
 

+import time
+

 from robot.api import logger
 from robot.libraries.BuiltIn import BuiltIn
 
 from .DropRateSearch import DropRateSearch
 from robot.api import logger
 from robot.libraries.BuiltIn import BuiltIn
 
 from .DropRateSearch import DropRateSearch

-from .constants import Constants
-from .ssh import SSH
+from .Constants import Constants
+from .ssh import SSH, exec_cmd_no_error

 from .topology import NodeType
 from .topology import NodeSubTypeTG
 from .topology import Topology
 from .MLRsearch.AbstractMeasurer import AbstractMeasurer
 from .MLRsearch.MultipleLossRatioSearch import MultipleLossRatioSearch
 from .MLRsearch.ReceiveRateMeasurement import ReceiveRateMeasurement
 from .topology import NodeType
 from .topology import NodeSubTypeTG
 from .topology import Topology
 from .MLRsearch.AbstractMeasurer import AbstractMeasurer
 from .MLRsearch.MultipleLossRatioSearch import MultipleLossRatioSearch
 from .MLRsearch.ReceiveRateMeasurement import ReceiveRateMeasurement

+from .PLRsearch.PLRsearch import PLRsearch

 
 __all__ = ['TGDropRateSearchImpl', 'TrafficGenerator', 'OptimizedSearch']
 
 
 
 __all__ = ['TGDropRateSearchImpl', 'TrafficGenerator', 'OptimizedSearch']
 
 

+def check_subtype(node):
+    """Return supported subtype of given node, or raise an exception.
+
+    Currently only one subtype is supported,
+    but we want our code to be ready for other ones.
+
+    :param node: Topology node to check. Can be None.
+    :type node: dict or NoneType
+    :returns: Subtype detected.
+    :rtype: NodeSubTypeTG
+    :raises RuntimeError: If node is not supported, message explains how.
+    """
+    if node.get('type') is None:
+        raise RuntimeError('Node type is not defined')
+    elif node['type'] != NodeType.TG:
+        raise RuntimeError('Node type is {typ!r}, not a TG'.format(
+            typ=node['type']))
+    elif node.get('subtype') is None:
+        raise RuntimeError('TG subtype is not defined')
+    elif node['subtype'] == NodeSubTypeTG.TREX:
+        return NodeSubTypeTG.TREX
+    raise RuntimeError('TG subtype {sub!r} is not supported'.format(
+        sub=node['subtype']))
+
+

 class TGDropRateSearchImpl(DropRateSearch):
     """Drop Rate Search implementation."""
 
 class TGDropRateSearchImpl(DropRateSearch):
     """Drop Rate Search implementation."""
 


@@ -36,21 +64,21 @@ class TGDropRateSearchImpl(DropRateSearch):
         super(TGDropRateSearchImpl, self).__init__()
 
     def measure_loss(self, rate, frame_size, loss_acceptance,
         super(TGDropRateSearchImpl, self).__init__()
 
     def measure_loss(self, rate, frame_size, loss_acceptance,

-                     loss_acceptance_type, traffic_type, skip_warmup=False):
+                     loss_acceptance_type, traffic_profile, skip_warmup=False):

         """Runs the traffic and evaluate the measured results.
 
         :param rate: Offered traffic load.
         :param frame_size: Size of frame.
         :param loss_acceptance: Permitted drop ratio or frames count.
         :param loss_acceptance_type: Type of permitted loss.
         """Runs the traffic and evaluate the measured results.
 
         :param rate: Offered traffic load.
         :param frame_size: Size of frame.
         :param loss_acceptance: Permitted drop ratio or frames count.
         :param loss_acceptance_type: Type of permitted loss.

-        :param traffic_type: Module name as a traffic type identifier.
+        :param traffic_profile: Module name as a traffic profile identifier.

             See resources/traffic_profiles/trex for implemented modules.
         :param skip_warmup: Start TRex without warmup traffic if true.
         :type rate: float
         :type frame_size: str
         :type loss_acceptance: float
         :type loss_acceptance_type: LossAcceptanceType
             See resources/traffic_profiles/trex for implemented modules.
         :param skip_warmup: Start TRex without warmup traffic if true.
         :type rate: float
         :type frame_size: str
         :type loss_acceptance: float
         :type loss_acceptance_type: LossAcceptanceType

-        :type traffic_type: str
+        :type traffic_profile: str

         :type skip_warmup: bool
         :returns: Drop threshold exceeded? (True/False)
         :rtype: bool
         :type skip_warmup: bool
         :returns: Drop threshold exceeded? (True/False)
         :rtype: bool


@@ -61,34 +89,24 @@ class TGDropRateSearchImpl(DropRateSearch):
         # to be able to use trex_stl-*()
         tg_instance = BuiltIn().get_library_instance(
             'resources.libraries.python.TrafficGenerator')
         # to be able to use trex_stl-*()
         tg_instance = BuiltIn().get_library_instance(
             'resources.libraries.python.TrafficGenerator')

-
-        if tg_instance.node['subtype'] is None:
-            raise RuntimeError('TG subtype not defined')
-        elif tg_instance.node['subtype'] == NodeSubTypeTG.TREX:
+        subtype = check_subtype(tg_instance.node)
+        if subtype == NodeSubTypeTG.TREX:

             unit_rate = str(rate) + self.get_rate_type_str()
             if skip_warmup:
             unit_rate = str(rate) + self.get_rate_type_str()
             if skip_warmup:

-                tg_instance.trex_stl_start_remote_exec(self.get_duration(),
-                                                       unit_rate, frame_size,
-                                                       traffic_type,
-                                                       warmup_time=0.0)
+                tg_instance.trex_stl_start_remote_exec(
+                    self.get_duration(), unit_rate, frame_size, traffic_profile,
+                    warmup_time=0.0)

             else:
             else:

-                tg_instance.trex_stl_start_remote_exec(self.get_duration(),
-                                                       unit_rate, frame_size,
-                                                       traffic_type)
+                tg_instance.trex_stl_start_remote_exec(
+                    self.get_duration(), unit_rate, frame_size, traffic_profile)

             loss = tg_instance.get_loss()
             sent = tg_instance.get_sent()
             if self.loss_acceptance_type_is_percentage():
                 loss = (float(loss) / float(sent)) * 100
             loss = tg_instance.get_loss()
             sent = tg_instance.get_sent()
             if self.loss_acceptance_type_is_percentage():
                 loss = (float(loss) / float(sent)) * 100

-
-            logger.trace("comparing: {} < {} {}".format(loss,
-                                                        loss_acceptance,
-                                                        loss_acceptance_type))
-            if float(loss) > float(loss_acceptance):
-                return False
-            else:
-                return True
-        else:
-            raise NotImplementedError("TG subtype not supported")
+            logger.trace("comparing: {los} < {acc} {typ}".format(
+                los=loss, acc=loss_acceptance, typ=loss_acceptance_type))
+            return float(loss) <= float(loss_acceptance)
+        return False

 
     def get_latency(self):
         """Returns min/avg/max latency.
 
     def get_latency(self):
         """Returns min/avg/max latency.


@@ -113,18 +131,25 @@ class TrafficGenerator(AbstractMeasurer):
     ROBOT_LIBRARY_SCOPE = 'TEST SUITE'
 
     def __init__(self):
     ROBOT_LIBRARY_SCOPE = 'TEST SUITE'
 
     def __init__(self):

+        # TODO: Number of fields will be reduced with CSIT-1378.
+        self._node = None
+        # T-REX interface order mapping
+        self._ifaces_reordered = False
+        # Result holding fields, to be removed.

         self._result = None
         self._loss = None
         self._sent = None
         self._latency = None
         self._received = None
         self._result = None
         self._loss = None
         self._sent = None
         self._latency = None
         self._received = None

-        self._node = None
-        # T-REX interface order mapping
-        self._ifaces_reordered = False
-        # Parameters not given by measure().
+        # Measurement input fields, needed for async stop result.
+        self._start_time = None
+        self._rate = None
+        # Other input parameters, not knowable from measure() signature.

         self.frame_size = None
         self.frame_size = None

-        self.traffic_type = None
+        self.traffic_profile = None

         self.warmup_time = None
         self.warmup_time = None

+        # TODO: Rename "xstats" to something opaque, so TRex is not privileged?
+        self._xstats = (None, None)

 
     @property
     def node(self):
 
     @property
     def node(self):


@@ -167,13 +192,14 @@ class TrafficGenerator(AbstractMeasurer):
         """
         return self._latency
 
         """
         return self._latency
 

-    def initialize_traffic_generator(self, tg_node, tg_if1, tg_if2,
-                                     tg_if1_adj_node, tg_if1_adj_if,
-                                     tg_if2_adj_node, tg_if2_adj_if,
-                                     test_type,
-                                     tg_if1_dst_mac=None, tg_if2_dst_mac=None):
+    def initialize_traffic_generator(
+            self, tg_node, tg_if1, tg_if2, tg_if1_adj_node, tg_if1_adj_if,
+            tg_if2_adj_node, tg_if2_adj_if, osi_layer, tg_if1_dst_mac=None,
+            tg_if2_dst_mac=None):

         """TG initialization.
 
         """TG initialization.
 

+        TODO: Document why do we need (and how do we use) _ifaces_reordered.
+

         :param tg_node: Traffic generator node.
         :param tg_if1: TG - name of first interface.
         :param tg_if2: TG - name of second interface.
         :param tg_node: Traffic generator node.
         :param tg_if1: TG - name of first interface.
         :param tg_if2: TG - name of second interface.


@@ -181,7 +207,7 @@ class TrafficGenerator(AbstractMeasurer):
         :param tg_if1_adj_if: TG if1 adjecent interface.
         :param tg_if2_adj_node: TG if2 adjecent node.
         :param tg_if2_adj_if: TG if2 adjecent interface.
         :param tg_if1_adj_if: TG if1 adjecent interface.
         :param tg_if2_adj_node: TG if2 adjecent node.
         :param tg_if2_adj_if: TG if2 adjecent interface.

-        :param test_type: 'L2', 'L3' or 'L7' - OSI Layer testing type.
+        :param osi_layer: 'L2', 'L3' or 'L7' - OSI Layer testing type.

         :param tg_if1_dst_mac: Interface 1 destination MAC address.
         :param tg_if2_dst_mac: Interface 2 destination MAC address.
         :type tg_node: dict
         :param tg_if1_dst_mac: Interface 1 destination MAC address.
         :param tg_if2_dst_mac: Interface 2 destination MAC address.
         :type tg_node: dict


@@ -191,19 +217,18 @@ class TrafficGenerator(AbstractMeasurer):
         :type tg_if1_adj_if: str
         :type tg_if2_adj_node: dict
         :type tg_if2_adj_if: str
         :type tg_if1_adj_if: str
         :type tg_if2_adj_node: dict
         :type tg_if2_adj_if: str

-        :type test_type: str
+        :type osi_layer: str

         :type tg_if1_dst_mac: str
         :type tg_if2_dst_mac: str
         :returns: nothing
         :raises RuntimeError: In case of issue during initialization.
         """
         :type tg_if1_dst_mac: str
         :type tg_if2_dst_mac: str
         :returns: nothing
         :raises RuntimeError: In case of issue during initialization.
         """

-        if tg_node['type'] != NodeType.TG:
-            raise RuntimeError('Node type is not a TG')
-        self._node = tg_node
+        subtype = check_subtype(tg_node)
+        if subtype == NodeSubTypeTG.TREX:
+            self._node = tg_node

 
 

-        if tg_node['subtype'] == NodeSubTypeTG.TREX:

             ssh = SSH()
             ssh = SSH()

-            ssh.connect(tg_node)
+            ssh.connect(self._node)

 
             (ret, _, _) = ssh.exec_command(
                 "sudo -E sh -c '{0}/resources/tools/trex/"
 
             (ret, _, _) = ssh.exec_command(
                 "sudo -E sh -c '{0}/resources/tools/trex/"


@@ -213,22 +238,22 @@ class TrafficGenerator(AbstractMeasurer):
             if int(ret) != 0:
                 raise RuntimeError('TRex installation failed.')
 
             if int(ret) != 0:
                 raise RuntimeError('TRex installation failed.')
 

-            if1_pci = Topology().get_interface_pci_addr(tg_node, tg_if1)
-            if2_pci = Topology().get_interface_pci_addr(tg_node, tg_if2)
-            if1_addr = Topology().get_interface_mac(tg_node, tg_if1)
-            if2_addr = Topology().get_interface_mac(tg_node, tg_if2)
+            if1_pci = Topology().get_interface_pci_addr(self._node, tg_if1)
+            if2_pci = Topology().get_interface_pci_addr(self._node, tg_if2)
+            if1_addr = Topology().get_interface_mac(self._node, tg_if1)
+            if2_addr = Topology().get_interface_mac(self._node, tg_if2)

 
 

-            if test_type == 'L2':
+            if osi_layer == 'L2':

                 if1_adj_addr = if2_addr
                 if2_adj_addr = if1_addr
                 if1_adj_addr = if2_addr
                 if2_adj_addr = if1_addr

-            elif test_type == 'L3':
+            elif osi_layer == 'L3':

                 if1_adj_addr = Topology().get_interface_mac(tg_if1_adj_node,
                                                             tg_if1_adj_if)
                 if2_adj_addr = Topology().get_interface_mac(tg_if2_adj_node,
                                                             tg_if2_adj_if)
                 if1_adj_addr = Topology().get_interface_mac(tg_if1_adj_node,
                                                             tg_if1_adj_if)
                 if2_adj_addr = Topology().get_interface_mac(tg_if2_adj_node,
                                                             tg_if2_adj_if)

-            elif test_type == 'L7':
-                if1_addr = Topology().get_interface_ip4(tg_node, tg_if1)
-                if2_addr = Topology().get_interface_ip4(tg_node, tg_if2)
+            elif osi_layer == 'L7':
+                if1_addr = Topology().get_interface_ip4(self._node, tg_if1)
+                if2_addr = Topology().get_interface_ip4(self._node, tg_if2)

                 if1_adj_addr = Topology().get_interface_ip4(tg_if1_adj_node,
                                                             tg_if1_adj_if)
                 if2_adj_addr = Topology().get_interface_ip4(tg_if2_adj_node,
                 if1_adj_addr = Topology().get_interface_ip4(tg_if1_adj_node,
                                                             tg_if1_adj_if)
                 if2_adj_addr = Topology().get_interface_ip4(tg_if2_adj_node,


@@ -247,11 +272,10 @@ class TrafficGenerator(AbstractMeasurer):
                 if1_adj_addr, if2_adj_addr = if2_adj_addr, if1_adj_addr
                 self._ifaces_reordered = True
 
                 if1_adj_addr, if2_adj_addr = if2_adj_addr, if1_adj_addr
                 self._ifaces_reordered = True
 

-            if test_type == 'L2' or test_type == 'L3':
+            if osi_layer == 'L2' or osi_layer == 'L3':

                 (ret, _, _) = ssh.exec_command(
                     "sudo sh -c 'cat << EOF > /etc/trex_cfg.yaml\n"
                 (ret, _, _) = ssh.exec_command(
                     "sudo sh -c 'cat << EOF > /etc/trex_cfg.yaml\n"

-                    "- port_limit: 2\n"
-                    "  version: 2\n"
+                    "- version: 2\n"

                     "  interfaces: [\"{0}\",\"{1}\"]\n"
                     "  port_info:\n"
                     "      - dest_mac: [{2}]\n"
                     "  interfaces: [\"{0}\",\"{1}\"]\n"
                     "  port_info:\n"
                     "      - dest_mac: [{2}]\n"


@@ -264,11 +288,10 @@ class TrafficGenerator(AbstractMeasurer):
                             "0x"+if1_addr.replace(":", ",0x"),
                             "0x"+if2_adj_addr.replace(":", ",0x"),
                             "0x"+if2_addr.replace(":", ",0x")))
                             "0x"+if1_addr.replace(":", ",0x"),
                             "0x"+if2_adj_addr.replace(":", ",0x"),
                             "0x"+if2_addr.replace(":", ",0x")))

-            elif test_type == 'L7':
+            elif osi_layer == 'L7':

                 (ret, _, _) = ssh.exec_command(
                     "sudo sh -c 'cat << EOF > /etc/trex_cfg.yaml\n"
                 (ret, _, _) = ssh.exec_command(
                     "sudo sh -c 'cat << EOF > /etc/trex_cfg.yaml\n"

-                    "- port_limit: 2\n"
-                    "  version: 2\n"
+                    "- version: 2\n"

                     "  interfaces: [\"{0}\",\"{1}\"]\n"
                     "  port_info:\n"
                     "      - ip: [{2}]\n"
                     "  interfaces: [\"{0}\",\"{1}\"]\n"
                     "  port_info:\n"
                     "      - ip: [{2}]\n"


@@ -284,49 +307,66 @@ class TrafficGenerator(AbstractMeasurer):
             if int(ret) != 0:
                 raise RuntimeError('TRex config generation error')
 
             if int(ret) != 0:
                 raise RuntimeError('TRex config generation error')
 

-            for _ in range(0, 3):
-                # kill TRex only if it is already running
-                ssh.exec_command(
-                    "sh -c 'pgrep t-rex && sudo pkill t-rex && sleep 3'")
+            self._startup_trex(osi_layer)

 
 

-                # configure TRex
-                (ret, _, _) = ssh.exec_command(
-                    "sh -c 'cd {0}/scripts/ && sudo ./trex-cfg'"\
-                    .format(Constants.TREX_INSTALL_DIR))
-                if int(ret) != 0:
-                    raise RuntimeError('trex-cfg failed')
-
-                # start TRex
-                if test_type == 'L2' or test_type == 'L3':
-                    (ret, _, _) = ssh.exec_command(
-                        "sh -c 'cd {0}/scripts/ && "
-                        "sudo nohup ./t-rex-64 -i -c 7 --iom 0 > /tmp/trex.log "
-                        "2>&1 &' > /dev/null"\
-                        .format(Constants.TREX_INSTALL_DIR))
-                elif test_type == 'L7':
-                    (ret, _, _) = ssh.exec_command(
-                        "sh -c 'cd {0}/scripts/ && "
-                        "sudo nohup ./t-rex-64 --astf -i -c 7 --iom 0 > "
-                        "/tmp/trex.log 2>&1 &' > /dev/null"\
-                        .format(Constants.TREX_INSTALL_DIR))
-                else:
-                    raise ValueError("Unknown Test Type")
-                if int(ret) != 0:
-                    ssh.exec_command("sh -c 'cat /tmp/trex.log'")
-                    raise RuntimeError('t-rex-64 startup failed')
-
-                # get TRex server info
-                (ret, _, _) = ssh.exec_command(
-                    "sh -c 'sleep 3; "
-                    "{0}/resources/tools/trex/trex_server_info.py'"\
-                    .format(Constants.REMOTE_FW_DIR),
-                    timeout=120)
-                if int(ret) == 0:
-                    # If we get info TRex is running
-                    return
-            # after max retries TRex is still not responding to API
-            # critical error occurred
-            raise RuntimeError('t-rex-64 startup failed')
+    def _startup_trex(self, osi_layer):
+        """Startup sequence for the TRex traffic generator.
+
+        :param osi_layer: 'L2', 'L3' or 'L7' - OSI Layer testing type.
+        :type osi_layer: str
+        :raises RuntimeError: If node subtype is not a TREX or startup failed.
+        """
+        # No need to check subtype, we know it is TREX.
+        for _ in range(0, 3):
+            # Kill TRex only if it is already running.
+            cmd = "sh -c 'pgrep t-rex && pkill t-rex && sleep 3 || true'"
+            exec_cmd_no_error(
+                self._node, cmd, sudo=True, message='Kill TRex failed!')
+
+            # Configure TRex.
+            ports = ''
+            for port in self._node['interfaces'].values():
+                ports += ' {pci}'.format(pci=port.get('pci_address'))
+
+            cmd = ("sh -c 'cd {dir}/scripts/ && "
+                   "./dpdk_nic_bind.py -u {ports} || true'"
+                   .format(dir=Constants.TREX_INSTALL_DIR, ports=ports))
+            exec_cmd_no_error(
+                self._node, cmd, sudo=True,
+                message='Unbind PCI ports from driver failed!')
+
+            cmd = ("sh -c 'cd {dir}/scripts/ && ./trex-cfg'"
+                   .format(dir=Constants.TREX_INSTALL_DIR))
+            exec_cmd_no_error(
+                self._node, cmd, sudo=True, message='Config TRex failed!')
+
+            # Start TRex.
+            cmd = ("sh -c 'cd {dir}/scripts/ && "
+                   "nohup ./t-rex-64 {mode} --prefix $(hostname)"
+                   " -i -c 7 > /tmp/trex.log 2>&1 &' > /dev/null"
+                   .format(dir=Constants.TREX_INSTALL_DIR,
+                           mode='--astf' if osi_layer == 'L7' else ''))
+            try:
+                exec_cmd_no_error(self._node, cmd, sudo=True)
+            except RuntimeError:
+                cmd = "sh -c 'cat /tmp/trex.log'"
+                exec_cmd_no_error(self._node, cmd, sudo=True,
+                                  message='Get TRex logs failed!')
+                raise RuntimeError('Start TRex failed!')
+
+            # Test if TRex starts successfuly.
+            cmd = ("sh -c '{dir}/resources/tools/trex/trex_server_info.py'"
+                   .format(dir=Constants.REMOTE_FW_DIR))
+            try:
+                exec_cmd_no_error(
+                    self._node, cmd, sudo=True, message='Test TRex failed!',
+                    retries=20)
+            except RuntimeError:
+                continue
+            return
+        # After max retries TRex is still not responding to API critical error
+        # occurred.
+        raise RuntimeError('Start TRex failed after multiple retries!')

 
     @staticmethod
     def is_trex_running(node):
 
     @staticmethod
     def is_trex_running(node):


@@ -338,8 +378,7 @@ class TrafficGenerator(AbstractMeasurer):
         :rtype: bool
         :raises RuntimeError: If node type is not a TG.
         """
         :rtype: bool
         :raises RuntimeError: If node type is not a TG.
         """

-        if node['type'] != NodeType.TG:
-            raise RuntimeError('Node type is not a TG')
+        # No need to check subtype, we know it is TREX.

 
         ssh = SSH()
         ssh.connect(node)
 
         ssh = SSH()
         ssh.connect(node)


@@ -356,9 +395,8 @@ class TrafficGenerator(AbstractMeasurer):
         :raises RuntimeError: If node type is not a TG,
             or if TRex teardown fails.
         """
         :raises RuntimeError: If node type is not a TG,
             or if TRex teardown fails.
         """

-        if node['type'] != NodeType.TG:
-            raise RuntimeError('Node type is not a TG')
-        if node['subtype'] == NodeSubTypeTG.TREX:
+        subtype = check_subtype(node)
+        if subtype == NodeSubTypeTG.TREX:

             ssh = SSH()
             ssh.connect(node)
             (ret, _, _) = ssh.exec_command(
             ssh = SSH()
             ssh.connect(node)
             (ret, _, _) = ssh.exec_command(


@@ -366,150 +404,217 @@ class TrafficGenerator(AbstractMeasurer):
             if int(ret) != 0:
                 raise RuntimeError('pkill t-rex failed')
 
             if int(ret) != 0:
                 raise RuntimeError('pkill t-rex failed')
 

-    @staticmethod
-    def trex_stl_stop_remote_exec(node):
+    def _parse_traffic_results(self, stdout):
+        """Parse stdout of scripts into fields of self.
+
+        Block of code to reuse, by sync start, or stop after async.
+        TODO: Is the output TG subtype dependent?
+
+        :param stdout: Text containing the standard output.
+        :type stdout: str
+        """
+        # last line from console output
+        line = stdout.splitlines()[-1]
+        self._result = line
+        logger.info('TrafficGen result: {0}'.format(self._result))
+        self._received = self._result.split(', ')[1].split('=')[1]
+        self._sent = self._result.split(', ')[2].split('=')[1]
+        self._loss = self._result.split(', ')[3].split('=')[1]
+        self._latency = []
+        self._latency.append(self._result.split(', ')[4].split('=')[1])
+        self._latency.append(self._result.split(', ')[5].split('=')[1])
+
+    def trex_stl_stop_remote_exec(self, node):

         """Execute script on remote node over ssh to stop running traffic.
 
         """Execute script on remote node over ssh to stop running traffic.
 

+        Internal state is updated with measurement results.
+

         :param node: TRex generator node.
         :type node: dict
         :param node: TRex generator node.
         :type node: dict

-        :returns: Nothing

         :raises RuntimeError: If stop traffic script fails.
         """
         :raises RuntimeError: If stop traffic script fails.
         """

+        # No need to check subtype, we know it is TREX.

         ssh = SSH()
         ssh.connect(node)
 
         ssh = SSH()
         ssh.connect(node)
 

-        (ret, _, _) = ssh.exec_command(
-            "sh -c '{}/resources/tools/trex/"
-            "trex_stateless_stop.py'".format(Constants.REMOTE_FW_DIR))
+        x_args = ""
+        for index, value in enumerate(self._xstats):
+            if value is not None:
+                # Nested quoting is fun.
+                value = value.replace("'", "\"")
+                x_args += " --xstat{i}='\"'\"'{v}'\"'\"'".format(
+                    i=index, v=value)
+        (ret, stdout, _) = ssh.exec_command(
+            "sh -c '{d}/resources/tools/trex/trex_stateless_stop.py{a}'"\
+            .format(d=Constants.REMOTE_FW_DIR, a=x_args))

 
         if int(ret) != 0:
             raise RuntimeError('TRex stateless runtime error')
 
 
         if int(ret) != 0:
             raise RuntimeError('TRex stateless runtime error')
 

-    def trex_stl_start_remote_exec(self, duration, rate, framesize,
-                                   traffic_type, async_call=False,
-                                   latency=True, warmup_time=5.0):
+        self._parse_traffic_results(stdout)
+
+    def trex_stl_start_remote_exec(
+            self, duration, rate, frame_size, traffic_profile, async_call=False,
+            latency=True, warmup_time=5.0, unidirection=False, tx_port=0,
+            rx_port=1):

         """Execute script on remote node over ssh to start traffic.
 
         """Execute script on remote node over ssh to start traffic.
 

+        In sync mode, measurement results are stored internally.
+        In async mode, initial data including xstats are stored internally.
+

         :param duration: Time expresed in seconds for how long to send traffic.
         :param rate: Traffic rate expressed with units (pps, %)
         :param duration: Time expresed in seconds for how long to send traffic.
         :param rate: Traffic rate expressed with units (pps, %)

-        :param framesize: L2 frame size to send (without padding and IPG).
-        :param traffic_type: Module name as a traffic type identifier.
+        :param frame_size: L2 frame size to send (without padding and IPG).
+        :param traffic_profile: Module name as a traffic profile identifier.

             See resources/traffic_profiles/trex for implemented modules.
         :param async_call: If enabled then don't wait for all incomming trafic.
         :param latency: With latency measurement.
         :param warmup_time: Warmup time period.
             See resources/traffic_profiles/trex for implemented modules.
         :param async_call: If enabled then don't wait for all incomming trafic.
         :param latency: With latency measurement.
         :param warmup_time: Warmup time period.

+        :param unidirection: Traffic is unidirectional. Default: False
+        :param tx_port: Traffic generator transmit port for first flow.
+            Default: 0
+        :param rx_port: Traffic generator receive port for first flow.
+            Default: 1

         :type duration: float
         :type rate: str
         :type duration: float
         :type rate: str

-        :type framesize: str
-        :type traffic_type: str
+        :type frame_size: str
+        :type traffic_profile: str

         :type async_call: bool
         :type latency: bool
         :type warmup_time: float
         :type async_call: bool
         :type latency: bool
         :type warmup_time: float

-        :returns: Nothing
+        :type unidirection: bool
+        :type tx_port: int
+        :type rx_port: int

         :raises RuntimeError: In case of TG driver issue.
         """
         :raises RuntimeError: In case of TG driver issue.
         """

+        # No need to check subtype, we know it is TREX.

         ssh = SSH()
         ssh.connect(self._node)
         ssh = SSH()
         ssh.connect(self._node)

+        reorder = self._ifaces_reordered  # Just to make the next line fit.
+        p_0, p_1 = (rx_port, tx_port) if reorder else (tx_port, rx_port)
+        # Values from Robot can introduce type unicode,
+        # we need to encode them, so that repr() does not lead with 'u'.
+        if isinstance(rate, unicode):
+            rate = rate.encode("utf-8")
+        if isinstance(duration, unicode):
+            duration = duration.encode("utf-8")
+        if isinstance(warmup_time, unicode):
+            warmup_time = warmup_time.encode("utf-8")
+        command = (
+            "sh -c '{tool}/resources/tools/trex/trex_stateless_profile.py"
+            " --profile {prof}/resources/traffic_profiles/trex/{traffic}.py"
+            " --duration {duration!r} --frame_size {frame_size} --rate {rate!r}"
+            " --warmup_time {warmup!r} --port_0 {p_0} --port_1 {p_1}").format(
+                tool=Constants.REMOTE_FW_DIR, prof=Constants.REMOTE_FW_DIR,
+                traffic=traffic_profile, duration=duration,
+                frame_size=frame_size, rate=rate, warmup=warmup_time, p_0=p_0,
+                p_1=p_1)
+        if async_call:
+            command += " --async_start"
+        if latency:
+            command += " --latency"
+        if unidirection:
+            command += " --unidirection"
+        command += "'"

 
 

-        _async = "--async" if async_call else ""
-        _latency = "--latency" if latency else ""
-        _p0, _p1 = (2, 1) if self._ifaces_reordered else (1, 2)
-
-        profile_path = ("{0}/resources/traffic_profiles/trex/"
-                        "{1}.py".format(Constants.REMOTE_FW_DIR,
-                                        traffic_type))

         (ret, stdout, _) = ssh.exec_command(
         (ret, stdout, _) = ssh.exec_command(

-            "sh -c "
-            "'{0}/resources/tools/trex/trex_stateless_profile.py "
-            "--profile {1} "
-            "--duration {2} "
-            "--frame_size {3} "
-            "--rate {4} "
-            "--warmup_time {5} "
-            "--port_0 {6} "
-            "--port_1 {7} "
-            "{8} "   # --async
-            "{9}'".  # --latency
-            format(Constants.REMOTE_FW_DIR, profile_path, duration, framesize,
-                   rate, warmup_time, _p0 - 1, _p1 - 1, _async, _latency),
-            timeout=float(duration) + 60)
+            command, timeout=float(duration) + 60)

 
         if int(ret) != 0:
             raise RuntimeError('TRex stateless runtime error')
         elif async_call:
             #no result
 
         if int(ret) != 0:
             raise RuntimeError('TRex stateless runtime error')
         elif async_call:
             #no result

+            self._start_time = time.time()
+            self._rate = float(rate[:-3]) if "pps" in rate else float(rate)

             self._received = None
             self._sent = None
             self._loss = None
             self._latency = None
             self._received = None
             self._sent = None
             self._loss = None
             self._latency = None

+            xstats = [None, None]
+            index = 0
+            for line in stdout.splitlines():
+                if "Xstats snapshot {i}: ".format(i=index) in line:
+                    xstats[index] = line[19:]
+                    index += 1
+                if index == 2:
+                    break
+            self._xstats = tuple(xstats)

         else:
         else:

-            # last line from console output
-            line = stdout.splitlines()[-1]
-
-            self._result = line
-            logger.info('TrafficGen result: {0}'.format(self._result))
-
-            self._received = self._result.split(', ')[1].split('=')[1]
-            self._sent = self._result.split(', ')[2].split('=')[1]
-            self._loss = self._result.split(', ')[3].split('=')[1]
-
-            self._latency = []
-            self._latency.append(self._result.split(', ')[4].split('=')[1])
-            self._latency.append(self._result.split(', ')[5].split('=')[1])
+            self._parse_traffic_results(stdout)
+            self._start_time = None
+            self._rate = None

 
     def stop_traffic_on_tg(self):
         """Stop all traffic on TG.
 
 
     def stop_traffic_on_tg(self):
         """Stop all traffic on TG.
 

-        :returns: Nothing
+        :returns: Structure containing the result of the measurement.
+        :rtype: ReceiveRateMeasurement

         :raises RuntimeError: If TG is not set.
         """
         :raises RuntimeError: If TG is not set.
         """

-        if self._node is None:
-            raise RuntimeError("TG is not set")
-        if self._node['subtype'] == NodeSubTypeTG.TREX:
+        subtype = check_subtype(self._node)
+        if subtype == NodeSubTypeTG.TREX:

             self.trex_stl_stop_remote_exec(self._node)
             self.trex_stl_stop_remote_exec(self._node)

+        return self.get_measurement_result()

 
 

-    def send_traffic_on_tg(self, duration, rate, framesize,
-                           traffic_type, warmup_time=5, async_call=False,
-                           latency=True):
+    def send_traffic_on_tg(
+            self, duration, rate, frame_size, traffic_profile, warmup_time=5,
+            async_call=False, latency=True, unidirection=False, tx_port=0,
+            rx_port=1):

         """Send traffic from all configured interfaces on TG.
 
         """Send traffic from all configured interfaces on TG.
 

+        In async mode, xstats is stored internally,
+        to enable getting correct result when stopping the traffic.
+        In both modes, stdout is returned,
+        but _parse_traffic_results only works in sync output.
+
+        Note that bidirectional traffic also contains flows
+        transmitted from rx_port and received in tx_port.
+        But some tests use asymmetric traffic, so those arguments are relevant.
+
+        Also note that traffic generator uses DPDK driver which might
+        reorder port numbers based on wiring and PCI numbering.
+        This method handles that, so argument values are invariant,
+        but you can see swapped valued in debug logs.
+
+        TODO: Is it better to have less descriptive argument names
+        just to make them less probable to be viewed as misleading or confusing?
+        See https://gerrit.fd.io/r/#/c/17625/11/resources/libraries/python\
+        /TrafficGenerator.py@406
+

         :param duration: Duration of test traffic generation in seconds.
         :param rate: Offered load per interface (e.g. 1%, 3gbps, 4mpps, ...).
         :param duration: Duration of test traffic generation in seconds.
         :param rate: Offered load per interface (e.g. 1%, 3gbps, 4mpps, ...).

-        :param framesize: Frame size (L2) in Bytes.
-        :param traffic_type: Module name as a traffic type identifier.
+        :param frame_size: Frame size (L2) in Bytes.
+        :param traffic_profile: Module name as a traffic profile identifier.

             See resources/traffic_profiles/trex for implemented modules.
         :param warmup_time: Warmup phase in seconds.
         :param async_call: Async mode.
         :param latency: With latency measurement.
             See resources/traffic_profiles/trex for implemented modules.
         :param warmup_time: Warmup phase in seconds.
         :param async_call: Async mode.
         :param latency: With latency measurement.

+        :param unidirection: Traffic is unidirectional. Default: False
+        :param tx_port: Traffic generator transmit port for first flow.
+            Default: 0
+        :param rx_port: Traffic generator receive port for first flow.
+            Default: 1

         :type duration: str
         :type rate: str
         :type duration: str
         :type rate: str

-        :type framesize: str
-        :type traffic_type: str
+        :type frame_size: str
+        :type traffic_profile: str

         :type warmup_time: float
         :type async_call: bool
         :type latency: bool
         :type warmup_time: float
         :type async_call: bool
         :type latency: bool

+        :type unidirection: bool
+        :type tx_port: int
+        :type rx_port: int

         :returns: TG output.
         :rtype: str
         :raises RuntimeError: If TG is not set, or if node is not TG,
             or if subtype is not specified.
         :raises NotImplementedError: If TG is not supported.
         """
         :returns: TG output.
         :rtype: str
         :raises RuntimeError: If TG is not set, or if node is not TG,
             or if subtype is not specified.
         :raises NotImplementedError: If TG is not supported.
         """

-
-        node = self._node
-        if node is None:
-            raise RuntimeError("TG is not set")
-
-        if node['type'] != NodeType.TG:
-            raise RuntimeError('Node type is not a TG')
-
-        if node['subtype'] is None:
-            raise RuntimeError('TG subtype not defined')
-        elif node['subtype'] == NodeSubTypeTG.TREX:
-            self.trex_stl_start_remote_exec(duration, rate, framesize,
-                                            traffic_type, async_call, latency,
-                                            warmup_time=warmup_time)
-        else:
-            raise NotImplementedError("TG subtype not supported")
+        subtype = check_subtype(self._node)
+        if subtype == NodeSubTypeTG.TREX:
+            self.trex_stl_start_remote_exec(
+                duration, rate, frame_size, traffic_profile, async_call,
+                latency, warmup_time, unidirection, tx_port, rx_port)

 
         return self._result
 
 
         return self._result
 


@@ -560,22 +665,52 @@ class TrafficGenerator(AbstractMeasurer):
             raise Exception("Traffic loss {} above loss acceptance: {}".format(
                 loss, loss_acceptance))
 
             raise Exception("Traffic loss {} above loss acceptance: {}".format(
                 loss, loss_acceptance))
 

-    def set_rate_provider_defaults(self, frame_size, traffic_type,
+    def set_rate_provider_defaults(self, frame_size, traffic_profile,

                                    warmup_time=0.0):
         """Store values accessed by measure().
 
         :param frame_size: Frame size identifier or value [B].
                                    warmup_time=0.0):
         """Store values accessed by measure().
 
         :param frame_size: Frame size identifier or value [B].

-        :param traffic_type: Module name as a traffic type identifier.
+        :param traffic_profile: Module name as a traffic profile identifier.

             See resources/traffic_profiles/trex for implemented modules.
         :param warmup_time: Traffic duration before measurement starts [s].
         :type frame_size: str or int
             See resources/traffic_profiles/trex for implemented modules.
         :param warmup_time: Traffic duration before measurement starts [s].
         :type frame_size: str or int

-        :type traffic_type: str
+        :type traffic_profile: str

         :type warmup_time: float
         """
         self.frame_size = frame_size
         :type warmup_time: float
         """
         self.frame_size = frame_size

-        self.traffic_type = str(traffic_type)
+        self.traffic_profile = str(traffic_profile)

         self.warmup_time = float(warmup_time)
 
         self.warmup_time = float(warmup_time)
 

+    def get_measurement_result(self, duration=None, transmit_rate=None):
+        """Return the result of last measurement as ReceiveRateMeasurement.
+
+        Separate function, as measurements can end either by time
+        or by explicit call, this is the common block at the end.
+
+        TODO: Fail on running or already reported measurement.
+
+        :param duration: Measurement duration [s] if known beforehand.
+            For explicitly stopped measurement it is estimated.
+        :param transmit_rate: Target aggregate transmit rate [pps].
+            If not given, computed assuming it was bidirectional.
+        :type duration: float or NoneType
+        :type transmit_rate: float or NoneType
+        :returns: Structure containing the result of the measurement.
+        :rtype: ReceiveRateMeasurement
+        """
+        if duration is None:
+            duration = time.time() - self._start_time
+            self._start_time = None
+        if transmit_rate is None:
+            # Assuming bi-directional traffic here.
+            transmit_rate = self._rate * 2.0
+        transmit_count = int(self.get_sent())
+        loss_count = int(self.get_loss())
+        measurement = ReceiveRateMeasurement(
+            duration, transmit_rate, transmit_count, loss_count)
+        measurement.latency = self.get_latency_int()
+        return measurement
+

     def measure(self, duration, transmit_rate):
         """Run bi-directional measurement, parse and return results.
 
     def measure(self, duration, transmit_rate):
         """Run bi-directional measurement, parse and return results.
 


@@ -591,17 +726,12 @@ class TrafficGenerator(AbstractMeasurer):
         """
         duration = float(duration)
         transmit_rate = float(transmit_rate)
         """
         duration = float(duration)
         transmit_rate = float(transmit_rate)

-        # Trex needs target Tr per stream, but reports aggregate Tx and Dx.
+        # TG needs target Tr per stream, but reports aggregate Tx and Dx.

         unit_rate = str(transmit_rate / 2.0) + "pps"
         self.send_traffic_on_tg(
         unit_rate = str(transmit_rate / 2.0) + "pps"
         self.send_traffic_on_tg(

-            duration, unit_rate, self.frame_size, self.traffic_type,
-            self.warmup_time, latency=True)
-        transmit_count = int(self.get_sent())
-        loss_count = int(self.get_loss())
-        measurement = ReceiveRateMeasurement(
-            duration, transmit_rate, transmit_count, loss_count)
-        measurement.latency = self.get_latency_int()
-        return measurement
+            duration, unit_rate, self.frame_size, self.traffic_profile,
+            warmup_time=self.warmup_time, latency=True)
+        return self.get_measurement_result(duration, transmit_rate)

 
 
 class OptimizedSearch(object):
 
 
 class OptimizedSearch(object):


@@ -609,7 +739,7 @@ class OptimizedSearch(object):
 
     @staticmethod
     def perform_optimized_ndrpdr_search(
 
     @staticmethod
     def perform_optimized_ndrpdr_search(

-            frame_size, traffic_type, minimum_transmit_rate,
+            frame_size, traffic_profile, minimum_transmit_rate,

             maximum_transmit_rate, packet_loss_ratio=0.005,
             final_relative_width=0.005, final_trial_duration=30.0,
             initial_trial_duration=1.0, number_of_intermediate_phases=2,
             maximum_transmit_rate, packet_loss_ratio=0.005,
             final_relative_width=0.005, final_trial_duration=30.0,
             initial_trial_duration=1.0, number_of_intermediate_phases=2,


@@ -617,7 +747,7 @@ class OptimizedSearch(object):
         """Setup initialized TG, perform optimized search, return intervals.
 
         :param frame_size: Frame size identifier or value [B].
         """Setup initialized TG, perform optimized search, return intervals.
 
         :param frame_size: Frame size identifier or value [B].

-        :param traffic_type: Module name as a traffic type identifier.
+        :param traffic_profile: Module name as a traffic profile identifier.

             See resources/traffic_profiles/trex for implemented modules.
         :param minimum_transmit_rate: Minimal bidirectional
             target transmit rate [pps].
             See resources/traffic_profiles/trex for implemented modules.
         :param minimum_transmit_rate: Minimal bidirectional
             target transmit rate [pps].


@@ -637,7 +767,7 @@ class OptimizedSearch(object):
             Default 1 is suitable for fairly stable tests,
             less stable tests might get better overal duration with 2 or more.
         :type frame_size: str or int
             Default 1 is suitable for fairly stable tests,
             less stable tests might get better overal duration with 2 or more.
         :type frame_size: str or int

-        :type traffic_type: str
+        :type traffic_profile: str

         :type minimum_transmit_rate: float
         :type maximum_transmit_rate: float
         :type packet_loss_ratio: float
         :type minimum_transmit_rate: float
         :type maximum_transmit_rate: float
         :type packet_loss_ratio: float


@@ -656,7 +786,7 @@ class OptimizedSearch(object):
         # to be able to use trex_stl-*()
         tg_instance = BuiltIn().get_library_instance(
             'resources.libraries.python.TrafficGenerator')
         # to be able to use trex_stl-*()
         tg_instance = BuiltIn().get_library_instance(
             'resources.libraries.python.TrafficGenerator')

-        tg_instance.set_rate_provider_defaults(frame_size, traffic_type)
+        tg_instance.set_rate_provider_defaults(frame_size, traffic_profile)

         algorithm = MultipleLossRatioSearch(
             measurer=tg_instance, final_trial_duration=final_trial_duration,
             final_relative_width=final_relative_width,
         algorithm = MultipleLossRatioSearch(
             measurer=tg_instance, final_trial_duration=final_trial_duration,
             final_relative_width=final_relative_width,


@@ -666,3 +796,47 @@ class OptimizedSearch(object):
         result = algorithm.narrow_down_ndr_and_pdr(
             minimum_transmit_rate, maximum_transmit_rate, packet_loss_ratio)
         return result
         result = algorithm.narrow_down_ndr_and_pdr(
             minimum_transmit_rate, maximum_transmit_rate, packet_loss_ratio)
         return result

+
+    @staticmethod
+    def perform_soak_search(
+            frame_size, traffic_profile, minimum_transmit_rate,
+            maximum_transmit_rate, plr_target=1e-7, tdpt=0.1,
+            initial_count=50, timeout=1800.0, trace_enabled=False):
+        """Setup initialized TG, perform soak search, return avg and stdev.
+
+        :param frame_size: Frame size identifier or value [B].
+        :param traffic_profile: Module name as a traffic profile identifier.
+            See resources/traffic_profiles/trex for implemented modules.
+        :param minimum_transmit_rate: Minimal bidirectional
+            target transmit rate [pps].
+        :param maximum_transmit_rate: Maximal bidirectional
+            target transmit rate [pps].
+        :param plr_target: Fraction of packets lost to achieve [1].
+        :param tdpt: Trial duration per trial.
+            The algorithm linearly increases trial duration with trial number,
+            this is the increment between succesive trials, in seconds.
+        :param initial_count: Offset to apply before the first trial.
+            For example initial_count=50 makes first trial to be 51*tdpt long.
+            This is needed because initial "search" phase of integrator
+            takes significant time even without any trial results.
+        :param timeout: The search will stop after this overall time [s].
+        :type frame_size: str or int
+        :type traffic_profile: str
+        :type minimum_transmit_rate: float
+        :type maximum_transmit_rate: float
+        :type plr_target: float
+        :type initial_count: int
+        :type timeout: float
+        :returns: Average and stdev of estimated bidirectional rate giving PLR.
+        :rtype: 2-tuple of float
+        """
+        tg_instance = BuiltIn().get_library_instance(
+            'resources.libraries.python.TrafficGenerator')
+        tg_instance.set_rate_provider_defaults(frame_size, traffic_profile)
+        algorithm = PLRsearch(
+            measurer=tg_instance, trial_duration_per_trial=tdpt,
+            packet_loss_ratio_target=plr_target,
+            trial_number_offset=initial_count, timeout=timeout,
+            trace_enabled=trace_enabled)
+        result = algorithm.search(minimum_transmit_rate, maximum_transmit_rate)
+        return result