Fix Tap failing tests

[csit.git] / resources / libraries / python / TrafficGenerator.py

diff --git a/resources/libraries/python/TrafficGenerator.py b/resources/libraries/python/TrafficGenerator.py
index aa839dd..ea587d7 100644 (file)
--- a/resources/libraries/python/TrafficGenerator.py
+++ b/resources/libraries/python/TrafficGenerator.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2018 Cisco and/or its affiliates.
+# Copyright (c) 2019 Cisco and/or its affiliates.

 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:


@@ -17,14 +17,15 @@ from robot.api import logger
 from robot.libraries.BuiltIn import BuiltIn
 
 from .DropRateSearch import DropRateSearch
 from robot.libraries.BuiltIn import BuiltIn
 
 from .DropRateSearch import DropRateSearch

-from .constants import Constants
+from .Constants import Constants

 from .ssh import SSH
 from .topology import NodeType
 from .topology import NodeSubTypeTG
 from .topology import Topology
 from .ssh import SSH
 from .topology import NodeType
 from .topology import NodeSubTypeTG
 from .topology import Topology

-from .search.AbstractRateProvider import AbstractRateProvider
-from .search.OptimizedSearchAlgorithm import OptimizedSearchAlgorithm
-from .search.ReceiveRateMeasurement import ReceiveRateMeasurement
+from .MLRsearch.AbstractMeasurer import AbstractMeasurer
+from .MLRsearch.MultipleLossRatioSearch import MultipleLossRatioSearch
+from .MLRsearch.ReceiveRateMeasurement import ReceiveRateMeasurement
+from .PLRsearch.PLRsearch import PLRsearch

 
 __all__ = ['TGDropRateSearchImpl', 'TrafficGenerator', 'OptimizedSearch']
 
 
 __all__ = ['TGDropRateSearchImpl', 'TrafficGenerator', 'OptimizedSearch']
 


@@ -101,15 +102,15 @@ class TGDropRateSearchImpl(DropRateSearch):
         return tg_instance.get_latency_int()
 
 
         return tg_instance.get_latency_int()
 
 

-class TrafficGenerator(AbstractRateProvider):
+class TrafficGenerator(AbstractMeasurer):

     """Traffic Generator.
 
     FIXME: Describe API."""
 
     # TODO: Decrease friction between various search and rate provider APIs.
     """Traffic Generator.
 
     FIXME: Describe API."""
 
     # TODO: Decrease friction between various search and rate provider APIs.

-    # FIXME: Remove "trex" from lines which could work with other TGs.
+    # TODO: Remove "trex" from lines which could work with other TGs.

 
 

-    # use one instance of TrafficGenerator for all tests in test suite
+    # Use one instance of TrafficGenerator for all tests in test suite

     ROBOT_LIBRARY_SCOPE = 'TEST SUITE'
 
     def __init__(self):
     ROBOT_LIBRARY_SCOPE = 'TEST SUITE'
 
     def __init__(self):


@@ -167,13 +168,14 @@ class TrafficGenerator(AbstractRateProvider):
         """
         return self._latency
 
         """
         return self._latency
 

-    def initialize_traffic_generator(self, tg_node, tg_if1, tg_if2,
-                                     tg_if1_adj_node, tg_if1_adj_if,
-                                     tg_if2_adj_node, tg_if2_adj_if,
-                                     test_type,
-                                     tg_if1_dst_mac=None, tg_if2_dst_mac=None):
+    def initialize_traffic_generator(
+            self, tg_node, tg_if1, tg_if2, tg_if1_adj_node, tg_if1_adj_if,
+            tg_if2_adj_node, tg_if2_adj_if, test_type, tg_if1_dst_mac=None,
+            tg_if2_dst_mac=None):

         """TG initialization.
 
         """TG initialization.
 

+        TODO: Document why do we need (and how do we use) _ifaces_reordered.
+

         :param tg_node: Traffic generator node.
         :param tg_if1: TG - name of first interface.
         :param tg_if2: TG - name of second interface.
         :param tg_node: Traffic generator node.
         :param tg_if1: TG - name of first interface.
         :param tg_if2: TG - name of second interface.


@@ -385,9 +387,10 @@ class TrafficGenerator(AbstractRateProvider):
         if int(ret) != 0:
             raise RuntimeError('TRex stateless runtime error')
 
         if int(ret) != 0:
             raise RuntimeError('TRex stateless runtime error')
 

-    def trex_stl_start_remote_exec(self, duration, rate, framesize,
-                                   traffic_type, async_call=False,
-                                   latency=True, warmup_time=5.0):
+    def trex_stl_start_remote_exec(
+            self, duration, rate, framesize, traffic_type, async_call=False,
+            latency=True, warmup_time=5.0, unidirection=False, tx_port=0,
+            rx_port=1):

         """Execute script on remote node over ssh to start traffic.
 
         :param duration: Time expresed in seconds for how long to send traffic.
         """Execute script on remote node over ssh to start traffic.
 
         :param duration: Time expresed in seconds for how long to send traffic.


@@ -398,6 +401,11 @@ class TrafficGenerator(AbstractRateProvider):
         :param async_call: If enabled then don't wait for all incomming trafic.
         :param latency: With latency measurement.
         :param warmup_time: Warmup time period.
         :param async_call: If enabled then don't wait for all incomming trafic.
         :param latency: With latency measurement.
         :param warmup_time: Warmup time period.

+        :param unidirection: Traffic is unidirectional. Default: False
+        :param tx_port: Traffic generator transmit port for first flow.
+            Default: 0
+        :param rx_port: Traffic generator receive port for first flow.
+            Default: 1

         :type duration: float
         :type rate: str
         :type framesize: str
         :type duration: float
         :type rate: str
         :type framesize: str


@@ -405,34 +413,33 @@ class TrafficGenerator(AbstractRateProvider):
         :type async_call: bool
         :type latency: bool
         :type warmup_time: float
         :type async_call: bool
         :type latency: bool
         :type warmup_time: float

-        :returns: Nothing
+        :type unidirection: bool
+        :type tx_port: int
+        :type rx_port: int

         :raises RuntimeError: In case of TG driver issue.
         """
         ssh = SSH()
         ssh.connect(self._node)
         :raises RuntimeError: In case of TG driver issue.
         """
         ssh = SSH()
         ssh.connect(self._node)

+        reorder = self._ifaces_reordered  # Just to make the next line fit.
+        p_0, p_1 = (rx_port, tx_port) if reorder else (tx_port, rx_port)
+        command = (
+            "sh -c '{tool}/resources/tools/trex/trex_stateless_profile.py"
+            " --profile {prof}/resources/traffic_profiles/trex/{traffic}.py"
+            " --duration {duration} --frame_size {framesize} --rate {rate}"
+            " --warmup_time {warmup} --port_0 {p_0} --port_1 {p_1}").format(
+                tool=Constants.REMOTE_FW_DIR, prof=Constants.REMOTE_FW_DIR,
+                traffic=traffic_type, duration=duration, framesize=framesize,
+                rate=rate, warmup=warmup_time, p_0=p_0, p_1=p_1)
+        if async_call:
+            command += " --async"
+        if latency:
+            command += " --latency"
+        if unidirection:
+            command += " --unidirection"
+        command += "'"

 
 

-        _async = "--async" if async_call else ""
-        _latency = "--latency" if latency else ""
-        _p0, _p1 = (2, 1) if self._ifaces_reordered else (1, 2)
-
-        profile_path = ("{0}/resources/traffic_profiles/trex/"
-                        "{1}.py".format(Constants.REMOTE_FW_DIR,
-                                        traffic_type))

         (ret, stdout, _) = ssh.exec_command(
         (ret, stdout, _) = ssh.exec_command(

-            "sh -c "
-            "'{0}/resources/tools/trex/trex_stateless_profile.py "
-            "--profile {1} "
-            "--duration {2} "
-            "--frame_size {3} "
-            "--rate {4} "
-            "--warmup_time {5} "
-            "--port_0 {6} "
-            "--port_1 {7} "
-            "{8} "   # --async
-            "{9}'".  # --latency
-            format(Constants.REMOTE_FW_DIR, profile_path, duration, framesize,
-                   rate, warmup_time, _p0 - 1, _p1 - 1, _async, _latency),
-            timeout=float(duration) + 60)
+            command, timeout=float(duration) + 60)

 
         if int(ret) != 0:
             raise RuntimeError('TRex stateless runtime error')
 
         if int(ret) != 0:
             raise RuntimeError('TRex stateless runtime error')


@@ -445,14 +452,11 @@ class TrafficGenerator(AbstractRateProvider):
         else:
             # last line from console output
             line = stdout.splitlines()[-1]
         else:
             # last line from console output
             line = stdout.splitlines()[-1]

-

             self._result = line
             logger.info('TrafficGen result: {0}'.format(self._result))
             self._result = line
             logger.info('TrafficGen result: {0}'.format(self._result))

-

             self._received = self._result.split(', ')[1].split('=')[1]
             self._sent = self._result.split(', ')[2].split('=')[1]
             self._loss = self._result.split(', ')[3].split('=')[1]
             self._received = self._result.split(', ')[1].split('=')[1]
             self._sent = self._result.split(', ')[2].split('=')[1]
             self._loss = self._result.split(', ')[3].split('=')[1]

-

             self._latency = []
             self._latency.append(self._result.split(', ')[4].split('=')[1])
             self._latency.append(self._result.split(', ')[5].split('=')[1])
             self._latency = []
             self._latency.append(self._result.split(', ')[4].split('=')[1])
             self._latency.append(self._result.split(', ')[5].split('=')[1])


@@ -468,11 +472,25 @@ class TrafficGenerator(AbstractRateProvider):
         if self._node['subtype'] == NodeSubTypeTG.TREX:
             self.trex_stl_stop_remote_exec(self._node)
 
         if self._node['subtype'] == NodeSubTypeTG.TREX:
             self.trex_stl_stop_remote_exec(self._node)
 

-    def send_traffic_on_tg(self, duration, rate, framesize,
-                           traffic_type, warmup_time=5, async_call=False,
-                           latency=True):
+    def send_traffic_on_tg(
+            self, duration, rate, framesize, traffic_type, warmup_time=5,
+            async_call=False, latency=True, unidirection=False, tx_port=0,
+            rx_port=1):

         """Send traffic from all configured interfaces on TG.
 
         """Send traffic from all configured interfaces on TG.
 

+        Note that bidirectional traffic also contains flows
+        transmitted from rx_port and received in tx_port.
+        But some tests use asymmetric traffic, so those arguments are relevant.
+
+        Also note that traffic generator uses DPDK driver which might
+        reorder port numbers based on wiring and PCI numbering.
+        This method handles that, so argument values are invariant,
+        but you can see swapped valued in debug logs.
+
+        TODO: Is it better to have less descriptive argument names
+        just to make them less probable to be viewed as misleading or confusing?
+        See https://gerrit.fd.io/r/#/c/17625/11/resources/libraries/python/TrafficGenerator.py@406
+

         :param duration: Duration of test traffic generation in seconds.
         :param rate: Offered load per interface (e.g. 1%, 3gbps, 4mpps, ...).
         :param framesize: Frame size (L2) in Bytes.
         :param duration: Duration of test traffic generation in seconds.
         :param rate: Offered load per interface (e.g. 1%, 3gbps, 4mpps, ...).
         :param framesize: Frame size (L2) in Bytes.


@@ -481,6 +499,11 @@ class TrafficGenerator(AbstractRateProvider):
         :param warmup_time: Warmup phase in seconds.
         :param async_call: Async mode.
         :param latency: With latency measurement.
         :param warmup_time: Warmup phase in seconds.
         :param async_call: Async mode.
         :param latency: With latency measurement.

+        :param unidirection: Traffic is unidirectional. Default: False
+        :param tx_port: Traffic generator transmit port for first flow.
+            Default: 0
+        :param rx_port: Traffic generator receive port for first flow.
+            Default: 1

         :type duration: str
         :type rate: str
         :type framesize: str
         :type duration: str
         :type rate: str
         :type framesize: str


@@ -488,6 +511,9 @@ class TrafficGenerator(AbstractRateProvider):
         :type warmup_time: float
         :type async_call: bool
         :type latency: bool
         :type warmup_time: float
         :type async_call: bool
         :type latency: bool

+        :type unidirection: bool
+        :type tx_port: int
+        :type rx_port: int

         :returns: TG output.
         :rtype: str
         :raises RuntimeError: If TG is not set, or if node is not TG,
         :returns: TG output.
         :rtype: str
         :raises RuntimeError: If TG is not set, or if node is not TG,


@@ -505,9 +531,9 @@ class TrafficGenerator(AbstractRateProvider):
         if node['subtype'] is None:
             raise RuntimeError('TG subtype not defined')
         elif node['subtype'] == NodeSubTypeTG.TREX:
         if node['subtype'] is None:
             raise RuntimeError('TG subtype not defined')
         elif node['subtype'] == NodeSubTypeTG.TREX:

-            self.trex_stl_start_remote_exec(duration, rate, framesize,
-                                            traffic_type, async_call, latency,
-                                            warmup_time=warmup_time)
+            self.trex_stl_start_remote_exec(
+                duration, rate, framesize, traffic_type, async_call, latency,
+                warmup_time, unidirection, tx_port, rx_port)

         else:
             raise NotImplementedError("TG subtype not supported")
 
         else:
             raise NotImplementedError("TG subtype not supported")
 


@@ -520,9 +546,20 @@ class TrafficGenerator(AbstractRateProvider):
         :raises Exception: If loss occured.
         """
         if self._loss is None:
         :raises Exception: If loss occured.
         """
         if self._loss is None:

-            raise Exception('The traffic generation has not been issued')
+            raise RuntimeError('The traffic generation has not been issued')

         if self._loss != '0':
         if self._loss != '0':

-            raise Exception('Traffic loss occurred: {0}'.format(self._loss))
+            raise RuntimeError('Traffic loss occurred: {0}'.format(self._loss))
+
+    def fail_if_no_traffic_forwarded(self):
+        """Fail if no traffic forwarded.
+
+        :returns: nothing
+        :raises Exception: If no traffic forwarded.
+        """
+        if self._received is None:
+            raise RuntimeError('The traffic generation has not been issued')
+        if self._received == '0':
+            raise RuntimeError('No traffic forwarded')

 
     def partial_traffic_loss_accepted(self, loss_acceptance,
                                       loss_acceptance_type):
 
     def partial_traffic_loss_accepted(self, loss_acceptance,
                                       loss_acceptance_type):


@@ -584,11 +621,11 @@ class TrafficGenerator(AbstractRateProvider):
         unit_rate = str(transmit_rate / 2.0) + "pps"
         self.send_traffic_on_tg(
             duration, unit_rate, self.frame_size, self.traffic_type,
         unit_rate = str(transmit_rate / 2.0) + "pps"
         self.send_traffic_on_tg(
             duration, unit_rate, self.frame_size, self.traffic_type,

-            self.warmup_time, latency=True)
+            warmup_time=self.warmup_time, latency=True)

         transmit_count = int(self.get_sent())
         transmit_count = int(self.get_sent())

-        drop_count = int(self.get_loss())
+        loss_count = int(self.get_loss())

         measurement = ReceiveRateMeasurement(
         measurement = ReceiveRateMeasurement(

-            duration, transmit_rate, transmit_count, drop_count)
+            duration, transmit_rate, transmit_count, loss_count)

         measurement.latency = self.get_latency_int()
         return measurement
 
         measurement.latency = self.get_latency_int()
         return measurement
 


@@ -598,38 +635,45 @@ class OptimizedSearch(object):
 
     @staticmethod
     def perform_optimized_ndrpdr_search(
 
     @staticmethod
     def perform_optimized_ndrpdr_search(

-            frame_size, traffic_type, fail_rate, line_rate,
-            allowed_drop_fraction=0.005, final_relative_width=0.005,
-            final_trial_duration=30.0, initial_trial_duration=1.0,
-            intermediate_phases=2, timeout=600.0):
+            frame_size, traffic_type, minimum_transmit_rate,
+            maximum_transmit_rate, packet_loss_ratio=0.005,
+            final_relative_width=0.005, final_trial_duration=30.0,
+            initial_trial_duration=1.0, number_of_intermediate_phases=2,
+            timeout=720.0, doublings=1):

         """Setup initialized TG, perform optimized search, return intervals.
 
         :param frame_size: Frame size identifier or value [B].
         :param traffic_type: Module name as a traffic type identifier.
             See resources/traffic_profiles/trex for implemented modules.
         """Setup initialized TG, perform optimized search, return intervals.
 
         :param frame_size: Frame size identifier or value [B].
         :param traffic_type: Module name as a traffic type identifier.
             See resources/traffic_profiles/trex for implemented modules.

-        :param fail_rate: Minimal target transmit rate [pps].
-        :param line_rate: Maximal target transmit rate [pps].
-        :param allowed_drop_fraction: Fraction of dropped packets for PDR [1].
+        :param minimum_transmit_rate: Minimal bidirectional
+            target transmit rate [pps].
+        :param maximum_transmit_rate: Maximal bidirectional
+            target transmit rate [pps].
+        :param packet_loss_ratio: Fraction of packets lost, for PDR [1].

         :param final_relative_width: Final lower bound transmit rate
             cannot be more distant that this multiple of upper bound [1].
         :param final_trial_duration: Trial duration for the final phase [s].
         :param initial_trial_duration: Trial duration for the initial phase
             and also for the first intermediate phase [s].
         :param final_relative_width: Final lower bound transmit rate
             cannot be more distant that this multiple of upper bound [1].
         :param final_trial_duration: Trial duration for the final phase [s].
         :param initial_trial_duration: Trial duration for the initial phase
             and also for the first intermediate phase [s].

-        :param intermediate_phases: Number of intermediate phases to perform
-            before the final phase [1].
+        :param number_of_intermediate_phases: Number of intermediate phases
+            to perform before the final phase [1].

         :param timeout: The search will fail itself when not finished
             before this overall time [s].
         :param timeout: The search will fail itself when not finished
             before this overall time [s].

+        :param doublings: How many doublings to do in external search step.
+            Default 1 is suitable for fairly stable tests,
+            less stable tests might get better overal duration with 2 or more.

         :type frame_size: str or int
         :type traffic_type: str
         :type frame_size: str or int
         :type traffic_type: str

-        :type fail_rate: float
-        :type line_rate: float
-        :type allowed_drop_fraction: float
+        :type minimum_transmit_rate: float
+        :type maximum_transmit_rate: float
+        :type packet_loss_ratio: float

         :type final_relative_width: float
         :type final_trial_duration: float
         :type initial_trial_duration: float
         :type final_relative_width: float
         :type final_trial_duration: float
         :type initial_trial_duration: float

-        :type intermediate_phases: int
+        :type number_of_intermediate_phases: int

         :type timeout: float
         :type timeout: float

-        :returns: Structure containing narrowed down intervals
+        :type doublings: int
+        :returns: Structure containing narrowed down NDR and PDR intervals

             and their measurements.
         :rtype: NdrPdrResult
         :raises RuntimeError: If total duration is larger than timeout.
             and their measurements.
         :rtype: NdrPdrResult
         :raises RuntimeError: If total duration is larger than timeout.


@@ -639,11 +683,55 @@ class OptimizedSearch(object):
         tg_instance = BuiltIn().get_library_instance(
             'resources.libraries.python.TrafficGenerator')
         tg_instance.set_rate_provider_defaults(frame_size, traffic_type)
         tg_instance = BuiltIn().get_library_instance(
             'resources.libraries.python.TrafficGenerator')
         tg_instance.set_rate_provider_defaults(frame_size, traffic_type)

-        algorithm = OptimizedSearchAlgorithm(
-            tg_instance, final_trial_duration=final_trial_duration,
+        algorithm = MultipleLossRatioSearch(
+            measurer=tg_instance, final_trial_duration=final_trial_duration,

             final_relative_width=final_relative_width,
             final_relative_width=final_relative_width,

-            intermediate_phases=intermediate_phases,
-            initial_trial_duration=initial_trial_duration, timeout=timeout)
+            number_of_intermediate_phases=number_of_intermediate_phases,
+            initial_trial_duration=initial_trial_duration, timeout=timeout,
+            doublings=doublings)

         result = algorithm.narrow_down_ndr_and_pdr(
         result = algorithm.narrow_down_ndr_and_pdr(

-            fail_rate, line_rate, allowed_drop_fraction)
+            minimum_transmit_rate, maximum_transmit_rate, packet_loss_ratio)
+        return result
+
+    @staticmethod
+    def perform_soak_search(
+            frame_size, traffic_type, minimum_transmit_rate,
+            maximum_transmit_rate, plr_target=1e-7, tdpt=0.2,
+            initial_count=50, timeout=1800.0):
+        """Setup initialized TG, perform soak search, return avg and stdev.
+
+        :param frame_size: Frame size identifier or value [B].
+        :param traffic_type: Module name as a traffic type identifier.
+            See resources/traffic_profiles/trex for implemented modules.
+        :param minimum_transmit_rate: Minimal bidirectional
+            target transmit rate [pps].
+        :param maximum_transmit_rate: Maximal bidirectional
+            target transmit rate [pps].
+        :param plr_target: Fraction of packets lost to achieve [1].
+        :param tdpt: Trial duration per trial.
+            The algorithm linearly increases trial duration with trial number,
+            this is the increment between succesive trials, in seconds.
+        :param initial_count: Offset to apply before the first trial.
+            For example initial_count=50 makes first trial to be 51*tdpt long.
+            This is needed because initial "search" phase of integrator
+            takes significant time even without any trial results.
+        :param timeout: The search will stop after this overall time [s].
+        :type frame_size: str or int
+        :type traffic_type: str
+        :type minimum_transmit_rate: float
+        :type maximum_transmit_rate: float
+        :type plr_target: float
+        :type initial_count: int
+        :type timeout: float
+        :returns: Average and stdev of estimated bidirectional rate giving PLR.
+        :rtype: 2-tuple of float
+        """
+        tg_instance = BuiltIn().get_library_instance(
+            'resources.libraries.python.TrafficGenerator')
+        tg_instance.set_rate_provider_defaults(frame_size, traffic_type)
+        algorithm = PLRsearch(
+            measurer=tg_instance, trial_duration_per_trial=tdpt,
+            packet_loss_ratio_target=plr_target,
+            trial_number_offset=initial_count, timeout=timeout)
+        result = algorithm.search(minimum_transmit_rate, maximum_transmit_rate)

         return result
         return result