PyYAML==3.11
pykwalify==1.5.0
scapy==2.3.1
-
+enum34==1.1.2
--- /dev/null
+# Copyright (c) 2016 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Drop rate search algorithms"""
+
+from abc import ABCMeta, abstractmethod
+from enum import Enum, unique
+
+@unique
+class SearchDirection(Enum):
+ """Direction of linear search"""
+
+ TOP_DOWN = 1
+ BOTTOM_UP = 2
+
+@unique
+class SearchResults(Enum):
+ """Result of the drop rate search"""
+
+ SUCCESS = 1
+ FAILURE = 2
+ SUSPICIOUS = 3
+
+@unique
+class RateType(Enum):
+ """Type of rate units"""
+
+ PERCENTAGE = 1
+ PACKETS_PER_SECOND = 2
+ BITS_PER_SECOND = 3
+
+@unique
+class LossAcceptanceType(Enum):
+ """Type of the loss acceptance criteria"""
+
+ FRAMES = 1
+ PERCENTAGE = 2
+
+class DropRateSearch(object):
+ """Abstract class with search algorithm implementation"""
+
+ __metaclass__ = ABCMeta
+
+ def __init__(self):
+ #duration of traffic run (binary, linear)
+ self._duration = 60
+ #initial start rate (binary, linear)
+ self._rate_start = 100
+ #step of the linear search, unit: RateType (self._rate_type)
+ self._rate_linear_step = 10
+ #linear search direction, permitted values: SearchDirection
+ self._search_linear_direction = SearchDirection.TOP_DOWN
+ #upper limit of search, unit: RateType (self._rate_type)
+ self._rate_max = 100
+ #lower limit of search, unit: RateType (self._rate_type)
+ self._rate_min = 1
+ #permitted values: RateType
+ self._rate_type = RateType.PERCENTAGE
+ #accepted loss during search, units: LossAcceptanceType
+ self._loss_acceptance = 0
+ #permitted values: LossAcceptanceType
+ self._loss_acceptance_type = LossAcceptanceType.FRAMES
+ #size of frames to send
+ self._frame_size = "64"
+ #binary convergence criterium type is self._rate_type
+ self._binary_convergence_threshhold = "0.01"
+ #numbers of traffic runs during one rate step
+ self._max_attempts = 1
+
+ #result of search
+ self._search_result = None
+ self._search_result_rate = None
+
+ @abstractmethod
+ def measure_loss(self, rate, frame_size, loss_acceptance,
+ loss_acceptance_type, traffic_type):
+ """Send traffic from TG and measure count of dropped frames
+
+ :param rate: offered traffic load
+ :param frame_size: size of frame
+ :param loss_acceptance: permitted drop ratio or frames count
+ :param loss_acceptance_type: type of permitted loss
+ :param traffic_type: traffic profile ([2,3]-node-L[2,3], ...)
+ :type rate: int
+ :type frame_size: str
+ :type loss_acceptance: float
+ :type loss_acceptance_type: LossAcceptanceType
+ :type traffic_type: str
+ :return: drop threshhold exceeded? (True/False)
+ :rtype bool
+ """
+ pass
+
+ def set_search_rate_boundaries(self, max_rate, min_rate):
+ """Set search boundaries: min,max
+
+ :param max_rate: upper value of search boundaries
+ :param min_rate: lower value of search boundaries
+ :type max_rate: float
+ :type min_rate: float
+ :return: nothing
+ """
+ if float(min_rate) <= 0:
+ raise ValueError("min_rate must be higher than 0")
+ elif float(min_rate) > float(max_rate):
+ raise ValueError("min_rate must be lower than max_rate")
+ else:
+ self._rate_max = float(max_rate)
+ self._rate_min = float(min_rate)
+
+ def set_search_linear_step(self, step_rate):
+ """Set step size for linear search
+
+ :param step_rate: linear search step size
+ :type step_rate: float
+ :return: nothing
+ """
+ self._rate_linear_step = float(step_rate)
+
+ def set_search_rate_type_percentage(self):
+ """Set rate type to percentage of linerate
+
+ :return: nothing
+ """
+ self._set_search_rate_type(RateType.PERCENTAGE)
+
+ def set_search_rate_type_bps(self):
+ """Set rate type to bits per second
+
+ :return: nothing
+ """
+ self._set_search_rate_type(RateType.BITS_PER_SECOND)
+
+ def set_search_rate_type_pps(self):
+ """Set rate type to packets per second
+
+ :return: nothing
+ """
+ self._set_search_rate_type(RateType.PACKETS_PER_SECOND)
+
+ def _set_search_rate_type(self, rate_type):
+ """Set rate type to one of RateType-s
+
+ :param rate_type: type of rate to set
+ :type rate_type: RateType
+ :return: nothing
+ """
+ if rate_type not in RateType:
+ raise Exception("rate_type unknown: {}".format(rate_type))
+ else:
+ self._rate_type = rate_type
+
+ def set_search_frame_size(self, frame_size):
+ """Set size of frames to send
+
+ :param frame_size: size of frames
+ :type frame_size: str
+ :return: nothing
+ """
+ self._frame_size = frame_size
+
+ def set_duration(self, duration):
+ """Set the duration of single traffic run
+
+ :param duration: number of seconds for traffic to run
+ :type duration: int
+ :return: nothing
+ """
+ self._duration = int(duration)
+
+ def get_duration(self):
+ """Return configured duration of single traffic run
+
+ :return: number of seconds for traffic to run
+ :rtype: int
+ """
+ return self._duration
+
+ def get_rate_type_str(self):
+ """Return rate type representation
+
+ :return: string representation of rate type
+ :rtype: str
+ """
+ if self._rate_type == RateType.PERCENTAGE:
+ return "%"
+ elif self._rate_type == RateType.BITS_PER_SECOND:
+ return "bps"
+ elif self._rate_type == RateType.PACKETS_PER_SECOND:
+ return "pps"
+ else:
+ raise ValueError("RateType unknown")
+
+ def linear_search(self, start_rate, traffic_type):
+ """Linear search of rate with loss below acceptance criteria
+
+ :param start_rate: initial rate
+ :param traffic_type: traffic profile
+ :type start_rate: float
+ :param traffic_type: str
+ :return: nothing
+ """
+
+ if not self._rate_min <= float(start_rate) <= self._rate_max:
+ raise ValueError("Start rate is not in min,max range")
+
+ rate = float(start_rate)
+ #the last but one step
+ prev_rate = None
+
+ #linear search
+ while True:
+ res = self.measure_loss(rate, self._frame_size,
+ self._loss_acceptance,
+ self._loss_acceptance_type,
+ traffic_type)
+ if self._search_linear_direction == SearchDirection.BOTTOM_UP:
+ #loss occured and it was above acceptance criteria
+ if res == False:
+ #if this is first run then we didn't find drop rate
+ if prev_rate == None:
+ self._search_result = SearchResults.FAILURE
+ self._search_result_rate = None
+ return
+ # else we found the rate, which is value from previous run
+ else:
+ self._search_result = SearchResults.SUCCESS
+ self._search_result_rate = prev_rate
+ return
+ #there was no loss / loss below acceptance criteria
+ elif res == True:
+ prev_rate = rate
+ rate += self._rate_linear_step
+ if rate > self._rate_max:
+ if prev_rate != self._rate_max:
+ #one last step with rate set to _rate_max
+ rate = self._rate_max
+ continue
+ else:
+ self._search_result = SearchResults.SUCCESS
+ self._search_result_rate = prev_rate
+ return
+ else:
+ continue
+ else:
+ raise RuntimeError("Unknown search result")
+
+ elif self._search_linear_direction == SearchDirection.TOP_DOWN:
+ #loss occured, decrease rate
+ if res == False:
+ prev_rate = rate
+ rate -= self._rate_linear_step
+ if rate < self._rate_min:
+ if prev_rate != self._rate_min:
+ #one last step with rate set to _rate_min
+ rate = self._rate_min
+ continue
+ else:
+ self._search_result = SearchResults.FAILURE
+ self._search_result_rate = None
+ return
+ else:
+ continue
+ #no loss => non/partial drop rate found
+ elif res == True:
+ self._search_result = SearchResults.SUCCESS
+ self._search_result_rate = rate
+ return
+ else:
+ raise RuntimeError("Unknown search result")
+ else:
+ raise Exception("Unknown search direction")
+
+ raise Exception("Wrong codepath")
+
+ def verify_search_result(self):
+ """Fail if search was not successful
+
+ :return: result rate
+ :rtype: float
+ """
+ if self._search_result == SearchResults.FAILURE:
+ raise Exception('Search FAILED')
+ elif self._search_result in [SearchResults.SUCCESS, SearchResults.SUSPICIOUS]:
+ return self._search_result_rate
+
+ def binary_search(self):
+ raise NotImplementedError
+
+ def combined_search(self):
+ raise NotImplementedError
"""Performance testing traffic generator library."""
from robot.api import logger
+from robot.libraries.BuiltIn import BuiltIn
+from robot.api.deco import keyword
from resources.libraries.python.ssh import SSH
from resources.libraries.python.topology import NodeType
from resources.libraries.python.topology import NodeSubTypeTG
from resources.libraries.python.topology import Topology
+from resources.libraries.python.DropRateSearch import DropRateSearch
-__all__ = ['TrafficGenerator']
+__all__ = ['TrafficGenerator', 'TGDropRateSearchImpl']
+
+class TGDropRateSearchImpl(DropRateSearch):
+ """Drop Rate Search implementation"""
+
+ def __init__(self):
+ super(TGDropRateSearchImpl, self).__init__()
+
+ def measure_loss(self, rate, frame_size, loss_acceptance,
+ loss_acceptance_type, traffic_type):
+
+ #we need instance of TrafficGenerator instantiated by Robot Framework
+ #to be able to use trex_stateless_remote_exec method
+ tg_instance = BuiltIn().get_library_instance('resources.libraries.python.TrafficGenerator')
+
+ if tg_instance._node['subtype'] is None:
+ raise Exception('TG subtype not defined')
+ elif tg_instance._node['subtype'] == NodeSubTypeTG.TREX:
+ unit_rate = str(rate) + self.get_rate_type_str()
+ tg_instance.trex_stateless_remote_exec(self.get_duration(), unit_rate,
+ frame_size, traffic_type)
+
+ #TODO:getters for tg_instance and loss_acceptance_type
+ logger.trace("comparing: {} < {} ".format(tg_instance._loss, loss_acceptance))
+ if float(tg_instance._loss) > float(loss_acceptance):
+ return False
+ else:
+ return True
+ else:
+ raise NotImplementedError("TG subtype not supported")
class TrafficGenerator(object):
"""Traffic Generator"""
+ #use one instance of TrafficGenerator for all tests in test suite
ROBOT_LIBRARY_SCOPE = 'TEST SUITE'
def __init__(self):
self._loss = None
self._sent = None
self._received = None
+ self._node = None
#T-REX interface order mapping
self._ifaces_reordered = 0
if tg_node['type'] != NodeType.TG:
raise Exception('Node type is not a TG')
+ self._node = tg_node
+
if tg_node['subtype'] == NodeSubTypeTG.TREX:
ssh = SSH()
ssh.connect(tg_node)
"- port_limit : 2\n"
" version : 2\n"
" interfaces : [\"{}\",\"{}\"]\n"
+ " port_bandwidth_gb : 10\n"
" port_info :\n"
" - dest_mac : [{}]\n"
" src_mac : [{}]\n"
logger.error('pkill t-rex failed: {0}'.format(stdout + stderr))
raise RuntimeError('pkill t-rex failed')
- def send_traffic_on(self, nodes_info, duration, rate,
+ def trex_stateless_remote_exec(self, duration, rate, framesize,
+ traffic_type):
+ """Execute stateless script on remote node over ssh
+
+ :param node: remote node
+ :param traffic_type: Traffic profile
+ :type node: dict
+ :type traffic_type: str
+ """
+ ssh = SSH()
+ ssh.connect(self._node)
+
+ _p0 = 1
+ _p1 = 2
+
+ if self._ifaces_reordered != 0:
+ _p0, _p1 = _p1, _p0
+
+ if traffic_type in ["3-node-xconnect", "3-node-bridge"]:
+ (ret, stdout, stderr) = ssh.exec_command(
+ "sh -c '/tmp/openvpp-testing/resources/tools/t-rex/"
+ "t-rex-stateless.py "
+ "-d {0} -r {1} -s {2} "
+ "--p{3}_src_start_ip 10.10.10.1 "
+ "--p{3}_src_end_ip 10.10.10.254 "
+ "--p{3}_dst_start_ip 20.20.20.1 "
+ "--p{4}_src_start_ip 20.20.20.1 "
+ "--p{4}_src_end_ip 20.20.20.254 "
+ "--p{4}_dst_start_ip 10.10.10.1'".\
+ format(duration, rate, framesize, _p0, _p1),\
+ timeout=int(duration)+60)
+ elif traffic_type in ["3-node-IPv4"]:
+ (ret, stdout, stderr) = ssh.exec_command(
+ "sh -c '/tmp/openvpp-testing/resources/tools/t-rex/"
+ "t-rex-stateless.py "
+ "-d {0} -r {1} -s {2} "
+ "--p{3}_src_start_ip 10.10.10.2 "
+ "--p{3}_src_end_ip 10.10.10.254 "
+ "--p{3}_dst_start_ip 20.20.20.2 "
+ "--p{4}_src_start_ip 20.20.20.2 "
+ "--p{4}_src_end_ip 20.20.20.254 "
+ "--p{4}_dst_start_ip 10.10.10.2'".\
+ format(duration, rate, framesize, _p0, _p1),\
+ timeout=int(duration)+60)
+ else:
+ raise NotImplementedError('Unsupported traffic type')
+
+ logger.trace(ret)
+ logger.trace(stdout)
+ logger.trace(stderr)
+
+ #last line from console output
+ line = stdout.splitlines()[-1]
+
+ self._result = line
+ logger.info('TrafficGen result: {0}'.format(self._result))
+
+ self._received = self._result.split(', ')[1].split('=')[1]
+ self._sent = self._result.split(', ')[2].split('=')[1]
+ self._loss = self._result.split(', ')[3].split('=')[1]
+
+ def send_traffic_on(self, node, duration, rate,
framesize, traffic_type):
"""Send traffic from all configured interfaces on TG
- :param nodes_info: Dictionary containing information on all nodes
- in topology.
+ :param node: Dictionary containing TG information
:param duration: Duration of test traffic generation in seconds
:param rate: Offered load per interface (e.g. 1%, 3gbps, 4mpps, ...)
:param framesize: Frame size (L2) in Bytes
:param traffic_type: Traffic profile
- :type nodes_info: dict
+ :type node: dict
:type duration: str
:type rate: str
:type framesize: str
:rtype: str
"""
- node = nodes_info["TG"]
-
if node['type'] != NodeType.TG:
raise Exception('Node type is not a TG')
if node['subtype'] is None:
raise Exception('TG subtype not defined')
-
- ssh = SSH()
- ssh.connect(node)
-
- if node['subtype'] == NodeSubTypeTG.TREX:
-
- _p0 = 1
- _p1 = 2
-
- if self._ifaces_reordered != 0:
- _p0, _p1 = _p1, _p0
-
- if traffic_type in ["3-node-xconnect", "3-node-bridge"]:
- (ret, stdout, stderr) = ssh.exec_command(
- "sh -c '/tmp/openvpp-testing/resources/tools/t-rex/"
- "t-rex-stateless.py "
- "-d {0} -r {1} -s {2} "
- "--p{3}_src_start_ip 10.10.10.1 "
- "--p{3}_src_end_ip 10.10.10.254 "
- "--p{3}_dst_start_ip 20.20.20.1 "
- "--p{4}_src_start_ip 20.20.20.1 "
- "--p{4}_src_end_ip 20.20.20.254 "
- "--p{4}_dst_start_ip 10.10.10.1'".\
- format(duration, rate, framesize, _p0, _p1),\
- timeout=int(duration)+60)
- elif traffic_type in ["3-node-IPv4"]:
- (ret, stdout, stderr) = ssh.exec_command(
- "sh -c '/tmp/openvpp-testing/resources/tools/t-rex/"
- "t-rex-stateless.py "
- "-d {0} -r {1} -s {2} "
- "--p{3}_src_start_ip 10.10.10.2 "
- "--p{3}_src_end_ip 10.10.10.254 "
- "--p{3}_dst_start_ip 20.20.20.2 "
- "--p{4}_src_start_ip 20.20.20.2 "
- "--p{4}_src_end_ip 20.20.20.254 "
- "--p{4}_dst_start_ip 10.10.10.2'".\
- format(duration, rate, framesize, _p0, _p1),\
- timeout=int(duration)+60)
- else:
- raise NotImplementedError('Unsupported traffic type')
-
+ elif node['subtype'] == NodeSubTypeTG.TREX:
+ self.trex_stateless_remote_exec(duration, rate, framesize,
+ traffic_type)
else:
raise NotImplementedError("TG subtype not supported")
- logger.trace(ret)
- logger.trace(stdout)
- logger.trace(stderr)
-
- for line in stdout.splitlines():
- pass
-
- self._result = line
- logger.info('TrafficGen result: {0}'.format(self._result))
-
- self._loss = self._result.split(', ')[3].split('=')[1]
-
return self._result
def no_traffic_loss_occured(self):
c.start(ports=[0, 1], mult=rate, duration=duration)
# block until done
- c.wait_on_traffic(ports=[0, 1])
+ c.wait_on_traffic(ports=[0, 1], timeout=(duration+30))
# read the stats after the test
stats = c.get_stats()
--- /dev/null
+# Copyright (c) 2016 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*** Settings ***
+| Resource | resources/libraries/robot/default.robot
+| Resource | resources/libraries/robot/interfaces.robot
+| Resource | resources/libraries/robot/bridge_domain.robot
+| Resource | resources/libraries/robot/performance.robot
+| Resource | resources/libraries/robot/counters.robot
+| Library | resources.libraries.python.TrafficGenerator
+| Library | resources.libraries.python.TrafficGenerator.TGDropRateSearchImpl
+| Library | resources.libraries.python.NodePath
+| Force Tags | 3_NODE_SINGLE_LINK_TOPO | PERFTEST | HW_ENV | PERFTEST_LONG
+| Suite Setup | 3-node Performance Suite Setup
+| Suite Teardown | 3-node Performance Suite Teardown
+| Test Setup | Setup all DUTs before test
+| Test Teardown | Run Keyword If Test Failed | Show statistics on all DUTs
+| Documentation | Throughput search suite (long running test suite based on RFC2544).
+
+*** Test Cases ***
+| Find NDR by using linear search and 64B frames through bridge domain in 3-node topology
+| | Given L2 bridge domain initialized in a 3-node circular topology
+| | Then Find NDR using linear search and pps | 64 | 4600000 | 100000
+| | ... | 3-node-bridge | 14000000 | 100000
+
+*** Keywords ***
+
+| 3-node Performance Suite Setup
+| | 3-node circular Topology Variables Setup
+| | Initialize traffic generator | ${tg} | ${tg_if1} | ${tg_if2}
+| | ... | ${dut1} | ${dut1_if1} | ${dut1_if2}
+| | ... | ${dut2} | ${dut2_if1} | ${dut2_if2}
+| | ... | L2
+
+| 3-node Performance Suite Teardown
+| | Teardown traffic generator | ${tg}
+
+| L2 bridge domain initialized in a 3-node circular topology
+| | Vpp l2bd forwarding setup | ${dut1} | ${dut1_if1} | ${dut1_if2}
+| | Vpp l2bd forwarding setup | ${dut2} | ${dut2_if1} | ${dut2_if2}
+| | All Vpp Interfaces Ready Wait | ${nodes}
+
+| Find NDR using linear search and pps
+| | [Arguments] | ${framesize} | ${start_rate} | ${step_rate}
+| | ... | ${topology_type} | ${min_rate} | ${max_rate}
+| | Set Duration | 60
+| | Set Search Rate Boundaries | 14000000 | 100000
+| | Set Search Linear Step | ${step_rate}
+| | Set Search Rate Type pps
+| | Linear Search | ${start_rate} | ${topology_type}
+| | ${result_rate}= | Verify Search Result
+| | Set Test Message | FINAL_RATE: ${result_rate} pps
+
+| Show statistics on all DUTs
+| | Sleep | 10 | Waiting for statistics to be collected
+| | Vpp show stats | ${dut1}
+| | Vpp show stats | ${dut2}
| Resource | resources/libraries/robot/counters.robot
| Library | resources.libraries.python.TrafficGenerator
| Library | resources.libraries.python.NodePath
-| Force Tags | 3_NODE_SINGLE_LINK_TOPO | PERFTEST | HW_ENV
+| Force Tags | 3_NODE_SINGLE_LINK_TOPO | PERFTEST | HW_ENV | PERFTEST_SHORT
| Suite Setup | 3-node Performance Suite Setup
| Suite Teardown | 3-node Performance Suite Teardown
| Test Setup | Setup all DUTs before test
| Traffic should pass with no loss
| | [Arguments] | ${duration} | ${rate} | ${framesize} | ${topology_type}
-| | Send traffic on | ${nodes} | ${duration}
+| | Send traffic on | ${tg} | ${duration}
| | ... | ${rate} | ${framesize} | ${topology_type}
| | No traffic loss occured
| Library | resources.libraries.python.InterfaceUtil
| Library | resources.libraries.python.IPv4Setup.Dut | ${nodes['DUT1']} | WITH NAME | dut1_v4
| Library | resources.libraries.python.IPv4Setup.Dut | ${nodes['DUT2']} | WITH NAME | dut2_v4
-| Force Tags | 3_NODE_SINGLE_LINK_TOPO | PERFTEST | HW_ENV
+| Force Tags | 3_NODE_SINGLE_LINK_TOPO | PERFTEST | HW_ENV | PERFTEST_SHORT
| Suite Setup | 3-node Performance Suite Setup
| Suite Teardown | 3-node Performance Suite Teardown
| Test Setup | Setup all DUTs before test
| Traffic should pass with no loss
| | [Arguments] | ${duration} | ${rate} | ${framesize} | ${topology_type}
-| | Send traffic on | ${nodes} | ${duration}
+| | Send traffic on | ${tg} | ${duration}
| | ... | ${rate} | ${framesize} | ${topology_type}
| | No traffic loss occured
| Library | resources.libraries.python.TrafficGenerator
| Library | resources.libraries.python.InterfaceUtil
| Library | resources.libraries.python.NodePath
-| Force Tags | 3_NODE_SINGLE_LINK_TOPO | PERFTEST | HW_ENV
+| Force Tags | 3_NODE_SINGLE_LINK_TOPO | PERFTEST | HW_ENV| PERFTEST_SHORT
| Suite Setup | 3-node Performance Suite Setup
| Suite Teardown | 3-node Performance Suite Teardown
| Test Setup | Setup all DUTs before test
| Traffic should pass with no loss
| | [Arguments] | ${duration} | ${rate} | ${framesize} | ${topology_type}
-| | Send traffic on | ${nodes} | ${duration}
+| | Send traffic on | ${tg} | ${duration}
| | ... | ${rate} | ${framesize} | ${topology_type}
| | No traffic loss occured
Code Review / csit.git / commitdiff