FIX: check if t-rex is running at test setup of all perf tests
[csit.git] / resources / libraries / python / TrafficGenerator.py
index a73e471..4e3a549 100644 (file)
@@ -1,4 +1,4 @@
-# Copyright (c) 2019 Cisco and/or its affiliates.
+# Copyright (c) 2020 Cisco and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
@@ -18,16 +18,18 @@ import time
 from robot.api import logger
 from robot.libraries.BuiltIn import BuiltIn
 
-from .DropRateSearch import DropRateSearch
 from .Constants import Constants
-from .ssh import exec_cmd_no_error, exec_cmd
-from .topology import NodeType
-from .topology import NodeSubTypeTG
-from .topology import Topology
+from .CpuUtils import CpuUtils
+from .DropRateSearch import DropRateSearch
 from .MLRsearch.AbstractMeasurer import AbstractMeasurer
 from .MLRsearch.MultipleLossRatioSearch import MultipleLossRatioSearch
 from .MLRsearch.ReceiveRateMeasurement import ReceiveRateMeasurement
 from .PLRsearch.PLRsearch import PLRsearch
+from .OptionString import OptionString
+from .ssh import exec_cmd_no_error, exec_cmd
+from .topology import NodeType
+from .topology import NodeSubTypeTG
+from .topology import Topology
 
 __all__ = [u"TGDropRateSearchImpl", u"TrafficGenerator", u"OptimizedSearch"]
 
@@ -277,6 +279,11 @@ class TrafficGenerator(AbstractMeasurer):
                 if1_adj_addr, if2_adj_addr = if2_adj_addr, if1_adj_addr
                 self._ifaces_reordered = True
 
+            master_thread_id, latency_thread_id, socket, threads = \
+                CpuUtils.get_affinity_trex(
+                    self._node, tg_if1, tg_if2,
+                    tg_dtc=Constants.TREX_CORE_COUNT)
+
             if osi_layer in (u"L2", u"L3"):
                 dst_mac0 = f"0x{if1_adj_addr.replace(u':', u',0x')}"
                 src_mac0 = f"0x{if1_addr.replace(u':', u',0x')}"
@@ -286,6 +293,7 @@ class TrafficGenerator(AbstractMeasurer):
                     self._node,
                     f"sh -c 'cat << EOF > /etc/trex_cfg.yaml\n"
                     f"- version: 2\n"
+                    f"  c: {len(threads)}\n"
                     f"  limit_memory: {Constants.TREX_LIMIT_MEMORY}\n"
                     f"  interfaces: [\"{if1_pci}\",\"{if2_pci}\"]\n"
                     f"  port_info:\n"
@@ -293,14 +301,21 @@ class TrafficGenerator(AbstractMeasurer):
                     f"        src_mac: [{src_mac0}]\n"
                     f"      - dest_mac: [{dst_mac1}]\n"
                     f"        src_mac: [{src_mac1}]\n"
+                    f"  platform :\n"
+                    f"      master_thread_id: {master_thread_id}\n"
+                    f"      latency_thread_id: {latency_thread_id}\n"
+                    f"      dual_if:\n"
+                    f"          - socket: {socket}\n"
+                    f"            threads: {threads}\n"
                     f"EOF'",
-                    sudo=True, message=u"TRex config generation error"
+                    sudo=True, message=u"TRex config generation!"
                 )
             elif osi_layer == u"L7":
                 exec_cmd_no_error(
                     self._node,
                     f"sh -c 'cat << EOF > /etc/trex_cfg.yaml\n"
                     f"- version: 2\n"
+                    f"  c: {len(threads)}\n"
                     f"  limit_memory: {Constants.TREX_LIMIT_MEMORY}\n"
                     f"  interfaces: [\"{if1_pci}\",\"{if2_pci}\"]\n"
                     f"  port_info:\n"
@@ -308,70 +323,93 @@ class TrafficGenerator(AbstractMeasurer):
                     f"        default_gw: [{if1_adj_addr}]\n"
                     f"      - ip: [{if2_addr}]\n"
                     f"        default_gw: [{if2_adj_addr}]\n"
+                    f"  platform :\n"
+                    f"      master_thread_id: {master_thread_id}\n"
+                    f"      latency_thread_id: {latency_thread_id}\n"
+                    f"      dual_if:\n"
+                    f"          - socket: {socket}\n"
+                    f"            threads: {threads}\n"
                     f"EOF'",
-                    sudo=True, message=u"TRex config generation error"
+                    sudo=True, message=u"TRex config generation!"
                 )
             else:
-                raise ValueError(u"Unknown Test Type")
+                raise ValueError(u"Unknown Test Type!")
 
-            self._startup_trex(osi_layer)
+            TrafficGenerator.startup_trex(
+                self._node, osi_layer, subtype=subtype
+            )
 
-    def _startup_trex(self, osi_layer):
+    @staticmethod
+    def startup_trex(tg_node, osi_layer, subtype=None):
         """Startup sequence for the TRex traffic generator.
 
+        :param tg_node: Traffic generator node.
         :param osi_layer: 'L2', 'L3' or 'L7' - OSI Layer testing type.
+        :param subtype: Traffic generator sub-type.
+        :type tg_node: dict
         :type osi_layer: str
+        :type subtype: NodeSubTypeTG
         :raises RuntimeError: If node subtype is not a TREX or startup failed.
         """
-        # No need to check subtype, we know it is TREX.
-        for _ in range(0, 3):
-            # Kill TRex only if it is already running.
-            cmd = u"sh -c \"pgrep t-rex && pkill t-rex && sleep 3 || true\""
-            exec_cmd_no_error(
-                self._node, cmd, sudo=True, message=u"Kill TRex failed!"
-            )
-
-            # Configure TRex.
-            ports = ''
-            for port in self._node[u"interfaces"].values():
-                ports += f" {port.get(u'pci_address')}"
-
-            cmd = f"sh -c \"cd {Constants.TREX_INSTALL_DIR}/scripts/ && " \
-                f"./dpdk_nic_bind.py -u {ports} || true\""
-            exec_cmd_no_error(
-                self._node, cmd, sudo=True,
-                message=u"Unbind PCI ports from driver failed!"
-            )
-
-            # Start TRex.
-            cmd = f"sh -c \"cd {Constants.TREX_INSTALL_DIR}/scripts/ && " \
-                f"nohup ./t-rex-64 -i -c {Constants.TREX_CORE_COUNT} --hdrh " \
-                f"{u' --astf' if osi_layer == u'L7' else u''} " \
-                f"--prefix $(hostname) {Constants.TREX_EXTRA_CMDLINE} " \
-                f"> /tmp/trex.log 2>&1 &\" > /dev/null"
-            try:
-                exec_cmd_no_error(self._node, cmd, sudo=True)
-            except RuntimeError:
-                cmd = u"sh -c \"cat /tmp/trex.log\""
+        if not subtype:
+            subtype = check_subtype(tg_node)
+        if subtype == NodeSubTypeTG.TREX:
+            for _ in range(0, 3):
+                # Kill TRex only if it is already running.
+                cmd = u"sh -c \"pgrep t-rex && pkill t-rex && sleep 3 || true\""
                 exec_cmd_no_error(
-                    self._node, cmd, sudo=True, message=u"Get TRex logs failed!"
+                    tg_node, cmd, sudo=True, message=u"Kill TRex failed!"
                 )
-                raise RuntimeError(u"Start TRex failed!")
 
-            # Test if TRex starts successfuly.
-            cmd = f"sh -c \"{Constants.REMOTE_FW_DIR}/resources/tools/trex/" \
-                f"trex_server_info.py\""
-            try:
+                # Configure TRex.
+                ports = ''
+                for port in tg_node[u"interfaces"].values():
+                    ports += f" {port.get(u'pci_address')}"
+
+                cmd = f"sh -c \"cd {Constants.TREX_INSTALL_DIR}/scripts/ && " \
+                    f"./dpdk_nic_bind.py -u {ports} || true\""
                 exec_cmd_no_error(
-                    self._node, cmd, sudo=True, message=u"Test TRex failed!",
-                    retries=20
+                    tg_node, cmd, sudo=True,
+                    message=u"Unbind PCI ports from driver failed!"
                 )
-            except RuntimeError:
-                continue
-            return
-        # After max retries TRex is still not responding to API critical error
-        # occurred.
-        raise RuntimeError(u"Start TRex failed after multiple retries!")
+
+                # Start TRex.
+                cd_cmd = f"cd '{Constants.TREX_INSTALL_DIR}/scripts/'"
+                trex_cmd = OptionString([u"nohup", u"./t-rex-64"])
+                trex_cmd.add(u"-i")
+                trex_cmd.add(u"--prefix $(hostname)")
+                trex_cmd.add(u"--hdrh")
+                trex_cmd.add(u"--no-scapy-server")
+                trex_cmd.add_if(u"--astf", osi_layer == u"L7")
+                # OptionString does not create double space if extra is empty.
+                trex_cmd.add(f"{Constants.TREX_EXTRA_CMDLINE}")
+                inner_command = f"{cd_cmd} && {trex_cmd} > /tmp/trex.log 2>&1 &"
+                cmd = f"sh -c \"{inner_command}\" > /dev/null"
+                try:
+                    exec_cmd_no_error(tg_node, cmd, sudo=True)
+                except RuntimeError:
+                    cmd = u"sh -c \"cat /tmp/trex.log\""
+                    exec_cmd_no_error(
+                        tg_node, cmd, sudo=True,
+                        message=u"Get TRex logs failed!"
+                    )
+                    raise RuntimeError(u"Start TRex failed!")
+
+                # Test if TRex starts successfully.
+                cmd = f"sh -c \"{Constants.REMOTE_FW_DIR}/resources/tools/" \
+                    f"trex/trex_server_info.py\""
+                try:
+                    exec_cmd_no_error(
+                        tg_node, cmd, sudo=True,
+                        message=u"Test TRex failed!", retries=20
+                    )
+                except RuntimeError:
+                    continue
+                return
+            # After max retries TRex is still not responding to API critical
+            # error occurred.
+            exec_cmd(tg_node, u"cat /tmp/trex.log", sudo=True)
+            raise RuntimeError(u"Start TRex failed after multiple retries!")
 
     @staticmethod
     def is_trex_running(node):
@@ -383,9 +421,7 @@ class TrafficGenerator(AbstractMeasurer):
         :rtype: bool
         :raises RuntimeError: If node type is not a TG.
         """
-        # No need to check subtype, we know it is TREX.
-
-        ret, _, _ = exec_cmd(node, u"pidof t-rex", sudo=True)
+        ret, _, _ = exec_cmd(node, u"pgrep t-rex", sudo=True)
         return bool(int(ret) == 0)
 
     @staticmethod
@@ -401,8 +437,11 @@ class TrafficGenerator(AbstractMeasurer):
         subtype = check_subtype(node)
         if subtype == NodeSubTypeTG.TREX:
             exec_cmd_no_error(
-                node, u"sh -c \"sudo pkill t-rex && sleep 3\"",
-                sudo=False, message=u"pkill t-rex failed"
+                node,
+                u"sh -c "
+                u"\"if pgrep t-rex; then sudo pkill t-rex && sleep 3; fi\"",
+                sudo=False,
+                message=u"pkill t-rex failed"
             )
 
     def _parse_traffic_results(self, stdout):
@@ -493,17 +532,19 @@ class TrafficGenerator(AbstractMeasurer):
             warmup_time = float(warmup_time)
         command = f"sh -c \"" \
             f"{Constants.REMOTE_FW_DIR}/resources/tools/trex/" \
-            f"trex_stateless_profile.py" \
-            f" --profile {Constants.REMOTE_FW_DIR}/resources/" \
-            f"traffic_profiles/trex/{traffic_profile}.py" \
-            f" --duration {duration!r} --frame_size {frame_size} " \
+            f"trex_stateless_profile.py " \
+            f"--profile {Constants.REMOTE_FW_DIR}/resources/" \
+            f"traffic_profiles/trex/{traffic_profile}.py " \
+            f"--duration {duration!r} --frame_size {frame_size} " \
             f"--rate {rate!r} --warmup_time {warmup_time!r} " \
-            f"--port_0 {p_0} --port_1 {p_1}" \
-            f" --traffic_directions {traffic_directions}"
+            f"--port_0 {p_0} --port_1 {p_1} " \
+            f"--traffic_directions {traffic_directions}"
         if async_call:
             command += u" --async_start"
         if latency:
             command += u" --latency"
+        if Constants.TREX_SEND_FORCE:
+            command += u" --force"
         command += u"\""
 
         stdout, _ = exec_cmd_no_error(