FIX: check if t-rex is running at test setup of all perf tests

[csit.git] / resources / libraries / python / TrafficGenerator.py

diff --git a/resources/libraries/python/TrafficGenerator.py b/resources/libraries/python/TrafficGenerator.py
index a73e471..4e3a549 100644 (file)
--- a/resources/libraries/python/TrafficGenerator.py
+++ b/resources/libraries/python/TrafficGenerator.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2019 Cisco and/or its affiliates.
+# Copyright (c) 2020 Cisco and/or its affiliates.

 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:


@@ -18,16 +18,18 @@ import time
 from robot.api import logger
 from robot.libraries.BuiltIn import BuiltIn
 
 from robot.api import logger
 from robot.libraries.BuiltIn import BuiltIn
 

-from .DropRateSearch import DropRateSearch

 from .Constants import Constants
 from .Constants import Constants

-from .ssh import exec_cmd_no_error, exec_cmd
-from .topology import NodeType
-from .topology import NodeSubTypeTG
-from .topology import Topology
+from .CpuUtils import CpuUtils
+from .DropRateSearch import DropRateSearch

 from .MLRsearch.AbstractMeasurer import AbstractMeasurer
 from .MLRsearch.MultipleLossRatioSearch import MultipleLossRatioSearch
 from .MLRsearch.ReceiveRateMeasurement import ReceiveRateMeasurement
 from .PLRsearch.PLRsearch import PLRsearch
 from .MLRsearch.AbstractMeasurer import AbstractMeasurer
 from .MLRsearch.MultipleLossRatioSearch import MultipleLossRatioSearch
 from .MLRsearch.ReceiveRateMeasurement import ReceiveRateMeasurement
 from .PLRsearch.PLRsearch import PLRsearch

+from .OptionString import OptionString
+from .ssh import exec_cmd_no_error, exec_cmd
+from .topology import NodeType
+from .topology import NodeSubTypeTG
+from .topology import Topology

 
 __all__ = [u"TGDropRateSearchImpl", u"TrafficGenerator", u"OptimizedSearch"]
 
 
 __all__ = [u"TGDropRateSearchImpl", u"TrafficGenerator", u"OptimizedSearch"]
 


@@ -277,6 +279,11 @@ class TrafficGenerator(AbstractMeasurer):
                 if1_adj_addr, if2_adj_addr = if2_adj_addr, if1_adj_addr
                 self._ifaces_reordered = True
 
                 if1_adj_addr, if2_adj_addr = if2_adj_addr, if1_adj_addr
                 self._ifaces_reordered = True
 

+            master_thread_id, latency_thread_id, socket, threads = \
+                CpuUtils.get_affinity_trex(
+                    self._node, tg_if1, tg_if2,
+                    tg_dtc=Constants.TREX_CORE_COUNT)
+

             if osi_layer in (u"L2", u"L3"):
                 dst_mac0 = f"0x{if1_adj_addr.replace(u':', u',0x')}"
                 src_mac0 = f"0x{if1_addr.replace(u':', u',0x')}"
             if osi_layer in (u"L2", u"L3"):
                 dst_mac0 = f"0x{if1_adj_addr.replace(u':', u',0x')}"
                 src_mac0 = f"0x{if1_addr.replace(u':', u',0x')}"


@@ -286,6 +293,7 @@ class TrafficGenerator(AbstractMeasurer):
                     self._node,
                     f"sh -c 'cat << EOF > /etc/trex_cfg.yaml\n"
                     f"- version: 2\n"
                     self._node,
                     f"sh -c 'cat << EOF > /etc/trex_cfg.yaml\n"
                     f"- version: 2\n"

+                    f"  c: {len(threads)}\n"

                     f"  limit_memory: {Constants.TREX_LIMIT_MEMORY}\n"
                     f"  interfaces: [\"{if1_pci}\",\"{if2_pci}\"]\n"
                     f"  port_info:\n"
                     f"  limit_memory: {Constants.TREX_LIMIT_MEMORY}\n"
                     f"  interfaces: [\"{if1_pci}\",\"{if2_pci}\"]\n"
                     f"  port_info:\n"


@@ -293,14 +301,21 @@ class TrafficGenerator(AbstractMeasurer):
                     f"        src_mac: [{src_mac0}]\n"
                     f"      - dest_mac: [{dst_mac1}]\n"
                     f"        src_mac: [{src_mac1}]\n"
                     f"        src_mac: [{src_mac0}]\n"
                     f"      - dest_mac: [{dst_mac1}]\n"
                     f"        src_mac: [{src_mac1}]\n"

+                    f"  platform :\n"
+                    f"      master_thread_id: {master_thread_id}\n"
+                    f"      latency_thread_id: {latency_thread_id}\n"
+                    f"      dual_if:\n"
+                    f"          - socket: {socket}\n"
+                    f"            threads: {threads}\n"

                     f"EOF'",
                     f"EOF'",

-                    sudo=True, message=u"TRex config generation error"
+                    sudo=True, message=u"TRex config generation!"

                 )
             elif osi_layer == u"L7":
                 exec_cmd_no_error(
                     self._node,
                     f"sh -c 'cat << EOF > /etc/trex_cfg.yaml\n"
                     f"- version: 2\n"
                 )
             elif osi_layer == u"L7":
                 exec_cmd_no_error(
                     self._node,
                     f"sh -c 'cat << EOF > /etc/trex_cfg.yaml\n"
                     f"- version: 2\n"

+                    f"  c: {len(threads)}\n"

                     f"  limit_memory: {Constants.TREX_LIMIT_MEMORY}\n"
                     f"  interfaces: [\"{if1_pci}\",\"{if2_pci}\"]\n"
                     f"  port_info:\n"
                     f"  limit_memory: {Constants.TREX_LIMIT_MEMORY}\n"
                     f"  interfaces: [\"{if1_pci}\",\"{if2_pci}\"]\n"
                     f"  port_info:\n"


@@ -308,70 +323,93 @@ class TrafficGenerator(AbstractMeasurer):
                     f"        default_gw: [{if1_adj_addr}]\n"
                     f"      - ip: [{if2_addr}]\n"
                     f"        default_gw: [{if2_adj_addr}]\n"
                     f"        default_gw: [{if1_adj_addr}]\n"
                     f"      - ip: [{if2_addr}]\n"
                     f"        default_gw: [{if2_adj_addr}]\n"

+                    f"  platform :\n"
+                    f"      master_thread_id: {master_thread_id}\n"
+                    f"      latency_thread_id: {latency_thread_id}\n"
+                    f"      dual_if:\n"
+                    f"          - socket: {socket}\n"
+                    f"            threads: {threads}\n"

                     f"EOF'",
                     f"EOF'",

-                    sudo=True, message=u"TRex config generation error"
+                    sudo=True, message=u"TRex config generation!"

                 )
             else:
                 )
             else:

-                raise ValueError(u"Unknown Test Type")
+                raise ValueError(u"Unknown Test Type!")

 
 

-            self._startup_trex(osi_layer)
+            TrafficGenerator.startup_trex(
+                self._node, osi_layer, subtype=subtype
+            )

 
 

-    def _startup_trex(self, osi_layer):
+    @staticmethod
+    def startup_trex(tg_node, osi_layer, subtype=None):

         """Startup sequence for the TRex traffic generator.
 
         """Startup sequence for the TRex traffic generator.
 

+        :param tg_node: Traffic generator node.

         :param osi_layer: 'L2', 'L3' or 'L7' - OSI Layer testing type.
         :param osi_layer: 'L2', 'L3' or 'L7' - OSI Layer testing type.

+        :param subtype: Traffic generator sub-type.
+        :type tg_node: dict

         :type osi_layer: str
         :type osi_layer: str

+        :type subtype: NodeSubTypeTG

         :raises RuntimeError: If node subtype is not a TREX or startup failed.
         """
         :raises RuntimeError: If node subtype is not a TREX or startup failed.
         """

-        # No need to check subtype, we know it is TREX.
-        for _ in range(0, 3):
-            # Kill TRex only if it is already running.
-            cmd = u"sh -c \"pgrep t-rex && pkill t-rex && sleep 3 || true\""
-            exec_cmd_no_error(
-                self._node, cmd, sudo=True, message=u"Kill TRex failed!"
-            )
-
-            # Configure TRex.
-            ports = ''
-            for port in self._node[u"interfaces"].values():
-                ports += f" {port.get(u'pci_address')}"
-
-            cmd = f"sh -c \"cd {Constants.TREX_INSTALL_DIR}/scripts/ && " \
-                f"./dpdk_nic_bind.py -u {ports} || true\""
-            exec_cmd_no_error(
-                self._node, cmd, sudo=True,
-                message=u"Unbind PCI ports from driver failed!"
-            )
-
-            # Start TRex.
-            cmd = f"sh -c \"cd {Constants.TREX_INSTALL_DIR}/scripts/ && " \
-                f"nohup ./t-rex-64 -i -c {Constants.TREX_CORE_COUNT} --hdrh " \
-                f"{u' --astf' if osi_layer == u'L7' else u''} " \
-                f"--prefix $(hostname) {Constants.TREX_EXTRA_CMDLINE} " \
-                f"> /tmp/trex.log 2>&1 &\" > /dev/null"
-            try:
-                exec_cmd_no_error(self._node, cmd, sudo=True)
-            except RuntimeError:
-                cmd = u"sh -c \"cat /tmp/trex.log\""
+        if not subtype:
+            subtype = check_subtype(tg_node)
+        if subtype == NodeSubTypeTG.TREX:
+            for _ in range(0, 3):
+                # Kill TRex only if it is already running.
+                cmd = u"sh -c \"pgrep t-rex && pkill t-rex && sleep 3 || true\""

                 exec_cmd_no_error(
                 exec_cmd_no_error(

-                    self._node, cmd, sudo=True, message=u"Get TRex logs failed!"
+                    tg_node, cmd, sudo=True, message=u"Kill TRex failed!"

                 )
                 )

-                raise RuntimeError(u"Start TRex failed!")

 
 

-            # Test if TRex starts successfuly.
-            cmd = f"sh -c \"{Constants.REMOTE_FW_DIR}/resources/tools/trex/" \
-                f"trex_server_info.py\""
-            try:
+                # Configure TRex.
+                ports = ''
+                for port in tg_node[u"interfaces"].values():
+                    ports += f" {port.get(u'pci_address')}"
+
+                cmd = f"sh -c \"cd {Constants.TREX_INSTALL_DIR}/scripts/ && " \
+                    f"./dpdk_nic_bind.py -u {ports} || true\""

                 exec_cmd_no_error(
                 exec_cmd_no_error(

-                    self._node, cmd, sudo=True, message=u"Test TRex failed!",
-                    retries=20
+                    tg_node, cmd, sudo=True,
+                    message=u"Unbind PCI ports from driver failed!"

                 )
                 )

-            except RuntimeError:
-                continue
-            return
-        # After max retries TRex is still not responding to API critical error
-        # occurred.
-        raise RuntimeError(u"Start TRex failed after multiple retries!")
+
+                # Start TRex.
+                cd_cmd = f"cd '{Constants.TREX_INSTALL_DIR}/scripts/'"
+                trex_cmd = OptionString([u"nohup", u"./t-rex-64"])
+                trex_cmd.add(u"-i")
+                trex_cmd.add(u"--prefix $(hostname)")
+                trex_cmd.add(u"--hdrh")
+                trex_cmd.add(u"--no-scapy-server")
+                trex_cmd.add_if(u"--astf", osi_layer == u"L7")
+                # OptionString does not create double space if extra is empty.
+                trex_cmd.add(f"{Constants.TREX_EXTRA_CMDLINE}")
+                inner_command = f"{cd_cmd} && {trex_cmd} > /tmp/trex.log 2>&1 &"
+                cmd = f"sh -c \"{inner_command}\" > /dev/null"
+                try:
+                    exec_cmd_no_error(tg_node, cmd, sudo=True)
+                except RuntimeError:
+                    cmd = u"sh -c \"cat /tmp/trex.log\""
+                    exec_cmd_no_error(
+                        tg_node, cmd, sudo=True,
+                        message=u"Get TRex logs failed!"
+                    )
+                    raise RuntimeError(u"Start TRex failed!")
+
+                # Test if TRex starts successfully.
+                cmd = f"sh -c \"{Constants.REMOTE_FW_DIR}/resources/tools/" \
+                    f"trex/trex_server_info.py\""
+                try:
+                    exec_cmd_no_error(
+                        tg_node, cmd, sudo=True,
+                        message=u"Test TRex failed!", retries=20
+                    )
+                except RuntimeError:
+                    continue
+                return
+            # After max retries TRex is still not responding to API critical
+            # error occurred.
+            exec_cmd(tg_node, u"cat /tmp/trex.log", sudo=True)
+            raise RuntimeError(u"Start TRex failed after multiple retries!")

 
     @staticmethod
     def is_trex_running(node):
 
     @staticmethod
     def is_trex_running(node):


@@ -383,9 +421,7 @@ class TrafficGenerator(AbstractMeasurer):
         :rtype: bool
         :raises RuntimeError: If node type is not a TG.
         """
         :rtype: bool
         :raises RuntimeError: If node type is not a TG.
         """

-        # No need to check subtype, we know it is TREX.
-
-        ret, _, _ = exec_cmd(node, u"pidof t-rex", sudo=True)
+        ret, _, _ = exec_cmd(node, u"pgrep t-rex", sudo=True)

         return bool(int(ret) == 0)
 
     @staticmethod
         return bool(int(ret) == 0)
 
     @staticmethod


@@ -401,8 +437,11 @@ class TrafficGenerator(AbstractMeasurer):
         subtype = check_subtype(node)
         if subtype == NodeSubTypeTG.TREX:
             exec_cmd_no_error(
         subtype = check_subtype(node)
         if subtype == NodeSubTypeTG.TREX:
             exec_cmd_no_error(

-                node, u"sh -c \"sudo pkill t-rex && sleep 3\"",
-                sudo=False, message=u"pkill t-rex failed"
+                node,
+                u"sh -c "
+                u"\"if pgrep t-rex; then sudo pkill t-rex && sleep 3; fi\"",
+                sudo=False,
+                message=u"pkill t-rex failed"

             )
 
     def _parse_traffic_results(self, stdout):
             )
 
     def _parse_traffic_results(self, stdout):


@@ -493,17 +532,19 @@ class TrafficGenerator(AbstractMeasurer):
             warmup_time = float(warmup_time)
         command = f"sh -c \"" \
             f"{Constants.REMOTE_FW_DIR}/resources/tools/trex/" \
             warmup_time = float(warmup_time)
         command = f"sh -c \"" \
             f"{Constants.REMOTE_FW_DIR}/resources/tools/trex/" \

-            f"trex_stateless_profile.py" \
-            f" --profile {Constants.REMOTE_FW_DIR}/resources/" \
-            f"traffic_profiles/trex/{traffic_profile}.py" \
-            f" --duration {duration!r} --frame_size {frame_size} " \
+            f"trex_stateless_profile.py " \
+            f"--profile {Constants.REMOTE_FW_DIR}/resources/" \
+            f"traffic_profiles/trex/{traffic_profile}.py " \
+            f"--duration {duration!r} --frame_size {frame_size} " \

             f"--rate {rate!r} --warmup_time {warmup_time!r} " \
             f"--rate {rate!r} --warmup_time {warmup_time!r} " \

-            f"--port_0 {p_0} --port_1 {p_1}" \
-            f" --traffic_directions {traffic_directions}"
+            f"--port_0 {p_0} --port_1 {p_1} " \
+            f"--traffic_directions {traffic_directions}"

         if async_call:
             command += u" --async_start"
         if latency:
             command += u" --latency"
         if async_call:
             command += u" --async_start"
         if latency:
             command += u" --latency"

+        if Constants.TREX_SEND_FORCE:
+            command += u" --force"

         command += u"\""
 
         stdout, _ = exec_cmd_no_error(
         command += u"\""
 
         stdout, _ = exec_cmd_no_error(