Fix Trex start configuration

Since https://gerrit.fd.io/r/c/csit/+/24056
csit-vpp perf tests are failing.

Turns out the reason is too large limit_memory value,
but other improvements applied during investigation
are included as well.

+ Cat trex.log if trex fails to start.
+ Improve trex startup command formatting (remove double spaces).
+ Reorder imports.

Change-Id: I2c1106ea6f88a1a275682e73eba212d08c7947c8
Signed-off-by: Vratko Polak <vrpolak@cisco.com>

diff --git a/resources/libraries/python/Constants.py b/resources/libraries/python/Constants.py
index e6f22bd..d6a94ab 100644 (file)
--- a/resources/libraries/python/Constants.py
+++ b/resources/libraries/python/Constants.py
@@ -184,8 +184,9 @@ class Constants:
     # TRex install directory
     TREX_INSTALL_DIR = u"/opt/trex-core-2.61"
 
     # TRex install directory
     TREX_INSTALL_DIR = u"/opt/trex-core-2.61"
 

-    # TRex limit memory
-    TREX_LIMIT_MEMORY = get_int_from_env(u"TREX_LIMIT_MEMORY ", 8192)
+    # TRex limit memory.
+    # 4096 pages (4 GB) is used just due to the current testbed settings.
+    TREX_LIMIT_MEMORY = get_int_from_env(u"TREX_LIMIT_MEMORY ", 4096)

 
     # TRex number of cores
     TREX_CORE_COUNT = get_int_from_env(u"TREX_CORE_COUNT", 7)
 
     # TRex number of cores
     TREX_CORE_COUNT = get_int_from_env(u"TREX_CORE_COUNT", 7)

diff --git a/resources/libraries/python/TrafficGenerator.py b/resources/libraries/python/TrafficGenerator.py
index a73e471..f0e2a17 100644 (file)
--- a/resources/libraries/python/TrafficGenerator.py
+++ b/resources/libraries/python/TrafficGenerator.py
@@ -18,16 +18,17 @@ import time
 from robot.api import logger
 from robot.libraries.BuiltIn import BuiltIn
 
 from robot.api import logger
 from robot.libraries.BuiltIn import BuiltIn
 

-from .DropRateSearch import DropRateSearch

 from .Constants import Constants
 from .Constants import Constants

-from .ssh import exec_cmd_no_error, exec_cmd
-from .topology import NodeType
-from .topology import NodeSubTypeTG
-from .topology import Topology
+from .DropRateSearch import DropRateSearch

 from .MLRsearch.AbstractMeasurer import AbstractMeasurer
 from .MLRsearch.MultipleLossRatioSearch import MultipleLossRatioSearch
 from .MLRsearch.ReceiveRateMeasurement import ReceiveRateMeasurement
 from .PLRsearch.PLRsearch import PLRsearch
 from .MLRsearch.AbstractMeasurer import AbstractMeasurer
 from .MLRsearch.MultipleLossRatioSearch import MultipleLossRatioSearch
 from .MLRsearch.ReceiveRateMeasurement import ReceiveRateMeasurement
 from .PLRsearch.PLRsearch import PLRsearch

+from .OptionString import OptionString
+from .ssh import exec_cmd_no_error, exec_cmd
+from .topology import NodeType
+from .topology import NodeSubTypeTG
+from .topology import Topology

 
 __all__ = [u"TGDropRateSearchImpl", u"TrafficGenerator", u"OptimizedSearch"]
 
 
 __all__ = [u"TGDropRateSearchImpl", u"TrafficGenerator", u"OptimizedSearch"]
 


@@ -344,11 +345,17 @@ class TrafficGenerator(AbstractMeasurer):
             )
 
             # Start TRex.
             )
 
             # Start TRex.

-            cmd = f"sh -c \"cd {Constants.TREX_INSTALL_DIR}/scripts/ && " \
-                f"nohup ./t-rex-64 -i -c {Constants.TREX_CORE_COUNT} --hdrh " \
-                f"{u' --astf' if osi_layer == u'L7' else u''} " \
-                f"--prefix $(hostname) {Constants.TREX_EXTRA_CMDLINE} " \
-                f"> /tmp/trex.log 2>&1 &\" > /dev/null"
+            cd_cmd = f"cd '{Constants.TREX_INSTALL_DIR}/scripts/'"
+            trex_cmd = OptionString([u"nohup", u"./t-rex-64"])
+            trex_cmd.add(u"-i")
+            trex_cmd.add(f"-c {Constants.TREX_CORE_COUNT}")
+            trex_cmd.add(u"--prefix $(hostname)")
+            trex_cmd.add(u"--hdrh")
+            trex_cmd.add_if(u"--astf", osi_layer == u"L7")
+            # OptionString does not create double space if extra is empty.
+            trex_cmd.add(f"{Constants.TREX_EXTRA_CMDLINE}")
+            inner_command = f"{cd_cmd} && {trex_cmd} > /tmp/trex.log 2>&1 &"
+            cmd = f"sh -c \"{inner_command}\" > /dev/null"

             try:
                 exec_cmd_no_error(self._node, cmd, sudo=True)
             except RuntimeError:
             try:
                 exec_cmd_no_error(self._node, cmd, sudo=True)
             except RuntimeError:


@@ -371,6 +378,7 @@ class TrafficGenerator(AbstractMeasurer):
             return
         # After max retries TRex is still not responding to API critical error
         # occurred.
             return
         # After max retries TRex is still not responding to API critical error
         # occurred.

+        exec_cmd(self._node, u"cat /tmp/trex.log", sudo=True)

         raise RuntimeError(u"Start TRex failed after multiple retries!")
 
     @staticmethod
         raise RuntimeError(u"Start TRex failed after multiple retries!")
 
     @staticmethod