fix(perf): Ignore negative count trials

Sometimes, TRex wrongly gives negative number of forwarded packets.
As json schema requires non-negative values, this is not acceptable.

The MRR export keyword is changed to raise an exception,
the robot keyword has added logic to explicitly skip without export.

MRR json can still fail validation if all 10 trials are skipped.
With this change, the JSON is exported as failed,
but Robot test is not failed (left TODO to add later if needed).

For ndrpdr and soak tests, ignoring such trials would just repeat them,
so in this case the simpler fix is to add repeats to TrafficGenerator.
If three consecutive trials give negative results, the test fails.

Ticket: csit-3983

Change-Id: I1e8abb2ece5bd457f1e2c0c0634f9d08e2697f12
Signed-off-by: Vratko Polak <[email protected]>

diff --git a/resources/libraries/python/TrafficGenerator.py b/resources/libraries/python/TrafficGenerator.py
index fe9db3c..3e0d854 100644 (file)
--- a/resources/libraries/python/TrafficGenerator.py
+++ b/resources/libraries/python/TrafficGenerator.py
@@ -1339,14 +1339,20 @@ class TrafficGenerator(AbstractMeasurer):
         intended_duration = float(intended_duration)
         time_start = time.monotonic()
         time_stop = time_start + intended_duration
-        if self.resetter:
-            self.resetter()
-        result = self._send_traffic_on_tg_with_ramp_up(
-            duration=intended_duration,
-            rate=intended_load,
-            async_call=False,
-        )
-        logger.debug(f"trial measurement result: {result!r}")
+        for _ in range(3):
+            if self.resetter:
+                self.resetter()
+            result = self._send_traffic_on_tg_with_ramp_up(
+                duration=intended_duration,
+                rate=intended_load,
+                async_call=False,
+            )
+            if result.receive_count >= 0:
+                break
+            logger.debug(f"Retry on negative count: {result.receive_count}")
+        else:
+            raise RuntimeError(f"Too many negative counts in a row!")
+        logger.debug(f"Trial measurement result: {result!r}")
         # In PLRsearch, computation needs the specified time to complete.
         if self.sleep_till_duration:
             while (sleeptime := time_stop - time.monotonic()) > 0.0:

diff --git a/resources/libraries/python/model/ExportJson.py b/resources/libraries/python/model/ExportJson.py
index 3658dc4..a83bf5e 100644 (file)
--- a/resources/libraries/python/model/ExportJson.py
+++ b/resources/libraries/python/model/ExportJson.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2024 Cisco and/or its affiliates.
+# Copyright (c) 2025 Cisco and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
@@ -367,6 +367,10 @@ class ExportJson():
         result_type = result_node["type"]
         if result_type == "unknown":
             # Device or something else not supported.
+            # Also could be MRR with all trials negative, write as a failure.
+            self.data["passed"] = False
+            self.data["message"] = "No result exported."
+            # TODO: Raise real Robot error after writing this JSON.
             return
 
         # Compute avg and stdev for mrr (rate and bandwidth).

diff --git a/resources/libraries/python/model/ExportResult.py b/resources/libraries/python/model/ExportResult.py
index f155848..b751e59 100644 (file)
--- a/resources/libraries/python/model/ExportResult.py
+++ b/resources/libraries/python/model/ExportResult.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2023 Cisco and/or its affiliates.
+# Copyright (c) 2025 Cisco and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
@@ -100,7 +100,7 @@ def append_mrr_value(mrr_value, mrr_unit, bandwidth_value=None,
         bandwidth_unit="bps"):
     """Store mrr value to proper place so it is dumped into json.
 
-    The value is appended only when unit is not empty.
+    Raises an error if the arguments would lead to invalid json.
 
     :param mrr_value: Forwarding rate from MRR trial.
     :param mrr_unit: Unit of measurement for the rate.
@@ -109,19 +109,22 @@ def append_mrr_value(mrr_value, mrr_unit, bandwidth_value=None,
     :type mrr_unit: str
     :type bandwidth_value: Optional[float]
     :type bandwidth_unit: Optional[str]
+    :raises RuntimeError: If mrr_unit is missing or any value is negative.
     """
     if not mrr_unit:
-        return
+        raise RuntimeError(f"Cannot be falsey: {mrr_unit=}")
     data = get_export_data()
     data["result"]["type"] = "mrr"
-
     for node_val, node_unit, node_name in ((mrr_value, mrr_unit, "rate"),
             (bandwidth_value, bandwidth_unit, "bandwidth")):
         if node_val is not None:
+            node_val = float(node_val)
+            if node_val < 0:
+                raise RuntimeError(f"Cannot be negative: {node_val=}")
             node = descend(descend(data["result"], "receive_rate"), node_name)
             node["unit"] = str(node_unit)
             values_list = descend(node, "values", list)
-            values_list.append(float(node_val))
+            values_list.append(node_val)
 
 
 def export_search_bound(text, value, unit, bandwidth=None):

diff --git a/resources/libraries/robot/performance/performance_utils.robot b/resources/libraries/robot/performance/performance_utils.robot
index 899bbb5..12d9488 100644 (file)
--- a/resources/libraries/robot/performance/performance_utils.robot
+++ b/resources/libraries/robot/performance/performance_utils.robot
@@ -510,6 +510,8 @@
 | | | # Out of several quantities for aborted traffic (duration stretching),
 | | | # the approximated receive rate is the best estimate we have.
 | | | ${value} = | Set Variable | ${result.forwarding_count}
+| | | Run Keyword If | ${value} < 0 | Log | Negative count ignored! | WARN
+| | | Continue For Loop If | ${value} < 0
 | | | ${value} = | Evaluate | ${value} / ${result.offered_duration}
 | | | ${bandwidth} | ${pps} = | Compute Bandwidth | ${value} / ${ppta}
 | | | Append Mrr Value | ${value} | ${export_mrr_unit} | ${bandwidth * 1e9}