Add 2048B file size cps rps tests in job specs for http-ldpreload-nginx-1_21_5.

[csit.git] / resources / tools / integrated / compare_perpatch.py

diff --git a/resources/tools/integrated/compare_perpatch.py b/resources/tools/integrated/compare_perpatch.py
index 9b04b7b..59ea7e5 100644 (file)
--- a/resources/tools/integrated/compare_perpatch.py
+++ b/resources/tools/integrated/compare_perpatch.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2023 Cisco and/or its affiliates.
+# Copyright (c) 2024 Cisco and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
@@ -56,7 +56,7 @@ def main() -> int:
         parent_results = {}
         current_results = {}
         parent_results = parse(f"csit_parent/{iteration}", fake_value=2.0)
-        parent_names = set(parent_results.keys())
+        parent_names = list(parent_results)
         if test_names is None:
             test_names = parent_names
         if not parent_names:
@@ -64,7 +64,7 @@ def main() -> int:
             break
         assert parent_names == test_names, f"{parent_names} != {test_names}"
         current_results = parse(f"csit_current/{iteration}", fake_value=1.0)
-        current_names = set(current_results.keys())
+        current_names = list(current_results)
         assert (
             current_names == parent_names
         ), f"{current_names} != {parent_names}"
@@ -77,7 +77,6 @@ def main() -> int:
             current_aggregate[name].extend(current_results[name])
     exit_code = 0
     for name in test_names:
-        print(f"Test name: {name}")
         parent_values = parent_aggregate[name]
         current_values = current_aggregate[name]
         print(f"Time-ordered MRR values for parent build: {parent_values}")
@@ -115,15 +114,14 @@ def main() -> int:
         # TODO: Version of classify that takes max_value and list of stats?
         # That matters if only stats (not list of floats) are given.
         classified_list = jumpavg.classify([parent_values, current_values])
-        if len(classified_list) < 2:
-            print(f"Test {name}: normal (no anomaly)")
-            continue
-        anomaly = classified_list[1].comment
-        if anomaly == "regression":
-            print(f"Test {name}: anomaly regression")
-            exit_code = 3  # 1 or 2 can be caused by other errors
-            continue
-        print(f"Test {name}: anomaly {anomaly}")
+        anomaly_name = "normal (no anomaly)"
+        if len(classified_list) > 1:
+            anomaly = classified_list[1].comment
+            anomaly_name = "anomaly progression"
+            if anomaly == "regression":
+                anomaly_name = "anomaly regression"
+                exit_code = 3  # 1 or 2 can be caused by other errors
+        print(f"Test name {name}: {anomaly_name}")
     print(f"Exit code: {exit_code}")
     return exit_code