Trending: Send alerts when regression occurs

Change-Id: I45d218ea323245199ad5093bd64c6a9bd862cad9
Signed-off-by: Tibor Frank <tifrank@cisco.com>

diff --git a/resources/tools/presentation/generator_CPTA.py b/resources/tools/presentation/generator_CPTA.py
index 9780d35..580b8f0 100644 (file)
--- a/resources/tools/presentation/generator_CPTA.py
+++ b/resources/tools/presentation/generator_CPTA.py
@@ -333,7 +333,7 @@ def _generate_all_charts(spec, input_data):
         job_name = graph["data"].keys()[0]
 
         csv_tbl = list()
-        res = list()
+        res = dict()
 
         # Transform the data
         logs.append(("INFO", "    Creating the data set for the {0} '{1}'.".
@@ -379,22 +379,22 @@ def _generate_all_charts(spec, input_data):
             for group in groups:
                 visible = list()
                 for tag in group:
-                    for test_name, test_data in chart_data.items():
+                    for tst_name, test_data in chart_data.items():
                         if not test_data:
                             logs.append(("WARNING",
                                          "No data for the test '{0}'".
-                                         format(test_name)))
+                                         format(tst_name)))
                             continue
-                        if tag in chart_tags[test_name]:
+                        if tag in chart_tags[tst_name]:
                             message = "index: {index}, test: {test}".format(
-                                index=index, test=test_name)
-                            test_name = test_name.split('.')[-1]
+                                index=index, test=tst_name)
                             try:
                                 trace, rslt = _generate_trending_traces(
                                     test_data,
                                     job_name=job_name,
                                     build_info=build_info,
-                                    name='-'.join(test_name.split('-')[2:-1]),
+                                    name='-'.join(tst_name.split('.')[-1].
+                                                  split('-')[2:-1]),
                                     color=COLORS[index])
                             except IndexError:
                                 message = "Out of colors: {}".format(message)
@@ -404,25 +404,24 @@ def _generate_all_charts(spec, input_data):
                                 continue
                             traces.extend(trace)
                             visible.extend([True for _ in range(len(trace))])
-                            res.append(rslt)
+                            res[tst_name] = rslt
                             index += 1
                             break
                 visibility.append(visible)
         else:
-            for test_name, test_data in chart_data.items():
+            for tst_name, test_data in chart_data.items():
                 if not test_data:
                     logs.append(("WARNING", "No data for the test '{0}'".
-                                 format(test_name)))
+                                 format(tst_name)))
                     continue
                 message = "index: {index}, test: {test}".format(
-                    index=index, test=test_name)
-                test_name = test_name.split('.')[-1]
+                    index=index, test=tst_name)
                 try:
                     trace, rslt = _generate_trending_traces(
                         test_data,
                         job_name=job_name,
                         build_info=build_info,
-                        name='-'.join(test_name.split('-')[2:-1]),
+                        name='-'.join(tst_name.split('.')[-1].split('-')[2:-1]),
                         color=COLORS[index])
                 except IndexError:
                     message = "Out of colors: {}".format(message)
@@ -431,7 +430,7 @@ def _generate_all_charts(spec, input_data):
                     index += 1
                     continue
                 traces.extend(trace)
-                res.append(rslt)
+                res[tst_name] = rslt
                 index += 1
 
         if traces:
@@ -535,7 +534,7 @@ def _generate_all_charts(spec, input_data):
                 testbed
             )
 
-    anomaly_classifications = list()
+    anomaly_classifications = dict()
 
     # Create the header:
     csv_tables = dict()
@@ -554,9 +553,12 @@ def _generate_all_charts(spec, input_data):
     for chart in spec.cpta["plots"]:
         result = _generate_chart(chart)
 
-        anomaly_classifications.extend(result["results"])
         csv_tables[result["job_name"]].extend(result["csv_table"])
 
+        if anomaly_classifications.get(result["job_name"], None) is None:
+            anomaly_classifications[result["job_name"]] = dict()
+        anomaly_classifications[result["job_name"]].update(result["results"])
+
     # Write the tables:
     for job_name, csv_table in csv_tables.items():
         file_name = spec.cpta["output-file"] + "-" + job_name + "-trending"
@@ -590,10 +592,16 @@ def _generate_all_charts(spec, input_data):
     # Evaluate result:
     if anomaly_classifications:
         result = "PASS"
-        for classification in anomaly_classifications:
-            if classification == "regression" or classification == "outlier":
-                result = "FAIL"
-                break
+        for job_name, job_data in anomaly_classifications.iteritems():
+            file_name = "{0}-regressions-{1}.txt".\
+                format(spec.cpta["output-file"], job_name)
+            with open(file_name, 'w') as txt_file:
+                for test_name, classification in job_data.iteritems():
+                    if classification == "regression":
+                        txt_file.write(test_name + '\n')
+                    if classification == "regression" or \
+                            classification == "outlier":
+                        result = "FAIL"
     else:
         result = "FAIL"
 

diff --git a/resources/tools/presentation/generator_alerts.py b/resources/tools/presentation/generator_alerts.py
index ed4318b..2c2b70d 100644 (file)
--- a/resources/tools/presentation/generator_alerts.py
+++ b/resources/tools/presentation/generator_alerts.py
@@ -362,10 +362,23 @@ class Alerting(object):
                     cores=params["cores"] +
                         " " * (max_len_cores - len(params["cores"])))
 
+            # Add list of regressions:
+            file_name = "{0}/cpta-regressions-{1}.txt".\
+                format(config["output-dir"],
+                       alert["urls"][idx].split('/')[-1])
+            try:
+                with open(file_name, 'r') as txt_file:
+                    file_content = txt_file.read()
+                    if file_content:
+                        text += "\nRegressions (full paths):\n\n"
+                        text += file_content
+            except IOError:
+                pass
+
         text += "\nFor detailed information visit: {url}\n".\
             format(url=alert["url-details"])
         file_name = "{0}/{1}".format(config["output-dir"],
-                                                config["output-file"])
+                                     config["output-file"])
         logging.info("Writing the file '{0}.txt' ...".format(file_name))
 
         try:

diff --git a/resources/tools/presentation/specification_CPTA.yaml b/resources/tools/presentation/specification_CPTA.yaml
index c6fc5fd..7e46018 100644 (file)
--- a/resources/tools/presentation/specification_CPTA.yaml
+++ b/resources/tools/presentation/specification_CPTA.yaml
@@ -188,7 +188,7 @@
     # 3n-hsw
     plot-performance-trending-all-3n-hsw:
       csit-vpp-perf-mrr-daily-master:
-        start: 763
+        start: 770
         end: "lastCompletedBuild"
         skip:
           - 781
@@ -236,7 +236,7 @@
 
     plot-performance-trending-vpp-3n-hsw:
       csit-vpp-perf-mrr-daily-master:
-        start: 763
+        start: 770
         end: "lastCompletedBuild"
         skip:
           - 781
@@ -287,7 +287,7 @@
     # 3n-skx
     plot-performance-trending-all-3n-skx:
       csit-vpp-perf-mrr-daily-master-3n-skx:
-        start: 513
+        start: 520
         end: "lastCompletedBuild"
         skip:
           - 531
@@ -340,7 +340,7 @@
 
     plot-performance-trending-vpp-3n-skx:
       csit-vpp-perf-mrr-daily-master-3n-skx:
-        start: 513
+        start: 520
         end: "lastCompletedBuild"
         skip:
           - 531
@@ -396,7 +396,7 @@
     # 2n-skx
     plot-performance-trending-all-2n-skx:
       csit-vpp-perf-mrr-daily-master-2n-skx:
-        start: 514
+        start: 520
         end: "lastCompletedBuild"
         skip:
           - 532
@@ -435,7 +435,7 @@
 
     plot-performance-trending-vpp-2n-skx:
       csit-vpp-perf-mrr-daily-master-2n-skx:
-        start: 514
+        start: 520
         end: "lastCompletedBuild"
         skip:
           - 532
@@ -666,6 +666,8 @@
         traceorder: "normal"  # "grouped" does not work: bug https://github.com/plotly/plotly.js/issues/1913
         tracegroupgap: 20
         bordercolor: "rgb(238, 238, 238)"
+      paper_bgcolor: "#fff"
+      plot_bgcolor: "#fff"
       hoverlabel:
         namelength: -1
 
@@ -694,7 +696,7 @@
 
     # 3n-hsw
     csit-vpp-perf-mrr-daily-master:
-      start: 763
+      start: 770
       end: "lastCompletedBuild"
       skip:
         - 781
@@ -742,7 +744,7 @@
 
     # 3n-skx
     csit-vpp-perf-mrr-daily-master-3n-skx:
-      start: 513
+      start: 520
       end: "lastCompletedBuild"
       skip:
         - 531
@@ -795,7 +797,7 @@
 
     # 2n-skx
     csit-vpp-perf-mrr-daily-master-2n-skx:
-      start: 514
+      start: 520
       end: "lastCompletedBuild"
       skip:
         - 532