CSIT-1041: Trending dashboard

Change-Id: I2b08c3c1859302437092456da4bb8f1ebe4756bf
Signed-off-by: Tibor Frank <tifrank@cisco.com>

diff --git a/docs/cpta/introduction/index.rst b/docs/cpta/introduction/index.rst
index 31da9ae..516e8b3 100644 (file)
--- a/docs/cpta/introduction/index.rst
+++ b/docs/cpta/introduction/index.rst
@@ -4,27 +4,29 @@ VPP MRR Performance Dashboard
 Description
 -----------
 
 Description
 -----------
 

-Dashboard tables list a summary of per test-case VPP MRR performance trend
-values and detected anomalies (Maximum Receive Rate - received packet rate
-under line rate load). Data comes from trending MRR jobs executed every 12 hrs
-(2:00, 14:00 UTC). Trend and anomaly calculations are done over a rolling
-window of <N> samples, currently with N=14 covering last 7 days. Separate
-tables are generated for tested VPP worker-thread-core combinations (1t1c,
-2t2c, 4t4c).
+Dashboard tables list a summary of per test-case VPP MRR performance trend 
+values and detected anomalies (Maximum Receive Rate - received packet rate 
+under line rate load). Data comes from trending MRR jobs executed every 12 
+hrs (2:00, 14:00 UTC). Trend, trend compliance and anomaly calculations are 
+based on a rolling window of <N> samples, currently N=14 covering last 7 days. 
+Separate tables are generated for tested VPP worker-thread-core combinations 
+(1t1c, 2t2c, 4t4c).

 
 Legend to table:
 
 
 Legend to table:
 

-    - "Test case": name of CSIT test case, naming convention here
-      `CSIT/csit-test-naming <https://wiki.fd.io/view/CSIT/csit-test-naming>`_
-    - "Thput trend [Mpps]": last value of trend over rolling window.
-    - "Anomaly value [Mpps]": in precedence - i) highest outlier if 3
-      consecutive outliers, ii) highest regression if regressions detected,
-      iii) highest progression if progressions detected, iv) nil if normal i.e.
-      within trend.
-    - "Anomaly vs. Trend [%]": anomaly value vs. trend value.
-    - "Classification": outlier, regression, progression, normal - observed
-      over a rolling window.
-    - "# Outliers": number of outliers detected.
+    - "Test Case": name of CSIT test case, naming convention on
+      `CSIT wiki <https://wiki.fd.io/view/CSIT/csit-test-naming>`_.
+    - "Throughput Trend [Mpps]": last value of trend calculated over a
+      rolling window.
+    - "Trend Compliance": calculated based on detected anomalies, listed in
+       precedence order - i) "failure" if 3 consecutive outliers,
+       ii) "regression" if any regressions, iii) "progression" if any
+       progressions, iv) "normal" if data compliant with trend.
+    - "Anomaly Value [Mpps]": i) highest outlier if "failure", ii) highest
+      regression if "regression", iii) highest progression if "progression",
+      iv) "-" if normal i.e. within trend.
+    - "Change [%]": "Anomaly Value" vs. "Throughput Trend", "-" if normal.
+    - "# Outliers": number of outliers detected within a rolling window.

 
 Tables are listed in sections 1.x. Followed by daily trending graphs in
 sections 2.x. Daily trending data used to generate the graphs is listed in
 
 Tables are listed in sections 1.x. Followed by daily trending graphs in
 sections 2.x. Daily trending data used to generate the graphs is listed in


@@ -44,4 +46,3 @@ VPP worker on 4t4c
 ------------------
 
 .. include:: ../../../_build/_static/vpp/performance-trending-dashboard-4t4c.rst
 ------------------
 
 .. include:: ../../../_build/_static/vpp/performance-trending-dashboard-4t4c.rst

-

diff --git a/resources/tools/presentation/generator_CPTA.py b/resources/tools/presentation/generator_CPTA.py
index 9ec196c..72aef53 100644 (file)
--- a/resources/tools/presentation/generator_CPTA.py
+++ b/resources/tools/presentation/generator_CPTA.py
@@ -164,26 +164,26 @@ def _evaluate_results(in_data, trimmed_data, window=10):
 
     if len(in_data) > 2:
         win_size = in_data.size if in_data.size < window else window
 
     if len(in_data) > 2:
         win_size = in_data.size if in_data.size < window else window

-        results = [0.0, ]
+        results = [0.66, ]

         median = in_data.rolling(window=win_size, min_periods=2).median()
         stdev_t = trimmed_data.rolling(window=win_size, min_periods=2).std()
         median = in_data.rolling(window=win_size, min_periods=2).median()
         stdev_t = trimmed_data.rolling(window=win_size, min_periods=2).std()

-        m_vals = median.values
-        s_vals = stdev_t.values
-        d_vals = in_data.values
-        t_vals = trimmed_data.values
-        for day in range(1, in_data.size):
-            if np.isnan(t_vals[day]) \
-                    or np.isnan(m_vals[day]) \
-                    or np.isnan(s_vals[day]) \
-                    or np.isnan(d_vals[day]):
+
+        first = True
+        for build_nr, value in in_data.iteritems():
+            if first:
+                first = False
+                continue
+            if np.isnan(trimmed_data[build_nr]) \
+                    or np.isnan(median[build_nr]) \
+                    or np.isnan(stdev_t[build_nr]) \
+                    or np.isnan(value):

                 results.append(0.0)
                 results.append(0.0)

-            elif d_vals[day] < (m_vals[day] - 3 * s_vals[day]):
+            elif value < (median[build_nr] - 3 * stdev_t[build_nr]):

                 results.append(0.33)
                 results.append(0.33)

-            elif (m_vals[day] - 3 * s_vals[day]) <= d_vals[day] <= \
-                    (m_vals[day] + 3 * s_vals[day]):
-                results.append(0.66)
-            else:
+            elif value > (median[build_nr] + 3 * stdev_t[build_nr]):

                 results.append(1.0)
                 results.append(1.0)

+            else:
+                results.append(0.66)

     else:
         results = [0.0, ]
         try:
     else:
         results = [0.0, ]
         try:


@@ -236,30 +236,23 @@ def _generate_trending_traces(in_data, build_info, period, moving_win_size=10,
         in_data = _select_data(in_data, period,
                                fill_missing=fill_missing,
                                use_first=use_first)
         in_data = _select_data(in_data, period,
                                fill_missing=fill_missing,
                                use_first=use_first)

-    # try:
-    #     data_x = ["{0}/{1}".format(key, build_info[str(key)][1].split("~")[-1])
-    #               for key in in_data.keys()]
-    # except KeyError:
-    #     data_x = [key for key in in_data.keys()]
-    hover_text = ["vpp-build: {0}".format(x[1].split("~")[-1])
-                  for x in build_info.values()]
-    data_x = [key for key in in_data.keys()]

 
 

+    data_x = [key for key in in_data.keys()]

     data_y = [val for val in in_data.values()]
     data_y = [val for val in in_data.values()]

+
+    hover_text = list()
+    for idx in data_x:
+        hover_text.append("vpp-build: {0}".
+                          format(build_info[str(idx)][1].split("~")[-1]))
+

     data_pd = pd.Series(data_y, index=data_x)
 
     t_data, outliers = find_outliers(data_pd, outlier_const=1.5)
     data_pd = pd.Series(data_y, index=data_x)
 
     t_data, outliers = find_outliers(data_pd, outlier_const=1.5)

-

     results = _evaluate_results(data_pd, t_data, window=moving_win_size)
 
     anomalies = pd.Series()
     anomalies_res = list()
     for idx, item in enumerate(in_data.items()):
     results = _evaluate_results(data_pd, t_data, window=moving_win_size)
 
     anomalies = pd.Series()
     anomalies_res = list()
     for idx, item in enumerate(in_data.items()):

-        # item_pd = pd.Series([item[1], ],
-        #                     index=["{0}/{1}".
-        #                     format(item[0],
-        #                            build_info[str(item[0])][1].split("~")[-1]),
-        #                            ])

         item_pd = pd.Series([item[1], ], index=[item[0], ])
         if item[0] in outliers.keys():
             anomalies = anomalies.append(item_pd)
         item_pd = pd.Series([item[1], ], index=[item[0], ])
         if item[0] in outliers.keys():
             anomalies = anomalies.append(item_pd)

diff --git a/resources/tools/presentation/generator_tables.py b/resources/tools/presentation/generator_tables.py
index 29e1006..29e29d0 100644 (file)
--- a/resources/tools/presentation/generator_tables.py
+++ b/resources/tools/presentation/generator_tables.py
@@ -355,7 +355,7 @@ def table_performance_comparison(table, input_data):
                  format(table.get("title", "")))
 
     # Transform the data
                  format(table.get("title", "")))
 
     # Transform the data

-    data = input_data.filter_data(table)
+    data = input_data.filter_data(table, continue_on_error=True)

 
     # Prepare the header of the tables
     try:
 
     # Prepare the header of the tables
     try:


@@ -544,7 +544,7 @@ def table_performance_comparison_mrr(table, input_data):
                  format(table.get("title", "")))
 
     # Transform the data
                  format(table.get("title", "")))
 
     # Transform the data

-    data = input_data.filter_data(table)
+    data = input_data.filter_data(table, continue_on_error=True)

 
     # Prepare the header of the tables
     try:
 
     # Prepare the header of the tables
     try:


@@ -668,14 +668,16 @@ def table_performance_trending_dashboard(table, input_data):
                  format(table.get("title", "")))
 
     # Transform the data
                  format(table.get("title", "")))
 
     # Transform the data

-    data = input_data.filter_data(table)
+    data = input_data.filter_data(table, continue_on_error=True)

 
     # Prepare the header of the tables
     header = ["Test case",
 
     # Prepare the header of the tables
     header = ["Test case",

-              "Thput trend [Mpps]",
-              "Anomaly [Mpps]",
+              "Throughput Trend [Mpps]",
+              "Trend Compliance",
+              "Anomaly Value [Mpps]",

               "Change [%]",
               "Change [%]",

-              "Classification"]
+              "#Outliers"
+              ]

     header_str = ",".join(header) + "\n"
 
     # Prepare data to the table:
     header_str = ",".join(header) + "\n"
 
     # Prepare data to the table:


@@ -688,55 +690,62 @@ def table_performance_trending_dashboard(table, input_data):
                                             "-".join(tst_data["name"].
                                                      split("-")[1:]))
                     tbl_dict[tst_name] = {"name": name,
                                             "-".join(tst_data["name"].
                                                      split("-")[1:]))
                     tbl_dict[tst_name] = {"name": name,

-                                          "data": list()}
+                                          "data": dict()}

                 try:
                 try:

-                    tbl_dict[tst_name]["data"]. \
-                        append(tst_data["result"]["throughput"])
+                    tbl_dict[tst_name]["data"][str(build)] =  \
+                        tst_data["result"]["throughput"]

                 except (TypeError, KeyError):
                     pass  # No data in output.xml for this test
 
     tbl_lst = list()
     for tst_name in tbl_dict.keys():
         if len(tbl_dict[tst_name]["data"]) > 2:
                 except (TypeError, KeyError):
                     pass  # No data in output.xml for this test
 
     tbl_lst = list()
     for tst_name in tbl_dict.keys():
         if len(tbl_dict[tst_name]["data"]) > 2:

-            sample_lst = tbl_dict[tst_name]["data"]
-            pd_data = pd.Series(sample_lst)
+
+            pd_data = pd.Series(tbl_dict[tst_name]["data"])

             win_size = pd_data.size \
                 if pd_data.size < table["window"] else table["window"]
             # Test name:
             name = tbl_dict[tst_name]["name"]
 
             win_size = pd_data.size \
                 if pd_data.size < table["window"] else table["window"]
             # Test name:
             name = tbl_dict[tst_name]["name"]
 

-            # Trend list:
-            trend_lst = list(pd_data.rolling(window=win_size, min_periods=2).
-                             median())
-            # Stdevs list:
-            t_data, _ = find_outliers(pd_data)
-            t_data_lst = list(t_data)
-            stdev_lst = list(t_data.rolling(window=win_size, min_periods=2).
-                             std())
+            median = pd_data.rolling(window=win_size, min_periods=2).median()
+            trimmed_data, _ = find_outliers(pd_data, outlier_const=1.5)
+            stdev_t = pd_data.rolling(window=win_size, min_periods=2).std()

 
             rel_change_lst = [None, ]
             classification_lst = [None, ]
 
             rel_change_lst = [None, ]
             classification_lst = [None, ]

-            for idx in range(1, len(trend_lst)):
+            median_lst = [None, ]
+            sample_lst = [None, ]
+            first = True
+            for build_nr, value in pd_data.iteritems():
+                if first:
+                    first = False
+                    continue

                 # Relative changes list:
                 # Relative changes list:

-                if not isnan(sample_lst[idx]) \
-                        and not isnan(trend_lst[idx])\
-                        and trend_lst[idx] != 0:
+                if not isnan(value) \
+                        and not isnan(median[build_nr]) \
+                        and median[build_nr] != 0:

                     rel_change_lst.append(
                     rel_change_lst.append(

-                        int(relative_change(float(trend_lst[idx]),
-                                            float(sample_lst[idx]))))
+                        int(relative_change(float(median[build_nr]),
+                                            float(value))))

                 else:
                     rel_change_lst.append(None)
                 else:
                     rel_change_lst.append(None)

+

                 # Classification list:
                 # Classification list:

-                if isnan(t_data_lst[idx]) or isnan(stdev_lst[idx]):
+                if isnan(trimmed_data[build_nr]) \
+                        or isnan(median[build_nr]) \
+                        or isnan(stdev_t[build_nr]) \
+                        or isnan(value):

                     classification_lst.append("outlier")
                     classification_lst.append("outlier")

-                elif sample_lst[idx] < (trend_lst[idx] - 3*stdev_lst[idx]):
+                elif value < (median[build_nr] - 3 * stdev_t[build_nr]):

                     classification_lst.append("regression")
                     classification_lst.append("regression")

-                elif sample_lst[idx] > (trend_lst[idx] + 3*stdev_lst[idx]):
+                elif value > (median[build_nr] + 3 * stdev_t[build_nr]):

                     classification_lst.append("progression")
                 else:
                     classification_lst.append("normal")
                     classification_lst.append("progression")
                 else:
                     classification_lst.append("normal")

+                sample_lst.append(value)
+                median_lst.append(median[build_nr])

 
 

-            last_idx = len(sample_lst) - 1
+            last_idx = len(classification_lst) - 1

             first_idx = last_idx - int(table["evaluated-window"])
             if first_idx < 0:
                 first_idx = 0
             first_idx = last_idx - int(table["evaluated-window"])
             if first_idx < 0:
                 first_idx = 0


@@ -752,28 +761,46 @@ def table_performance_trending_dashboard(table, input_data):
             else:
                 classification = None
 
             else:
                 classification = None
 

+            nr_outliers = 0
+            consecutive_outliers = 0
+            failure = False
+            for item in classification_lst[first_idx:]:
+                if item == "outlier":
+                    nr_outliers += 1
+                    consecutive_outliers += 1
+                    if consecutive_outliers == 3:
+                        failure = True
+                else:
+                    consecutive_outliers = 0
+

             idx = len(classification_lst) - 1
             while idx:
                 if classification_lst[idx] == classification:
                     break
                 idx -= 1
 
             idx = len(classification_lst) - 1
             while idx:
                 if classification_lst[idx] == classification:
                     break
                 idx -= 1
 

-            trend = round(float(trend_lst[-2]) / 1000000, 2) \
-                if not isnan(trend_lst[-2]) else ''
+            if failure:
+                classification = "failure"
+            elif classification == "outlier":
+                classification = "normal"
+
+            trend = round(float(median_lst[-1]) / 1000000, 2) \
+                if not isnan(median_lst[-1]) else ''

             sample = round(float(sample_lst[idx]) / 1000000, 2) \
                 if not isnan(sample_lst[idx]) else ''
             rel_change = rel_change_lst[idx] \
                 if rel_change_lst[idx] is not None else ''
             tbl_lst.append([name,
                             trend,
             sample = round(float(sample_lst[idx]) / 1000000, 2) \
                 if not isnan(sample_lst[idx]) else ''
             rel_change = rel_change_lst[idx] \
                 if rel_change_lst[idx] is not None else ''
             tbl_lst.append([name,
                             trend,

-                            sample,
-                            rel_change,
-                            classification])
+                            classification,
+                            '-' if classification == "normal" else sample,
+                            '-' if classification == "normal" else rel_change,
+                            nr_outliers])

 
     # Sort the table according to the classification
     tbl_sorted = list()
 
     # Sort the table according to the classification
     tbl_sorted = list()

-    for classification in ("regression", "progression", "outlier", "normal"):
-        tbl_tmp = [item for item in tbl_lst if item[4] == classification]
+    for classification in ("failure", "regression", "progression", "normal"):
+        tbl_tmp = [item for item in tbl_lst if item[2] == classification]

         tbl_tmp.sort(key=lambda rel: rel[0])
         tbl_sorted.extend(tbl_tmp)
 
         tbl_tmp.sort(key=lambda rel: rel[0])
         tbl_sorted.extend(tbl_tmp)
 


@@ -832,7 +859,7 @@ def table_performance_trending_dashboard_html(table, input_data):
     # Table header:
     tr = ET.SubElement(dashboard, "tr", attrib=dict(bgcolor="#6699ff"))
     for idx, item in enumerate(csv_lst[0]):
     # Table header:
     tr = ET.SubElement(dashboard, "tr", attrib=dict(bgcolor="#6699ff"))
     for idx, item in enumerate(csv_lst[0]):

-        alignment = "left" if idx == 0 else "right"
+        alignment = "left" if idx == 0 else "center"

         th = ET.SubElement(tr, "th", attrib=dict(align=alignment))
         th.text = item
 
         th = ET.SubElement(tr, "th", attrib=dict(align=alignment))
         th.text = item
 


@@ -845,10 +872,10 @@ def table_performance_trending_dashboard_html(table, input_data):
         for c_idx, item in enumerate(row):
             alignment = "left" if c_idx == 0 else "center"
             td = ET.SubElement(tr, "td", attrib=dict(align=alignment))
         for c_idx, item in enumerate(row):
             alignment = "left" if c_idx == 0 else "center"
             td = ET.SubElement(tr, "td", attrib=dict(align=alignment))

-            if c_idx == 4:
+            if c_idx == 2:

                 if item == "regression":
                     td.set("bgcolor", "#eca1a6")
                 if item == "regression":
                     td.set("bgcolor", "#eca1a6")

-                elif item == "outlier":
+                elif item == "failure":

                     td.set("bgcolor", "#d6cbd3")
                 elif item == "progression":
                     td.set("bgcolor", "#bdcebe")
                     td.set("bgcolor", "#d6cbd3")
                 elif item == "progression":
                     td.set("bgcolor", "#bdcebe")