CSIT-1104: Trending: Speed-up plots generation

[csit.git] / resources / tools / presentation / generator_tables.py

diff --git a/resources/tools/presentation/generator_tables.py b/resources/tools/presentation/generator_tables.py
index 36933cc..5246952 100644 (file)
--- a/resources/tools/presentation/generator_tables.py
+++ b/resources/tools/presentation/generator_tables.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2017 Cisco and/or its affiliates.
+# Copyright (c) 2018 Cisco and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
@@ -21,9 +21,8 @@ import prettytable
 import pandas as pd
 
 from string import replace
-from math import isnan
 from collections import OrderedDict
-from numpy import nan
+from numpy import nan, isnan
 from xml.etree import ElementTree as ET
 
 from errors import PresentationError
@@ -63,6 +62,8 @@ def table_details(table, input_data):
                  format(table.get("title", "")))
 
     # Transform the data
+    logging.info("    Creating the data set for the {0} '{1}'.".
+                 format(table.get("type", ""), table.get("title", "")))
     data = input_data.filter_data(table)
 
     # Prepare the header of the tables
@@ -129,10 +130,14 @@ def table_merged_details(table, input_data):
                  format(table.get("title", "")))
 
     # Transform the data
+    logging.info("    Creating the data set for the {0} '{1}'.".
+                 format(table.get("type", ""), table.get("title", "")))
     data = input_data.filter_data(table)
     data = input_data.merge_data(data)
     data.sort_index(inplace=True)
 
+    logging.info("    Creating the data set for the {0} '{1}'.".
+                 format(table.get("type", ""), table.get("title", "")))
     suites = input_data.filter_data(table, data_set="suites")
     suites = input_data.merge_data(suites)
 
@@ -226,6 +231,8 @@ def table_performance_improvements(table, input_data):
         return None
 
     # Transform the data
+    logging.info("    Creating the data set for the {0} '{1}'.".
+                 format(table.get("type", ""), table.get("title", "")))
     data = input_data.filter_data(table)
 
     # Prepare the header of the tables
@@ -357,6 +364,8 @@ def table_performance_comparison(table, input_data):
                  format(table.get("title", "")))
 
     # Transform the data
+    logging.info("    Creating the data set for the {0} '{1}'.".
+                 format(table.get("type", ""), table.get("title", "")))
     data = input_data.filter_data(table, continue_on_error=True)
 
     # Prepare the header of the tables
@@ -595,6 +604,8 @@ def table_performance_comparison_mrr(table, input_data):
                  format(table.get("title", "")))
 
     # Transform the data
+    logging.info("    Creating the data set for the {0} '{1}'.".
+                 format(table.get("type", ""), table.get("title", "")))
     data = input_data.filter_data(table, continue_on_error=True)
 
     # Prepare the header of the tables
@@ -727,16 +738,18 @@ def table_performance_trending_dashboard(table, input_data):
                  format(table.get("title", "")))
 
     # Transform the data
+    logging.info("    Creating the data set for the {0} '{1}'.".
+                 format(table.get("type", ""), table.get("title", "")))
     data = input_data.filter_data(table, continue_on_error=True)
 
     # Prepare the header of the tables
-    header = ["        Test Case",
+    header = ["Test Case",
               "Trend [Mpps]",
-              "  Short-Term   Change [%]",
-              "  Long-Term   Change [%]",
-              "  Regressions [#]",
-              "  Progressions [#]",
-              "  Outliers [#]"
+              "Short-Term Change [%]",
+              "Long-Term Change [%]",
+              "Regressions [#]",
+              "Progressions [#]",
+              "Outliers [#]"
               ]
     header_str = ",".join(header) + "\n"
 
@@ -752,7 +765,7 @@ def table_performance_trending_dashboard(table, input_data):
                                             "-".join(tst_data["name"].
                                                      split("-")[1:]))
                     tbl_dict[tst_name] = {"name": name,
-                                          "data": dict()}
+                                          "data": OrderedDict()}
                 try:
                     tbl_dict[tst_name]["data"][str(build)] =  \
                         tst_data["result"]["throughput"]
@@ -764,18 +777,16 @@ def table_performance_trending_dashboard(table, input_data):
         if len(tbl_dict[tst_name]["data"]) > 2:
 
             pd_data = pd.Series(tbl_dict[tst_name]["data"])
-            last_key = pd_data.keys()[-1]
-            win_size = min(pd_data.size, table["window"])
-            win_first_idx = pd_data.size - win_size
-            key_14 = pd_data.keys()[win_first_idx]
-            long_win_size = min(pd_data.size, table["long-trend-window"])
-
             data_t, _ = split_outliers(pd_data, outlier_const=1.5,
-                                       window=win_size)
-
+                                       window=table["window"])
+            last_key = data_t.keys()[-1]
+            win_size = min(data_t.size, table["window"])
+            win_first_idx = data_t.size - win_size
+            key_14 = data_t.keys()[win_first_idx]
+            long_win_size = min(data_t.size, table["long-trend-window"])
             median_t = data_t.rolling(window=win_size, min_periods=2).median()
             stdev_t = data_t.rolling(window=win_size, min_periods=2).std()
-            median_first_idx = pd_data.size - long_win_size
+            median_first_idx = median_t.size - long_win_size
             try:
                 max_median = max(
                     [x for x in median_t.values[median_first_idx:-win_size]
@@ -791,15 +802,10 @@ def table_performance_trending_dashboard(table, input_data):
             except KeyError:
                 median_t_14 = nan
 
-            # Test name:
-            name = tbl_dict[tst_name]["name"]
-
             # Classification list:
             classification_lst = list()
-            for build_nr, value in pd_data.iteritems():
-
-                if isnan(data_t[build_nr]) \
-                        or isnan(median_t[build_nr]) \
+            for build_nr, value in data_t.iteritems():
+                if isnan(median_t[build_nr]) \
                         or isnan(stdev_t[build_nr]) \
                         or isnan(value):
                     classification_lst.append("outlier")
@@ -822,11 +828,8 @@ def table_performance_trending_dashboard(table, input_data):
                 rel_change_long = round(
                     ((last_median_t - max_median) / max_median) * 100, 2)
 
-            logging.info("rel_change_last : {}".format(rel_change_last))
-            logging.info("rel_change_long : {}".format(rel_change_long))
-
             tbl_lst.append(
-                [name,
+                [tbl_dict[tst_name]["name"],
                  '-' if isnan(last_median_t) else
                  round(last_median_t / 1000000, 2),
                  '-' if isnan(rel_change_last) else rel_change_last,