CSIT-891: Add data sources for 1801 report

Change-Id: Iaf362be95b882b23fdaf478260e058b0495bbf5d
Signed-off-by: Tibor Frank <tifrank@cisco.com>

diff --git a/resources/tools/presentation/generator_tables.py b/resources/tools/presentation/generator_tables.py
index 6458926..443c11c 100644 (file)
--- a/resources/tools/presentation/generator_tables.py
+++ b/resources/tools/presentation/generator_tables.py
@@ -22,7 +22,7 @@ import prettytable
 from string import replace
 
 from errors import PresentationError
 from string import replace
 
 from errors import PresentationError

-from utils import mean, stdev, relative_change
+from utils import mean, stdev, relative_change, remove_outliers

 
 
 def generate_tables(spec, data):
 
 
 def generate_tables(spec, data):


@@ -401,17 +401,17 @@ def table_performance_comparison(table, input_data):
     for tst_name in tbl_dict.keys():
         item = [tbl_dict[tst_name]["name"], ]
         if tbl_dict[tst_name]["ref-data"]:
     for tst_name in tbl_dict.keys():
         item = [tbl_dict[tst_name]["name"], ]
         if tbl_dict[tst_name]["ref-data"]:

-            item.append(round(mean(tbl_dict[tst_name]["ref-data"]) / 1000000,
-                              2))
-            item.append(round(stdev(tbl_dict[tst_name]["ref-data"]) / 1000000,
-                              2))
+            item.append(round(mean(remove_outliers(
+                tbl_dict[tst_name]["ref-data"], 1.5)) / 1000000, 2))
+            item.append(round(stdev(remove_outliers(
+                tbl_dict[tst_name]["ref-data"], 1.5)) / 1000000, 2))

         else:
             item.extend([None, None])
         if tbl_dict[tst_name]["cmp-data"]:
         else:
             item.extend([None, None])
         if tbl_dict[tst_name]["cmp-data"]:

-            item.append(round(mean(tbl_dict[tst_name]["cmp-data"]) / 1000000,
-                              2))
-            item.append(round(stdev(tbl_dict[tst_name]["cmp-data"]) / 1000000,
-                              2))
+            item.append(round(mean(remove_outliers(
+                tbl_dict[tst_name]["cmp-data"], 1.5)) / 1000000, 2))
+            item.append(round(stdev(remove_outliers(
+                tbl_dict[tst_name]["cmp-data"], 1.5)) / 1000000, 2))

         else:
             item.extend([None, None])
         if item[1] is not None and item[3] is not None:
         else:
             item.extend([None, None])
         if item[1] is not None and item[3] is not None:

diff --git a/resources/tools/presentation/utils.py b/resources/tools/presentation/utils.py
index cf3005c..7037404 100644 (file)
--- a/resources/tools/presentation/utils.py
+++ b/resources/tools/presentation/utils.py
@@ -14,6 +14,8 @@
 """General purpose utilities.
 """
 
 """General purpose utilities.
 """
 

+import numpy as np
+

 from os import walk
 from os.path import join
 from math import sqrt
 from os import walk
 from os.path import join
 from math import sqrt


@@ -60,6 +62,29 @@ def relative_change(nr1, nr2):
     return float(((nr2 - nr1) / nr1) * 100)
 
 
     return float(((nr2 - nr1) / nr1) * 100)
 
 

+def remove_outliers(input_data, outlier_const):
+    """
+
+    :param input_data: Data from which the outliers will be removed.
+    :param outlier_const: Outlier constant.
+    :type input_data: list
+    :type outlier_const: float
+    :returns: The input list without outliers.
+    :rtype: list
+    """
+
+    data = np.array(input_data)
+    upper_quartile = np.percentile(data, 75)
+    lower_quartile = np.percentile(data, 25)
+    iqr = (upper_quartile - lower_quartile) * outlier_const
+    quartile_set = (lower_quartile - iqr, upper_quartile + iqr)
+    result_lst = list()
+    for y in data.tolist():
+        if quartile_set[0] <= y <= quartile_set[1]:
+            result_lst.append(y)
+    return result_lst
+
+

 def get_files(path, extension=None, full_path=True):
     """Generates the list of files to process.
 
 def get_files(path, extension=None, full_path=True):
     """Generates the list of files to process.