C-Dash: Add detailed views to comparison tables

[csit.git] / csit.infra.dash / app / cdash / report / graphs.py

diff --git a/csit.infra.dash/app/cdash/report/graphs.py b/csit.infra.dash/app/cdash/report/graphs.py
index 44c57d4..0627411 100644 (file)
--- a/csit.infra.dash/app/cdash/report/graphs.py
+++ b/csit.infra.dash/app/cdash/report/graphs.py
@@ -14,11 +14,11 @@
 """Implementation of graphs for iterative data.
 """
 
 """Implementation of graphs for iterative data.
 """
 

-

 import plotly.graph_objects as go
 import pandas as pd
 
 from copy import deepcopy
 import plotly.graph_objects as go
 import pandas as pd
 
 from copy import deepcopy

+from numpy import percentile

 
 from ..utils.constants import Constants as C
 from ..utils.utils import get_color, get_hdrh_latencies
 
 from ..utils.constants import Constants as C
 from ..utils.utils import get_color, get_hdrh_latencies


@@ -74,7 +74,7 @@ def select_iterative_data(data: pd.DataFrame, itm:dict) -> pd.DataFrame:
 
 
 def graph_iterative(data: pd.DataFrame, sel: list, layout: dict,
 
 
 def graph_iterative(data: pd.DataFrame, sel: list, layout: dict,

-        normalize: bool=False) -> tuple:
+        normalize: bool=False, remove_outliers: bool=False) -> tuple:

     """Generate the statistical box graph with iterative data (MRR, NDR and PDR,
     for PDR also Latencies).
 
     """Generate the statistical box graph with iterative data (MRR, NDR and PDR,
     for PDR also Latencies).
 


@@ -83,15 +83,19 @@ def graph_iterative(data: pd.DataFrame, sel: list, layout: dict,
     :param layout: Layout of plot.ly graph.
     :param normalize: If True, the data is normalized to CPU frequency
         Constants.NORM_FREQUENCY.
     :param layout: Layout of plot.ly graph.
     :param normalize: If True, the data is normalized to CPU frequency
         Constants.NORM_FREQUENCY.

-    :param data: pandas.DataFrame
-    :param sel: list
-    :param layout: dict
-    :param normalize: bool
+    :param remove_outliers: If True the outliers are removed before
+        generating the table.
+    :type data: pandas.DataFrame
+    :type sel: list
+    :type layout: dict
+    :type normalize: bool
+    :type remove_outliers: bool

     :returns: Tuple of graphs - throughput and latency.
     :rtype: tuple(plotly.graph_objects.Figure, plotly.graph_objects.Figure)
     """
 
     :returns: Tuple of graphs - throughput and latency.
     :rtype: tuple(plotly.graph_objects.Figure, plotly.graph_objects.Figure)
     """
 

-    def get_y_values(data, y_data_max, param, norm_factor, release=str()):
+    def get_y_values(data, y_data_max, param, norm_factor, release=str(),
+                     remove_outliers=False):

         if param == "result_receive_rate_rate_values":
             if release == "rls2402":
                 y_vals_raw = data["result_receive_rate_rate_avg"].to_list()
         if param == "result_receive_rate_rate_values":
             if release == "rls2402":
                 y_vals_raw = data["result_receive_rate_rate_avg"].to_list()


@@ -100,6 +104,17 @@ def graph_iterative(data: pd.DataFrame, sel: list, layout: dict,
         else:
             y_vals_raw = data[param].to_list()
         y_data = [(y * norm_factor) for y in y_vals_raw]
         else:
             y_vals_raw = data[param].to_list()
         y_data = [(y * norm_factor) for y in y_vals_raw]

+
+        if remove_outliers:
+            try:
+                q1 = percentile(y_data, 25, method=C.COMP_PERCENTILE_METHOD)
+                q3 = percentile(y_data, 75, method=C.COMP_PERCENTILE_METHOD)
+                irq = q3 - q1
+                lif = q1 - C.COMP_OUTLIER_TYPE * irq
+                uif = q3 + C.COMP_OUTLIER_TYPE * irq
+                y_data = [i for i in y_data if i >= lif and i <= uif]
+            except TypeError:
+                pass

         try:
             y_data_max = max(max(y_data), y_data_max)
         except TypeError:
         try:
             y_data_max = max(max(y_data), y_data_max)
         except TypeError:


@@ -142,7 +157,12 @@ def graph_iterative(data: pd.DataFrame, sel: list, layout: dict,
         y_units.update(itm_data[C.UNIT[ttype]].unique().tolist())
 
         y_data, y_tput_max = get_y_values(
         y_units.update(itm_data[C.UNIT[ttype]].unique().tolist())
 
         y_data, y_tput_max = get_y_values(

-            itm_data, y_tput_max, C.VALUE_ITER[ttype], norm_factor, itm["rls"]
+            itm_data,
+            y_tput_max,
+            C.VALUE_ITER[ttype],
+            norm_factor,
+            itm["rls"],
+            remove_outliers

         )
 
         nr_of_samples = len(y_data)
         )
 
         nr_of_samples = len(y_data)


@@ -192,7 +212,8 @@ def graph_iterative(data: pd.DataFrame, sel: list, layout: dict,
                 itm_data,
                 y_band_max,
                 C.VALUE_ITER[f"{ttype}-bandwidth"],
                 itm_data,
                 y_band_max,
                 C.VALUE_ITER[f"{ttype}-bandwidth"],

-                norm_factor
+                norm_factor,
+                remove_outliers=remove_outliers

             )
             if not all(pd.isna(y_band)):
                 y_band_units.update(
             )
             if not all(pd.isna(y_band)):
                 y_band_units.update(


@@ -221,7 +242,8 @@ def graph_iterative(data: pd.DataFrame, sel: list, layout: dict,
                 itm_data,
                 y_lat_max,
                 C.VALUE_ITER["latency"],
                 itm_data,
                 y_lat_max,
                 C.VALUE_ITER["latency"],

-                1 / norm_factor
+                1 / norm_factor,
+                remove_outliers=remove_outliers

             )
             if not all(pd.isna(y_lat)):
                 customdata = list()
             )
             if not all(pd.isna(y_lat)):
                 customdata = list()