C-Dash: add "download row data" feature to comparison tables

Change-Id: Iaf5a57402e954453c237fb1e082da6c620cff59f
Signed-off-by: Tibor Frank <tifrank@cisco.com>

diff --git a/csit.infra.dash/app/cdash/comparisons/layout.py b/csit.infra.dash/app/cdash/comparisons/layout.py
index 82b5b2d..9c89a55 100644 (file)
--- a/csit.infra.dash/app/cdash/comparisons/layout.py
+++ b/csit.infra.dash/app/cdash/comparisons/layout.py
@@ -616,7 +616,7 @@ class Layout:
                             ),
                             dbc.Button(
                                 id="plot-btn-download",
-                                children="Download Data",
+                                children="Download Table",
                                 class_name="me-1",
                                 color="info",
                                 style={
@@ -624,7 +624,18 @@ class Layout:
                                     "padding": "0rem 1rem"
                                 }
                             ),
-                            dcc.Download(id="download-iterative-data")
+                            dcc.Download(id="download-iterative-data"),
+                            dbc.Button(
+                                id="plot-btn-download-raw",
+                                children="Download Raw Data",
+                                class_name="me-1",
+                                color="info",
+                                style={
+                                    "text-transform": "none",
+                                    "padding": "0rem 1rem"
+                                }
+                            ),
+                            dcc.Download(id="download-raw-data")
                         ],
                         className=\
                             "d-grid gap-0 d-md-flex justify-content-md-end"
@@ -1046,6 +1057,38 @@ class Layout:
 
             return dcc.send_data_frame(table.to_csv, C.COMP_DOWNLOAD_FILE_NAME)
 
+        @app.callback(
+            Output("download-raw-data", "data"),
+            State("store-selected", "data"),
+            Input("plot-btn-download-raw", "n_clicks"),
+            prevent_initial_call=True
+        )
+        def _download_raw_comparison_data(selected: dict, _: int) -> dict:
+            """Download the data.
+
+            :param selected: Selected tests.
+            :type selected: dict
+            :returns: dict of data frame content (base64 encoded) and meta data
+                used by the Download component.
+            :rtype: dict
+            """
+
+            if not selected:
+                raise PreventUpdate
+
+            _, table = comparison_table(
+                    data=self._data,
+                    selected=selected,
+                    normalize=False,
+                    remove_outliers=False,
+                    raw_data=True
+                )
+
+            return dcc.send_data_frame(
+                table.dropna(how="all", axis=1).to_csv,
+                f"raw_{C.COMP_DOWNLOAD_FILE_NAME}"
+            )
+
         @app.callback(
             Output("offcanvas-documentation", "is_open"),
             Input("btn-documentation", "n_clicks"),

diff --git a/csit.infra.dash/app/cdash/comparisons/tables.py b/csit.infra.dash/app/cdash/comparisons/tables.py
index 0c247e8..18f9404 100644 (file)
--- a/csit.infra.dash/app/cdash/comparisons/tables.py
+++ b/csit.infra.dash/app/cdash/comparisons/tables.py
@@ -27,7 +27,8 @@ def select_comp_data(
         data: pd.DataFrame,
         selected: dict,
         normalize: bool=False,
-        remove_outliers: bool=False
+        remove_outliers: bool=False,
+        raw_data: bool=False
     ) -> pd.DataFrame:
     """Select data for a comparison table.
 
@@ -38,10 +39,13 @@ def select_comp_data(
         Constants.NORM_FREQUENCY.
     :param remove_outliers: If True the outliers are removed before
         generating the table.
+    :param raw_data: If True, returns data as it is in parquets without any
+        processing. It is used for "download raw data" feature.
     :type data: pandas.DataFrame
     :type selected: dict
     :type normalize: bool
     :type remove_outliers: bool
+    :type raw_data: bool
     :returns: A data frame with selected data.
     :rtype: pandas.DataFrame
     """
@@ -161,13 +165,14 @@ def select_comp_data(
                     norm_factor = C.NORM_FREQUENCY / C.FREQUENCY[itm["tbed"]]
             else:
                 norm_factor = 1.0
-            tmp_df = _calculate_statistics(
-                tmp_df,
-                itm["ttype"].lower(),
-                itm["driver"],
-                norm_factor,
-                remove_outliers=remove_outliers
-            )
+            if not raw_data:
+                tmp_df = _calculate_statistics(
+                    tmp_df,
+                    itm["ttype"].lower(),
+                    itm["driver"],
+                    norm_factor,
+                    remove_outliers=remove_outliers
+                )
 
         lst_df.append(tmp_df)
 
@@ -190,7 +195,8 @@ def comparison_table(
         selected: dict,
         normalize: bool,
         format: str="html",
-        remove_outliers: bool=False
+        remove_outliers: bool=False,
+        raw_data: bool=False
     ) -> tuple:
     """Generate a comparison table.
 
@@ -206,11 +212,14 @@ def comparison_table(
           units.
     :param remove_outliers: If True the outliers are removed before
         generating the table.
+    :param raw_data: If True, returns data as it is in parquets without any
+        processing. It is used for "download raw data" feature.
     :type data: pandas.DataFrame
     :type selected: dict
     :type normalize: bool
     :type format: str
     :type remove_outliers: bool
+    :type raw_data: bool
     :returns: A tuple with the tabe title and the comparison table.
     :rtype: tuple[str, pandas.DataFrame]
     """
@@ -241,9 +250,32 @@ def comparison_table(
                     })
         return selection
 
+    # Select reference data
     r_sel = deepcopy(selected["reference"]["selection"])
-    c_params = selected["compare"]
     r_selection = _create_selection(r_sel)
+    r_data = select_comp_data(
+        data, r_selection, normalize, remove_outliers, raw_data
+    )
+
+    # Select compare data
+    c_sel = deepcopy(selected["reference"]["selection"])
+    c_params = selected["compare"]
+    if c_params["parameter"] in ("core", "frmsize", "ttype"):
+        c_sel[c_params["parameter"]] = [c_params["value"], ]
+    else:
+        c_sel[c_params["parameter"]] = c_params["value"]
+    c_selection = _create_selection(c_sel)
+    c_data = select_comp_data(
+        data, c_selection, normalize, remove_outliers, raw_data
+    )
+
+    if raw_data:
+        r_data["ref/cmp"] = "reference"
+        c_data["ref/cmp"] = "compare"
+        return str(), pd.concat([r_data, c_data], ignore_index=True, copy=False)
+
+    if r_data.empty or c_data.empty:
+        return str(), pd.DataFrame()
 
     if format == "html" and "Latency" not in r_sel["ttype"]:
         unit_factor, s_unit_factor = (1e6, "M")
@@ -266,22 +298,6 @@ def comparison_table(
         r_name = "|".join(r_name)
     c_name = c_params["value"]
 
-    # Select reference data
-    r_data = select_comp_data(data, r_selection, normalize, remove_outliers)
-
-    # Select compare data
-    c_sel = deepcopy(selected["reference"]["selection"])
-    if c_params["parameter"] in ("core", "frmsize", "ttype"):
-        c_sel[c_params["parameter"]] = [c_params["value"], ]
-    else:
-        c_sel[c_params["parameter"]] = c_params["value"]
-
-    c_selection = _create_selection(c_sel)
-    c_data = select_comp_data(data, c_selection, normalize, remove_outliers)
-
-    if r_data.empty or c_data.empty:
-        return str(), pd.DataFrame()
-
     l_name, l_r_mean, l_r_std, l_c_mean, l_c_std, l_rc_mean, l_rc_std, unit = \
         list(), list(), list(), list(), list(), list(), list(), set()
     for _, row in r_data.iterrows():